From 3f5ac6caae580f9c73e3ff02e80031579b8fb939 Mon Sep 17 00:00:00 2001 From: "migrations-devops[bot]" Date: Sun, 30 Aug 2026 01:21:17 +0000 Subject: [PATCH] Sync plugin v2.42.0 from migrations-ai-skills --- plugin/.cortex-plugin/plugin.json | 25 +- plugin/VERSION | 2 +- plugin/agents/business_logic.md | 51 + .../agents/data_driven.md | 40 +- .../agents/edge_cases.md | 34 +- plugin/agents/general-task.md | 543 +++++++ plugin/agents/task-invalidate.md | 100 ++ plugin/agents/test_case_verifier.md | 159 ++ plugin/commands/dash.md | 46 - plugin/hooks/dev-build-reminder.txt | 3 + plugin/hooks/install-dependencies.ps1 | 51 +- plugin/hooks/install-dependencies.sh | 44 +- plugin/hooks/session-context.ps1 | 24 +- plugin/hooks/session-context.sh | 18 +- plugin/hooks/tests/session-context.test.ps1 | 65 + plugin/hooks/tests/session-context.test.sh | 54 + plugin/mcp-server/README.md | 23 +- .../migration/ENTRY.app.md} | 75 +- plugin/skills/migration/SKILL.md | 15 +- plugin/skills/migration/assessment/SKILL.md | 49 +- .../assessment/effort-estimate/SKILL.md | 37 + .../assessment/scripts/Base_estimates.csv | 19 - .../scripts/Base_estimates.redshift.csv | 22 - .../assessment/scripts/effort_estimation.py | 1289 +---------------- .../scripts/generate_multi_report.py | 87 +- .../snowconvert_reports/ARCHITECTURE.md | 2 +- .../loaders/project_config.py | 5 - .../snowconvert_reports/type_coverage.py | 103 +- .../migration/code-conversion-only/SKILL.md | 52 +- .../connection/db2-connection/SKILL.md | 120 ++ .../db2-connection/references/REFERENCE.md | 172 +++ .../connection/snowflake-connection/SKILL.md | 63 + .../migration/data-infrastructure/SKILL.md | 57 +- .../advanced-operations-reference.md | 182 +++ .../metadata-storage-mode-reference.md | 64 + .../references/worker-config-reference.md | 33 + .../data-infrastructure/teardown/SKILL.md | 215 +-- .../data-infrastructure/worker-spcs/SKILL.md | 2 + .../skills/migration/extensibility/TASKS.md | 102 +- .../skills/migration/migrate-objects/SKILL.md | 20 +- .../actions/data-migration/RUN.md | 14 +- .../actions/data-migration/SKILL.md | 152 +- .../references/task-model-reference.md | 44 +- .../references/troubleshooting-reference.md | 73 + .../references/workflow-config-reference.md | 49 +- .../actions/etl-stabilization/SKILL.md | 49 +- .../etl-stabilization/dbt-fixer/SKILL.md | 9 +- .../etl-stabilization/dbt-test-gen/SKILL.md | 2 +- .../orchestration-fixer/SKILL.md | 43 + .../orchestration-test-gen/SKILL.md | 18 +- .../agent-prompts/dbt-context-mapper.md | 1 + .../etl-stabilization/reference/examples.md | 4 +- .../reference/protocols/phase-execution.md | 1 + .../reference/templates/ROADMAP_TEMPLATE.md | 22 +- .../reference/templates/batch-artifacts.md | 17 +- .../reference/templates/fix-log-format.md | 24 +- .../etl-stabilization/reference/tools.md | 2 +- .../reference/troubleshooting.md | 1 + .../scripts/check_sync_leaks.py | 109 ++ .../scripts/path_resolver.py | 27 +- .../etl-stabilization/scripts/scan_unit.py | 17 +- .../scripts/tests/__init__.py | 0 .../fixtures/session_status_sanitized.json | 23 + .../scripts/tests/helpers.py | 127 ++ .../scripts/tests/test_check_sync_leaks.py | 77 + .../scripts/tests/test_scan_unit.py | 75 + .../scripts/tests/test_track_status.py | 606 ++++++++ .../etl-stabilization/scripts/track_status.py | 685 ++++++--- .../migrate-objects/actions/finish_objects.md | 9 +- .../migrate-objects/autonomous/SKILL.md | 504 +++++++ .../baseline-capture/CAPTURE.md | 10 +- .../migrate-objects/baseline-capture/SWARM.md | 41 +- .../baseline-capture/agents/business_logic.md | 56 - .../testbed-generator/SKILL.md | 42 +- .../prompts/critics/joins_critic.md | 32 + .../prompts/critics/spec_critic.md | 28 + .../prompts/critics/verdict-contract.md | 59 + .../testbed-generator/scripts/assembler.py | 11 +- .../scripts/critic_backbone.py | 414 ++++++ .../scripts/critic_benchmark.py | 95 ++ .../testbed-generator/scripts/critic_gate.py | 214 +++ .../testbed-generator/scripts/critic_index.py | 281 ++++ .../scripts/critic_normalize.py | 119 ++ .../testbed-generator/scripts/manifest.py | 86 +- .../testbed-generator/scripts/run_pipeline.py | 203 ++- .../testbed-generator/scripts/subcommands.py | 35 +- .../migrate-etl/etl-validate/SKILL.md | 23 +- .../migrate-objects/migrate-object/DEPLOY.md | 17 +- .../migrate-object/DIAGNOSE_FIX.md | 416 ++---- .../migrate-object/RUN_TESTS.md | 44 +- .../migrate-objects/migrate-object/SKILL.md | 7 +- .../migrate-object/VALIDATE_VIEW.md | 34 +- .../migrate-objects/migrate-object/VERIFY.md | 16 +- .../references/BTEQ_TEST_YAML.md | 2 +- .../references/collaboration-model.md | 3 +- .../migrate-objects/references/overrides.md | 15 +- .../migrate-objects/references/sql-queries.md | 39 +- .../references/testing-framework-perms.md | 2 +- .../references/troubleshooting.md | 11 +- .../rule-engine/apply/BATCH.md | 2 +- .../extract-code-units/SKILL.md | 35 +- plugin/skills/migration/sas/INTEGRATION.md | 62 + plugin/skills/migration/sas/README.md | 55 + plugin/skills/migration/sas/SKILL.md | 36 + .../sas/assess-sas-migration/SKILL.md | 284 ++++ .../references/complexity-analysis.md | 82 ++ .../references/sizing-model.md | 72 + .../references/threshold-calibration.md | 102 ++ .../references/wave-planning.md | 100 ++ .../templates/assessment-report.md | 286 ++++ .../templates/effort-staffing-report.md | 94 ++ .../sas/assess-sas-migration/tool/README.md | 74 + .../assess-sas-migration/tool/assess_sas.py | 155 ++ .../sas/assess-sas-migration/tool/config.json | 10 + .../tool/requirements.txt | 2 + .../tool/sas_analyzer/__init__.py | 14 + .../assets/snowconvert_ai_logo.svg | 15 + .../sas_analyzer/assets/snowflake_logo.svg | 5 + .../tool/sas_analyzer/classifier.py | 122 ++ .../tool/sas_analyzer/constants.py | 98 ++ .../tool/sas_analyzer/dependency.py | 200 +++ .../tool/sas_analyzer/html_report.py | 419 ++++++ .../tool/sas_analyzer/parser.py | 387 +++++ .../tool/sas_analyzer/reporter.py | 261 ++++ .../tool/sas_analyzer/scorer.py | 177 +++ .../sas/convert-sas-to-snowflake/SKILL.md | 760 ++++++++++ .../references/auto-validation.md | 321 ++++ .../references/batch-mode.md | 168 +++ .../references/checkpoint-logging.md | 119 ++ .../references/classification-logic.md | 133 ++ .../references/common-patterns.md | 497 +++++++ .../references/comparison-rules.md | 180 +++ .../references/consolidation-patterns.md | 433 ++++++ .../references/conversion-rules.md | 225 +++ .../references/data-steps.md | 505 +++++++ .../references/e2e-orchestration-test.md | 188 +++ .../references/function-mappings.md | 442 ++++++ .../references/large-file-rules.md | 121 ++ .../references/macros.md | 572 ++++++++ .../references/mermaid-diagrams.md | 279 ++++ .../references/multi-block-orchestration.md | 197 +++ .../references/proc-sql.md | 305 ++++ .../references/proc-steps.md | 723 +++++++++ .../references/pyspark-fallback.md | 216 +++ .../references/python-datascience.md | 340 +++++ .../references/schema-inference.md | 168 +++ .../references/snowflake-compile.md | 213 +++ .../references/snowflake-execution.md | 281 ++++ .../references/state-tracker-schema.md | 189 +++ .../references/synthetic-data-rules.md | 219 +++ .../references/validation-execution.md | 279 ++++ .../references/vendor-function-mappings.md | 85 ++ .../references/vendor-passthrough.md | 54 + .../templates/consolidation-report.md | 270 ++++ .../templates/conversion-report.md | 713 +++++++++ .../templates/output-mixed.md | 332 +++++ .../templates/output-sql.md | 179 +++ .../validate-sas-conversion/SKILL.md | 552 +++++++ .../workflows/steps-7-validation.md | 164 +++ .../workflows/steps-8-10-post-conversion.md | 411 ++++++ .../workflows/validation-pipeline.md | 353 +++++ .../migrate-sas7bdat-to-snowflake/SKILL.md | 226 +++ .../assets/control_table.sql | 31 + .../assets/directory_table_setup.sql | 23 + .../assets/loader_sproc.sql | 209 +++ .../assets/task_setup.sql | 68 + .../references/architecture.md | 139 ++ .../references/testing.md | 105 ++ .../sas/references/block-tiering-spec.md | 131 ++ plugin/skills/migration/setup/SKILL.md | 2 + .../setup/configure-snowflake-target.md | 5 + .../setup/configure-source-connection.md | 1 + .../migration/setup/data-strategy/SKILL.md | 20 + .../migration/setup/data-validation/SKILL.md | 3 +- .../references/workflow-config-reference.md | 60 +- .../migration/setup/discover-extras/SKILL.md | 293 ++++ .../discover-extras/agents/investigate.md | 129 ++ .../discover-extras/cookbook-template.md | 50 + .../migration/validate-objects/SKILL.md | 16 +- .../migration/validate-objects/actions/RUN.md | 22 +- .../references/validation-levels-reference.md | 2 +- .../actions/validate_tables.md | 126 +- 182 files changed, 22687 insertions(+), 2515 deletions(-) create mode 100644 plugin/agents/business_logic.md rename plugin/{skills/migration/migrate-objects/baseline-capture => }/agents/data_driven.md (55%) rename plugin/{skills/migration/migrate-objects/baseline-capture => }/agents/edge_cases.md (51%) create mode 100644 plugin/agents/general-task.md create mode 100644 plugin/agents/task-invalidate.md create mode 100644 plugin/agents/test_case_verifier.md delete mode 100644 plugin/commands/dash.md create mode 100644 plugin/hooks/dev-build-reminder.txt create mode 100644 plugin/hooks/tests/session-context.test.ps1 create mode 100644 plugin/hooks/tests/session-context.test.sh rename plugin/{commands/migrate.md => skills/migration/ENTRY.app.md} (51%) create mode 100644 plugin/skills/migration/assessment/effort-estimate/SKILL.md delete mode 100644 plugin/skills/migration/assessment/scripts/Base_estimates.csv delete mode 100644 plugin/skills/migration/assessment/scripts/Base_estimates.redshift.csv create mode 100644 plugin/skills/migration/connection/db2-connection/SKILL.md create mode 100644 plugin/skills/migration/connection/db2-connection/references/REFERENCE.md create mode 100644 plugin/skills/migration/connection/snowflake-connection/SKILL.md create mode 100644 plugin/skills/migration/data-infrastructure/references/advanced-operations-reference.md create mode 100644 plugin/skills/migration/data-infrastructure/references/metadata-storage-mode-reference.md create mode 100644 plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/check_sync_leaks.py create mode 100644 plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/__init__.py create mode 100644 plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/fixtures/session_status_sanitized.json create mode 100644 plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/helpers.py create mode 100644 plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/test_check_sync_leaks.py create mode 100644 plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/test_scan_unit.py create mode 100644 plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/test_track_status.py create mode 100644 plugin/skills/migration/migrate-objects/autonomous/SKILL.md delete mode 100644 plugin/skills/migration/migrate-objects/baseline-capture/agents/business_logic.md create mode 100644 plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/prompts/critics/joins_critic.md create mode 100644 plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/prompts/critics/spec_critic.md create mode 100644 plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/prompts/critics/verdict-contract.md create mode 100644 plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_backbone.py create mode 100644 plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_benchmark.py create mode 100644 plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_gate.py create mode 100644 plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_index.py create mode 100644 plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_normalize.py create mode 100644 plugin/skills/migration/sas/INTEGRATION.md create mode 100644 plugin/skills/migration/sas/README.md create mode 100644 plugin/skills/migration/sas/SKILL.md create mode 100644 plugin/skills/migration/sas/assess-sas-migration/SKILL.md create mode 100644 plugin/skills/migration/sas/assess-sas-migration/references/complexity-analysis.md create mode 100644 plugin/skills/migration/sas/assess-sas-migration/references/sizing-model.md create mode 100644 plugin/skills/migration/sas/assess-sas-migration/references/threshold-calibration.md create mode 100644 plugin/skills/migration/sas/assess-sas-migration/references/wave-planning.md create mode 100644 plugin/skills/migration/sas/assess-sas-migration/templates/assessment-report.md create mode 100644 plugin/skills/migration/sas/assess-sas-migration/templates/effort-staffing-report.md create mode 100644 plugin/skills/migration/sas/assess-sas-migration/tool/README.md create mode 100644 plugin/skills/migration/sas/assess-sas-migration/tool/assess_sas.py create mode 100644 plugin/skills/migration/sas/assess-sas-migration/tool/config.json create mode 100644 plugin/skills/migration/sas/assess-sas-migration/tool/requirements.txt create mode 100644 plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/__init__.py create mode 100644 plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/assets/snowconvert_ai_logo.svg create mode 100644 plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/assets/snowflake_logo.svg create mode 100644 plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/classifier.py create mode 100644 plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/constants.py create mode 100644 plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/dependency.py create mode 100644 plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/html_report.py create mode 100644 plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/parser.py create mode 100644 plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/reporter.py create mode 100644 plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/scorer.py create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/SKILL.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/auto-validation.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/batch-mode.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/checkpoint-logging.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/classification-logic.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/common-patterns.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/comparison-rules.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/consolidation-patterns.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/conversion-rules.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/data-steps.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/e2e-orchestration-test.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/function-mappings.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/large-file-rules.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/macros.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/mermaid-diagrams.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/multi-block-orchestration.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/proc-sql.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/proc-steps.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/pyspark-fallback.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/python-datascience.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/schema-inference.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/snowflake-compile.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/snowflake-execution.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/state-tracker-schema.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/synthetic-data-rules.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/validation-execution.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/vendor-function-mappings.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/references/vendor-passthrough.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/templates/consolidation-report.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/templates/conversion-report.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/templates/output-mixed.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/templates/output-sql.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/validate-sas-conversion/SKILL.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/workflows/steps-7-validation.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/workflows/steps-8-10-post-conversion.md create mode 100644 plugin/skills/migration/sas/convert-sas-to-snowflake/workflows/validation-pipeline.md create mode 100644 plugin/skills/migration/sas/migrate-sas7bdat-to-snowflake/SKILL.md create mode 100644 plugin/skills/migration/sas/migrate-sas7bdat-to-snowflake/assets/control_table.sql create mode 100644 plugin/skills/migration/sas/migrate-sas7bdat-to-snowflake/assets/directory_table_setup.sql create mode 100644 plugin/skills/migration/sas/migrate-sas7bdat-to-snowflake/assets/loader_sproc.sql create mode 100644 plugin/skills/migration/sas/migrate-sas7bdat-to-snowflake/assets/task_setup.sql create mode 100644 plugin/skills/migration/sas/migrate-sas7bdat-to-snowflake/references/architecture.md create mode 100644 plugin/skills/migration/sas/migrate-sas7bdat-to-snowflake/references/testing.md create mode 100644 plugin/skills/migration/sas/references/block-tiering-spec.md create mode 100644 plugin/skills/migration/setup/discover-extras/SKILL.md create mode 100644 plugin/skills/migration/setup/discover-extras/agents/investigate.md create mode 100644 plugin/skills/migration/setup/discover-extras/cookbook-template.md diff --git a/plugin/.cortex-plugin/plugin.json b/plugin/.cortex-plugin/plugin.json index 0988afb..3b3824f 100644 --- a/plugin/.cortex-plugin/plugin.json +++ b/plugin/.cortex-plugin/plugin.json @@ -1,15 +1,25 @@ { "name": "snowflake-migration", - "version": "2.41.0", - "description": "End-to-end database migration to Snowflake — assessment, conversion, validation, and deployment", - "author": { "name": "db-eng-migrations" }, - "skills": ["skills/migration"], + "version": "2.42.0", + "description": "End-to-end database migration to Snowflake \u2014 assessment, conversion, validation, and deployment", + "author": { + "name": "db-eng-migrations" + }, + "skills": [ + "skills/migration" + ], "mcpServers": { "mcp": { "type": "stdio", "command": "scai", - "args": ["mcp", "run"], - "env": { } + "args": [ + "mcp", + "run" + ], + "env": { + "CUSTOM_SNOWCONVERT_DATABASE": "${CUSTOM_SNOWCONVERT_DATABASE:-}", + "CUSTOM_SNOWFLAKE_DATABASE_FOR_METADATA": "${CUSTOM_SNOWFLAKE_DATABASE_FOR_METADATA:-}" + } } }, "hooks": { @@ -35,5 +45,6 @@ "matcher": "*" } ] - } + }, + "buildChannel": "preview" } diff --git a/plugin/VERSION b/plugin/VERSION index 2d4c52e..345a83e 100644 --- a/plugin/VERSION +++ b/plugin/VERSION @@ -1 +1 @@ -2.41.0 +2.42.0 diff --git a/plugin/agents/business_logic.md b/plugin/agents/business_logic.md new file mode 100644 index 0000000..237ad0d --- /dev/null +++ b/plugin/agents/business_logic.md @@ -0,0 +1,51 @@ +--- +name: business_logic +description: Produce test_cases rows for an existing step-based YAML stub that cover IF/ELSE branches, CASE WHEN arms, happy paths, and error paths. Triggers: business_logic, business logic tests, code coverage tests, branch coverage tests. +license: Proprietary. See License-Skills for complete terms +--- + +You produce **`test_cases:` rows** for the object in your prompt that +exercise every code path in the source SQL. + +> You are NOT writing a YAML file. The stub YAML already exists (created by `scai test seed`). Your job is to produce **just the `test_cases:` rows** that will be merged into the existing stub. +> +> See [`../skills/migration/migrate-objects/references/step-based-yaml.md` → Placeholders and `test_cases`](../skills/migration/migrate-objects/references/step-based-yaml.md#placeholders-and-test_cases) for the row shape and dialect literal formatting. + +## Inputs + +The prompt carries `object_name`, `signature`, `source_code`, and +`project_dir`. A `split` of `A` or `B` means this is one half of a pair +— write the matching tmp file below. + +## Instructions + +Analyze the source code and produce rows that: + +- Exercise each `IF` / `ELSEIF` / `ELSE` branch. +- Cover each `CASE WHEN` arm. +- Hit the happy path with typical values. +- Trigger early-return conditions. +- Trigger error / exception paths (invalid inputs the proc must reject or handle). + +**Do not query the source database.** Generate rows purely from code analysis. For parameter values that depend on data (e.g. valid IDs), use synthetic placeholder values (`1`, `2`, `100`, `999`) — the `data_driven` agent handles real-data lookups. + +When `split` is `A`, focus on happy paths and main branches. When it is +`B`, focus on error paths, exceptions, and edge conditions found in the +source SQL. + +## Output + +Write your rows to `/.scai/tmp/_business_logic.yml`, +or `_business_logic_a.yml` / `_business_logic_b.yml` when `split` is set. + +The file must contain only valid YAML starting with `test_cases:`. Also print the rows to stdout as a backup. + +```yaml +test_cases: + - [1, 100.00] # happy path - main IF branch + - [1, 1500.00] # high-value branch - CASE WHEN amount > 1000 + - [-1, 10.00] # error path - negative ID + - [null, 10.00] # NULL guard - COALESCE branch +``` + +Each row is a JSON-ish array of literals matching the proc's parameter order. Add a trailing `# ...` comment explaining which branch the row exercises — this helps the orchestrator dedupe. diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/agents/data_driven.md b/plugin/agents/data_driven.md similarity index 55% rename from plugin/skills/migration/migrate-objects/baseline-capture/agents/data_driven.md rename to plugin/agents/data_driven.md index 063cda5..bb57c41 100644 --- a/plugin/skills/migration/migrate-objects/baseline-capture/agents/data_driven.md +++ b/plugin/agents/data_driven.md @@ -1,26 +1,22 @@ --- -name: data-driven-test-agent -description: Produces `test_cases:` rows for an existing step-based YAML stub by querying realistic parameter values from the source database. The most important swarm agent — real data produces the highest-confidence baselines. Triggers: data-driven tests, real data test cases, query source for test values. -parent_skill: baseline-capture +name: data_driven +description: Produce test_cases rows for an existing step-based YAML stub from realistic source-database values. Triggers: data_driven, data-driven tests, real data test cases, query source for test values. +license: Proprietary. See License-Skills for complete terms --- -# Agent: Data-Driven Test Cases - -You produce **`test_cases:` rows** for ``, using realistic values queried from the source database. - -This is the most important swarm agent — real data produces the highest-confidence baselines. +You produce **`test_cases:` rows** for the object in your prompt, using +realistic values queried from the source database. > You are NOT writing a YAML file. The stub YAML already exists (created by `scai test seed`). Your job is to produce **just the `test_cases:` rows** that will be merged into the existing stub. > -> See [`../../references/step-based-yaml.md` → Placeholders and `test_cases`](../../references/step-based-yaml.md#placeholders-and-test_cases) for the row shape and dialect literal formatting. +> See [`../skills/migration/migrate-objects/references/step-based-yaml.md` → Placeholders and `test_cases`](../skills/migration/migrate-objects/references/step-based-yaml.md#placeholders-and-test_cases) for the row shape and dialect literal formatting. ## Inputs -- **Object signature**: `` -- **Source code**: `` -- **Referenced tables**: `` -- **Source connection name**: `` -- **Project directory**: `` +The prompt carries `object_name`, `signature`, `source_code`, +`referenced_tables`, `source_connection`, and `project_dir`. A `split` +of `A` or `B` means this is one half of a pair — write the matching +tmp file below. ## Instructions @@ -29,6 +25,10 @@ This is the most important swarm agent — real data produces the highest-confid 3. Build positional arrays matching the proc's parameter order. Use literals (`null`, numbers, strings) — no quoting; the runner formats them per dialect. 4. Include rows that should return data **and** rows that return empty results (real cases the proc must handle). +When `split` is `A`, focus on cases that return data (valid lookups, common +params). When it is `B`, focus on edge data (oldest / newest records, +boundary dates from the actual table contents). + ### Testbed Fallback (No Live Source Connection) When `query_source` is unavailable (e.g. Teradata migrations without a live connection): @@ -39,7 +39,8 @@ When `query_source` is unavailable (e.g. Teradata migrations without a live conn ## Output -Write your rows to: `/.scai/tmp/_data_driven.yml` +Write your rows to `/.scai/tmp/_data_driven.yml`, +or `_data_driven_a.yml` / `_data_driven_b.yml` when `split` is set. The file must contain only valid YAML starting with `test_cases:`. Also print the rows to stdout as a backup. @@ -51,12 +52,3 @@ test_cases: ``` Each row is a JSON-ish array of literals: `null` for SQL NULL, unquoted numbers, strings in double-quotes if they contain colons / special chars (otherwise unquoted is fine in YAML). - -## When the orchestrator splits data-driven into A/B - -For complex objects, two data-driven agents may be spawned: - -- **Agent 1A** — focus on cases that return data (valid lookups, common params). -- **Agent 1B** — focus on edge data (oldest / newest records, boundary dates from the actual table contents). - -Each writes its own tmp file (`_data_driven_a.yml` vs `_b.yml`); the orchestrator merges them. diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/agents/edge_cases.md b/plugin/agents/edge_cases.md similarity index 51% rename from plugin/skills/migration/migrate-objects/baseline-capture/agents/edge_cases.md rename to plugin/agents/edge_cases.md index 93462f3..de3b89b 100644 --- a/plugin/skills/migration/migrate-objects/baseline-capture/agents/edge_cases.md +++ b/plugin/agents/edge_cases.md @@ -1,22 +1,21 @@ --- -name: edge-cases-test-agent -description: Produces `test_cases:` rows for an existing step-based YAML stub focusing on edge cases and boundary values — NULLs, zeros, empty strings, type limits, overflow, precision boundaries. No source DB access required. Triggers: edge case tests, boundary value tests, null handling tests. -parent_skill: baseline-capture +name: edge_cases +description: Produce test_cases rows for an existing step-based YAML stub covering NULLs, zeros, empty strings, type limits, overflow, and precision boundaries. Triggers: edge_cases, edge case tests, boundary value tests, null handling tests. +license: Proprietary. See License-Skills for complete terms --- -# Agent: Edge Cases & Boundaries - -You produce **`test_cases:` rows** for `` focusing on edge cases and boundary values. +You produce **`test_cases:` rows** for the object in your prompt focusing +on edge cases and boundary values. > You are NOT writing a YAML file. The stub YAML already exists (created by `scai test seed`). Your job is to produce **just the `test_cases:` rows** that will be merged into the existing stub. > -> See [`../../references/step-based-yaml.md` → Placeholders and `test_cases`](../../references/step-based-yaml.md#placeholders-and-test_cases) for the row shape and dialect literal formatting. +> See [`../skills/migration/migrate-objects/references/step-based-yaml.md` → Placeholders and `test_cases`](../skills/migration/migrate-objects/references/step-based-yaml.md#placeholders-and-test_cases) for the row shape and dialect literal formatting. ## Inputs -- **Object signature**: `` -- **Source code**: `` -- **Project directory**: `` +The prompt carries `object_name`, `signature`, `source_code`, and +`project_dir`. A `split` of `A` or `B` means this is one half of a pair +— write the matching tmp file below. ## Instructions @@ -36,9 +35,13 @@ Produce rows covering: - Date boundaries: min/max SQL dates, year/month boundaries. - Values likely to trigger overflow or truncation in either dialect. +When `split` is `A`, focus on NULL handling and zero / empty values. +When it is `B`, focus on type limits, overflow, and precision boundaries. + ## Output -Write your rows to: `/.scai/tmp/_edge_cases.yml` +Write your rows to `/.scai/tmp/_edge_cases.yml`, +or `_edge_cases_a.yml` / `_edge_cases_b.yml` when `split` is set. The file must contain only valid YAML starting with `test_cases:`. Also print the rows to stdout as a backup. @@ -52,12 +55,3 @@ test_cases: ``` Each row is a JSON-ish array of literals matching the proc's parameter order. - -## When the orchestrator splits edge cases into A/B - -For complex objects, two edge-case agents may be spawned: - -- **Agent 2A** — focus on NULL handling and zero / empty values. -- **Agent 2B** — focus on type limits, overflow, and precision boundaries. - -Each writes its own tmp file (`_edge_cases_a.yml` vs `_b.yml`); the orchestrator merges them. diff --git a/plugin/agents/general-task.md b/plugin/agents/general-task.md new file mode 100644 index 0000000..a7308f0 --- /dev/null +++ b/plugin/agents/general-task.md @@ -0,0 +1,543 @@ +--- +name: general-task +description: Walks ONE object end-to-end through the migration machine — resolves its next task, runs it, stamps the outcome, and repeats until the object is done or needs a human. Dispatched by the autonomous migration loop with a single object id. +license: Proprietary. See License-Skills for complete terms +--- + +You migrate **one object**, through as many tasks as it takes. The first prompt +carries `objectId`, `agentId`, `projectDir`, `pluginDir`, +`snowflakeConnection`, and `snowflakeDatabase`. Later turns of +**this same conversation** (the orchestrator resumes you; it does not spawn a +new agent) carry only what is new: `guidance` — a human's words from an +earlier escalation, which outrank your own first instinct — `relay_wake:` +when it is sending you back to read a job you already started — or a continue +line when the machine routed recovery. Pick up from §2; do not treat a later +turn as a new object. + +`agentId` is your identity for the run: **pass it on every call that writes.** One MCP +server serves you and every other agent on this wave, so it is the only thing that +tells your writes from theirs. Claiming records it on your object, and every later +write of yours has to carry the same id — a write aimed at an object you do not hold is +refused, and so is an id the server did not issue. It is four hex digits, e.g. `a3f2`; +never invent one, never substitute `objectId`, and never use `0000` (the +orchestrator's). + +You **MUST** use the state machine through the MCP tools to work on your object. If you're stuck, do not invent things or +go off the rail. More below on escalations. + +Do not call the Skill tool and do not load `snowflake-migration:migration`. +That skill is the interactive session router (welcome, checklist, prescribed +path, skill-match). You already have this definition. Task skills are files: +Read `/skills/migration/`. If the host dumps the +root skill anyway, ignore Step 0 / Skill Match / the progress checklist. + +## 1. Attach and claim + +``` +configure(project_dir="", agent_id="") +transition_status(status="begin", where="id = ''", agent_id="") +``` + +`configure` is first, always: it names you so a live job is announced to this +conversation. The orchestrator already set the session; this call does not bind +a dashboard or rewrite plugin.yml. Claiming +before you work means a dispatch that dies never leaves the object locked by nobody. + +Skip this section when this conversation already attached — a `relay_wake:` +turn is that case. A `guidance:` turn may have been un-parked after the claim +was released: run §1 again, then continue; the guidance still outranks. + +Your `agentId` belongs on that first call too. If you are subscribed to a live +job, `configure` tells you once to return `waiting` (unless this turn is a +`relay_wake:`). It changes nothing else about the call. + +## 2. Loop + +``` +migration_status(mode="next_task", object_ids=[""], agent_id="") +``` + +That call is your board: one id, one next task. `my_objects_board`, +`my_objects_summary`, `my_objects_details`, and `next_objects` are the +parent's wave views — they name other objects and will not advance yours. + +The response is keyed by object id. Read `response[""]` and take the +first case that matches: + +| Case | Do | +|---|---| +| `error` | The id is unknown or the registry read failed. Return `stuck`. | +| `nextTask: null`, not `errored`, not `blocked` | Terminal — go to **Finish**. | +| `wait` | The machine parked you on another object's `migrateData` / `validateData`. Return `waiting`. The orchestrator resumes this conversation with `relay_wake:` when that job succeeds. Do not call `job_status(wake=true)` on their job — `next_task` already registered the wait. | +| `blocked: true` without `wait`, or `nextTaskBlocked` without `wait` | Report every `blockedOn` / `nextTaskBlockedOn` entry with its `reason`, return `partial`. Blocks are the orchestrator's call and will not clear by looping. A `reason` you can disprove is still not yours to fix: put what you found in `notes` and leave it. The blocking object is outside your dispatch: it has its own walk, and the block clears when that walk finishes, not when you act on it. Two writers on one object race on the same files and registry entries — never `deploy` it, never run SQL that changes it, never edit its files, never stamp a task on its behalf. When this arrives on an advance response, call `next_task` with your `agent_id` before you return: that is what registers a job-backed `wait` if there is one. | +| `prompt` | The machine wants an answer you have no user to give. **Escalate**, with the question as your `asks`. | +| `needsHuman: true` | Already parked on someone else's escalation. Leave it, return `stuck`, pass its `asks` through unchanged — a second row for one question is just noise in the queue. | +| otherwise | Run it, stamp it, loop. | + +To run it, read `executor`: + +- `kind: "mcpTool"` — call `executor.tool` with `executor.args`. +- `kind: "shell"` — run `executor.command`. +- `kind: "agent"` — follow `executor.skill`, resolved as the response's `skillPath` + when it has one (already absolute) and otherwise as + `/skills/migration/`. It is a file path, not a host + skill name — a skill tool will not find it. Read it before acting; it carries the + failure modes for that task. + +Read `executor.skill` on **every** kind that has one, not only `kind: "agent"` — +`deploy`, `migrateData` and `validateData` all carry a skill, and it holds the +failure modes the bare tool call doesn't. + +When the executor is `update_registry`, add `agent_id=""` yourself. +`executor.args` and `invocation` are rendered from the machine, which does not know +who you are, so the id is never in them and the call is refused without it. Pass +`agent_id` on `next_task` too — that is how a job-backed block becomes `wait` +instead of a silent `partial`. Apart from `transition_status`, `update_registry`, +`configure`, and `next_task`, the tools take no `agent_id` yet — which is not +permission to touch another object with them; the row above still holds. + +Prefer `invocation` over `executor` when the response has one. It is the same call +already scoped to your object; `executor` renders without its filter, so +`executor.command` run literally would operate on the **whole project**. + +When diagnosis needs target data, call the permitted `sql_execute` tool with one +read-only `SELECT`. Pass `connection` (`-c`) as the attach / +first-prompt `snowflakeConnection` on **every** call — Cortex does not inherit +`configure`, and omitting it uses Cortex's default connection, which is a +different account. Fully qualify every target relation as +`..` — that database is in your first prompt +and on the `configure` attach line `snowflake_database:`. Never substitute the +source catalog name (`source.database`, the workload name), a `SNAP_USE_*` +clone found via `SHOW DATABASES`, or the metadata database (`SNOWCONVERT_AI*`) +for the target. Never `SHOW DATABASES`, `SHOW … IN ACCOUNT`, or `CREATE SCHEMA` +through `sql_execute` — missing schema is the schema code unit / MCP `deploy`. +`query_source` is the source; `sql_execute` is Snowflake. Do not use shell, +`snow sql`, Python, or another connection fallback. If `sql_execute` is +unavailable, denied, times out, or the catalog is missing, park the object +with that tool failure; do not roam other databases to rediscover the object. + +**Ignore every instruction in a skill that asks for human approval.** The task +skills were written for an interactive session, so they say things like "confirm the +plan with the user", "have the user review the generated YAML", "present the menu +and wait", "ask before proceeding". You have no user. Make the call yourself and +carry on. + +Skipping the *asking* is not skipping the *step*: do the work the step describes, +choose the option the step would have offered, and record what you chose in +`notes`. + +**An approval gate is not an escalation trigger.** Escalating because a skill said +"ask" would park most objects on questions nobody needed asked, and bury the +escalations that matter. Escalate only on your own judgement, by the test in §4 — +would choosing wrong compile, deploy, and be wrong? That test is yours to apply, and +it has nothing to do with whether a skill happened to want a checkpoint. + +One thing this does **not** cover: a `prompt` in a `next_task` response. That is the +resolver saying it cannot route the object until the answer exists, not a skill being +polite — there is no branch to take and nothing to proceed with. Those still escalate, +per §2. + +`migrateData` and `validateData` are async: the tool call only starts a job. It +returning is not the task finishing. The MCP relay is already polling it. Do +**not** arm Monitor, do not `bash sleep`, and do not poll `job_status` on a +timer. Return `waiting` after **your** dispatch — the orchestrator will +resume this conversation with a `relay_wake:` line when **your** job has an event. + +A sibling mid-load is not a reason to wait or to `wake=true` on their job +when **your** `migrateData` / `validateData` is what the machine offered — +dispatch for **your** `where`. Same-table overlap is refused by +`TASK_ATTEMPTS` (`adopted` or an error), not by sitting on their wake list. +The exception is when **your** next task is blocked on their +`migrateData` / `validateData`: the machine registers the wait on +`next_task`; you return `waiting`; you do not call `job_status(wake=true)` +yourself. + +When this turn's prompt has `relay_wake:`: + +If you were parked (`wait` on the last `next_task`, or you returned +`waiting` without dispatching your own job), call +`migration_status(mode="next_task", object_ids=[""], agent_id="")`. +Do not treat a successful blocker as "dispatch `migrateData` for your +`where`". If still blocked, the machine re-registers; return `waiting` +again. If unblocked, run the pending task. + +If you had dispatched your own job, call +`job_status(job_id, details=true)` on **this object's** job — the `job_id` +from your `migrate_data` / `validate_data` dispatch, or the job whose +`params.object_ids` contains your object. Do not pick the latest +`data_migration` in the project. + +Then call `migration_status(mode="next_task", object_ids=[""], agent_id="")` +for your object. The machine reads the cloud oracle (`TASK_ATTEMPTS` + +`TABLE_PROGRESS`). Do not `transition_status(advance)` for `migrateData` / +`validateData`. + +If the finished job is a sibling's — your object id is not in +`params.object_ids` — and your own migrate/validate is still pending +(you were not parked on that sibling), ignore its verdict and dispatch +`migrate_data` / `validate_data` for **your** `where`. + +The verdict on **your** job is `job_status`'s own `failed` / `status` / +`summary`: `failed: true` is a failed load, whatever the per-table rows +underneath it say. A 1–3s sleep after killing a process is fine; sleeping to +"give the workflow time" is not. + +Classify the terminal evidence before choosing the failure path. A stable source / +target row mismatch is an observed data result, not an infrastructure outage. +Invalid identifiers or compilation errors in generated validation SQL, and a tool +invocation that fails identically with the same inputs, are deterministic tool +failures. Do not rerun any of them unchanged. Use the machine's SQL or dependency +recovery route when one exists; otherwise **escalate** with the exact mismatch or +error and concrete repair options. A retry becomes valid only after code, +configuration, data, or the failing service condition has changed. + +When a result is too large and the runtime spills it to a file, the verdict was in +the small payload you already read. A file full of passing rows is not a pass — +those rows can be a previous run's. + +Then stamp the outcome. The machine advances on its own when it can observe the +work — a registry field written, the object present in Snowflake, the expected +file on disk. Stamp yourself when it can't: a judged result, or any failure. A +stamp is a claim about reality and it is attributed — every `transition_status` +call is recorded against your agent id in `ORCHESTRATION.TASK_EVENTS`, and the run +report names the outcomes the machine could not observe for itself. + +When a task's status comes from a file, it advances when the file exists: produce +the artifact its skill describes. A stamp is refused there, and `bypass` will not +move it either — `bypass` waives a task's unmet *preconditions*, never the task. + +``` +transition_status(status="advance", task="", outcome="completed|failed", + error="sql|infra", where="id = ''", + agent_id="") +``` + +Outcomes and error classes are defined in +[extensibility/TASKS.md](../skills/migration/extensibility/TASKS.md#outcome-vocabulary): +`sql` routes into the fix loop, `infra` is a transient failure the orchestrator +may reset only after remediation or evidence the condition cleared. A wait on +another object is `blocked` / `blockedOn` on the next walk — do not stamp it. + +The response hands back the object's next task and its executor, which is what you +act on — no second `next_task` call needed. Read `nextTaskBlocked` before you do: +when it is set, the task you were just handed has unmet preconditions, +`nextTaskBlockedOn` names them, and this is the `blocked` row of the table above +arriving on a different payload. Do not run it. Call `next_task` with your +`agent_id` so a job-backed block can carry `wait`. + +A deploy that fails to compile is the machine's to route. Make the bounded +pre-deploy fixes its skill lists, and when the error is not one of those, stamp +`error="sql"`: `applyRules` → `fixCode` carries the diagnosis guidance and turns the +fix into a rule other objects reuse. Hand-patching the SQL against Snowflake instead +skips both, so the same converted defect arrives unfixed on the next object with that +pattern. + +When you do edit converted SQL, change only what the error implicated. `EXECUTE AS`, +the SnowConvert `COMMENT` provenance block, and the database qualifier are not part +of a syntax error — dropping them silently changes who the object runs as, loses the +link back to its source, and can pin it to a database the target does not have. + +`plannedSteps` is the machine's estimate of the path still ahead. Read it on the +first iteration and keep it — it is how you know whether you are progressing or +circling. + +## 3. Do not loop forever + +Repeating is this agent's whole risk. **The decisive check is whether `nextTask` +comes back the same as the previous iteration:** + +- **Same deterministic evidence, unchanged inputs** — stop before another call. + This includes the same row mismatch and the same generated-SQL or tool error. + Escalate with the evidence and state what must be repaired before rerunning. +- **`inFixLoop: true`** — expected. `applyRules → fixCode → retryEntry` is a cycle + by design only when the cycle changed code or another relevant input. Keep going, + bounded by the thresholds in + [migrate-object/SKILL.md](../skills/migration/migrate-objects/migrate-object/SKILL.md#escalation-criteria): + same primary error three times, churning errors after five iterations. +- **`inFixLoop` absent or false** — stop. You stamped an outcome and the machine + did not register it, so the task's status source cannot observe what you did. A + second attempt fails identically. Escalate, and say in `asks` that the status + source is not picking up the work. + +Cap the whole walk regardless: your first `plannedSteps` count plus five, and never +more than 30 iterations. The path is finite and `plannedSteps` measures it; the fix +loop is the only legitimate source of extra passes and it is already capped. +Hitting the cap is an escalation, not a quiet return — say how many tasks you +completed and which one you were on. + +## 4. Judgment + +One test before you write: **did I choose among meanings, or did I only make +it compile?** + +| Verb | Meaning | Object | Person | +|---|---|---|---| +| `escalate` | I must not choose. Wrong pick compiles and ships the wrong answer, and I cannot name a reversible default. | parks | must answer | +| `note` | I did choose. I can name the inverse. Work continues. | does not park | reviews later | +| `override_accept_case` | A note with a case key. RESULTS stays FAIL; the oracle absorbs that `params_hash`. Same review queue as `note`. When the project requires independent override-accept, a `test_case_verifier` makes the call, not you. | does not park | reviews later | + +Only compile — quoting, identifier, a type Snowflake needs to parse — is +neither. Stamp `error="sql"` and fix. No note. + +Chose among meanings, and the skill names a default **or** I can undo it: +**note**, keep going. Chose among meanings, and I must not: **escalate** +first, before writing the wrong answer. A named `runTests` case with a +`params_hash` is `override_accept_case` only after the fix loop has tried +and a code change would be illogical — not on the first FAIL. + +Exhaustion still escalates. That is not this test: §3, the thresholds in +[migrate-object/SKILL.md](../skills/migration/migrate-objects/migrate-object/SKILL.md#escalation-criteria), +the same `nextTask` coming back, the walk cap. + +`note` is not a license to change Snowflake off the converted file. A live +`ALTER` / `DROP` / `CREATE` that is not in `snowflake/` is not the inverse: +the next `deploy` overwrites it. Edit the file, deploy, retry. Do not `note` +as a substitute for that fix. After a file edit that chose among meanings — +dropping source CHECKs so COPY can run is a choice — **do note**. + +``` +transition_status(status="note", task="", + asks=["", ""], + choice="", + reason="", + change="", + undo="", + where="id = ''", agent_id="") +``` + +`asks`, `choice`, `reason`, `change`, and `undo` are **required** — the call is +refused without them. `choice` must be one of the `asks`. `undo` is what stops +this from being a silent mutation: if you cannot name the inverse, escalate +instead. The object does not park. The row surfaces on +`migration_status(mode="escalations")` as an unreviewed note for a later look. + +A first `runTests` FAIL that is not a dependency is `error="sql"` — enter +the fix loop and try to make the cases pass. Follow +[DIAGNOSE_FIX.md](../skills/migration/migrate-objects/migrate-object/DIAGNOSE_FIX.md): +conversion bugs, YAML shape, prefixes, CAST / normalize when that still +means what the source means. Do not spawn a verifier on first sight of +GETDATE or a ±1 day drift. + +If the FAIL is a defect on **another in-scope code unit** you depend on +(or that depends on you) and that unit already reads done, do not edit +its files and do not paper over it here. Spawn **one** foreground +[`task-invalidate`](task-invalidate.md) (`run_in_background=false`, +`subagent_type="task-invalidate"`) with fresh context: + +``` +Reopen these code units, following your agent definition. + +codeUnitIds: +task: +reason: +waiterId: +waiterTask: +projectDir: +pluginDir: +``` + +Do not pass your `agentId`. Do not `begin` for that child. Wait. Then +**return** — do not keep walking: + +```json +{ + "objectId": "", + "task": "runTests", + "tasksCompleted": [], + "result": "reopened", + "reopenedCodeUnits": ["", ""] +} +``` + +The parent first-sends the reopened code unit(s). You stay claimed; you +drop the slot until that walk finishes. `status='invalidate'` under your +id is refused. + +`override_accept_case` is reserved for a named case whose `params_hash` +is still FAIL **after** that attempt, and only when a code change would +be illogical (the source computes from wall-clock time; freezing "now" +or deleting the expression would lie). Recapturing baselines is not a +fix for a clock moving. Do not stamp `runTests` completed, do not delete +the YAML case (including leftover / overflow / short-input rows), +and do not write a fake PASS. Old `VALIDATION.RESULTS` FAIL rows +are not a reason to drop a case. A case with no `params_hash` that +meets the same test is `note`. + +Do not call `override_accept_case` yourself — you have been iterating on +the same failures, and the server refuses it under your `agentId` and +refuses an omitted `agent_id`. Spawn **one** foreground +[`test_case_verifier`](test_case_verifier.md) (`run_in_background=false`, +`subagent_type="test_case_verifier"`) with fresh context for **all** +remaining hashes on this object. Do not pass `fork_conversation_history`. +Do not pass your `agentId`. Do not tell it the verdict or hand it a +ready-made call — that child reads each case and decides; a prompt that +already names the limitation turns it into a clerk. Only if the project +set `require_independent_override_accept: false` may you call it yourself. + +``` +Verify these failing runTests cases, following your agent definition. + +objectId: +projectDir: +pluginDir: +params_hashes: + +``` + +Wait. For each `cases[]` row: `accepted` — keep going; `rejected` — that +hash is a real FAIL on the SQL you just showed. Stamp `error="sql"` and +fix. After that fix, if the same hash still FAILs, spawn a **new** +verifier for the remaining hashes — the first child judged the pre-fix +SQL. A second override-accept is allowed; do not escalate solely because +the first verifier rejected. `override_accept_case` under your walker id +is still refused; the new child uses its own minted id. + +Otherwise call it yourself: + +``` +transition_status(status="override_accept_case", task="runTests", + params_hash="", + asks=["override-accept the named case", ""], + choice="override-accept the named case", + reason="", + where="id = ''", agent_id="") +``` + +`task` must be `runTests`. `params_hash`, `asks`, `choice`, and `reason` +are required. `test_name` is display only (defaults to `source.name`); the +oracle joins `(procedure, params_hash, target)`. The event is +`override_accepted`. `0000` is refused. + +Do **not** escalate a decision because it is hard, or because you would have to +read more source to settle it. Read more. And do not escalate what already has a +route: a SQL bug is `outcome="failed", error="sql"` and goes to the fix loop, a +flaky environment is `error="infra"`. A missing dependency is not a stamp — +return `waiting` / `partial` and let the walk derive `blockedOn`. +An unchanged row mismatch or repeatable generated-SQL/tool failure is not flaky; if +the machine has no repair route for it, park it instead of relabeling it `infra`. +Every escalation costs a person's attention, and the autonomous loop dispatches +several of you at once — one that could have been answered by reading the source +crowds out one that genuinely needs a human. + +When you escalate a design decision, **omit `cause`**: there is no underlying +failure class, because nothing failed. State the readings as your `asks` and put +what makes them ambiguous in `reason`. + +### The call + +``` +transition_status(status="escalate", task="", + asks=["", ""], + cause="sql|infra", + reason="", + where="id = ''", agent_id="") +``` + +`asks` is **required** — the call is refused without it. State real options a +person can choose between ("rebuild the component in SQL" / "skip the object"), +never "please advise". They outlive you: the object parks with no `failed` +transition, so it will not route into the fix loop and no other agent picks it up +until someone answers. + +`outcome` and `error` are implied and **refused** — escalating always means +`failed` / `human`. + +`cause` and `reason` are optional but worth giving. `cause` is the class of the +failure that *led* here, not the escalation's own class: `sql` when you hit a SQL +bug you couldn't fix, `infra` when the environment kept failing — and nothing at +all when the escalation is a design decision (including a missing dependency a +person must register, stub, or mark out of scope). `reason` is the recurring error, the iteration history, or +what makes the readings ambiguous. Together they are what the person reads before +choosing, and they make "which of our escalations are really SQL problems" +answerable. + +You will be resumed with the answer as your `guidance`. It is not a +suggestion — a human chose between the options you gave them, so it outranks the +reading you would have picked. + +Then return `stuck`, with the same `asks` in your JSON. + +## 5. Finish + +A **verified** terminal — every enabled task completed, none skipped — is the +machine's to close. It stamps `extensions.isDone`, merges your files to main, and +releases the claim. You do not need a wake just to call `finish`. + +A last task that **failed** stays failed. Do not finish it. A terminal reached by +skipping work is not a verified close — leave it claimed and return `stuck` or +escalate. + +If you are still live when the walk goes terminal, you may call: + +``` +transition_status(status="finish", where="id = ''", agent_id="") +``` + +That is a no-op when the machine already closed the object. Safe from a subagent +while others work — the handler builds the commit in a throwaway index, pushes the +SHA, and rebases under an advisory lock without ever checking out. + +Relay `git_activity`. If the response carries a +`stash_warning`, put it in `notes` — an autostash pop left conflict markers and a +human needs to know. + +If `finish` refuses and `refused[].reason` names a pending or blocked task, +that task is next: + +| Then | Do | +|---|---| +| The machine offers the named task | Call `next_task` again, or run the executor it already handed you. | +| The named task is locked and you already produced its artifact or job | **Escalate**, and say in `asks` that the status source is not picking up the work. `next_task` said terminal and `finish` still sees the task — recapturing, `git log` on the snap repo, and Snowflake metadata tables will not make the resolver observe it. | + +An object that ended `blocked`, `stuck`, or capped is **not** terminal. Leave it +claimed and return `stuck`. The orchestrator relays; it does not finish for you. + +## 6. Return + +Your final message is one JSON object, nothing else — no prose, no fence. + +```json +{ + "objectId": "", + "task": "", + "tasksCompleted": ["convert", "deploy"], + "result": "completed|partial|stuck|waiting|reopened", + "reopenedCodeUnits": [""], + "failed": {"error": "sql|infra|human", "why": ""}, + "blocked": {"on": "", "reason": ""}, + "evidence": "", + "asks": [""], + "notes": "" +} +``` + +`completed` means the object reached a verified terminal (the machine closes +it; you may have called `finish` yourself). `partial` means +it blocked on something that is not a relay job, or failed with a class that +routes. `stuck` means a human has to act — and `asks` is required for it. +`waiting` means you are done until a `relay_wake:` resume: you dispatched +your own async data job, or `next_task` returned `wait` for a dependency. +`reopened` means you spawned `task-invalidate` and exited so the parent can +first-send the named `reopenedCodeUnits`. Omit `failed` / `blocked` when they +don't apply. + +The parent reads only `objectId`, `tasksCompleted`, `result`, and +`reopenedCodeUnits`. The other +keys are for you; they are not a channel to the dispatcher. Keep `evidence` +and `notes` short. Never paste whole SQL files, full test output, or your +reasoning. Report faithfully — a `completed` you cannot substantiate is worse +than an honest `stuck`. + +## Never + +| Don't | Why | +|---|---| +| Touch any object but `objectId` | Other agents are live on the rest of the wave. | +| `transition_status` with `bypass` / `reset` / `skip` | Overrides belong to the orchestrator, with the user. | +| `data_infrastructure` up or down | Shared by every slot; the orchestrator owns its lifecycle. | +| `configure(...)` with anything but `project_dir` and your `agent_id` | Everything else there is shared session config — one process serves the whole wave, so a database or a wave you set lands under every sibling. `agent_id` is the exception because it is not config: it tells the server who is asking and mutates nothing anyone reads. That distinction is the whole test — not "two parameters are allowed now". `subagent_mode` and `require_independent_override_accept` in particular are the orchestrator's alone. | +| Pass any `agent_id` but the `agentId` in your prompt | It is bound to your object's claim. `0000` is the orchestrator's, and an id the server never issued is refused. When `require_independent_override_accept` is on, `override_accept_case` under your id or with no `agent_id` is refused — spawn a `test_case_verifier`. `status='invalidate'` under your id is refused — spawn a `task-invalidate`. A first reject does not block a second verifier after a later code change. | +| Delete a YAML test case | Overlay or fix the converted SQL. Old RESULTS rows and dialect error-code mismatches are not a reason to drop a case. | +| Reach for `deploy` / `migrate_data` / `validate_data` outside `objectId` | They do not check the claim yet, so nothing stops you — which makes this yours to get right, not the server's. | +| Ask a question or wait for input | Nothing you write reaches a human mid-run. Decide and `note`, or escalate if §4 applies. | +| `git add` / `commit` / `push` / `rebase` / branch switching by hand | `transition_status` does the git work correctly and under a lock; a file you staged is one housekeeping commit away from permanent. | diff --git a/plugin/agents/task-invalidate.md b/plugin/agents/task-invalidate.md new file mode 100644 index 0000000..ae10300 --- /dev/null +++ b/plugin/agents/task-invalidate.md @@ -0,0 +1,100 @@ +--- +name: task-invalidate +description: Independently reopen a code unit at a resume task by writing TASK_INVALIDATIONS watermarks. Triggers: task-invalidate, invalidate tasks, reopen code unit, resume-point watermark. +license: Proprietary. See License-Skills for complete terms +--- + +You reopen **one or more in-scope code units** at a named resume task, in one +turn. The prompt carries `codeUnitIds`, `task` (the resume point), `reason`, +the waiter's `objectId` / verification task when the walker also needs its +tests re-run, and `projectDir`. That context is the walker's diagnosis. It is +not a write. You decide the `where` and `task` from the registry and the walk. + +## 1. Attach + +``` +configure(project_dir="") +``` + +Pass nothing else — attach only, no dashboard or session rewrite. The response +mints an `agent_id`; that is yours. Pass it on every `invalidate`. Do not +`begin`. Do not edit any file. Do not run SQL that changes a code unit. + +## 2. Read + +For each `codeUnitId` (and the waiter, if named): + +``` +query_registry(where="id = ''", fields="id,source,dependencies,inScope,extensions") +migration_status(mode="next_task", object_ids=[""]) +``` + +Hold `source.objectType`, `dependsOn` / `requiredBy`, and whether `isDone` is +set. The resume `task` must be a node on **that** type's walk (`validateView` +on a view, `runTests` on a procedure — not the other way around). + +## 3. Decide + +You exist so a walker cannot stamp another code unit's walk. Invalidate when: + +| What you found | Do | +|---|---| +| The defect is on this in-scope code unit, and `task` is the first stale step | **invalidate** at that task | +| The waiter also needs its verification re-run (`runTests` / `validateView`) | **invalidate** the waiter at that verification task too | +| The named id is out of scope, missing, or `task` is not on its walk | **reject** that id — do not write | +| Someone is actively walking the target as their own code unit and it is not the waiter | **reject** — do not stomp a live walk | + +Do not pick an earlier resume than the diagnosis needs. Reopening a view at +`validateView` leaves `deploy` completed. Reopening a procedure at `runTests` +does not recapture baselines. + +## 4. Act + +One call per code unit, same minted `agent_id`: + +``` +transition_status(status="invalidate", task="", + reason="", + where="id = ''", + agent_id="") +``` + +`task` and `reason` are required. `where` is the code unit to reopen — never +the walker's claim as a substitute for the defective unit. The server writes +the resume task **and every later task** on that type's walk, clears +`extensions.isDone`, and unparks. You do not stamp, finish, or spawn anyone. + +A refusal that says to spawn `task-invalidate` means you passed the walker's +id — attach again without `agent_id` and use the mint. + +## 5. Return + +Your final message is one JSON object, nothing else — no prose, no fence. + +```json +{ + "result": "invalidated|rejected", + "codeUnits": [ + { + "id": "", + "task": "", + "verdict": "invalidated|rejected", + "why": "" + } + ] +} +``` + +Include every id you were given. Then **exit**. The walker returns to the +parent; you do not resume anyone. + +## Never + +| Don't | Why | +|---|---| +| Pass the walker's `agentId` | It is the claim holder and the call is refused. | +| `configure` with anything but `project_dir` | Shared session config. `agent_id` here would skip the mint. | +| `begin` / stamp / finish / edit SQL | You only write watermarks. | +| Touch a code unit that is not in the prompt | Other agents are live on the rest of the wave. | +| Spawn a subagent | The write is yours. | +| Stay running after the JSON | The parent first-sends the reopened code unit(s). | diff --git a/plugin/agents/test_case_verifier.md b/plugin/agents/test_case_verifier.md new file mode 100644 index 0000000..5a1e801 --- /dev/null +++ b/plugin/agents/test_case_verifier.md @@ -0,0 +1,159 @@ +--- +name: test_case_verifier +description: Independently review the remaining failing runTests cases on one object and override-accept a case only when a code change would be illogical. Triggers: test_case_verifier, verify failing cases, independent override-accept, review params_hash. +license: Proprietary. See License-Skills for complete terms +--- + +You review **every remaining failing runTests case on one object**, in one +turn. The prompt carries `objectId`, `projectDir`, and a list of +`params_hash` values. It may also carry a verdict, a reason, a sibling +finding, or a ready-made `override_accept_case` call. Those are the +walker's opinion. They are not evidence. Read each case and decide. + +## 1. Attach + +``` +configure(project_dir="") +``` + +Pass nothing else — attach only, no dashboard or session rewrite. The response +mints an `agent_id`; that is yours. Pass it +on every `override_accept_case`. + +## 2. Read the cases + +Do not decide from the prompt. For this `objectId` and each `params_hash`: + +``` +query_registry(where="id = ''", fields="id,source,files,target") +``` + +Hold `files.source.path`, `files.converted.path`, `files.artifacts.path`, +and `source.{database,schema,name}`. Then read, in this order: + +1. `.VALIDATION.LATEST` — attach names + `metadata_database`. Same catalog as ORCHESTRATION / RULE_ENGINE + (`scai test validate` writes one VARIANT row per case into + `VALIDATION.RESULTS`; LATEST is the newest run per case). It is + **not** `snowflake_database` (the migration target). There is no + `/test-results/results.json`. + + `sql_execute` one read-only `SELECT`. Pass `connection` from attach + `snowflake_connection:`. Take `status`, `error_message`, `parameters`, + and `differences` from the rows. A prompt summary that disagrees with + LATEST is wrong. + + ``` + SELECT params_hash, status, parameters, error_message, differences, + baseline_rows, actual_rows, match_type + FROM .VALIDATION.LATEST + WHERE UPPER(procedure_name) IN ( + UPPER('..'), + UPPER('.') + ) + AND params_hash IN ('', …) + ``` + +2. The test YAML under `//test/` (glob + `*.yml`). Find the `test_cases` row for each hash / params. +3. The source SQL at `files.source.path` and the converted SQL at + `files.converted.path`. +4. Every view or column the diffs name that is not defined in the proc. + `query_registry` for that name and read its source and converted SQL + the same way. + +A second `sql_execute` of one read-only `SELECT` is allowed when a +definition is not on disk. Fully qualify +`..`. +Do not re-run the procedure, and do not use shell, `snow sql`, Python, +`SHOW DATABASES`, or another catalog +as a fallback. If the tool is denied, decide from the rows and files you have. +Do not edit any file. + +## 3. Decide each case + +Ask first: **would a converted-SQL or YAML edit make this case pass +without changing what the source means?** If yes, it is a real FAIL — +**reject**. The walker still owns the fix. You exist so an accept is not +a way around a fixable bug. + +The only accept is a case where that edit would be **illogical**: the +source (or a view it reads) computes a column from wall-clock time +(`GETDATE`, `CURRENT_TIMESTAMP`, `CURRENT_DATE`, `DATEDIFF` / +`DATEDIFF_BIG` against now, an `AgeDays`-style age from today), the cell +diffs are the drift that clock movement produces, and freezing "now", +hardcoding the baseline, or deleting the expression would lie about the +source. Recapturing baselines is not a fix for a clock moving. + +Be lenient when the source errored and Snowflake succeeded — still use +best judgement, but those cases are not as important. + +| What you found | Verdict | +|---|---| +| A faithful code or YAML edit would make the case pass | **reject** — fixable | +| That clock expression is in the source or a view it reads, the cell diffs match clock drift, and no faithful edit exists | **accept** | +| Source errored and Snowflake succeeded | **lenient** — still use judgement; not as important | +| `ERROR`, missing object, unknown identifier, row-count mismatch, extra or missing columns | **reject** — SQL or dependency | +| The column is stored, or a deterministic expression with no clock | **reject** — SQL bug | +| You cannot find the clock expression in the SQL you read | **reject** — not shown | + +A parent verdict, a sibling discovery, or a prompt that already filled in +`reason` does not move a row. Decide each hash on its own; one accept +does not cover the rest. + +## 4. Act + +**accept** — write `asks` / `choice` / `reason` from the SQL you read, not +from the prompt. One call per accepted hash, same minted `agent_id`: + +``` +transition_status(status="override_accept_case", task="runTests", + params_hash="", + asks=["override-accept: ", "treat as SQL bug"], + choice="override-accept: ", + reason="", + where="id = ''", + agent_id="") +``` + +`task` must be `runTests`. `params_hash`, `asks`, `choice`, and `reason` +are required. `test_name` is display only. RESULTS still shows FAIL; the +oracle absorbs the hash. Do not stamp `runTests` completed and do not +delete the YAML case. + +**reject** — do not call `override_accept_case` for that hash. The walker +stamps `error="sql"` or escalates from your return. + +You do not park the object, stamp a task, or spawn another agent. + +## 5. Return + +Your final message is one JSON object, nothing else — no prose, no fence. + +```json +{ + "objectId": "", + "cases": [ + { + "params_hash": "", + "verdict": "accepted|rejected", + "why": "" + } + ] +} +``` + +Include every hash you were given. + +## Never + +| Don't | Why | +|---|---| +| Accept because the prompt said to | The walker already believed it; you exist so that belief is checked. | +| Accept when a faithful edit would pass | That is the fixer's job, not an overlay. | +| Pass the walker's `agentId` | It is the claim holder and the call is refused. | +| `configure` with anything but `project_dir` | Shared session config. `agent_id` here would skip the mint. | +| Stamp `runTests` or delete the YAML case | Overlay only; RESULTS stays FAIL. | +| Edit converted SQL or the YAML | You decide; the walker writes. | +| Touch any object but `objectId` | Other agents are live on the rest of the wave. | +| Spawn a subagent | The read is yours. One spawn reviews every remaining hash. | diff --git a/plugin/commands/dash.md b/plugin/commands/dash.md deleted file mode 100644 index 635bc05..0000000 --- a/plugin/commands/dash.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -description: Open the migration dashboard in the browser -allowed-tools: Bash(open:*) ---- - -The migration dashboard is a **read-only web view** of this project — object -inventory, wave/deployment progress, and data-migration status — served -locally on `127.0.0.1`. Point the user at it so they can watch progress while -the agent works. - -## 1. Find the dashboard URL - -The dashboard is opt-in and only runs when a `dashboard_port` has been set. -Resolve the actual port (do **not** guess — there is no fixed default unless -`-1` was chosen): - -- The `configure()` response returns the **bound URL** whenever the dashboard - is running — use it verbatim. -- Otherwise read the persisted `dashboard_port` from `.scai/config/plugin.yml` - (it's saved there so the dashboard auto-starts on later sessions). Map it to - a URL: `-1` → `http://127.0.0.1:7878/`; a positive integer → that port. A - value of `-2` (auto-scan) has no fixed port — get the real one from the - `configure()` response, not the file. - -## 2. Open it and tell the user what it's for - -Open the resolved URL in the default browser: - -!`open ` - -Then tell the user it's now open and that it shows live migration progress -(objects, waves, and data migration/validation status), refreshing as work -proceeds. - -## If no dashboard is running - -If no `dashboard_port` is set, enable it by calling -`configure(dashboard_port=)`: - -- `-2` — auto-scan for a free port starting at `7878` (recommended) -- `-1` — fixed port `7878` -- any positive integer — bind that exact port - -The choice is persisted to `.scai/config/plugin.yml` and reused next session. -The `configure()` response returns the bound URL — open it as in step 2. For -the full setup flow, see `setup/SKILL.md`. diff --git a/plugin/hooks/dev-build-reminder.txt b/plugin/hooks/dev-build-reminder.txt new file mode 100644 index 0000000..10fb50b --- /dev/null +++ b/plugin/hooks/dev-build-reminder.txt @@ -0,0 +1,3 @@ + +[snowflake-migration] Do not work around plugin bugs. Stop and ask the user to file a Jira ticket. + diff --git a/plugin/hooks/install-dependencies.ps1 b/plugin/hooks/install-dependencies.ps1 index a604235..23137cd 100644 --- a/plugin/hooks/install-dependencies.ps1 +++ b/plugin/hooks/install-dependencies.ps1 @@ -40,12 +40,29 @@ if (!(Test-Path $VersionFile)) { } $Version = (Get-Content $VersionFile -Raw).Trim() -if (-not $env:SCAI_CHANNEL) { $env:SCAI_CHANNEL = "preview" } +if (-not $env:SCAI_CHANNEL) { $env:SCAI_CHANNEL = "stable" } Log "SessionStart hook running (v$Version, plugin root: $PluginRoot)" $cortexCh = if ($env:CORTEX_CHANNEL) { $env:CORTEX_CHANNEL } else { "(not set)" } Log "SCAI_CHANNEL=$($env:SCAI_CHANNEL), CORTEX_CHANNEL=$cortexCh" +# Optional runtime override (opt-in): pin a specific scai version via the shared +# config file. Absent config = default behavior (update to the channel's latest). +$ScaiVersionPin = "" +$MigrationConfig = Join-Path $env:USERPROFILE ".snowflake\migration-plugin\config.json" +if (Test-Path $MigrationConfig) { + try { + $cfg = Get-Content $MigrationConfig -Raw | ConvertFrom-Json + $v = $cfg.scai.version + if ($v -is [string] -and $v) { + $ScaiVersionPin = $v + Log "Config pins scai.version=$ScaiVersionPin" + } + } catch { + Log "WARNING: could not parse $MigrationConfig — ignoring, using default scai version" + } +} + # System dependencies # uv @@ -81,16 +98,32 @@ if (Get-Command "brew" -ErrorAction SilentlyContinue) { # scai CLI (bundles the migration MCP server binary) $start = Get-Date if (Get-Command "scai" -ErrorAction SilentlyContinue) { - Log "scai already installed, running explicit update..." - try { - scai update 2>&1 | ForEach-Object { Log $_ } - } catch { - Log "scai update failed: $_" + if ($ScaiVersionPin) { + Log "scai already installed, pinning to v$ScaiVersionPin..." + try { + scai update $ScaiVersionPin 2>&1 | ForEach-Object { Log $_ } + } catch { + Log "scai pin to v$ScaiVersionPin failed: $_" + } + $elapsed = [math]::Round(((Get-Date) - $start).TotalSeconds) + Log "scai pinned to v$ScaiVersionPin (${elapsed}s)" + } else { + Log "scai already installed, running explicit update..." + try { + scai update 2>&1 | ForEach-Object { Log $_ } + } catch { + Log "scai update failed: $_" + } + $elapsed = [math]::Round(((Get-Date) - $start).TotalSeconds) + Log "scai up to date (${elapsed}s)" } - $elapsed = [math]::Round(((Get-Date) - $start).TotalSeconds) - Log "scai up to date (${elapsed}s)" } else { - Log "Installing scai CLI..." + if ($ScaiVersionPin) { + Log "Installing scai CLI (pinned v$ScaiVersionPin)..." + $env:SCAI_VERSION = $ScaiVersionPin + } else { + Log "Installing scai CLI..." + } try { irm https://snowconvert.snowflake.com/storage/windows/prod/cli/install.ps1 | iex $elapsed = [math]::Round(((Get-Date) - $start).TotalSeconds) diff --git a/plugin/hooks/install-dependencies.sh b/plugin/hooks/install-dependencies.sh index 08c5b55..588f45e 100755 --- a/plugin/hooks/install-dependencies.sh +++ b/plugin/hooks/install-dependencies.sh @@ -39,11 +39,36 @@ if [ -z "$VERSION" ]; then exit 1 fi -export SCAI_CHANNEL="${SCAI_CHANNEL:-preview}" +export SCAI_CHANNEL="${SCAI_CHANNEL:-stable}" log "SessionStart hook running (v$VERSION, plugin root: $PLUGIN_ROOT)" log "SCAI_CHANNEL=$SCAI_CHANNEL, CORTEX_CHANNEL=${CORTEX_CHANNEL:-(not set)}" +# Optional runtime override (opt-in): pin a specific scai version via the shared +# config file. Absent config = default behavior (update to the channel's latest). +SCAI_VERSION_PIN="" +MIGRATION_CONFIG="$HOME/.snowflake/migration-plugin/config.json" +if [ -f "$MIGRATION_CONFIG" ]; then + if command -v python3 &>/dev/null; then + # `if VAR=$(...)` keeps `set -e` from aborting when the JSON is unparseable. + if SCAI_VERSION_PIN=$(python3 - "$MIGRATION_CONFIG" <<'PY' +import json, sys +with open(sys.argv[1], encoding="utf-8") as f: + cfg = json.load(f) +v = (cfg.get("scai") or {}).get("version") +print(v if isinstance(v, str) else "") +PY + ); then + [ -n "$SCAI_VERSION_PIN" ] && log "Config pins scai.version=$SCAI_VERSION_PIN" + else + SCAI_VERSION_PIN="" + log "WARNING: could not parse $MIGRATION_CONFIG — ignoring, using default scai version" + fi + else + log "WARNING: python3 not found — ignoring $MIGRATION_CONFIG, using default scai version" + fi +fi + # System dependencies # uv @@ -73,12 +98,19 @@ fi # scai CLI (bundles the migration MCP server binary) start=$SECONDS if command -v scai &>/dev/null; then - log "scai already installed, running explicit update..." - scai update 2>&1 | tee -a "$LOG" >&2 || log "scai update failed" - log "scai up to date ($(( SECONDS - start ))s)" + if [ -n "$SCAI_VERSION_PIN" ]; then + log "scai already installed, pinning to v$SCAI_VERSION_PIN..." + scai update "$SCAI_VERSION_PIN" 2>&1 | tee -a "$LOG" >&2 || log "scai pin to v$SCAI_VERSION_PIN failed" + log "scai pinned to v$SCAI_VERSION_PIN ($(( SECONDS - start ))s)" + else + log "scai already installed, running explicit update..." + scai update 2>&1 | tee -a "$LOG" >&2 || log "scai update failed" + log "scai up to date ($(( SECONDS - start ))s)" + fi else - log "Installing scai CLI..." - curl -fsSL https://snowconvert.snowflake.com/storage/linux/prod/cli/install.sh | bash 2>&1 | tee -a "$LOG" >&2 + log "Installing scai CLI${SCAI_VERSION_PIN:+ (pinned v$SCAI_VERSION_PIN)}..." + # SCAI_VERSION empty = install latest; set = pin. install.sh honors it. + curl -fsSL https://snowconvert.snowflake.com/storage/linux/prod/cli/install.sh | SCAI_VERSION="$SCAI_VERSION_PIN" bash 2>&1 | tee -a "$LOG" >&2 log "Installed scai CLI ($(( SECONDS - start ))s)" fi diff --git a/plugin/hooks/session-context.ps1 b/plugin/hooks/session-context.ps1 index 4871348..71c3012 100644 --- a/plugin/hooks/session-context.ps1 +++ b/plugin/hooks/session-context.ps1 @@ -2,13 +2,27 @@ # SPDX-License-Identifier: Apache-2.0 # # UserPromptSubmit hook (Windows) — see session-context.sh for full rationale. -# Injects, ONCE per session, whether a migration project exists here plus -# routing guidance, on the user's first message. UserPromptSubmit reaches the -# model (SessionStart additionalContext does not, in coco). +# Injects, ONCE per session, migration-project context plus routing guidance +# on the user's first message. Dev-channel plugin builds also receive their +# release-safety guardrail here. UserPromptSubmit reaches the model +# (SessionStart additionalContext does not, in coco). $stdin = "" try { $stdin = [Console]::In.ReadToEnd() } catch {} +$pluginRoot = if ($env:CLAUDE_PLUGIN_ROOT) { + $env:CLAUDE_PLUGIN_ROOT +} else { + Split-Path -Parent $PSScriptRoot +} +$devReminder = $null +try { + $manifest = Get-Content -Raw (Join-Path $pluginRoot '.cortex-plugin/plugin.json') | ConvertFrom-Json + if ($manifest.buildChannel -ceq 'dev') { + $devReminder = Get-Content -Raw (Join-Path $pluginRoot 'hooks/dev-build-reminder.txt') + } +} catch {} + $sid = "" if ($stdin -match '"session_id"\s*:\s*"([^"]*)"') { $sid = $Matches[1] } @@ -39,6 +53,10 @@ if (Test-Path (Join-Path $dir '.scai/config/project.yml')) { Write-Output "" } +if ($devReminder) { + Write-Output $devReminder +} + if ($marker) { New-Item -ItemType Directory -Force -Path $markerDir | Out-Null New-Item -ItemType File -Force -Path $marker | Out-Null diff --git a/plugin/hooks/session-context.sh b/plugin/hooks/session-context.sh index 394bd63..382cba6 100755 --- a/plugin/hooks/session-context.sh +++ b/plugin/hooks/session-context.sh @@ -2,10 +2,11 @@ # Copyright 2026 Snowflake Inc. # SPDX-License-Identifier: Apache-2.0 # -# UserPromptSubmit hook — ONCE per session, injects a single fact (does a -# migration project exist in this directory?) plus which way to route, so the -# agent orients on the user's first message. Tool-level instructions -# (configure, migration_status) belong to the migration skill, not here. +# UserPromptSubmit hook — ONCE per session, injects the migration-project +# context plus routing guidance so the agent orients on the user's first +# message. Dev-channel plugin builds also receive their release-safety +# guardrail here. Tool-level instructions (configure, migration_status) belong +# to the migration skill, not here. # # Why UserPromptSubmit, not SessionStart: coco only logs/displays SessionStart # additionalContext (agentService.executeSessionStartHooks logs it; the CLI @@ -19,6 +20,10 @@ input="$(cat 2>/dev/null)" +plugin_root="${CLAUDE_PLUGIN_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}" +manifest="$plugin_root/.cortex-plugin/plugin.json" +dev_reminder="$plugin_root/hooks/dev-build-reminder.txt" + sid="$(printf '%s' "$input" | sed -n 's/.*"session_id"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n1)" marker="" if [ -n "$sid" ]; then @@ -52,6 +57,11 @@ else EOF fi +if [ -r "$dev_reminder" ] && [ -r "$manifest" ] \ + && grep -Eq '"buildChannel"[[:space:]]*:[[:space:]]*"dev"' "$manifest"; then + cat "$dev_reminder" +fi + # Record that this session has received the context so later turns stay silent. if [ -n "$marker" ]; then mkdir -p "$(dirname "$marker")" 2>/dev/null && : > "$marker" 2>/dev/null diff --git a/plugin/hooks/tests/session-context.test.ps1 b/plugin/hooks/tests/session-context.test.ps1 new file mode 100644 index 0000000..bd5fc5d --- /dev/null +++ b/plugin/hooks/tests/session-context.test.ps1 @@ -0,0 +1,65 @@ +$ErrorActionPreference = 'Stop' + +$scriptDirectory = Split-Path -Parent $MyInvocation.MyCommand.Path +$hook = Join-Path (Split-Path -Parent $scriptDirectory) 'session-context.ps1' +$devReminder = Join-Path (Split-Path -Parent $scriptDirectory) 'dev-build-reminder.txt' +$testRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("session-context-test-" + [guid]::NewGuid()) +New-Item -ItemType Directory -Path $testRoot | Out-Null + +function Fail([string]$message) { + throw "FAIL: $message" +} + +function Invoke-Hook([string]$buildChannel, [string]$sessionId, [bool]$compactManifest = $false) { + $pluginRoot = Join-Path $testRoot "plugin-$buildChannel" + $manifestDirectory = Join-Path $pluginRoot '.cortex-plugin' + New-Item -ItemType Directory -Path $manifestDirectory -Force | Out-Null + New-Item -ItemType Directory -Path (Join-Path $pluginRoot 'hooks') -Force | Out-Null + $manifest = if ($buildChannel -eq 'none') { + '{"name":"snowflake-migration"}' + } elseif ($compactManifest) { + "{`"name`":`"snowflake-migration`",`"buildChannel`":`"$buildChannel`",`"version`":`"1.0.0`"}" + } else { + "{`"buildChannel`":`"$buildChannel`"}" + } + Set-Content -NoNewline -Path (Join-Path $manifestDirectory 'plugin.json') -Value $manifest + Copy-Item $devReminder (Join-Path $pluginRoot 'hooks/dev-build-reminder.txt') + $markerDirectory = Join-Path $testRoot "markers-$sessionId" + New-Item -ItemType Directory -Path $markerDirectory -Force | Out-Null + $previousPluginRoot = $env:CLAUDE_PLUGIN_ROOT + $previousTempDirectory = $env:TMPDIR + try { + $env:CLAUDE_PLUGIN_ROOT = $pluginRoot + $env:TMPDIR = $markerDirectory + return ('{"session_id":"' + $sessionId + '","cwd":"' + $testRoot + '"}' | pwsh -NoProfile -File $hook) + } finally { + $env:CLAUDE_PLUGIN_ROOT = $previousPluginRoot + $env:TMPDIR = $previousTempDirectory + } +} + +try { + $devOutput = (Invoke-Hook 'dev' 'dev-session') -join "`n" + if ($devOutput -notmatch 'Do not work around plugin bugs\.') { + Fail 'dev manifest did not inject the bug guardrail' + } + + $compactDevOutput = (Invoke-Hook 'dev' 'compact-dev-session' $true) -join "`n" + if ($compactDevOutput -notmatch 'Do not work around plugin bugs\.') { + Fail 'compact dev manifest did not inject the bug guardrail' + } + + $previewOutput = (Invoke-Hook 'preview' 'preview-session') -join "`n" + if ($previewOutput -match 'Do not work around plugin bugs\.') { + Fail 'preview manifest injected the bug guardrail' + } + + $noChannelOutput = (Invoke-Hook 'none' 'no-channel-session') -join "`n" + if ($noChannelOutput -match 'Do not work around plugin bugs\.') { + Fail 'manifest without buildChannel injected the bug guardrail' + } + + Write-Output 'session-context.ps1 tests passed' +} finally { + Remove-Item -Recurse -Force $testRoot +} diff --git a/plugin/hooks/tests/session-context.test.sh b/plugin/hooks/tests/session-context.test.sh new file mode 100644 index 0000000..f29c41b --- /dev/null +++ b/plugin/hooks/tests/session-context.test.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env sh +set -eu + +script_dir="$(cd "$(dirname "$0")" && pwd)" +hook="$script_dir/../session-context.sh" +dev_reminder="$script_dir/../dev-build-reminder.txt" +test_root="$(mktemp -d)" +trap 'rm -rf "$test_root"' EXIT + +fail() { + echo "FAIL: $1" >&2 + exit 1 +} + +run_hook() { + build_channel="$1" + session_id="$2" + manifest_layout="${3:-multiline}" + plugin_root="$test_root/plugin-$build_channel" + marker_root="$test_root/markers-$session_id" + mkdir -p "$plugin_root/.cortex-plugin" "$plugin_root/hooks" + if [ "$build_channel" = none ]; then + printf '{\n "name": "snowflake-migration"\n}\n' > "$plugin_root/.cortex-plugin/plugin.json" + elif [ "$manifest_layout" = compact ]; then + printf '{"name":"snowflake-migration","buildChannel":"%s","version":"1.0.0"}\n' "$build_channel" > "$plugin_root/.cortex-plugin/plugin.json" + else + printf '{\n "buildChannel": "%s"\n}\n' "$build_channel" > "$plugin_root/.cortex-plugin/plugin.json" + fi + cp "$dev_reminder" "$plugin_root/hooks/dev-build-reminder.txt" + mkdir -p "$marker_root" + TMPDIR="$marker_root" CLAUDE_PLUGIN_ROOT="$plugin_root" sh "$hook" </`, and claims the object. `status="done"` commits, fast-forward-merges into main, pushes, and marks the claim completed. Returns structured JSON errors on dirty tree, missing config, or merge conflict. | | `update_registry` | Update registry fields | | `query_registry` | Query the project registry with SQL-like filters | +| `register_units` | Register custom (`kind=custom`) units: one unit (`custom_kind`+`name`), a list (`entries`), or investigation findings (`expected_slugs`) | | `app_info` | Report the desktop app's installed version and last app-update-check time. **App-only** — hidden from non-app callers (`SCAI_CALLER != aim-app`), so it doesn't ship to the standalone CLI. No Snowflake connection or project needed. | ### Rule engine (need Snowflake connection) @@ -48,9 +61,9 @@ The server can host a small read-only HTML dashboard on `127.0.0.1` (no data lea |------|-------------| | `deploy` | Deploy objects via `scai code deploy` (single `object_name` or `where` filter) | | `query_source` | Run a SQL query against the source database via `scai query` | -| `migrate_data` | Two-mode tool. `mode="setup"` generates a per-`where` workflow YAML at `artifacts/data_migration/workflows/.yaml` (forwarding `where` to scai's `--where`) and persists the other params under `data_migration:` in `plugin.yml` as defaults; the agent reviews/edits before running. `mode="run"` takes the `workflow_path` and starts the migration (`scai data orchestrator setup`, `scai data worker start`, then `scai data migrate create-workflow`) in the background. | +| `migrate_data` | Two-mode tool. `mode="setup"` requires `where` and generates a per-`where` workflow YAML at `artifacts/data_migration/workflows/.yaml` (forwarding `where` to scai's `--where`); `where` is **not** stored on the session. Wave-level knobs persist under `data_migration:` in `plugin.yml`. The agent reviews/edits before running. `mode="run"` takes the `workflow_path` and binds objects from that file's `tables:` list (pure dispatch against the shared orchestrator + worker). | | `validate_data` | Validate migrated data between source and Snowflake. `mode="setup"` generates workflow YAML; `mode="run"` executes it; `mode="revalidate"` retries failed partitions from a finished parent workflow. Uses cloud validation (SPCS) when configured. | -| `job_status` | Report long-running job state. `monitor=true` starts the background relay and returns a `watch_command` for the Monitor tool; `details=true` attaches the full status payload plus parsed CSV failure reports (`details.reports.files`). | +| `job_status` | Report long-running job state. `monitor=true` starts the background relay and returns `orchestrator_watch` (wake instructions for the parent) plus a per-job `watch_command`; `wait=true` blocks until the next wake line (`0000` / omitted `agent_id` also needs `confirm=true`, otherwise a reminder is returned and the call does not block); `wake=true` re-attaches a walker to a job it already dispatched (dependents do not use this — `next_task` registers that wait); `details=true` attaches the full status payload plus parsed CSV failure reports (`details.reports.files`). | ## Building diff --git a/plugin/commands/migrate.md b/plugin/skills/migration/ENTRY.app.md similarity index 51% rename from plugin/commands/migrate.md rename to plugin/skills/migration/ENTRY.app.md index cdf46f1..14fdaf5 100644 --- a/plugin/commands/migrate.md +++ b/plugin/skills/migration/ENTRY.app.md @@ -1,11 +1,71 @@ +# Database Migration to Snowflake + +## Working with the MCP state machine + +The built-in MCP server runs a state machine that guides you through user flows and will ask you to "tell the user x" or "ask the user y". This is expected — follow its lead. It is the official MCP server of Snowflake migrations. + +## Step 1: Detect current state + +Call `migration_status`. It returns JSON with `project_exists`, `directory_empty`, `by_type`, `stage_totals`, `routing`, and `highest_stage_reached`. + +**If `project_exists` is false**, this is a new project — go to the setup skill `setup/SKILL.md` (read it under the skills root given in ``). + +**If `project_exists` is true**, give the user a brief prose summary of where the migration stands, built from the `routing` booleans and `by_type` counts. The UI panel already shows the stage-by-stage numbers, so your job is the sentence the panel cannot write: what phase this project is in and what stands out. Aim for this level of detail: + +- **Early setup:** "Your project is initialized and connected to SQL Server. 147 objects are registered but haven't been converted yet — we're still in the setup phase." +- **Mid-migration:** "Setup is complete and assessment is done. You're in the migration phase: 12 of 47 objects deployed so far, in Wave 2." +- **Near completion:** "Almost there — 45 of 47 objects are deployed and tested. 2 procedures are still failing tests." + +Then go to Step 2. + +Reading `by_type` correctly matters: + +- **Type absent** (`total` absent or 0): don't mention it. A project with no functions has no function story. +- **Type present but not in the current wave** (`total` > 0, wave-scoped counts absent or 0): acknowledge it anyway — e.g. "you also have 1 Informatica ETL workflow staged for a later wave." Don't let the current wave hide work that exists elsewhere in the project. +- Never describe a project as "-only" (e.g. "table-only") when `by_type` lists any other type with `total` > 0. +- Counts that never incremented are absent from the JSON — treat them as 0. +- **BTEQ scripts have no deploy step.** They run their converted SQL inside the test itself, so report them only by `tested`, never "deployed"; `by_type.bteq` carries no `deployed` count. + +## Step 2: Ask the user + +> "What would you like to do? You can: +> 1. **Continue with migration plan** — I'll start or pick up where we left off +> 2. **Something specific** — tell me what you need" + +If the user picks **Continue** (or says "continue", "next", etc.) → **Prescribed Path**. +If the user describes a **specific request** → **Skill Match**. + --- -description: Snowflake AIM Migration Agent -allowed-tools: Bash(open:*) + +## Prescribed Path + + +Use `routing` from the status JSON to delegate to the next step: + +| Condition | Sub-skill | +|-----------|-----------| +| `routing.project_exists` = false | Load `./setup/SKILL.md` | +| `routing.code_conversion_only` = true | Load `./code-conversion-only/SKILL.md` | +| `routing.assessed` = false | Load `./setup/SKILL.md` | +| `routing.assessed` = true | Load `./migrate-objects/SKILL.md` | + +Each sub-skill handles its own internal routing based on the full `routing` object. + + --- -Help me using the reference skills below. +## State Queries -$ARGUMENTS +These answer common questions about project state without loading a sub-skill: + +- **"What is the current state?"** — Call `migration_status(mode="summary")` and give the prose summary described in Step 1. +- **"What should I work on next?"** — Call `migration_status(mode="my_objects_summary")` to get per-`(task, object_type)` counts plus `errored_count` and `done_count`. Present the list and **ask the user what they want to work on — do not pick for them**. When the user picks a group, drill down with `migration_status(mode="my_objects_details", group=)`. +- **"Show me objects that match rule X"** — Load `migrate-objects/rule-engine/propagate/SKILL.md`. +- **"How do I extend / customize the migration plugin?"** / **"How do I override task X?"** — Load `extensibility/TASKS.md` for the full reference: overridable task ids, per-task contracts, and the project-local + `$AIM_SKILL_EXT_DIR` paths. Optionally call `migration_status(mode='extensions')` to show which overrides are active. + +--- + +## Skill Match Match the user's request to the most relevant skill and load it. @@ -20,6 +80,7 @@ Match the user's request to the most relevant skill and load it. - **setup** — full setup, steps 1–5: connect, init, register, convert, assess → `./setup/SKILL.md` - **midway-entry** — existing project with source + pre-converted Snowflake SQL (SQL Server / Redshift only) → `./setup/midway-entry.md` - **configure-snowflake-target** — set or change the Snowflake connection and target database for object migration. Triggers: "change the target database", "deploy to a different database", "switch Snowflake connection" → `./setup/configure-snowflake-target.md` + - **snowflake-connection** — create or repair a Snowflake target authenticator. Use for Microsoft Entra ID / Azure AD / OIDC (`oauth_authorization_code`); do not use `externalbrowser` for Entra → `./connection/snowflake-connection/SKILL.md` - **configure-testing** — pick or change the testing path (source-data vs synthetic) for procedure/function equivalence tests. Triggers: "change testing path", "switch to synthetic tests", "use query logs" → `./setup/configure-testing.md` - **data-validation-setup** — configure cloud data validation: schema, metrics, row-level checks → `./setup/data-validation/SKILL.md` - **data-infrastructure-teardown** — suspend SPCS service + compute pool, stop local worker (cost-saving) → `./data-infrastructure/teardown/SKILL.md` @@ -56,3 +117,9 @@ Match the user's request to the most relevant skill and load it. If no skill matches, say so explicitly, then help with your own knowledge. + +## Rules + +1. **Always detect first** — call `migration_status` before routing. +2. **Follow sub-skill instructions** — complete each sub-skill fully before returning. +3. **Confirm transitions** — ask the user before moving to the next stage. diff --git a/plugin/skills/migration/SKILL.md b/plugin/skills/migration/SKILL.md index 3b38aeb..9edb641 100644 --- a/plugin/skills/migration/SKILL.md +++ b/plugin/skills/migration/SKILL.md @@ -73,6 +73,7 @@ Present the narrative summary followed by the progress checklist, then continue ## Prescribed Path + Use `routing` from the status JSON to delegate to the next step: | Condition | Sub-skill | @@ -83,6 +84,7 @@ Use `routing` from the status JSON to delegate to the next step: | `routing.assessed` = true | Load `./migrate-objects/SKILL.md` | Each sub-skill handles its own internal routing based on the full `routing` object. + --- @@ -95,7 +97,9 @@ These answer common questions about project state without loading a sub-skill: - **"Show me objects that match rule X"** — Load `./migrate-objects/rule-engine/propagate/SKILL.md`. -- **"How do I extend / customize the migration plugin?"** / **"How do I override task X?"** — Load `./extensibility/TASKS.md` for the full reference: overridable task ids, per-task contracts, and the project-local + `$AIM_SKILL_EXT_DIR` paths. Optionally call `migration_status(mode='extensions')` to show which overrides are active. +- **"How do I extend / customize the migration plugin?"** / **"How do I override task X?"** — Load `./extensibility/TASKS.md` for the full reference: overridable task ids, per-task contracts, the project-local + `$AIM_SKILL_EXT_DIR` paths, and registering code units with `kind=custom` plus a free-form `customKind` discriminator. Optionally call `migration_status(mode='extensions')` to show which overrides are active. + +- **"My migration also has FiveTran / SSAS / dbt / Airflow / Oracle PACKAGE bodies / scripts the engine doesn't generate"** / **"How do I track in the migration?"** — Load `./setup/discover-extras/SKILL.md`. Registers each asset as a code unit with `kind=custom` and a `customKind` discriminator, then writes a `.scai/skills/.md` cookbook with the customer so every object of that kind runs the same playbook (see `./extensibility/TASKS.md` → "Custom code units"). --- @@ -110,10 +114,18 @@ Match the user's request to the most relevant skill and load it. - If the request is ambiguous between siblings, ask one clarifying question. - If no skill matches, fall back to the section below. +### SAS (Preview — parallel track, not SnowConvert) +- **sas** (Preview) — SAS → Snowflake: assess portfolios, convert `.sas` programs, or load `.sas7bdat` from a stage → `./sas/SKILL.md` + - **assess-sas-migration** — portfolio complexity, dependency DAG, migration waves → `./sas/assess-sas-migration/SKILL.md` + - **convert-sas-to-snowflake** — convert SAS programs to Snowflake SQL / stored procedures → `./sas/convert-sas-to-snowflake/SKILL.md` + - **migrate-sas7bdat-to-snowflake** — bulk-load `.sas7bdat` from a stage into tables → `./sas/migrate-sas7bdat-to-snowflake/SKILL.md` + - **validate-sas-conversion** — validate an existing SAS conversion → `./sas/convert-sas-to-snowflake/validate-sas-conversion/SKILL.md` + ### Setup & onboarding - **setup** — full setup, steps 1–5: connect, init, register, convert, assess → `./setup/SKILL.md` - **midway-entry** — existing project with source + pre-converted Snowflake SQL (SQL Server / Redshift only) → `./setup/midway-entry.md` - **configure-snowflake-target** — set or change the Snowflake connection and target database for object migration. Triggers: "change the target database", "deploy to a different database", "switch Snowflake connection" → `./setup/configure-snowflake-target.md` + - **snowflake-connection** — create or repair a Snowflake target authenticator. Use for Microsoft Entra ID / Azure AD / OIDC (`oauth_authorization_code`); do not use `externalbrowser` for Entra → `./connection/snowflake-connection/SKILL.md` - **configure-testing** — pick or change the testing path (source-data vs synthetic) for procedure/function equivalence tests. Triggers: "change testing path", "switch to synthetic tests", "use query logs" → `./setup/configure-testing.md` - **data-validation-setup** — configure cloud data validation: schema, metrics, row-level checks → `./setup/data-validation/SKILL.md` - **data-infrastructure-teardown** — suspend SPCS service + compute pool, stop local worker (cost-saving) → `./data-infrastructure/teardown/SKILL.md` @@ -145,6 +157,7 @@ Match the user's request to the most relevant skill and load it. ### Customization - **task-overrides** — replace the skill that runs for any built-in task with the user's own `SKILL.md`, scoped to the project or to a global directory via `$AIM_SKILL_EXT_DIR`. Triggers: "extend the plugin", "customize task X", "swap out the skill for Y", "override registerCode/convertCode/deploy/...". Reference: `./extensibility/TASKS.md` +- **discover-extras** — register assets the conversion engine doesn't generate (FiveTran, dbt, Airflow, Informatica, SSAS cubes, Oracle PACKAGE bodies, custom shell scripts) as code units with `kind=custom` and a free-form `customKind` discriminator (any string outside the reserved `databaseObject` / `script` / `etl` / `custom` set) so they flow through orchestration alongside built-in units. Triggers: "I have FiveTran / SSAS / dbt / a script that touches the database", "register custom asset", "track in the migration". Skill: `./setup/discover-extras/SKILL.md`. Reference: `./extensibility/TASKS.md` (Custom code units). ## Fallback diff --git a/plugin/skills/migration/assessment/SKILL.md b/plugin/skills/migration/assessment/SKILL.md index 43a0817..101d0f7 100644 --- a/plugin/skills/migration/assessment/SKILL.md +++ b/plugin/skills/migration/assessment/SKILL.md @@ -1,6 +1,6 @@ --- name: assessment -description: Analyzes workloads to be migrated to Snowflake using SnowConvert assessment reports. Routes to specialized sub-skills for high-quality assessments. Use this skill when user wants to do an assessment of their code or ETL workload, waves generation, object exclusion, anti-patterns, sql dynamic and/or ETL analysis (SSIS) +description: Analyzes workloads to be migrated to Snowflake using SnowConvert assessment reports. Routes to specialized sub-skills for high-quality assessments. Use this skill when user wants to do an assessment of their code or ETL workload, waves generation, object exclusion, effort estimates, anti-patterns, sql dynamic and/or ETL analysis (SSIS) version: 0.1.0 license: Proprietary. See License-Skills for complete terms --- @@ -49,6 +49,7 @@ Show one compact confirmation that lists what will run. This is the **only** con I will run: 1. Waves (dependency analysis + deployment partitioning) 2. Anti-Patterns (SQL Server only) +3. Effort Estimates (SQL Server and Redshift only) 4. Dynamic SQL Patterns 5. ETL/SSIS Assessment (only if present) 6. Informatica Assessment (only if present) @@ -118,11 +119,15 @@ This prompt is **MANDATORY** — do not skip or default. Record `informatica.tar No prompts. Note the sub-skill is in scope. -### 5.5 Anti-patterns (SQL Server only, no inputs) +### 5.5 Effort estimate (no inputs) + +No prompts. Note the sub-skill is in scope. + +### 5.6 Anti-patterns (SQL Server only, no inputs) No prompts. In scope **only when the project's source dialect is SQL Server** — `scai assessment anti-patterns` self-gates and aborts (error `ASM0024`) on other dialects. For non-SQL-Server projects, treat it as out of scope and synthesize a `"skipped"` result in Step 6. -### 5.6 Snapshot the inputs +### 5.7 Snapshot the inputs Lay out the resolved values in your working context like this (text only — do not write to disk): @@ -136,6 +141,7 @@ assessment_inputs: prioritization_globs: [, ...] wave_ordering: category | dependency exclusion: {} + effort_estimate: {} anti_patterns: {} dynamic_sql: review_mode: generate-only | auto-review-all | skip @@ -216,6 +222,33 @@ Report back JSON only: } ``` +### 6.2b effort-estimate-runner prompt + +``` +Read and follow plugin/skills/migration/assessment/effort-estimate/SKILL.md. +You are running in sub-agent mode — do NOT ask the user any questions. + +Context (from parent): +- project_dir: + +Steps: +1. Call configure() with project_dir above. Snowflake credentials are not needed for effort estimate analysis. +2. Run `scai assessment effort-estimate` from . +3. Locate the timestamped effort-estimates-*.json the CLI wrote under + /artifacts/assessment/. If the CLI aborts because the + source dialect is unsupported (error ASM0031), report status "skipped" + with that reason — do NOT treat it as an error. + +Report back JSON only: +{ + "sub_skill": "effort-estimate", + "status": "ok" | "skipped" | "error", + "output_json": "" | null, + "summary": "", + "error": "" | null +} +``` + ### 6.3 dynamic-sql-runner prompt ``` @@ -373,7 +406,7 @@ For every dispatched sub-skill, validate: For sub-skills excluded by the Step 3 scope (or set to `review_mode: skip` in Step 4), synthesize `{status: "skipped", output_json: null, summary: ""}` so Step 8 has a complete row for every sub-skill. -Build a `results` table indexed by sub-skill name (`waves-generator`, `object-exclusion-detection`, `anti-patterns`, `analyzing-sql-dynamic-patterns`, `etl-assessment`, `informatica-assessment`). Carry it into Step 7 and Step 8. +Build a `results` table indexed by sub-skill name (`waves-generator`, `object-exclusion-detection`, `effort-estimate`, `anti-patterns`, `analyzing-sql-dynamic-patterns`, `etl-assessment`, `informatica-assessment`). Carry it into Step 7 and Step 8. ## Step 7: Generate Unified HTML Report @@ -432,13 +465,14 @@ If the report command fails, record the failure and proceed to Step 8 anyway — ## Step 8: Surface Results + Retry -Print a status table from the `results` collected in Step 6, one line per sub-skill, in this order: `waves-generator`, `object-exclusion-detection`, `anti-patterns`, `analyzing-sql-dynamic-patterns`, `etl-assessment`. +Print a status table from the `results` collected in Step 6, one line per sub-skill, in this order: `waves-generator`, `object-exclusion-detection`, `effort-estimate`, `anti-patterns`, `analyzing-sql-dynamic-patterns`, `etl-assessment`. Format: ``` waves-generator ok () object-exclusion-detection ok () +effort-estimate ok () anti-patterns ok () analyzing-sql-dynamic-patterns FAIL etl-assessment skip @@ -519,6 +553,10 @@ Detect user intent and load the appropriate sub-skill: - Triggers: "temporary objects", "staging objects", "deprecated", "exclude objects", "test objects", "cleanup" - Load: `object_exclusion_detection/SKILL.md` +**Effort Estimates** - Generate migration effort estimates from SnowConvert reports: +- Triggers: "effort estimate", "effort estimates", "migration effort", "FDE hours", "how long will migration take" +- Load: `effort-estimate/SKILL.md` + **Anti-Patterns** - Surface migration converns from existing SnowConvert findings (SQL Server only): - Triggers: "anti-patterns", "anti patterns", "risk analysis", "performance risks", "collation risks", "semantic risks", "architecture blockers" - Load: `anti-patterns/SKILL.md` @@ -818,6 +856,7 @@ If any answer is "No", go back and use the correct script. - `waves-generator/SKILL.md` - Algorithm details, partition creation - `object_exclusion_detection/SKILL.md` - Pattern definitions, naming conventions +- `effort-estimate/SKILL.md` - FDE hour estimates from SnowConvert reports (SQL Server, Redshift) - `anti-patterns/SKILL.md` - Curated SnowConvert issue-code catalog → customer-facing risk buckets (SQL Server only) - `analyzing-sql-dynamic-patterns/SKILL.md` - Pattern classification, complexity scoring - `etl-assessment/SKILL.md` - SSIS package analysis, control flow, data flow pipelines diff --git a/plugin/skills/migration/assessment/effort-estimate/SKILL.md b/plugin/skills/migration/assessment/effort-estimate/SKILL.md new file mode 100644 index 0000000..219d5e7 --- /dev/null +++ b/plugin/skills/migration/assessment/effort-estimate/SKILL.md @@ -0,0 +1,37 @@ +--- +name: effort-estimate +description: Generates migration effort estimates by running `scai assessment effort-estimate`, which writes JSON for the assessment multi-report. Supports SQL Server and Redshift. +parent_skill: assessment +license: Proprietary. See License-Skills for complete terms +--- + +# Effort Estimates + +Thin wrapper over `scai assessment effort-estimate`. SCAI reads SnowConvert reports, calculates FDE hours, and writes one timestamped JSON the parent assessment multi-report consumes. + +- **Supported dialects:** SQL Server, Redshift. Any other dialect aborts with `ASM0031` — report `skipped`, not an error. + +## Run + +```bash +scai assessment effort-estimate +``` + +Writes `/artifacts/assessment/effort-estimates-YYYYMMDD_HHMMSS.json`. Return that path to the parent; the multi-report auto-discovers it from `--project-dir`. Never parse or rewrite the file. + +## Sub-agent contract + +On entry, call `configure` with `project_dir` from the parent's context block. Take no user prompts. On completion return **JSON only**: + +```json +{ + "sub_skill": "effort-estimate", + "status": "ok", + "output_json": "", + "summary": "", + "error": null +} +``` + +- Unsupported dialect (`ASM0031`): `status` `"skipped"`, `output_json` `null`, `error` `null`. +- Any other failure: `status` `"error"`, `output_json` `null`, `error` `""`. diff --git a/plugin/skills/migration/assessment/scripts/Base_estimates.csv b/plugin/skills/migration/assessment/scripts/Base_estimates.csv deleted file mode 100644 index 6621839..0000000 --- a/plugin/skills/migration/assessment/scripts/Base_estimates.csv +++ /dev/null @@ -1,19 +0,0 @@ -Section,Key,Migration Component,Object Type,Quantity Rule,Baseline Hours,Small,Medium,Large,Comments -meta,workload_small_max_objects,,,,,500,,,Inclusive maximum object count for a Small workload tier -meta,workload_medium_max_objects,,,,,1500,,,Inclusive maximum object count for a Medium workload tier; Large is above this -rate,tables_views_flat,,,,4,,,,Flat hours for all tables or all views when at least one object exists -rate,code_conversion_per_object,,,,1,,,,Hours per function or stored procedure (conversion and unit testing) -rate,data_migration_flat,,,,8,,,,Constant hours for DMVA setup and data refresh rows -phase,optimization,Optimization,Refactoring & Optimization,flat_budget,,4,8,16,Fixed budget scaled by workload tier -phase,integration_testing,Integration Testing,Integration Testing (Create And Execute Test Cases),flat_budget,,8,16,32,Fixed budget scaled by workload tier -phase,integration_testing_bug_fix,Integration Testing Bug Fix,Stabilization & Bug Fixing,flat_budget,,4,8,16,Fixed budget scaled by workload tier -phase,uat,UAT,UAT,flat_budget,,4,8,16,Fixed budget scaled by workload tier -phase,delivery,Delivery,Stabilization & Bug Fixing,flat_budget,,4,8,16,Fixed budget scaled by workload tier -calculator,data_migration_dmva,Data Migration,Data Migration and Validation using DMVA,constant,8,,,,Includes setup and not actual data movement time -calculator,data_migration_refresh,Data Migration,Data Refresh and Validation,constant,8,,,,Includes setup and not actual data movement time -calculator,code_tables,Code Conversion,Tables,ddl_table_count,4,,,,Flat 4h for all tables regardless of count -calculator,code_views,Code Conversion,Views,ddl_view_count,4,,,,Flat 4h for all views regardless of count -calculator,code_functions,Code Conversion,Functions,ddl_function_count,1,,,,1h per object -calculator,code_procedures,Code Conversion,Stored Procedures,ddl_procedure_count,1,,,,1h per object -calculator,test_functions,Code Conversion Testing,Unit Testing Functions,ddl_function_count,1,,,,1h per object -calculator,test_procedures,Code Conversion Testing,Unit Testing Stored Procedures,ddl_procedure_count,1,,,,1h per object diff --git a/plugin/skills/migration/assessment/scripts/Base_estimates.redshift.csv b/plugin/skills/migration/assessment/scripts/Base_estimates.redshift.csv deleted file mode 100644 index a43adc8..0000000 --- a/plugin/skills/migration/assessment/scripts/Base_estimates.redshift.csv +++ /dev/null @@ -1,22 +0,0 @@ -Section,Key,Migration Component,Object Type,Quantity Rule,Baseline Hours,Small,Medium,Large,Comments -meta,workload_small_max_objects,,,,,500,,,Inclusive maximum object count for a Small workload tier -meta,workload_medium_max_objects,,,,,1500,,,Inclusive maximum object count for a Medium workload tier; Large is above this -meta,conversion_weighted,,,,,1,,,When 1 code conversion is charged only for what SnowConvert did not convert: per-object types scale by (1 - LoC conversion rate); flat categories cost 0h once every object converted and the full budget while any object needs manual work; objects with no parseable LoCConversionPercentage count as fully manual. Unit testing is never weighted -rate,tables_views_flat,,,,4,,,,Flat hours for all tables or all views when at least one object exists -rate,code_conversion_per_object,,,,1,,,,Hours per function or stored procedure (conversion and unit testing) -rate,data_migration_flat,,,,8,,,,Constant hours for DMVA setup and data refresh rows -phase,optimization,Optimization,Refactoring & Optimization,flat_budget,,4,8,16,Fixed budget scaled by workload tier -phase,integration_testing,Integration Testing,Integration Testing (Create And Execute Test Cases),flat_budget,,8,16,32,Fixed budget scaled by workload tier -phase,integration_testing_bug_fix,Integration Testing Bug Fix,Stabilization & Bug Fixing,flat_budget,,4,8,16,Fixed budget scaled by workload tier -phase,uat,UAT,UAT,flat_budget,,4,8,16,Fixed budget scaled by workload tier -phase,delivery,Delivery,Stabilization & Bug Fixing,flat_budget,,4,8,16,Fixed budget scaled by workload tier -calculator,data_migration_dmva,Data Migration,Data Migration and Validation using DMVA,constant,8,,,,Includes setup and not actual data movement time -calculator,data_migration_refresh,Data Migration,Data Refresh and Validation,constant,8,,,,Includes setup and not actual data movement time -calculator,code_tables,Code Conversion,Tables,ddl_table_count,4,,,,Flat 4h for all tables regardless of count -calculator,code_external_tables,Code Conversion,External Tables,ddl_external_table_count,4,,,,Flat 4h for all external tables regardless of count -calculator,code_views,Code Conversion,Views,ddl_view_count,4,,,,Flat 4h for all views regardless of count -calculator,code_materialized_views,Code Conversion,Materialized Views,ddl_materialized_view_count,4,,,,Flat 4h for all materialized views regardless of count -calculator,code_functions,Code Conversion,Functions,ddl_function_count,1,,,,1h per object -calculator,code_procedures,Code Conversion,Stored Procedures,ddl_procedure_count,1,,,,1h per object -calculator,test_functions,Code Conversion Testing,Unit Testing Functions,ddl_function_count,1,,,,1h per object -calculator,test_procedures,Code Conversion Testing,Unit Testing Stored Procedures,ddl_procedure_count,1,,,,1h per object diff --git a/plugin/skills/migration/assessment/scripts/effort_estimation.py b/plugin/skills/migration/assessment/scripts/effort_estimation.py index 9097fea..d51d869 100644 --- a/plugin/skills/migration/assessment/scripts/effort_estimation.py +++ b/plugin/skills/migration/assessment/scripts/effort_estimation.py @@ -10,51 +10,19 @@ # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the permissions and limitations under the License. +# See the License for the specific language governing permissions and +# limitations under the License. -"""Migration effort estimation using flat phase budgets and LOC-based partial formulas.""" +"""Load and render versioned migration effort-estimate artifacts.""" from __future__ import annotations -import csv -import logging +import json import math -import re -from collections import Counter +import sys from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, FrozenSet, List, Optional, Tuple - -logger = logging.getLogger(__name__) - -# Maps display names in Base_estimates CSV → TopLevelCodeUnits Category -_OBJECT_TYPE_MAP = { - "tables": "TABLE", - "views": "VIEW", - "functions": "FUNCTION", - "stored procedures": "PROCEDURE", -} - -# Categories excluded from object counts and effort (not migratable DDL inventory) -_SKIP_EFFORT_CATEGORIES = frozenset({ - "OUT OF SCOPE", - "SESSION / BATCH CONTROL", -}) - -_SKIP_DDL_CATEGORIES = set(_SKIP_EFFORT_CATEGORIES) - -# Migration effort calculator (flat budgets + LOC-based partials) -_TABLE_FLAT_HOURS = 80.0 -_PROC_SUCCESS_FLAT_HOURS = 160.0 -_SYNONYM_FLAT_HOURS = 4.0 -_SUCCESS_REVIEW_HOURS = 0.1 -_ISSUE_SEVERITY_HOURS = {"high": 0.5, "critical": 1.5} -# Fallback dataclass defaults only. Base_estimates.csv's `meta` rows -# (workload_small_max_objects / workload_medium_max_objects) are the single -# source of truth and override these whenever the CSV loads successfully; keep -# both in sync if the bundled CSV's tier thresholds ever change. -_WORKLOAD_SIZE_SMALL_MAX = 500 -_WORKLOAD_SIZE_MEDIUM_MAX = 1500 +from typing import Any, Dict, List, Optional _HOURS_PER_WORK_DAY = 8.0 @@ -72,1112 +40,42 @@ "

" ) _DDL_EXCLUDED_DISPLAY_TYPES = frozenset({"Index", "Flow Control"}) -_DATA_MIGRATION_FLAT_HOURS = 8.0 -DEFAULT_BASE_ESTIMATES_CSV = Path(__file__).parent / "Base_estimates.csv" -_FIXED_BUDGET_COMPONENTS = frozenset( - { - "Fixed Budget", - "Data Migration", - "Optimization", - "Integration Testing", - "Integration Testing Bug Fix", - "UAT", - "Delivery", - } -) - -_CATEGORY_DISPLAY = { - "TABLE": "Table", - "EXTERNAL TABLE": "External Table", - "VIEW": "View", - "MATERIALIZED VIEW": "Materialized View", - "PROCEDURE": "Procedure", - "FUNCTION": "Function", - "INDEX": "Index", - "SYNONYM": "Synonym", - "SCHEMA": "Schema", - "TYPE": "Type", - "DATABASE": "Database", -} - -_DDL_NOTES = { - "Synonym": "Replace with Snowflake aliases or views; flat effort in Fixed Budget", -} - -_CALCULATOR_TO_DDL = { - "tables": "Table", - "views": "View", - "functions": "Function", - "stored procedures": "Procedure", -} - - -# SnowConvert ``SourceLanguage`` values and common aliases for SQL Server / T-SQL. -_SQL_SERVER_DIALECT_VALUES = frozenset({ - "transact", - "sql server", - "sqlserver", - "t-sql", - "tsql", - "t sql", - "mssql", - "ms sql", - "microsoft sql server", -}) - -# SnowConvert ``SourceLanguage`` values and common aliases for Amazon Redshift. -_REDSHIFT_DIALECT_VALUES = frozenset({ - "redshift", - "amazon redshift", - "aws redshift", -}) - -# effort dialect key → the frozenset of source_language values that map to it -_DIALECT_VALUE_SETS: Dict[str, frozenset] = { - "sqlserver": _SQL_SERVER_DIALECT_VALUES, - "redshift": _REDSHIFT_DIALECT_VALUES, -} - -# Each dialect's rates are tuned independently, so each gets its own bundled CSV. -_DIALECT_BASE_ESTIMATES = { - "sqlserver": "Base_estimates.csv", - "redshift": "Base_estimates.redshift.csv", -} - - -def _normalize_dialect(dialect: str) -> str: - return re.sub(r"[\s_\-]+", " ", (dialect or "").strip().lower()) - - -def resolve_effort_dialect(source_language: str) -> Optional[str]: - """Map a project ``source_language`` to a supported effort dialect key. - - Returns ``"sqlserver"``, ``"redshift"``, or ``None`` for unsupported dialects. - Matches the whole normalized string against a set (no substring matching), so a - word merely containing a dialect name never enables the feature. - """ - normalized = _normalize_dialect(source_language) - if not normalized: - return None - for key, values in _DIALECT_VALUE_SETS.items(): - if normalized in values: - return key - return None - - -def read_project_source_language(project_dir: Optional[Path]) -> str: - """Read ``source_language`` from ``{project_dir}/.scai/config/project.yml``. - - ``project.yml`` is a flat ``key: value`` document, so a dependency-free line reader - is used (PyYAML is not available in the assessment environment). Returns ``""`` when - ``project_dir`` is falsy, the file is absent/unreadable, or the key is missing. - """ - if not project_dir: - return "" - yml = Path(project_dir) / ".scai" / "config" / "project.yml" - if not yml.exists(): - return "" - try: - for line in yml.read_text(encoding="utf-8-sig").splitlines(): - stripped = line.strip() - if not stripped or stripped.startswith("#") or ":" not in stripped: - continue - key, _, value = stripped.partition(":") - if key.strip() == "source_language": - return value.strip().strip('"').strip("'") - except OSError: - return "" - return "" - - -def _default_base_estimates_for(dialect_key: str) -> Path: - """Resolve the bundled Base_estimates CSV for a dialect key.""" - filename = _DIALECT_BASE_ESTIMATES.get(dialect_key, "Base_estimates.csv") - return Path(__file__).parent / filename - - -def is_effort_estimation_supported(project_dir: Path) -> bool: - """Return True when the project's ``source_language`` is a supported effort dialect. - - Dialect is read exclusively from ``{project_dir}/.scai/config/project.yml``. Runs - without a project directory are unsupported and produce no effort tab. - """ - return resolve_effort_dialect(read_project_source_language(project_dir)) is not None - - -def classify_workload_size( - total_objects: int, - config: Optional[EffortEstimateConfig] = None, -) -> str: - """Map total workload object count to Small / Medium / Large tier.""" - cfg = config or get_effort_estimate_config() - if total_objects <= cfg.workload_small_max: - return "small" - if total_objects <= cfg.workload_medium_max: - return "medium" - return "large" - - -def workload_size_label( - tier: str, - config: Optional[EffortEstimateConfig] = None, -) -> str: - """Human-readable workload tier label for overview UI.""" - cfg = config or get_effort_estimate_config() - labels = { - "small": f"Small (up to {cfg.workload_small_max:,} objects)", - "medium": ( - f"Medium ({cfg.workload_small_max + 1:,}" - f"–{cfg.workload_medium_max:,} objects)" - ), - "large": f"Large (more than {cfg.workload_medium_max:,} objects)", - } - return labels.get(tier, tier.title()) - - -def count_workload_objects(ddl_summary: Dict[str, Dict[str, Any]]) -> int: - """Count migratable objects used for workload tier sizing.""" - return sum( - st.get("total", 0) - for name, st in ddl_summary.items() - if name not in _DDL_EXCLUDED_DISPLAY_TYPES - ) - - -def compute_tiered_phase_budgets( - total_objects: int, - config: Optional[EffortEstimateConfig] = None, -) -> Dict[Tuple[str, str], float]: - """Return phase fixed budgets scaled by workload size tier. - - Keyed by (component, object_type) lowercased display labels — not the CSV - ``Key`` slug — so callers can look up a phase the same way it's rendered. - """ - cfg = config or get_effort_estimate_config() - tier = classify_workload_size(total_objects, cfg) - return { - (phase.component.lower(), phase.object_type.lower()): phase.hours_by_tier[tier] - for phase in cfg.phase_budgets - } - - -def _parse_tier_to_min_pct(tier_str: str) -> int: - t = tier_str.strip().lower() - if "full" in t: - return 100 - if "partially" in t: - return 0 - m = re.search(r"(\d+)\s*[-–]\s*\d+\s*%", t) - if m: - return int(m.group(1)) - return 0 - - -def _loc_pct_is_measured(loc_pct_str: str) -> bool: - """True when the report actually carries a parseable LoC conversion percentage. - - ``_parse_loc_pct`` infers 100% from ``ConversionStatus == Success`` when the cell is - missing, which is fine for tier bucketing but must not drive an effort discount — - otherwise absent data reads as perfect conversion and zeroes the budget. - """ - return bool(loc_pct_str) and bool(re.sub(r"[^0-9.]", "", str(loc_pct_str))) - - -def _parse_loc_pct(loc_pct_str: str, conversion_status: str) -> float: - if loc_pct_str: - cleaned = re.sub(r"[^0-9.]", "", str(loc_pct_str)) - if cleaned: - return float(cleaned) - status = str(conversion_status).strip().lower() - if status == "success": - return 100.0 - if status in ("failure", "notsupported", "not supported"): - return 0.0 - return 0.0 - - -def _conversion_bucket(category: str, pct: float) -> str: - """Map an object to a tier bucket key used for quantity counting.""" - cat = category.upper() - if cat in ("TABLE", "VIEW", "FUNCTION"): - return "full" if pct >= 100 else "partial" - if cat == "PROCEDURE": - if pct >= 100: - return "full" - if pct >= 75: - return "75-99" - if pct >= 50: - return "50-75" - if pct >= 25: - return "25-50" - return "0-25" - return "partial" - - -def _normalize_status(status: str) -> str: - s = (status or "").strip().lower() - if s == "success": - return "Success" - if s in ("partial", "action required", "actionrequired"): - return "Partial" - if s in ("notsupported", "not supported", "failure"): - return "Unsupported" - return "Partial" @dataclass class CalculatorRow: component: str object_type: str - quantity: Any # int, float, or display str + quantity: Any baseline_hours: float total_baseline_hours: float fde_hours: float + unweighted_fde_hours: float comments: str = "" - # Pre-weighting hours, so the summary can report what conversion automation saved. - unweighted_fde_hours: float = 0.0 - -@dataclass(frozen=True) -class PhaseBudgetTemplate: - key: str - component: str - object_type: str - hours_by_tier: Dict[str, float] - -@dataclass(frozen=True) -class CalculatorRowTemplate: - key: str - component: str - object_type: str - quantity_rule: str - baseline_hours: float - comments: str = "" - - -@dataclass -class EffortEstimateConfig: - workload_small_max: int = _WORKLOAD_SIZE_SMALL_MAX - workload_medium_max: int = _WORKLOAD_SIZE_MEDIUM_MAX - tables_views_flat_hours: float = 4.0 - code_conversion_per_object_hours: float = 1.0 - data_migration_flat_hours: float = _DATA_MIGRATION_FLAT_HOURS - conversion_weighted: bool = False - phase_budgets: Tuple[PhaseBudgetTemplate, ...] = () - calculator_rows: Tuple[CalculatorRowTemplate, ...] = () - - -_CONFIG_CACHE: Dict[str, EffortEstimateConfig] = {} - - -def _parse_config_float(value: Any, default: float = 0.0) -> float: - raw = str(value or "").strip() - if not raw: - return default +def load_effort_assessment(path: Path) -> Optional[Dict[str, Any]]: + """Load an effort artifact and rehydrate its calculator rows for HTML renderers.""" try: - return float(raw) - except (TypeError, ValueError): - logger.warning( - "Base_estimates.csv: could not parse numeric value %r; using default %s", - value, - default, - ) - return default - - -def _parse_config_int(value: Any, default: int = 0) -> int: - return int(_parse_config_float(value, float(default))) - - -def load_effort_estimate_config(csv_path: Path) -> EffortEstimateConfig: - """Load calculator templates, rates, and workload tier thresholds from CSV.""" - path = Path(csv_path) - cache_key = str(path.resolve()) - if cache_key in _CONFIG_CACHE: - return _CONFIG_CACHE[cache_key] - - cfg = EffortEstimateConfig() - phase_rows: List[PhaseBudgetTemplate] = [] - calculator_rows: List[CalculatorRowTemplate] = [] - - with open(path, newline="", encoding="utf-8-sig") as f: - reader = csv.DictReader(f) - for row in reader: - section = (row.get("Section") or "").strip().lower() - key = (row.get("Key") or "").strip() - if not section: - continue - - if section == "meta": - if key == "workload_small_max_objects": - cfg.workload_small_max = _parse_config_int( - row.get("Small"), cfg.workload_small_max - ) - elif key == "workload_medium_max_objects": - cfg.workload_medium_max = _parse_config_int( - row.get("Medium"), cfg.workload_medium_max - ) - elif key == "conversion_weighted": - cfg.conversion_weighted = _parse_config_int(row.get("Small")) == 1 - continue - - if section == "rate": - baseline = _parse_config_float(row.get("Baseline Hours")) - if key == "tables_views_flat": - cfg.tables_views_flat_hours = baseline - elif key == "code_conversion_per_object": - cfg.code_conversion_per_object_hours = baseline - elif key == "data_migration_flat": - cfg.data_migration_flat_hours = baseline - continue - - if section == "phase": - phase_rows.append( - PhaseBudgetTemplate( - key=key, - component=(row.get("Migration Component") or "").strip(), - object_type=(row.get("Object Type") or "").strip(), - hours_by_tier={ - "small": _parse_config_float(row.get("Small")), - "medium": _parse_config_float(row.get("Medium")), - "large": _parse_config_float(row.get("Large")), - }, - ) - ) - continue - - if section == "calculator": - calculator_rows.append( - CalculatorRowTemplate( - key=key, - component=(row.get("Migration Component") or "").strip(), - object_type=(row.get("Object Type") or "").strip(), - quantity_rule=(row.get("Quantity Rule") or "").strip().lower(), - baseline_hours=_parse_config_float(row.get("Baseline Hours")), - comments=(row.get("Comments") or "").strip(), - ) - ) - - cfg.phase_budgets = tuple(phase_rows) - cfg.calculator_rows = tuple(calculator_rows) - _CONFIG_CACHE[cache_key] = cfg - return cfg - - -def get_effort_estimate_config( - csv_path: Optional[Path] = None, -) -> EffortEstimateConfig: - """Return cached effort config, defaulting to bundled Base_estimates.csv.""" - return load_effort_estimate_config(csv_path or DEFAULT_BASE_ESTIMATES_CSV) - - -def _find_toplevel_code_units_csv(reports_dir: Path) -> Optional[Path]: - reports_dir = Path(reports_dir) - dirs = [reports_dir] - sub = reports_dir / "SnowConvert" - if sub.exists(): - dirs.append(sub) - for d in dirs: - for pattern in ("TopLevelCodeUnits.NA.csv", "TopLevelCodeUnits.*.csv"): - matches = list(d.glob(pattern)) - if matches: - return matches[0] - return None - - -def _find_report_csv(reports_dir: Path, base_name: str) -> Optional[Path]: - reports_dir = Path(reports_dir) - dirs = [reports_dir] - sub = reports_dir / "SnowConvert" - if sub.exists(): - dirs.append(sub) - for d in dirs: - for pattern in (f"{base_name}.NA.csv", f"{base_name}.*.csv"): - matches = sorted(d.glob(pattern), key=lambda p: p.stat().st_mtime, reverse=True) - if matches: - return matches[0] - return None - - -def _csv_field(row: dict, *names: str) -> str: - for name in names: - for key, val in row.items(): - if key.strip().lower() == name.lower(): - return str(val or "").strip() - return "" - - -def _severity_bucket(severity: str) -> str: - s = (severity or "").strip().lower() - if s in ("none", "info"): - return "none_info" - if s == "low": - return "low" - if s == "medium": - return "medium" - if s == "high": - return "high" - if s == "critical": - return "critical" - return "none_info" - - -def _new_ddl_row() -> Dict[str, Any]: - return { - "total": 0, - "success": 0, - "partial": 0, - "unsupported": 0, - "lines_of_code": 0, - "issues_none_info": 0, - "issues_low": 0, - "issues_medium": 0, - "issues_high": 0, - "issues_critical": 0, - "effort_hours": 0.0, - "notes": "", - "pct_auto": 0.0, - } - - -def merge_issue_counts_into_ddl( - ddl_summary: Dict[str, Dict[str, Any]], - issues_path: Path, - code_unit_map: Dict[str, str], -) -> None: - """Attach Issues.csv severity counts to each DDL object type.""" - with open(issues_path, newline="", encoding="utf-8-sig") as f: - for row in csv.DictReader(f): - cu_id = _csv_field(row, "CodeUnitId", "Code Unit Id") - category = code_unit_map.get(cu_id, "") - if not category: - continue - display = _CATEGORY_DISPLAY.get(category, category.title()) - if display not in ddl_summary: - continue - bucket = _severity_bucket(_csv_field(row, "Severity")) - key = f"issues_{bucket}" - if key in ddl_summary[display]: - ddl_summary[display][key] += 1 - - -def build_top_ddl_issues(issues_path: Optional[Path], limit: int = 10) -> List[Dict[str, Any]]: - if not issues_path or not issues_path.exists(): - return [] - counts: Counter[str] = Counter() - meta: Dict[str, Dict[str, str]] = {} - with open(issues_path, newline="", encoding="utf-8-sig") as f: - for row in csv.DictReader(f): - code = _csv_field(row, "Code") - if not code: - continue - counts[code] += 1 - if code not in meta: - meta[code] = { - "code": code, - "name": _csv_field(row, "Name"), - "severity": _csv_field(row, "Severity") or "—", - } - result = [] - for code, count in counts.most_common(limit): - entry = dict(meta[code]) - entry["occurrences"] = count - result.append(entry) - return result - - -def assign_ddl_effort_from_calculator( - ddl_summary: Dict[str, Dict[str, Any]], - calculator_rows: List[CalculatorRow], -) -> None: - """Sum FDE hours from calculator tiers into DDL object types.""" - for row in calculator_rows: - comp = row.component.strip().lower() - if not comp.startswith("code conversion"): - continue - m = re.match(r"^(.+?)\s*\(", row.object_type, re.I) - if not m: - continue - display = _CALCULATOR_TO_DDL.get(m.group(1).strip().lower()) - if display and display in ddl_summary: - ddl_summary[display]["effort_hours"] += row.fde_hours - for display, note in _DDL_NOTES.items(): - if display in ddl_summary and not ddl_summary[display]["notes"]: - ddl_summary[display]["notes"] = note - - -def _exclude_effort_category(category: str) -> bool: - return category.upper() in _SKIP_EFFORT_CATEGORIES - - -def _partial_effort_hours(category: str, loc: int) -> float: - """Partial-conversion effort (50% testing-framework discount already applied).""" - if category == "PROCEDURE": - return 0.25 if loc < 50 else 0.50 - if loc < 50: - return 0.25 - if loc <= 200: - return 0.50 - return 1.25 - - -def count_quantities_from_code_units( - csv_path: Path, -) -> Tuple[Dict[str, int], Dict[str, Any], Dict[str, str]]: - """Count unique CodeUnitId per tier and build DDL summary (migration inventory rules). - - Excludes SESSION / BATCH CONTROL and OUT OF SCOPE rows. Each CodeUnitId is - counted once (duplicate CSV rows from batch/session artifacts are ignored). - - Also returns a CodeUnitId → Category map built in this same pass, so callers - merging in Issues.csv counts (see ``merge_issue_counts_into_ddl``) don't need - a second full scan of the same CSV just to look up categories. - """ - tier_counts: Dict[str, int] = {} - ddl: Dict[str, Dict[str, Any]] = {} - code_unit_categories: Dict[str, str] = {} - seen_ids: set[str] = set() - - with open(csv_path, newline="", encoding="utf-8-sig") as f: - for row in csv.DictReader(f): - category = _csv_field(row, "Category").upper() - if not category or _exclude_effort_category(category) or category in ("INDEX", "FLOW CONTROL"): - continue - cu_id = _csv_field(row, "CodeUnitId", "Code Unit Id") - if not cu_id or cu_id in seen_ids: - continue - seen_ids.add(cu_id) - code_unit_categories[cu_id] = category - - loc_pct_cell = _csv_field(row, "LoCConversionPercentage") - pct = _parse_loc_pct(loc_pct_cell, _csv_field(row, "ConversionStatus")) - pct_measured = _loc_pct_is_measured(loc_pct_cell) - status = _normalize_status(_csv_field(row, "ConversionStatus")) - try: - loc = int(_csv_field(row, "Lines of Code", "LinesOfCode") or 0) - except ValueError: - loc = 0 - - if category in _OBJECT_TYPE_MAP.values(): - bucket = _conversion_bucket(category, pct) - obj_label = next( - (k for k, v in _OBJECT_TYPE_MAP.items() if v == category), - category.lower(), - ) - tier_label = _tier_label_for_bucket(bucket) - key = f"{obj_label} ({tier_label})".lower() - tier_counts[key] = tier_counts.get(key, 0) + 1 - - cat_key = _CATEGORY_DISPLAY.get(category, category.title()) - if cat_key not in ddl: - ddl[cat_key] = _new_ddl_row() - ddl[cat_key]["total"] += 1 - if status == "Success": - ddl[cat_key]["success"] += 1 - elif status == "Unsupported": - ddl[cat_key]["unsupported"] += 1 - else: - ddl[cat_key]["partial"] += 1 - ddl[cat_key]["lines_of_code"] += loc - ddl[cat_key].setdefault("_objects", []).append( - { - "category": category, - "status": status, - "loc": loc, - "pct": pct, - "pct_measured": pct_measured, - } - ) - - for stats in ddl.values(): - total = stats["total"] - stats["pct_auto"] = round(stats["success"] / total, 4) if total else 0.0 - - return tier_counts, ddl, code_unit_categories - - -def _per_object_ddl_effort(display: str, obj: Dict[str, Any]) -> float: - """Incremental per-object DDL effort for object types without a flat table/view budget.""" - cat = obj["category"] - loc = obj["loc"] - status = obj["status"] - effort = 0.0 - if status == "Partial": - effort = _partial_effort_hours(cat, loc) - elif status == "Success" and display not in ("Procedure",): - effort = _SUCCESS_REVIEW_HOURS - return effort - - -def _table_view_flat_effort( - stats: Dict[str, Any], - config: Optional[EffortEstimateConfig] = None, -) -> float: - """Flat conversion effort for all tables or all views when at least one object exists.""" - cfg = config or get_effort_estimate_config() - return cfg.tables_views_flat_hours if stats.get("total", 0) else 0.0 - - -def _code_conversion_object_effort( - stats: Dict[str, Any], - config: Optional[EffortEstimateConfig] = None, -) -> float: - """Flat per-object code conversion effort (functions and stored procedures).""" - cfg = config or get_effort_estimate_config() - return round(stats.get("total", 0) * cfg.code_conversion_per_object_hours, 2) - - -# Code-conversion calculator quantity rules → the DDL display type they cost. -_CODE_CONVERSION_RULE_TO_DISPLAY = { - "ddl_table_count": "Table", - "ddl_view_count": "View", - "ddl_external_table_count": "External Table", - "ddl_materialized_view_count": "Materialized View", - "ddl_function_count": "Function", - "ddl_procedure_count": "Procedure", -} - -# DDL display types priced as a single flat budget for the whole category. -_ALWAYS_FLAT_CODE_CONVERSION_TYPES = frozenset({"Table", "View"}) - -# Flat only for dialects whose CSV declares the matching calculator row. Pricing these -# flat without that row moves the DDL breakdown without moving the calculator total the -# Overview card reads, so the two tables would disagree. -_CSV_GATED_FLAT_CODE_CONVERSION_TYPES = frozenset( - {"External Table", "Materialized View"} -) - -_FLAT_CATEGORY_LABELS = { - "External Table": "external tables", - "Materialized View": "materialized views", -} - - -def _flat_code_conversion_types(config: EffortEstimateConfig) -> FrozenSet[str]: - """DDL display types this dialect prices as one flat budget for the whole category.""" - declared = { - _CODE_CONVERSION_RULE_TO_DISPLAY.get(tmpl.quantity_rule) - for tmpl in config.calculator_rows - } - return _ALWAYS_FLAT_CODE_CONVERSION_TYPES | ( - _CSV_GATED_FLAT_CODE_CONVERSION_TYPES & declared - ) - - -def _object_manual_fractions(stats: Dict[str, Any]) -> List[float]: - """Per-object share still needing manual work, clamped to [0, 1]. - - An object whose LoC conversion percentage was never measured counts as fully manual: - the alternative credits automation for data the report does not contain. - """ - fractions: List[float] = [] - for o in stats.get("_objects", []): - if not o.get("pct_measured", False): - fractions.append(1.0) - continue - fractions.append(min(1.0, max(0.0, 1.0 - (o.get("pct", 0.0) / 100.0)))) - return fractions - - -def _type_manual_fraction(stats: Dict[str, Any], flat_budget: bool = False) -> float: - """Share of an object type's budget still needing manual work. - - ``1.0`` means nothing auto-converted (full budget), ``0.0`` means SnowConvert - converted everything (no manual effort). - - Per-object types use the mean, which is identical to summing - ``baseline × (1 - conversion_rate)`` over the objects. - - Flat-budget types are all-or-nothing. Their rate is one budget for the whole - category "regardless of count", so a mean would price the same unconverted object at - the full budget when it stands alone and at ~0 among converted siblings — 4.0h vs - 0.04h for one failed table among 99 successes, and exactly 0.0h among 999. - """ - fractions = _object_manual_fractions(stats) - if not fractions: - return 1.0 - if flat_budget: - return 1.0 if any(f > 0.0 for f in fractions) else 0.0 - return sum(fractions) / len(fractions) - - -def _ddl_notes_for_display( - display: str, - config: EffortEstimateConfig, - flat_types: FrozenSet[str], -) -> str: - if display == "Table": - return ( - f"Flat {config.tables_views_flat_hours:.0f}h for all tables " - "(half business day)" - ) - if display == "View": - return ( - f"Flat {config.tables_views_flat_hours:.0f}h for all views " - "(half business day)" - ) - if display in _CSV_GATED_FLAT_CODE_CONVERSION_TYPES and display in flat_types: - return ( - f"Flat {config.tables_views_flat_hours:.0f}h for all " - f"{_FLAT_CATEGORY_LABELS[display]}" - ) - if display == "Function": - return f"{config.code_conversion_per_object_hours:g}h per function" - if display == "Procedure": - return f"{config.code_conversion_per_object_hours:g}h per stored procedure" - return "" - - -def compute_fixed_budget_items( - ddl_summary: Dict[str, Dict[str, Any]], - total_objects: Optional[int] = None, - config: Optional[EffortEstimateConfig] = None, -) -> Dict[str, float]: - """Flat migration budgets shown separately from per-object DDL effort.""" - cfg = config or get_effort_estimate_config() - if total_objects is None: - total_objects = count_workload_objects(ddl_summary) - tier = classify_workload_size(total_objects, cfg) - items: Dict[str, float] = {} - if ddl_summary.get("Synonym", {}).get("total", 0): - items["Synonym conversion"] = _SYNONYM_FLAT_HOURS - items["Data migration (DMVA)"] = cfg.data_migration_flat_hours - items["Data refresh and validation"] = cfg.data_migration_flat_hours - for phase in cfg.phase_budgets: - label = f"{phase.component} — {phase.object_type}" - items[label] = phase.hours_by_tier[tier] - return items - - -def compute_ddl_effort( - ddl_summary: Dict[str, Dict[str, Any]], - config: Optional[EffortEstimateConfig] = None, -) -> None: - """Apply per-object DDL effort for the conversion assessment table.""" - cfg = config or get_effort_estimate_config() - flat_types = _flat_code_conversion_types(cfg) - for display in list(ddl_summary.keys()): - if display in _DDL_EXCLUDED_DISPLAY_TYPES: - ddl_summary.pop(display, None) - continue - - stats = ddl_summary[display] - stats["effort_hours"] = 0.0 - objects = stats.get("_objects", []) - - note = _DDL_NOTES.get(display) or _ddl_notes_for_display(display, cfg, flat_types) - if note: - stats["notes"] = note - - if display in flat_types: - stats["effort_hours"] = _table_view_flat_effort(stats, cfg) - elif display in ("Function", "Procedure"): - stats["effort_hours"] = _code_conversion_object_effort(stats, cfg) - else: - for obj in objects: - stats["effort_hours"] += _per_object_ddl_effort(display, obj) - stats["effort_hours"] += ( - stats.get("issues_high", 0) * _ISSUE_SEVERITY_HOURS["high"] - + stats.get("issues_critical", 0) * _ISSUE_SEVERITY_HOURS["critical"] - ) - - # Scale code-conversion effort by the un-converted share so the breakdown - # matches the weighted calculator (fully auto-converted types read 0h). - if cfg.conversion_weighted and display in _CODE_CONVERSION_RULE_TO_DISPLAY.values(): - stats["effort_hours"] *= _type_manual_fraction( - stats, flat_budget=display in flat_types - ) - - stats["effort_hours"] = round(stats["effort_hours"], 2) - - -def _calc_row( - component: str, - object_type: str, - quantity: Any, - baseline: float, - total_baseline: float, - fde: float, - comments: str = "", -) -> CalculatorRow: - return CalculatorRow( - component=component, - object_type=object_type, - quantity=quantity, - baseline_hours=baseline, - total_baseline_hours=round(total_baseline, 2), - fde_hours=round(fde, 2), - comments=comments, - unweighted_fde_hours=round(fde, 2), - ) - - -def _calculator_code_conversion_fde( - fde_mode: str, - tier_key: str, - qty: int, - tier_counts: Dict[str, int], - ddl_summary: Dict[str, Dict[str, Any]], -) -> float: - """Compute FDE hours for one code-conversion calculator row.""" - if fde_mode == "flat_zero": - return 0.0 - if fde_mode == "flat_table": - return _TABLE_FLAT_HOURS if ddl_summary.get("Table", {}).get("total", 0) else 0.0 - if fde_mode == "flat_proc": - return _PROC_SUCCESS_FLAT_HOURS if qty else 0.0 - if fde_mode == "success_review": - return qty * _SUCCESS_REVIEW_HOURS - if fde_mode == "partial_view": - return sum( - _partial_effort_hours("VIEW", o["loc"]) - for o in ddl_summary.get("View", {}).get("_objects", []) - if o["status"] == "Partial" - ) - if fde_mode == "partial_function": - return sum( - _partial_effort_hours("FUNCTION", o["loc"]) - for o in ddl_summary.get("Function", {}).get("_objects", []) - if o["status"] == "Partial" - ) - if fde_mode == "partial_proc_tier": - fde = 0.0 - for o in ddl_summary.get("Procedure", {}).get("_objects", []): - if o["status"] != "Partial": - continue - bucket = _conversion_bucket("PROCEDURE", o["pct"]) - label = _tier_label_for_bucket(bucket).lower() - if tier_key.endswith(f"({label})"): - fde += _partial_effort_hours("PROCEDURE", o["loc"]) - return fde - return 0.0 - - -def _resolve_calculator_quantity( - rule: str, - ddl_summary: Dict[str, Dict[str, Any]], - tier_counts: Dict[str, int], -) -> Tuple[Any, int]: - if rule == "constant": - return "Constant", 1 - if rule == "ddl_table_count": - qty = ddl_summary.get("Table", {}).get("total", 0) - return qty, qty - if rule == "ddl_view_count": - qty = ddl_summary.get("View", {}).get("total", 0) - return qty, qty - if rule == "ddl_external_table_count": - qty = ddl_summary.get("External Table", {}).get("total", 0) - return qty, qty - if rule == "ddl_materialized_view_count": - qty = ddl_summary.get("Materialized View", {}).get("total", 0) - return qty, qty - if rule == "ddl_function_count": - qty = ddl_summary.get("Function", {}).get("total", 0) - return qty, qty - if rule == "ddl_procedure_count": - qty = ddl_summary.get("Procedure", {}).get("total", 0) - return qty, qty - logger.warning( - "Base_estimates.csv: unrecognized Quantity Rule %r; row will render as 0. " - "Known rules: constant, ddl_table_count, ddl_view_count, " - "ddl_external_table_count, ddl_materialized_view_count, ddl_function_count, " - "ddl_procedure_count.", - rule, - ) - return 0, 0 - - -def _resolve_calculator_fde( - rule: str, - qty_num: int, - baseline: float, - comments: str, - config: EffortEstimateConfig, -) -> float: - if "included" in comments.lower(): - return 0.0 - if rule == "constant": - return baseline - if rule in ( - "ddl_table_count", - "ddl_view_count", - "ddl_external_table_count", - "ddl_materialized_view_count", - ): - return baseline if qty_num > 0 else 0.0 - if rule in ("ddl_function_count", "ddl_procedure_count"): - return round(baseline * qty_num, 2) - return round(baseline * qty_num, 2) - - -def build_effort_calculator( - tier_counts: Dict[str, int], - ddl_summary: Dict[str, Dict[str, Any]], - config: EffortEstimateConfig, -) -> List[CalculatorRow]: - """Build migration calculator rows from Base_estimates.csv templates.""" - rows: List[CalculatorRow] = [] - total_objects = count_workload_objects(ddl_summary) - workload_tier = classify_workload_size(total_objects, config) - tier_comment = f"{workload_size_label(workload_tier, config)} flat budget" - flat_types = _flat_code_conversion_types(config) - - for tmpl in config.calculator_rows: - qty_display, qty_num = _resolve_calculator_quantity( - tmpl.quantity_rule, - ddl_summary, - tier_counts, - ) - baseline = tmpl.baseline_hours - naive_fde = _resolve_calculator_fde( - tmpl.quantity_rule, qty_num, baseline, tmpl.comments, config - ) - fde = naive_fde - # Charge manual effort only for the portion SnowConvert did not auto-convert. - # Applies to code conversion only — converted objects still need unit testing. - if config.conversion_weighted and tmpl.component.strip().lower() == "code conversion": - display = _CODE_CONVERSION_RULE_TO_DISPLAY.get(tmpl.quantity_rule) - if display: - fraction = _type_manual_fraction( - ddl_summary.get(display, {}), - flat_budget=display in flat_types, - ) - fde = round(naive_fde * fraction, 2) - total_baseline = ( - baseline - if tmpl.quantity_rule == "constant" - else round(baseline * qty_num, 2) - ) - row = _calc_row( - tmpl.component, - tmpl.object_type, - qty_display, - baseline, - total_baseline, - fde, - tmpl.comments, - ) - row.unweighted_fde_hours = round(naive_fde, 2) - rows.append(row) - - for phase in config.phase_budgets: - hours = phase.hours_by_tier[workload_tier] - rows.append( - _calc_row( - phase.component, - phase.object_type, - "Flat budget", - hours, - hours, - hours, - tier_comment, - ) - ) - - return rows - - -def _tier_label_for_bucket(bucket: str) -> str: - mapping = { - "full": "Full Converted", - "partial": "Partially Converted", - "75-99": "75-99% Converted", - "50-75": "50-75% Converted", - "25-50": "25-50% Converted", - "0-25": "0-25% Converted", - } - return mapping.get(bucket, bucket) - - -def _is_fixed_budget_calculator_row(row: CalculatorRow) -> bool: - return row.component.strip() in _FIXED_BUDGET_COMPONENTS - - -def _code_conversion_fde(rows: List[CalculatorRow]) -> float: - return round( - sum(r.fde_hours for r in rows if r.component.strip() == "Code Conversion"), - 2, - ) - - -def _code_conversion_testing_fde(rows: List[CalculatorRow]) -> float: - return round( - sum( - r.fde_hours - for r in rows - if r.component.strip().lower() == "code conversion testing" - ), - 2, - ) - - -def summarize_calculator( - rows: List[CalculatorRow], - ddl_summary: Dict[str, Any], - config: Optional[EffortEstimateConfig] = None, -) -> Dict[str, Any]: - """Aggregate totals for overview summary cards.""" - cfg = config or get_effort_estimate_config() - total_baseline = round(sum(r.total_baseline_hours for r in rows), 2) - ddl_objects = count_workload_objects(ddl_summary) - workload_size_tier = classify_workload_size(ddl_objects, cfg) - fixed_budget_items = compute_fixed_budget_items(ddl_summary, ddl_objects, cfg) - - conversion_fde = _code_conversion_fde(rows) - conversion_naive = round( - sum( - r.unweighted_fde_hours - for r in rows - if r.component.strip() == "Code Conversion" - ), - 2, - ) - hours_saved = round(max(0.0, conversion_naive - conversion_fde), 2) - # Share of the conversion budget automation removed. Derived from the same two - # figures as hours_saved, so the headline can never disagree with the hours beside - # it — unlike ddl_auto_pct, which counts objects by ConversionStatus and is free to - # diverge from the LoC-based weighting. - conversion_automated_pct = ( - round(hours_saved / conversion_naive, 4) if conversion_naive else 0.0 - ) - testing_fde = _code_conversion_testing_fde(rows) - # "Synonym conversion" is a fixed-budget line item with no matching - # Base_estimates.csv calculator row (no Quantity Rule fits a per-synonym flat - # budget), so it must be folded in here explicitly or its hours never reach - # fixed_budget_fde_hours / total_fde_hours despite showing in the tooltip. - synonym_fde = fixed_budget_items.get("Synonym conversion", 0.0) - fixed_budget_fde = round( - sum(r.fde_hours for r in rows if _is_fixed_budget_calculator_row(r)) + synonym_fde, - 2, - ) - ddl_fde = round(conversion_fde + testing_fde, 2) - total_fde = round(sum(r.fde_hours for r in rows) + synonym_fde, 2) + with Path(path).open(encoding="utf-8") as stream: + assessment = json.load(stream) + assessment["calculator_rows"] = [ + CalculatorRow(**row) for row in assessment.get("calculator_rows") or [] + ] + return assessment + except (OSError, json.JSONDecodeError, TypeError, ValueError, AttributeError) as exc: + print(f"Warning: Could not load effort estimates data: {exc}", file=sys.stderr) + return None - ddl_success = sum(s["success"] for s in ddl_summary.values()) - ddl_auto_pct = round(ddl_success / ddl_objects, 4) if ddl_objects else 0.0 - return { - "total_baseline_hours": total_baseline, - "total_fde_hours": total_fde, - "ddl_fde_hours": ddl_fde, - "conversion_fde_hours": conversion_fde, - "code_conversion_naive_hours": conversion_naive, - "hours_saved_by_automation": hours_saved, - "conversion_automated_pct": conversion_automated_pct, - "conversion_weighted": cfg.conversion_weighted, - "testing_fde_hours": testing_fde, - "fixed_budget_fde_hours": fixed_budget_fde, - "fixed_budget_items": fixed_budget_items, - "workload_size_tier": workload_size_tier, - "workload_object_count": ddl_objects, - "workload_small_max": cfg.workload_small_max, - "workload_medium_max": cfg.workload_medium_max, - "ddl_objects": ddl_objects, - "ddl_auto_pct": ddl_auto_pct, - "ddl_summary": ddl_summary, +def workload_size_label(tier: str, small_max: int, medium_max: int) -> str: + """Human-readable workload tier label using thresholds from the artifact.""" + labels = { + "small": f"Small (up to {small_max:,} objects)", + "medium": f"Medium ({small_max + 1:,}–{medium_max:,} objects)", + "large": f"Large (more than {medium_max:,} objects)", } + return labels.get(tier, tier.title()) def _esc(text: Any) -> str: @@ -1198,20 +96,24 @@ def _days_label(days: int) -> str: def _render_fixed_budget_info_icon( - fixed_items: Dict[str, float], + fixed_items: Dict[str, Optional[float]], workload_tier: str, total_objects: int, + small_max: int, + medium_max: int, ) -> str: """Info icon with hover tooltip listing fixed budget line items.""" if not fixed_items: return "" header = ( f"
" - f"{_esc(workload_size_label(workload_tier))} · {total_objects:,} objects" + f"{_esc(workload_size_label(workload_tier, small_max, medium_max))} · " + f"{total_objects:,} objects" f"
" ) tooltip_lines = header + "".join( - f"
{_esc(label)}: {hours:g} h
" + f"
{_esc(label)}: " + f"{f'{hours:g} h' if hours is not None else 'N/A'}
" for label, hours in fixed_items.items() ) return ( @@ -1243,12 +145,18 @@ def render_overview_section_b_html(assessment: Dict[str, Any]) -> str: fixed_items = s.get("fixed_budget_items", {}) workload_tier = s.get("workload_size_tier", "small") workload_objects = s.get("workload_object_count", s.get("ddl_objects", 0)) + small_max = s.get("workload_small_max", 500) + medium_max = s.get("workload_medium_max", 1500) fixed_info_icon = _render_fixed_budget_info_icon( - fixed_items, workload_tier, workload_objects + fixed_items, + workload_tier, + workload_objects, + small_max, + medium_max, ) workload_subtitle = ( f"
" - f"{_esc(workload_size_label(workload_tier))}
" + f"{_esc(workload_size_label(workload_tier, small_max, medium_max))}" ) total_days = _hours_to_rounded_days(s.get("total_fde_hours", 0)) @@ -1278,7 +186,7 @@ def render_overview_section_b_html(assessment: Dict[str, Any]) -> str:
-
{_days_label(total_days)}
+
{_esc(_days_label(total_days))}
Total Effort
@@ -1325,6 +233,24 @@ def render_overview_section_b_html(assessment: Dict[str, Any]) -> str: """ +def _new_ddl_row() -> Dict[str, Any]: + return { + "total": 0, + "success": 0, + "partial": 0, + "unsupported": 0, + "lines_of_code": 0, + "issues_none_info": 0, + "issues_low": 0, + "issues_medium": 0, + "issues_high": 0, + "issues_critical": 0, + "effort_hours": 0.0, + "notes": "", + "pct_auto": 0.0, + } + + def _render_ddl_assessment_table( ddl_summary: Dict[str, Dict[str, Any]], testing_fde: float = 0.0, @@ -1440,10 +366,10 @@ def _render_top_issues_section(top_issues: List[Dict[str, Any]]) -> str:
""" -def _render_effort_formulas_legend(s: Dict[str, Any]) -> str: +def _render_effort_formulas_legend(summary: Dict[str, Any]) -> str: """Collapsible legend explaining how each effort figure is derived.""" weighting_note = "" - if s.get("conversion_weighted"): + if summary.get("conversion_weighted"): weighting_note = ( "

Automated conversion: code-conversion effort is charged " "only for the share SnowConvert did not convert automatically. Per-object " @@ -1451,7 +377,7 @@ def _render_effort_formulas_legend(s: Dict[str, Any]) -> str: "Flat-category budgets (tables, views) are all-or-nothing: a category costs 0h " "once every object in it auto-converted, and its full budget while any object " "still needs manual work. " - f"On this workload automation avoided ≈ {s.get('hours_saved_by_automation', 0):,.0f} h " + f"On this workload automation avoided ≈ {summary.get('hours_saved_by_automation', 0):,.0f} h " "of manual conversion.

" ) return f""" @@ -1460,7 +386,7 @@ def _render_effort_formulas_legend(s: Dict[str, Any]) -> str:

DDL: Tables = flat 4h total; Views = flat 4h total; Functions and stored procedures = 1h each for conversion (plus 1h each for unit testing in the calculator).

{weighting_note} -

Fixed Budget: Data migration setup plus phase budgets scaled by workload size — Small (≤{s.get('workload_small_max', 500):,} objects), Medium ({s.get('workload_small_max', 500) + 1:,}–{s.get('workload_medium_max', 1500):,}), Large (>{s.get('workload_medium_max', 1500):,}). Rates are configured in Base_estimates.csv.

+

Fixed Budget: Data migration setup plus phase budgets scaled by workload size — Small (≤{summary.get('workload_small_max', 500):,} objects), Medium ({summary.get('workload_small_max', 500) + 1:,}–{summary.get('workload_medium_max', 1500):,}), Large (>{summary.get('workload_medium_max', 1500):,}).

Sources: SnowConvert conversion statistics — object conversion rates and issue severity counts.

""" @@ -1469,15 +395,15 @@ def _render_effort_formulas_legend(s: Dict[str, Any]) -> str: def render_effort_tab_html(assessment: Dict[str, Any]) -> str: """Full effort page aligned with the migration assessment workbook sections.""" rows = assessment["calculator_rows"] - s = assessment["summary"] - ddl = s.get("ddl_summary", {}) + summary = assessment["summary"] + ddl = summary.get("ddl_summary", {}) top_issues = assessment.get("top_issues", []) - ddl_effort = s.get("ddl_fde_hours", 0) + ddl_effort = summary.get("ddl_fde_hours", 0) calc_body = _render_calculator_rows(rows) issues_section = _render_top_issues_section(top_issues) - formulas_legend = _render_effort_formulas_legend(s) + formulas_legend = _render_effort_formulas_legend(summary) return f"""
@@ -1495,15 +421,15 @@ def render_effort_tab_html(assessment: Dict[str, Any]) -> str:
-
{s.get('ddl_objects', 0):,}
+
{summary.get('ddl_objects', 0):,}
Total DDL Objects
-
{s.get('ddl_auto_pct', 0) * 100:.1f}%
+
{summary.get('ddl_auto_pct', 0) * 100:.1f}%
DDL Auto-Converted
-
{s.get('total_fde_hours', 0):,.1f} h
+
{summary.get('total_fde_hours', 0):,.1f} h
Total Effort
@@ -1531,7 +457,7 @@ def render_effort_tab_html(assessment: Dict[str, Any]) -> str: Effort (h) Notes - {_render_ddl_assessment_table(ddl, s.get("testing_fde_hours", 0))} + {_render_ddl_assessment_table(ddl, summary.get("testing_fde_hours", 0))}
@@ -1553,7 +479,7 @@ def render_effort_tab_html(assessment: Dict[str, Any]) -> str: {calc_body} TOTAL - {s.get('total_fde_hours', 0):,.1f} + {summary.get('total_fde_hours', 0):,.1f} @@ -1563,72 +489,3 @@ def render_effort_tab_html(assessment: Dict[str, Any]) -> str: {formulas_legend}
""" - - -def _warn_on_unmeasured_conversion( - ddl_summary: Dict[str, Dict[str, Any]], - csv_path: Path, -) -> None: - """Warn when conversion weighting ran without LoC conversion data to weight by.""" - objects = [o for s in ddl_summary.values() for o in s.get("_objects", [])] - if not objects: - return - unmeasured = sum(1 for o in objects if not o.get("pct_measured", False)) - if not unmeasured: - return - logger.warning( - "%s: %d of %d code units have no parseable LoCConversionPercentage. Conversion " - "weighting charges full manual effort for those objects rather than crediting " - "automation for data the report does not contain.", - csv_path.name, - unmeasured, - len(objects), - ) - - -def build_effort_assessment( - reports_dir: Path, - base_estimates_csv: Optional[Path] = None, - project_dir: Optional[Path] = None, -) -> Optional[Dict[str, Any]]: - """Build the full effort assessment payload for a supported source dialect. - - The dialect is resolved from ``{project_dir}/.scai/config/project.yml`` only. - ``base_estimates_csv`` overrides the CSV; when omitted the per-dialect bundled CSV - is used (``Base_estimates.redshift.csv`` for Redshift, ``Base_estimates.csv`` for - SQL Server). Returns ``None`` for unsupported dialects so the tab is omitted - entirely rather than rendered half-populated. - """ - source_dialect = read_project_source_language(project_dir) - dialect_key = resolve_effort_dialect(source_dialect) - if dialect_key is None: - return None - - csv_path = _find_toplevel_code_units_csv(reports_dir) - if not csv_path: - return None - - config_csv = base_estimates_csv or _default_base_estimates_for(dialect_key) - config = load_effort_estimate_config(config_csv) - tier_counts, ddl_summary, code_unit_categories = count_quantities_from_code_units(csv_path) - issues_path = _find_report_csv(reports_dir, "Issues") - if issues_path: - merge_issue_counts_into_ddl(ddl_summary, issues_path, code_unit_categories) - - calculator_rows = build_effort_calculator(tier_counts, ddl_summary, config) - compute_ddl_effort(ddl_summary, config) - top_issues = build_top_ddl_issues(issues_path) - summary = summarize_calculator(calculator_rows, ddl_summary, config) - - if config.conversion_weighted: - _warn_on_unmeasured_conversion(ddl_summary, csv_path) - - for stats in ddl_summary.values(): - stats.pop("_objects", None) - - return { - "source_dialect": source_dialect, - "calculator_rows": calculator_rows, - "summary": summary, - "top_issues": top_issues, - } diff --git a/plugin/skills/migration/assessment/scripts/generate_multi_report.py b/plugin/skills/migration/assessment/scripts/generate_multi_report.py index 3644af6..d08f354 100644 --- a/plugin/skills/migration/assessment/scripts/generate_multi_report.py +++ b/plugin/skills/migration/assessment/scripts/generate_multi_report.py @@ -96,11 +96,10 @@ print(f"Warning: Anti-patterns report generator not available: {e}", file=sys.stderr) ANTI_PATTERNS_SUPPORT = False -# Effort estimation (dialect-gated calculator tab: SQL Server, Redshift) +# Effort estimation artifact loader and renderers try: from effort_estimation import ( - build_effort_assessment, - is_effort_estimation_supported, + load_effort_assessment, render_effort_tab_html, render_overview_section_b_html, ) @@ -1025,7 +1024,7 @@ def generate_multi_report( anti_patterns_json: Path = None, informatica_json: Path = None, informatica_source_dir: Path = None, - base_estimates_csv: Path = None, + effort_estimates_json: Path = None, project_dir: Path = None, ) -> None: """Generate multi-tab HTML report""" @@ -1097,8 +1096,18 @@ def generate_multi_report( print(f"Warning: Could not load anti-patterns data: {e}", file=sys.stderr) has_anti_patterns = False - if not exclusion_data and not dynamic_sql_data and not waves_info and not ssis_data and not informatica_data and not has_anti_patterns: - raise ValueError("At least one data source (exclusion, dynamic SQL, waves, SSIS, Informatica, or anti-patterns) must be provided") + effort_assessment = None + if effort_estimates_json and EFFORT_SUPPORT: + print(f"Loading effort estimates data from {effort_estimates_json}...") + effort_assessment = load_effort_assessment(effort_estimates_json) + if effort_assessment: + print( + f" - Effort estimates: {effort_assessment['summary']['total_fde_hours']:,.1f} " + f"FDE hours ({effort_assessment['source_dialect']})" + ) + + if not exclusion_data and not dynamic_sql_data and not waves_info and not ssis_data and not informatica_data and not has_anti_patterns and not effort_estimates_json: + raise ValueError("At least one data source (exclusion, dynamic SQL, waves, SSIS, Informatica, anti-patterns, or effort estimates) must be provided") # Process exclusion data — schema produced by `scai assessment object-exclusion` # is the single source of truth; field names below match that schema directly. @@ -1209,30 +1218,6 @@ def tag_objects_for_export(objects: List, category: str) -> List: overview_stats['source_dialect'] = scai_lang print(f" - Using source dialect from SQL Dynamic: {scai_lang}") - # Dialect-gated effort calculator (SQL Server, Redshift). The dialect is read from - # {project_dir}/.scai/config/project.yml only, so runs without --project-dir get no effort tab. - effort_assessment = None - if EFFORT_SUPPORT and snowconvert_reports_dir and project_dir: - reports_path = Path(snowconvert_reports_dir) - project_path = Path(project_dir) - if is_effort_estimation_supported(project_path): - effort_assessment = build_effort_assessment( - reports_path, - base_estimates_csv, # None → per-dialect bundled CSV is resolved - project_dir=project_path, - ) - if effort_assessment: - print( - f" - Effort estimates: {effort_assessment['summary']['total_fde_hours']:,.1f} " - f"FDE hours ({effort_assessment['source_dialect']})" - ) - else: - print( - " - Warning: supported dialect but effort assessment could not be built " - "(missing TopLevelCodeUnits report?)", - file=sys.stderr, - ) - # Set default tab to overview if available if waves_json: default_tab = 'overview' @@ -2671,6 +2656,11 @@ def _callers_count(item: Dict[str, Any]) -> int: background: #D6E6FF; color: #1A6CE7; }} + /* Only nav row carrying a badge: without nowrap the flex label gives way + to it and the wrapped line is clipped by the fixed 30px height. */ + .nav-link[data-tab="effort-estimates"] {{ + white-space: nowrap; + }} .nav-sublist {{ padding: 4px 0 8px 0; }} @@ -6501,11 +6491,9 @@ def main(): ) parser.add_argument( - '--base-estimates', + '--effort-estimates-json', type=Path, - help='Path to Base_estimates CSV file with per-object-type hourly rates. ' - 'Defaults to the bundled CSV for the project source dialect ' - '(Base_estimates.csv for SQL Server, Base_estimates.redshift.csv for Redshift).' + help='Path to effort-estimates JSON produced by `scai assessment effort-estimate`.' ) parser.add_argument( @@ -6516,6 +6504,7 @@ def main(): ) args = parser.parse_args() + explicit_effort_estimates_json = args.effort_estimates_json # --project-dir auto-discovery: fill in registry-dir and snowconvert-reports-dir # from the conventional layout if they weren't set explicitly. @@ -6568,6 +6557,18 @@ def main(): if ap_candidates: args.anti_patterns_json = ap_candidates[-1] print(f"Using anti-patterns JSON: {args.anti_patterns_json}", file=sys.stderr) + if not args.effort_estimates_json: + effort_dir = args.project_dir / "artifacts" / "assessment" + if effort_dir.is_dir(): + effort_candidates = sorted( + effort_dir.glob("effort-estimates-*.json") + ) + if effort_candidates: + args.effort_estimates_json = effort_candidates[-1] + print( + f"Using effort estimates JSON: {args.effort_estimates_json}", + file=sys.stderr, + ) if not args.informatica_json: # Check common locations for Informatica analysis output candidates = [ @@ -6580,8 +6581,8 @@ def main(): print(f"Using Informatica JSON: {args.informatica_json}", file=sys.stderr) break - if not args.exclusion_json and not args.dynamic_sql_json and not args.waves_json and not args.ssis_json and not args.informatica_json and not args.anti_patterns_json and not args.registry_dir: - print("Error: At least one data source (--exclusion-json, --dynamic-sql-json, --waves-json, --ssis-json, --informatica-json, --anti-patterns-json, --registry-dir, or --project-dir) must be provided", file=sys.stderr) + if not args.exclusion_json and not args.dynamic_sql_json and not args.waves_json and not args.ssis_json and not args.informatica_json and not args.anti_patterns_json and not args.effort_estimates_json and not args.registry_dir: + print("Error: At least one data source (--exclusion-json, --dynamic-sql-json, --waves-json, --ssis-json, --informatica-json, --anti-patterns-json, --effort-estimates-json, --registry-dir, or --project-dir) must be provided", file=sys.stderr) print_usage() sys.exit(1) @@ -6605,10 +6606,14 @@ def main(): print(f"Error: Anti-Patterns JSON file not found: {args.anti_patterns_json}", file=sys.stderr) sys.exit(1) - # Validated here rather than at load time: load_effort_estimate_config() opens the - # path unguarded, so a typo reaching the render would abort the whole report. - if args.base_estimates and not args.base_estimates.exists(): - print(f"Error: Base estimates CSV not found: {args.base_estimates}", file=sys.stderr) + if ( + explicit_effort_estimates_json + and not explicit_effort_estimates_json.exists() + ): + print( + f"Error: Effort estimates JSON file not found: {explicit_effort_estimates_json}", + file=sys.stderr, + ) sys.exit(1) # Registry-driven waves data. Two modes: @@ -6669,7 +6674,7 @@ def main(): anti_patterns_json=args.anti_patterns_json, informatica_json=args.informatica_json, informatica_source_dir=getattr(args, 'informatica_source_dir', None), - base_estimates_csv=getattr(args, 'base_estimates', None), + effort_estimates_json=getattr(args, 'effort_estimates_json', None), project_dir=getattr(args, 'project_dir', None), ) except Exception as e: diff --git a/plugin/skills/migration/assessment/scripts/snowconvert_reports/ARCHITECTURE.md b/plugin/skills/migration/assessment/scripts/snowconvert_reports/ARCHITECTURE.md index 90fa7ec..6bf3195 100644 --- a/plugin/skills/migration/assessment/scripts/snowconvert_reports/ARCHITECTURE.md +++ b/plugin/skills/migration/assessment/scripts/snowconvert_reports/ARCHITECTURE.md @@ -282,7 +282,7 @@ ETL doesn't subclass `Element`. It composes a richer domain model: |---|---|---| | CSV parsing, encoding | `snowconvert_reports/loaders/csv_reader.py` | Single implementation for all sub-skills | | Report file discovery | `snowconvert_reports/services/report_finder.py` | Consistent glob patterns | -| Project facts (`project_name`) | `snowconvert_reports/loaders/project_config.py` | Reads `.scai/config/project.yml`, which scai owns and writes. Duplicates the flat-YAML scan in `effort_estimation.py` rather than sharing it — that module deliberately imports nothing from here (`ai/CLAUDE.md` pitfall #8) | +| Project facts (`project_name`) | `snowconvert_reports/loaders/project_config.py` | Reads the scai-owned `.scai/config/project.yml` for assessment-name fallback and metadata tooling | | Report display name (`assessment.json`) | `snowconvert_reports/services/assessment_metadata.py` | Composes both sources behind `resolve_assessment_name()`, so the name offered at the `SKILL.md` prompt and the name rendered in the report cannot disagree. The only module here that **writes** | | Data models (raw rows) | `snowconvert_reports/models/` | One frozen dataclass per CSV file type | | Effort calculation | `snowconvert_reports/services/issue_effort_service.py` | Unified EWI/non-EWI logic | diff --git a/plugin/skills/migration/assessment/scripts/snowconvert_reports/loaders/project_config.py b/plugin/skills/migration/assessment/scripts/snowconvert_reports/loaders/project_config.py index 20a260c..118df0e 100644 --- a/plugin/skills/migration/assessment/scripts/snowconvert_reports/loaders/project_config.py +++ b/plugin/skills/migration/assessment/scripts/snowconvert_reports/loaders/project_config.py @@ -32,11 +32,6 @@ def _read_flat_yaml_value(project_dir: Optional[Path], key: str) -> str: A line reader rather than a YAML parse because PyYAML is not available — ``assessment/pyproject.toml`` declares ``dependencies = []``. Returns ``""`` on every failure so callers never need a ``try``/``except``. - - ``effort_estimation.py`` holds a near-identical scan for ``source_language``. - It is duplicated rather than shared: that module deliberately imports nothing - from ``snowconvert_reports`` (``ai/CLAUDE.md`` pitfall #8), and its copy is - under test. """ if not project_dir: return "" diff --git a/plugin/skills/migration/assessment/scripts/snowconvert_reports/type_coverage.py b/plugin/skills/migration/assessment/scripts/snowconvert_reports/type_coverage.py index bb98b29..19f4ca6 100644 --- a/plugin/skills/migration/assessment/scripts/snowconvert_reports/type_coverage.py +++ b/plugin/skills/migration/assessment/scripts/snowconvert_reports/type_coverage.py @@ -96,15 +96,35 @@ def is_clean(self) -> bool: "SQL Server TIMESTAMP is a synonym for ROWVERSION, " "not a datetime; it migrates as BINARY." ) -_WIDENED_NUMERIC = ( - "Precision and scale are widened on the target, so the converted DDL " - "deliberately differs from the source." -) _SUB_MICROSECOND = ( "The 7th fractional-second digit truncates on readback. On high-precision " "columns that surfaces as level 2 and 3 differences, which are not data loss." ) _INTERVAL = "Compared as a native INTERVAL by default; the handling is configurable." +_ORACLE_CHAR_ANTI_CASE = ( + "Snowflake stores CHAR internally as VARCHAR/TEXT; DM aligns the dict " + "target with runtime rather than SCAI's DDL literal." +) +_ORACLE_RAW_ANTI_CASE = ( + "SCAI drops the size when converting RAW; DM keeps catalog " + "char_length and emits BINARY(n) because it is strictly more " + "informative than SCAI's bare BINARY." +) +_ORACLE_BARE_NUMBER = ( + "SCAI emits NUMBER(38,18) for bare NUMBER; DM cannot distinguish bare " + "NUMBER from NUMBER(*) via catalog and keeps the default helper. Per-column " + "validationCustomTypeRules is the escape hatch." +) +_ORACLE_DATE_AS_TIMESTAMP = ( + "Oracle DATE carries century/year/month/day/hour/minute/second; Snowflake " + "DATE drops the time component, so migrations preserve it as TIMESTAMP_NTZ." +) +_ORACLE_FIXED_ROWID_SIZE = ( + "Emits as VARCHAR(18) per SCAI OraSimpleDataTypeReplacer fixed size." +) +_ORACLE_FIXED_UROWID_SIZE = ( + "Emits as VARCHAR(4000) per SCAI (Oracle SQL Reference default width)." +) def _row(name: str, target: str, validation: str, note: str = "") -> TypeCoverage: @@ -120,8 +140,8 @@ def _row(name: str, target: str, validation: str, note: str = "") -> TypeCoverag _row("SMALLINT", "NUMBER", _S), _row("INT", "NUMBER", _S), _row("BIGINT", "NUMBER", _S), - _row("DECIMAL", "NUMBER(p+2, s+4)", _S, _WIDENED_NUMERIC), - _row("NUMERIC", "NUMBER(p+2, s+4)", _S, _WIDENED_NUMERIC), + _row("DECIMAL", "NUMBER", _S), + _row("NUMERIC", "NUMBER", _S), _row("MONEY", "NUMBER", _S), _row("SMALLMONEY", "NUMBER", _S), _row("FLOAT", "FLOAT", _S), @@ -209,9 +229,80 @@ def _row(name: str, target: str, validation: str, note: str = "") -> TypeCoverag ), ) +_ORACLE: tuple[TypeCoverage, ...] = ( + _row("NUMBER", "NUMBER", _S, _ORACLE_BARE_NUMBER), + _row("INTEGER", "NUMBER", _S), + _row("INT", "NUMBER", _S), + _row("SMALLINT", "NUMBER", _S), + _row("DECIMAL", "NUMBER", _S), + _row("NUMERIC", "NUMBER", _S), + _row("FLOAT", "FLOAT", _S), + _row("REAL", "FLOAT", _S), + _row("BINARY_FLOAT", "FLOAT", _S), + _row("BINARY_DOUBLE", "FLOAT", _S), + _row("CHAR", "VARCHAR", _S, _ORACLE_CHAR_ANTI_CASE), + _row("VARCHAR", "VARCHAR", _S), + _row("VARCHAR2", "VARCHAR", _S), + _row("NCHAR", "VARCHAR", _S), + _row("NVARCHAR2", "VARCHAR", _S), + _row("CLOB", "VARCHAR", _S), + _row("NCLOB", "VARCHAR", _S), + _row("LONG", "VARCHAR", _S), + _row("RAW", "BINARY", _S, _ORACLE_RAW_ANTI_CASE), + _row("LONG RAW", "BINARY", _S), + _row("BLOB", "BINARY", _S), + _row("BFILE", "VARCHAR", _S), + _row("ROWID", "VARCHAR", _S, _ORACLE_FIXED_ROWID_SIZE), + _row("UROWID", "VARCHAR", _S, _ORACLE_FIXED_UROWID_SIZE), + _row("DATE", "TIMESTAMP_NTZ", _S, _ORACLE_DATE_AS_TIMESTAMP), + _row("TIMESTAMP", "TIMESTAMP_NTZ", _S), + _row("TIMESTAMP WITH TIME ZONE", "TIMESTAMP_TZ", _S), + _row("TIMESTAMP WITH LOCAL TIME ZONE", "TIMESTAMP_LTZ", _S), + _row("INTERVAL YEAR TO MONTH", "INTERVAL", _S, _INTERVAL), + _row("INTERVAL DAY TO SECOND", "INTERVAL", _S, _INTERVAL), + _row("BOOLEAN", "BOOLEAN", _S), + _row("BOOL", "BOOLEAN", _S), + _row("JSON", "VARIANT", _S), + _row("XMLTYPE", "VARIANT", _S), + _row("SDO_GEOMETRY", "GEOGRAPHY", _SCHEMA, _CONTENTS_NEVER_COMPARED), + _row("VECTOR", "VECTOR", VALIDATION_UNDOCUMENTED, _UNDOCUMENTED), +) + +_AZURE_SYNAPSE: tuple[TypeCoverage, ...] = ( + _row("BIT", "BOOLEAN", _S), + _row("TINYINT", "NUMBER", _S), + _row("SMALLINT", "NUMBER", _S), + _row("INT", "NUMBER", _S), + _row("BIGINT", "NUMBER", _S), + _row("DECIMAL", "NUMBER", _S), + _row("NUMERIC", "NUMBER", _S), + _row("MONEY", "NUMBER", _S), + _row("SMALLMONEY", "NUMBER", _S), + _row("FLOAT", "FLOAT", _S), + _row("REAL", "FLOAT", _S), + _row("CHAR", "VARCHAR", _S), + _row("VARCHAR", "VARCHAR", _S), + _row("NCHAR", "VARCHAR", _S, "Row comparison applies TRIM when the target is VARCHAR."), + _row("NVARCHAR", "VARCHAR", _S), + _row("SYSNAME", "VARCHAR", _S), + _row("DATE", "DATE", _S), + _row("TIME", "TIME", _S), + _row("DATETIME", "TIMESTAMP_NTZ", _S), + _row("SMALLDATETIME", "TIMESTAMP_NTZ", _S), + _row("DATETIME2", "TIMESTAMP_NTZ", VALIDATION_DRIFT, _SUB_MICROSECOND), + _row("DATETIMEOFFSET", "TIMESTAMP_TZ", VALIDATION_DRIFT, _SUB_MICROSECOND), + _row("BINARY", "BINARY", _S), + _row("VARBINARY", "BINARY", _S), + _row("UNIQUEIDENTIFIER", "VARCHAR", _S, + "Stored as an uppercase UUID string, so case-sensitive joins on it need review."), +) + + COVERAGE: dict[str, tuple[TypeCoverage, ...]] = { "sqlserver": _SQLSERVER, "redshift": _REDSHIFT, + "oracle": _ORACLE, + "azure_synapse": _AZURE_SYNAPSE, } # The six rows where the published page and the shipped orchestrator disagree. diff --git a/plugin/skills/migration/code-conversion-only/SKILL.md b/plugin/skills/migration/code-conversion-only/SKILL.md index 103b5a9..efa9f5a 100644 --- a/plugin/skills/migration/code-conversion-only/SKILL.md +++ b/plugin/skills/migration/code-conversion-only/SKILL.md @@ -1,6 +1,6 @@ --- name: code-conversion-only -description: Convert local source code to Snowflake SQL for code-conversion-only source systems such as Sybase IQ, Azure Synapse, Spark SQL, Databricks SQL, BigQuery, Greenplum, Netezza, Vertica, Hive, and IBM DB2. Optionally repoints Power BI reports. Use when configure returns project_type code_conversion_only for these sources. +description: Convert local source code to Snowflake SQL for code-conversion-only source systems such as Sybase IQ, Azure Synapse, Spark SQL, Databricks SQL, BigQuery, Greenplum, Netezza, Vertica, and Hive. Optionally repoints Power BI reports. Use when configure returns project_type code_conversion_only for these sources. parent_skill: migration license: Proprietary. See License-Skills for complete terms --- @@ -34,7 +34,7 @@ If `configure()` does not already have source_language configured, use the CLI d If the user says "PostgreSQL & Based Languages" but does not specify one, ask whether they mean PostgreSQL, Greenplum, or Netezza. -> **Note on PostgreSQL:** New PostgreSQL projects default to `project_type: full_migration` and are routed through `setup/SKILL.md`. PostgreSQL appears in this table only for **legacy** projects whose `project.yml` already has `project_type: code_conversion_only` persisted from before full-pipeline support shipped. If you reach this skill for PostgreSQL, the legacy project configuration is being honored. +> **Note on PostgreSQL and IBM DB2:** New PostgreSQL and DB2 projects default to `project_type: full_migration` and are routed through `setup/SKILL.md`. They appear in this table only for **legacy** projects whose `project.yml` already has `project_type: code_conversion_only` persisted from before full-pipeline support shipped. If you reach this skill for PostgreSQL or DB2, the legacy project configuration is being honored. ## Workflow @@ -75,29 +75,57 @@ scai code add -i --skip-split --json If the user explicitly wants to replace existing source files, add `--overwrite`. -### Step 3: Check for Power BI Reports +### Step 3: Check for ETL Code + +Any ETL imported by `scai code add` lands in `source/_etl/`. That is where `convert` reads it from. + +Check whether `source/_etl/` exists and contains ETL files — `.dtsx` for SSIS, `.xml` for Informatica PowerCenter (use whichever portable form fits the host). + +- If it is **missing or empty**, there is no ETL to convert; proceed to Step 4. +- If it contains **SSIS** packages only, no conversion-target prompt is needed; proceed to Step 4. +- If it contains **Informatica** PowerCenter XML, ask the remaining ETL questions up front, in one sequence, before running the conversion: + 1. Conversion target, via `ask_user_question` (`multiSelect = false`): + > "How should Informatica mappings be converted? + > 1. **dbt** (default): each mapping becomes a dbt model orchestrated by Snowflake Tasks. + > 2. **Snowflake Scripting** (preview): each mapping becomes a standalone Snowflake stored procedure the Task graph calls. Stabilization and deploy are skipped for this preview flavor." + 2. If the answer is **dbt**, also ask (`multiSelect = false`): "Consolidate dbt model chains to reduce the number of generated model files?". On yes, set `CONSOLIDATE_DBT = true` for Step 5. + 3. If the answer is **Snowflake Scripting**, set `SCRIPTING_MODE = true` for Step 5. + + Persist the choice with the MCP `configure` tool: `etl_informatica_target = "dbt"` or `"scripting"`. + +These questions are still required: the conversion target is not something `scai code add` can infer from the imported files. Ask them on every conversion, re-runs included — `scai code convert` defaults to dbt, so a missing flag in Step 5 silently changes the output. + +### Step 4: Check for Power BI Reports Ask the user: > "Do you have Power BI reports (`.pbit` files) you'd like to repoint to Snowflake?" -If **yes**, load `../powerbi-repointing/SKILL.md`. It collects `PBIT_PATH` and tells you to append `--powerbi-repointing ` to the convert command in Step 4. Return here when complete. +If **yes**, load `../powerbi-repointing/SKILL.md`. It collects `PBIT_PATH` and tells you to append `--powerbi-repointing ` to the convert command in Step 5. Return here when complete. -If **no**, proceed to Step 4. `PBIT_PATH` remains unset; do not pass `--powerbi-repointing` to scai. +If **no**, proceed to Step 5. `PBIT_PATH` remains unset; do not pass `--powerbi-repointing` to scai. -### Step 4: Convert +### Step 5: Convert -Before running, tell the user what the conversion will cover. If `PBIT_PATH` was set, mention Power BI repointing. +Before running, tell the user what the conversion will cover. If `PBIT_PATH` was set, mention Power BI repointing. If `SCRIPTING_MODE` was set, mention the Snowflake Scripting target. -Always include `--json` so the agent can parse the result envelope. Append `--powerbi-repointing ` only if `PBIT_PATH` was set. +Always include `--json` so the agent can parse the result envelope. Start from the base command and append one flag per decision already recorded in the steps above — nothing else: ```bash scai code convert --json ``` -Substitute `` with the actual folder path you stored. Do not emit the literal `` token to the shell. +| Append | When | +|--------|------| +| `--informatica-to-snowflake-scripting` | `SCRIPTING_MODE` was set in Step 3 (Informatica target is Snowflake Scripting) | +| `--consolidate-dbt-model-chains` | `CONSOLIDATE_DBT` was set in Step 3 (Informatica target is dbt and the user chose to consolidate model chains) | +| `--powerbi-repointing ` | `PBIT_PATH` was set in Step 4 | + +The two Informatica flags are mutually exclusive — they come from the same single-select answer, so at most one can apply. Either combines with `--powerbi-repointing`. If none of the conditions hold, run the base command as-is. + +Substitute `` with the actual folder path you stored. Do not emit literal placeholder tokens to the shell. -Per-EWI details (code, description, severity) are written to `reports/SnowConvert/Issues.*.csv`; read those files in Step 5 when working on **Review EWIs** or **Resolve EWIs with Cortex Code**. +Per-EWI details (code, description, severity) are written to `reports/SnowConvert/Issues.*.csv`; read those files in Step 6 when working on **Review EWIs** or **Resolve EWIs with Cortex Code**. For Power BI output paths and the CHECKPOINT addendum, see `../powerbi-repointing/SKILL.md`. @@ -107,10 +135,12 @@ For Power BI output paths and the CHECKPOINT addendum, see `../powerbi-repointin |--------|-------------| | `-x, --show-ewis` | Show detailed EWI breakdown | | `--overwrite-working-directory` | Overwrite output files in `snowflake/` and registry | +| `--informatica-to-snowflake-scripting` | Convert Informatica mappings to standalone Snowflake stored procedures (Snowflake Scripting) instead of dbt projects. Preview flavor. | +| `--consolidate-dbt-model-chains` | Consolidate Informatica dbt model chains to reduce the number of generated model files. Applies when the Informatica target is dbt. | For Power BI options, see `../powerbi-repointing/SKILL.md`. -### Step 5: Ask the user +### Step 6: Ask the user After conversion, ask: diff --git a/plugin/skills/migration/connection/db2-connection/SKILL.md b/plugin/skills/migration/connection/db2-connection/SKILL.md new file mode 100644 index 0000000..d53bd04 --- /dev/null +++ b/plugin/skills/migration/connection/db2-connection/SKILL.md @@ -0,0 +1,120 @@ +--- +name: db2-connection +description: Connect to a source IBM DB2 (LUW) database for migration to Snowflake using scai CLI. Triggers: db2, ibm db2, source connection, source database, connect to db2, add db2 connection. +license: Proprietary. See License-Skills for complete terms +--- + +# DB2 Connection Skill + +## On Entry + +Tell the user: +> **Setting up DB2 connection** — I'll configure and test a connection to your source IBM DB2 (LUW) database. I'll need a few connection details. + +## Prerequisites + +- Network access to the DB2 host (self-hosted LUW or cloud-hosted) +- DB2 username and password with read access to the system catalog (`SYSCAT`) +- The `scai` CLI installed and available + +## Required Connection Details + +### Standard Auth (only auth method supported in MVP) + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `-s, --source-connection` | Yes | Friendly name for this source connection | +| `--auth` | Yes | `standard` | +| `--host` | Yes | DB2 host | +| `--port` | No | TCP port (default: `50000`) | +| `--database` | Yes | Database name | +| `--user` | Yes | DB2 username | +| `--password` | Yes | DB2 password | +| `--connection-timeout` | No | Connection timeout in seconds | + +> DB2 is read through the `ibm_db` DB-API driver, which stages Parquet — there is no ODBC install required on the worker host. + +## Workflow + +### Step 1: Ask How to Provide Credentials + +Ask the user: +> "I need the following to connect to DB2: +> - **Host** +> - **Port** (default `50000`) +> - **Database name** +> - **Username** and **password** +> +> How would you like to provide these?" + +Options: +1. **1Password** — Credentials stored in 1Password vault +2. **Enter manually** — Provide values directly + +### Step 2: Route Based on Answer + +| User says | Action | +|-----------|--------| +| "1Password" | Follow `../1PASSWORD.md` (DB2 section) | +| "Enter manually" | Proceed to Step 3 | +| Other credential manager | Check if `../references/.md` exists; if not, ask user to explain their setup | + +### Step 3: Add the Connection + +```bash +scai connection add-db2 \ + -s \ + --auth standard \ + --host \ + --port \ + --database \ + --user \ + --password +``` + +- Omit `--port` if the server uses the default (`50000`). + +### Step 4: Save and Test Source Connection + +Call the `configure` tool with `source_connection` set to ``. The MCP server runs `scai connection test` internally and only persists the connection if the test passes. + +- **On success:** the response includes `connection_test: ok`. +- **On failure:** the tool returns an error containing the scai message. Surface it to the user, help them fix the issue, then re-run `configure(source_connection=)`. + +**Common errors:** + +| Error | Cause | Solution | +|-------|-------|----------| +| `Operation timed out` | Network / firewall | Check VPN, security groups, firewall rules | +| `SQL30081N` (communication error) | Wrong host/port or server down | Verify host, port (default `50000`), and that DB2 is listening | +| `SQL1013N` (database not found) | Wrong database name | Verify the database alias / name on the server | +| `SQL30082N` (auth failed) | Bad credentials | Re-check username and password | + +## CHECKPOINT + +Confirm with user: +- [ ] `configure` returned `connection_test: ok` +- [ ] Connection appears in `scai connection list -l db2 --json` +- [ ] Source connection saved to session config + +## On Completion + +After the CHECKPOINT passes, tell the user: +> **Connection configured** — Successfully connected to DB2 using connection ``. + +Then return to the calling skill. + +## Security Rules + +- **NEVER** log or display secrets (passwords) in plain text. +- **NEVER** include secrets in command-line arguments that might be logged. Prefer a credential manager (e.g. 1Password `op run`, or use /secrets capability from Cortex Code to store them). + +## Quick Reference + +| Action | Command | +|--------|---------| +| Add connection | `scai connection add-db2 -s NAME --auth standard --host HOST --port 50000 --database DB --user USER --password PASS` | +| Test connection | `configure(source_connection=NAME)` (runs the test internally) | +| List connections | `scai connection list -l db2 --json` | +| Set default | `scai connection set-default -l db2 -c NAME` | +| Extract code | `scai code extract -s NAME --json` | diff --git a/plugin/skills/migration/connection/db2-connection/references/REFERENCE.md b/plugin/skills/migration/connection/db2-connection/references/REFERENCE.md new file mode 100644 index 0000000..e35fd77 --- /dev/null +++ b/plugin/skills/migration/connection/db2-connection/references/REFERENCE.md @@ -0,0 +1,172 @@ +# DB2 Connection Reference + +Detailed reference for IBM DB2 (LUW) connection options, authentication, extraction, and troubleshooting. + +## Connection Options + +### Required Parameters + +| Parameter | Flag | Description | +|-----------|------|-------------| +| Connection name | `-s, --source-connection` | Unique identifier for this connection | +| Authentication | `--auth` | Authentication method: `standard` | +| Host | `--host` | DB2 hostname or IP address | +| Database | `--database` | Database name (bound at connection time) | +| Username | `--user` | DB2 username | +| Password | `--password` | DB2 password | + +### Optional Parameters + +| Parameter | Flag | Default | Description | +|-----------|------|---------|-------------| +| Port | `--port` | 50000 | TCP port number | +| Connection timeout | `--connection-timeout` | 30 | Timeout in seconds | + +## Authentication Methods + +### Standard Authentication (Username/Password) + +The only supported authentication method for DB2. + +```bash +scai connection add-db2 \ + -s my-db2 \ + --auth standard \ + --host db2-server.example.com \ + --port 50000 \ + --database SAMPLE \ + --user myuser \ + --password mypassword +``` + +**When to use:** +- All DB2 connections (only method available in MVP) + +## Data Exchange Worker (cloud migration / validation) + +The worker connects via the `ibm_db` DB-API driver using the same credentials as +`scai connection add-db2`. DB2 is read off a DB-API cursor and staged as **Parquet** +(not CSV) — Parquet preserves NULL-vs-empty-string and multi-byte data that a CSV +extract would corrupt. The generated worker TOML: + +```toml +[connections.source.db2] +user = "" +password = "" +database = "" +host = "" +port = 50000 +``` + +Ensure the DB2 user has: + +- `SELECT` on all tables being migrated or validated +- Read access to `SYSCAT.COLUMNS` / `SYSCAT.TABLES` for schema discovery + +Network: the worker host must reach the DB2 server on the configured port. For SPCS +workers, ensure `EXTERNAL_ACCESS_INTEGRATIONS` covers the database host. DB2 requires +the `ibm_db` driver, which is bundled with the worker — no separate ODBC install. + +Worker TOML `[connections.source.db2].database` must match `source.databaseName` in the +migration/validation workflow YAML. + +## scai TOML Fields (`~/.snowflake/snowct/db2.toml`) + +Populated by `scai connection add-db2`. The MCP server reads this file when generating +the DEW worker config. + +| Field | Required | Description | +|---|---|---| +| `auth_method` | Yes | Always `standard` | +| `user` | Yes | DB2 username | +| `password` | Yes | DB2 password | +| `host` | Yes | DB2 host | +| `port` | No | TCP port (default: `50000`) | +| `database` | Yes | Database name (bound at connection time) | +| `connection_timeout` | No | Timeout in seconds (if set) | + +## Workflow YAML + +`sourcePlatform` for DB2 validation workflows: + +```yaml +sourcePlatform: db2 +``` + +## `whereClauseCriteria` + +DB2 folds unquoted identifiers to uppercase and uses double quotes for case-sensitive names: + +```yaml +tables: + - source: + schemaName: MYSCHEMA + name: MYTABLE + whereClauseCriteria: '"ID" > 1000' +``` + +## Partition Column Guidance + +DB2 has no portable system row identifier suitable for partitioning. Use: + +- A monotonic integer primary key (e.g. `ID BIGINT`) — most common and reliable +- A timestamp column with a narrow range — for time-partitioned loads +- No partition column — omit `columnNamesToPartitionBy` for a single-partition full extract + +## Troubleshooting + +### Communication Error (SQL30081N) + +**Symptoms:** +- `SQL30081N ... communication error` +- Connection hangs then times out + +**Solutions:** +1. Verify host and port (default `50000`) +2. Check VPN / firewall / security-group rules allow the port +3. Confirm the DB2 instance is listening: `db2 get dbm cfg | grep SVCENAME` +4. Test network path: `nc -zv 50000` + +### Authentication Failed (SQL30082N) + +**Solutions:** +1. Verify username and password are correct +2. DB2 usernames map to OS/LDAP accounts — verify the account is valid on the server + +### Database Not Found (SQL1013N) + +**Solutions:** +1. Verify the database name / alias (`db2 list database directory`) +2. Confirm the database is cataloged on the target instance + +### Insufficient Privileges + +**Solutions:** +1. Grant read access: `GRANT SELECT ON . TO ;` +2. Ensure the user can read `SYSCAT` catalog views (granted to `PUBLIC` by default) + +## Verifying Connectivity + +```bash +scai connection test -l db2 -c --json +``` + +Manual network test: + +```bash +nc -zv 50000 +``` + +## Stored Connection Format + +SCAI stores DB2 connections in TOML format with these fields: + +| TOML key | Description | +|----------|-------------| +| `auth_method` | Always `standard` | +| `user` | DB2 username | +| `host` | DB2 hostname or IP | +| `database` | Database name | +| `port` | Port number (default `50000`) | +| `password` | Encrypted password | +| `connection_timeout` | Timeout in seconds (if set) | diff --git a/plugin/skills/migration/connection/snowflake-connection/SKILL.md b/plugin/skills/migration/connection/snowflake-connection/SKILL.md new file mode 100644 index 0000000..34404d3 --- /dev/null +++ b/plugin/skills/migration/connection/snowflake-connection/SKILL.md @@ -0,0 +1,63 @@ +--- +name: snowflake-connection +description: Choose and configure a Snowflake target authenticator. Use when creating or repairing a Snowflake connection, especially Microsoft Entra ID / Azure AD / OIDC SSO, or when `externalbrowser` fails against an OIDC IdP. +license: Proprietary. See License-Skills for complete terms +--- + +# Snowflake target connection + +`scai` opens Snowflake through Snowflake.Data. Pick the authenticator that matches the identity provider. Do **not** guess; ask if the user is unsure. + +## When to use which authenticator + +| Situation | Authenticator | Notes | +|-----------|---------------|--------| +| Microsoft Entra ID, Azure AD, or any OIDC IdP | `oauth_authorization_code` | Browser PKCE flow. Required for Entra. | +| Snowflake SAML SSO / classic IdP SSO | `externalbrowser` | Do **not** use this for Entra OIDC. | +| Password + MFA | omit / `username_password_mfa` | Existing default. | +| Programmatic access token | `programmatic_access_token` | Headless / CI. | +| Key-pair | `snowflake_jwt` | Headless / CI. | +| Token already in hand (SPCS) | `oauth` | Non-interactive. | +| Service principal | `oauth_client_credentials` | Non-interactive. | + +If a connection using `externalbrowser` fails with an OIDC / Entra / AADSTS error, switch it to `oauth_authorization_code`. Do not keep retrying SAML. + +Full field reference: `Snowflake.SnowConvertDesktop/Snowflake.SnowConvert.Cli/docs/entra-oidc-oauth.md`. + +## Entra / OIDC (`oauth_authorization_code`) + +Requires a desktop session (system browser + loopback listener). Refuse it in containers, CI, or headless Linux (`DISPLAY` / `WAYLAND_DISPLAY` unset). Suggest PAT or key-pair instead. + +Required `connections.toml` keys: + +- `account`, `user` — `user` is required so Snowflake.Data can cache and refresh tokens +- `authenticator = "oauth_authorization_code"` +- `oauth_client_id`, `oauth_client_secret` +- `oauth_scope` — forwarded verbatim; do not trim or reorder +- `oauth_authorization_url` and `oauth_token_request_url` — both HTTPS, set together +- `oauth_redirect_uri` — **required whenever those external endpoints are set**. Fixed absolute loopback URI registered **exactly** in Entra (scheme, host, port, path). Example: `http://127.0.0.1:8080/` + +Do not omit `oauth_redirect_uri` for Entra. The connector's random-port `127.0.0.1` callback is not Entra-compatible (`localhost` any-port does not apply to `127.0.0.1`). + +Example: + +```toml +[entra_oidc] +authenticator = "oauth_authorization_code" +account = "myorg-myaccount" +user = "first.last@example.com" +oauth_client_id = "" +oauth_client_secret = "" +oauth_scope = "" +oauth_authorization_url = "https://login.microsoftonline.com//oauth2/v2.0/authorize" +oauth_token_request_url = "https://login.microsoftonline.com//oauth2/v2.0/token" +oauth_redirect_uri = "http://127.0.0.1:8080/" +``` + +Select it with `scai … --connection entra_oidc` (or the project's `snowflake_connection`). + +## Limits the agent must not ignore + +- **Data validation and test generation do not support this authenticator.** Their DTO cannot carry client, endpoints, scope, or redirect. Use PAT, key-pair, or password for those jobs (`CNX0037`). +- **Browser single-flight is process-local** and covers SCAI Jobs/Databases native opens only. Do not start overlapping Authorization Code data-migration / SMA opens; cache the first login, then run them one at a time. +- Never log client secrets, authorization codes, tokens, or full authorize URLs. diff --git a/plugin/skills/migration/data-infrastructure/SKILL.md b/plugin/skills/migration/data-infrastructure/SKILL.md index 6e82708..39409f0 100644 --- a/plugin/skills/migration/data-infrastructure/SKILL.md +++ b/plugin/skills/migration/data-infrastructure/SKILL.md @@ -19,6 +19,8 @@ Before starting any configuration, tell the user verbatim: > > **The same infrastructure is used for validation.** You choose what level of validation to run: **schema validation**, **metrics validation**, and **row-level validation**. +> **Metadata storage mode (trial accounts):** On Snowflake accounts **without Hybrid Table** support, the orchestrator falls back to standard/`TRANSIENT` metadata tables automatically. That path preserves correctness but **reduces task-queue throughput under contention** and effectively **limits how many workers you should run in parallel** — start with fewer workers and scale up carefully. This is **not** the same as L3 “hybrid” row validation. See [Metadata storage mode reference](./references/metadata-storage-mode-reference.md). + > **Always use the official tooling for data movement and validation.** Route table migration through `migrate_data` and validation through `validate_data`. Do not suggest ad-hoc extract/copy/compare scripts — the DMVF orchestrator and workers handle partitioning, loading, and multi-level validation. > **Supported sources**: SQL Server, Redshift, Oracle, Teradata, PostgreSQL @@ -50,6 +52,21 @@ Use the `data_infrastructure` tool (modes `up` / `down` / `status`) to manage th ## Idempotency +**One project setting, not one choice per action.** The orchestrator/worker +placement selected in Step 0 is shared by migration and validation and remains +in effect until the user explicitly asks to change it. Do not re-ask local vs +SPCS when validation begins, and do not ask a per-object subagent to start, +stop, repair, or reconfigure infrastructure. Subagents only dispatch against +the setup already in place; if infrastructure is unavailable, they return the +remediation to the main agent. + +If the user explicitly asks to change placement or a persisted data strategy, +the **main agent** owns that setup change. Re-enter this skill for placement or +worker changes, and load [`../setup/data-strategy/SKILL.md`](../setup/data-strategy/SKILL.md) +for migration/validation strategy changes. Complete the change once, persist +it, bring infrastructure up as needed, and only then resume per-object +dispatch. + If this sub-skill has already been completed in the current project — i.e., the project's `.scai/config/dew_configuration.toml` (path relative to the SCAI project root) exists with no remaining `` values — the infrastructure is **configured**, but that does **not** mean it is **running**. Completion is durable; "up" is ephemeral: an SPCS orchestrator auto-suspends when idle, and a freshly-resumed session owns no local worker process. A config-only check (`scai data doctor` alone) passes while the service is suspended and the worker is dead — so dispatch would then sit at `Tables=0/N` with no error. Do **not** stop at a config check: bring the shared infrastructure back up. Tell the user verbatim: "Data infrastructure already configured — bringing the shared orchestrator + worker back up (this also runs `scai data doctor` to confirm nothing has drifted)." Say it out loud rather than acting silently, because this runs live checks against Snowflake and the source. Then call `data_infrastructure(mode="up")` — it runs the [Level 1 Data Doctor](./references/data-doctor-reference.md#level-1-infrastructure-no-workflow-yaml) gate first (iterate until it stops reporting `doctor_failures`), then starts/resumes the orchestrator and (unless this project runs no local worker) the worker; relay its `cost_reminder`. **Also state the placement it resolved** — the response carries the project's recorded decision: `orchestrator_placement` (`local` = on this machine, no compute pool; `spcs` = SPCS compute pool) and `worker_placement` (`local` | `spcs` | `none` for Iceberg / externally-managed). `up` persists this the first time it resolves it, so on later resumes it is the project's saved choice — relayed consistently rather than re-inferred silently. If `orchestrator_placement` is `local` and they meant SPCS, they can switch by re-running with `compute_pool=""`. Return to the caller without re-prompting. Only run a bare Level 1 Data Doctor instead when you specifically need a config-drift check *without* bringing infrastructure up. @@ -88,7 +105,7 @@ Ask: | Answer | Next step | |--------|-----------| | **Local** | **Skip Question 0** (no compute pool). Bring it up later with `data_infrastructure(mode="up")` (omit `compute_pool`). | -| **SPCS** | Set up the compute pool in **Question 0**; bring it up with `data_infrastructure(mode="up", compute_pool="")`. | +| **SPCS** | Now that SPCS is chosen, call `configure(needs_compute_pools=true)` so the response lists the accessible pools, then resolve the pool in **Question 0**. Bring it up with `data_infrastructure(mode="up", compute_pool="")`. | ### 0.c — Where should each worker run? @@ -106,14 +123,24 @@ Having chosen placement in Step 0, gather the remaining Snowflake-side details * If the user chose **local** in Step 0.b, **do not ask Question 0** — continue from **Question 1**. -Ask the user: +The orchestrator on SPCS needs a compute pool. **Always ask** whether the user already has one or needs to create one — but first surface the pools their role can already reach, so they pick from a short list instead of recalling a name. Call `configure(needs_compute_pools=true)`; its response carries an `existing_compute_pools:` line listing them as `NAME (state)` (the compute-pool analogue of the `existing_connections:` line from `needs_source_connection`). + +Then ask, presenting those pools as a short list (name + `state`): + +> The Orchestrator needs a compute pool to run on SPCS. Here are the pools your role can access: +> `` (one per line) +> Do you already have one to use, or do you need to create one? + +Handle the `existing_compute_pools:` value: -> The Orchestrator needs a compute pool to run inside your Snowpark Container Services. Do you already have a compute pool, or will you need to set one up? +- **Pools listed** → present them and let the user pick, or create a new one. +- **`none`** → the role sees no pool; say so and go straight to the create path. +- **`(unavailable: …)`** → couldn't reach Snowflake; tell the user, then fall back to asking them to name an existing pool or create one. | Answer | Action | |--------|--------| -| **I have one** | Verify it is active (see below), then save it and continue to Question 1. | -| **I need to set one up** | Route to → `./compute-pool-setup/SKILL.md` — guide the user through creating and configuring a compute pool. After that skill completes, return here and run the steps below. | +| **Use an existing pool** | Confirm its `state` (from the list) is **ACTIVE** or **IDLE**. If `SUSPENDED`, run `ALTER COMPUTE POOL RESUME;` and re-check; if `STARTING`, wait a moment and re-check. Then save it and continue to Question 1. | +| **Create a new one** | Route to → `./compute-pool-setup/SKILL.md` — guide the user through creating and configuring a compute pool. After that skill completes, return here and run the steps below. | **Bring the shared infrastructure up on this pool:** @@ -203,8 +230,28 @@ Return control to the calling skill. --- +## Advanced operations (when the customer asks) + +These are **optional** — not part of default setup. Load [Advanced operations reference](./references/advanced-operations-reference.md) when the user mentions: + +| Topic | Trigger phrases | Summary | +|-------|-----------------|---------| +| **Rate limiting** | Source overloaded, throttle extractions/loads, pause one workflow | SQL rules in `DATA_MIGRATION.RATE_LIMIT`; soft cap on worker task pulls | +| **Preflight migration** | Dry-run, smoke test pipeline, test before full load | `preflight: true` in DM YAML → transient `PREFLIGHT_` schema (not doctor, not Preliminary) | +| **Incremental validation** | Re-validate only changed partitions, ongoing DV | `synchronization` + watermark/checksum in DV YAML; baseline run required first | +| **Re-validation** | Retry failed validation partitions | `validate_data(mode="revalidate", workflow_name=…)` after parent workflow finishes | +| **Custom L3 normalization** | Formatting drift, case/spatial/period compare | `validationCustomNormalizationRules` in DV YAML — not DM `columnTypeMappings` | +| **Checksum blind spots** | Incremental sync missed a column change | Some types excluded/rounded from checksum — see advanced ref | +| **Non-hybrid metadata (trial)** | Trial account, HYBRID TABLE unsupported, slow pulls with many workers | Standard/`TRANSIENT` fallback — limit workers; see metadata storage mode ref | + +Do not suggest ad-hoc throttling via polling env vars. Rate limits are metadata SQL, not workflow YAML fields. + +--- + ## Reference +- [Metadata storage mode reference](./references/metadata-storage-mode-reference.md) — Hybrid vs standard metadata (trial fallback, worker scaling) +- [Advanced operations reference](./references/advanced-operations-reference.md) — rate limiting, preflight, incremental DV, revalidate - [Data Doctor reference](./references/data-doctor-reference.md) - [Worker Config Reference](./references/worker-config-reference.md) - [Teardown (cost-saving suspend)](./teardown/SKILL.md) diff --git a/plugin/skills/migration/data-infrastructure/references/advanced-operations-reference.md b/plugin/skills/migration/data-infrastructure/references/advanced-operations-reference.md new file mode 100644 index 0000000..67ec533 --- /dev/null +++ b/plugin/skills/migration/data-infrastructure/references/advanced-operations-reference.md @@ -0,0 +1,182 @@ +# Advanced operations reference + +Use this when a customer asks about **protecting the source**, **bounded migration dry-runs**, **incremental validation**, **re-validating failed partitions**, **throttling concurrent tasks**, **custom L3 normalization**, or **why checksum/incremental sync missed a change**. These are optional — default migrate/validate flows do not require them. + +**Related references:** [Metadata storage mode (Hybrid vs standard)](./metadata-storage-mode-reference.md), [DM workflow config](../../migrate-objects/actions/data-migration/references/workflow-config-reference.md), [DV workflow config](../../setup/data-validation/references/workflow-config-reference.md), [validate_tables.md](../../validate-objects/actions/validate_tables.md). + +--- + +## Rate limiting (protect source or shared resources) + +**When the customer asks:** source DB is overloaded during migration; they need to cap concurrent extractions/loads without stopping workers entirely; they want to pause one workflow while others continue. + +**What it is:** Scope-pattern rules in Snowflake table `DATA_MIGRATION.RATE_LIMIT` (or your `CUSTOM_SNOWFLAKE_SCHEMA_FOR_DATA_MIGRATION_METADATA` schema). Each rule limits how many **executing** tasks whose `SCOPE` matches a SQL `LIKE` pattern can run at once. + +**Agent guidance:** + +1. Prefer explaining rate limits **before** lowering `max_parallel_tasks` globally or stopping workers — workers can stay up while matching tasks wait in `pending`. +2. `TARGET_CONCURRENT_TASKS` is a **target**, not a hard ceiling — eligibility is snapshot-based, so many workers polling together can overshoot before claims settle. Worst case is roughly **2× the target** (target `5` → up to ~10 executing). Set the target **below** a hard source limit with headroom for your worker count. **`TARGET_CONCURRENT_TASKS = 0`** is the one exact value — it reliably pauses matching scopes. +3. Enforcement applies to **worker (DEA) single-task pulls** only — not orchestrator batch pulls. Workers with `max_parallel_tasks > 1` batch-fetch bypass rate limiting; keep default single-task fetch when using limits. +4. Insert rules with SQL in a Snowflake session (admin on migration metadata schema): + +```sql +-- At most 5 concurrent loading tasks (any table): +INSERT INTO SNOWCONVERT_AI.DATA_MIGRATION.RATE_LIMIT (SCOPE_PATTERN, TARGET_CONCURRENT_TASKS) +VALUES ('Table[%]::Loading', 5); + +-- Cap one workflow's partition loads: +INSERT INTO SNOWCONVERT_AI.DATA_MIGRATION.RATE_LIMIT (SCOPE_PATTERN, WORKFLOW_ID, TARGET_CONCURRENT_TASKS) +VALUES ('Table[%]::Partition[%]::Loading', , 2); + +-- Pause all tasks matching a pattern: +INSERT INTO SNOWCONVERT_AI.DATA_MIGRATION.RATE_LIMIT (SCOPE_PATTERN, TARGET_CONCURRENT_TASKS) +VALUES ('Table[MY_DB.%]::%', 0); +``` + +| Column | Meaning | +|--------|---------| +| `SCOPE_PATTERN` | `LIKE` pattern on task scope (e.g. `Table[DB.SCHEMA.TABLE]::Partition[3]::Extraction`) | +| `WORKFLOW_ID` | Optional — restrict rule to one workflow | +| `AFFINITY` | Optional — restrict to one worker affinity | +| `TARGET_CONCURRENT_TASKS` | Target concurrent executing matches (~2× worst-case overshoot; `0` = exact pause) | +| `ENABLED` | Set `FALSE` to disable without deleting | + +Remove with `DELETE FROM … RATE_LIMIT WHERE …` or disable with `ENABLED = FALSE`. Empty table = no limiting. + +**Not in workflow YAML** — rate limits are metadata-table SQL, not `migrate_data` / `validate_data` parameters. + +--- + +## Preflight workflows (bounded migration dry-run) + +**When the customer asks:** test connectivity and pipeline end-to-end on a small slice before a full migration; validate types/partitioning without writing to production target schemas. + +**What it is:** A **data migration** workflow flag — not validation, not `scai data doctor`. Each table runs as **one partition**; targets land in transient schema `PREFLIGHT_` instead of the configured target schema. + +**Not the same as:** + +| Concept | Purpose | +|---------|---------| +| **Preflight workflow** (`preflight: true`) | Bounded DM dry-run to transient schema | +| **Preliminary migration type** | Row-limited full migration via `whereClauseCriteria` to real target | +| **`scai data doctor`** | Infra/config health before start — blocks local start on fail | + +**Agent guidance:** + +1. Offer preflight when the user wants a **pipeline smoke test**, not when they only need row sampling to production (use **Preliminary** + `whereClauseCriteria` instead). +2. Set in workflow YAML at top level (Step 2a edit after `migrate_data(mode="setup")`): + +```yaml +preflight: true +preflightKeepSchema: false # true = leave PREFLIGHT_ for manual inspection +``` + +3. Run with normal `migrate_data(mode="run", workflow_path=...)`. Inspect transient schema objects; production target schemas are not used. +4. For a real migration, turn `preflight` off (or generate a new workflow without it) before production load. + +See [workflow-config-reference.md](../../migrate-objects/actions/data-migration/references/workflow-config-reference.md#preflight-bounded-dry-run). + +--- + +## Incremental data validation + +**When the customer asks:** re-run validation without scanning every partition; validate only what changed since last run; ongoing validation after initial full pass. + +**What it is:** DV **`synchronization`** block (`watermark` or `checksum`) — same JSON shape as DM incremental sync, but **read-only** (no data movement, no duplicate rows on target). + +**Agent guidance:** + +1. Capture mode at setup: `validation_type=incremental` + `sync_strategy=watermark|checksum` via `validate_data(mode="setup", …)` (or `progress_setup(mode="data_validation")` wizard). Setup patches `defaultTableConfiguration.synchronization.strategy`. +2. **Prerequisites:** table partitioned (`columnNamesToPartitionBy`); **at least one prior full validation** completed for baseline metadata. First incremental-configured run still validates everything (establishes baseline). +3. Edit YAML for `watermarkColumn` or `checksumExpression` as needed — see [DV workflow config reference](../../setup/data-validation/references/workflow-config-reference.md#incremental-validation-synchronization). +4. Later unchanged runs may report **Not validated** (skipped partitions) — that is expected, not a failure. +5. DV ignores DM-only sync fields (`trackModifications`, `trackDeletions`) — do not copy DM incremental examples that rely on them. +6. **Checksum blind spots:** default partition checksums skip or round some types (SQL Server `text`/`ntext`/`image`, Oracle LOBs, float rounding, etc.) — a change only in those columns may **not** trigger re-validation. See [Checksum / incremental sync — types that may not trigger re-sync](#checksum--incremental-sync--types-that-may-not-trigger-re-sync). + +--- + +## Re-validation (retry failed partitions) + +**When the customer asks:** validation finished but some tables/partitions failed; fix data and retry without re-running the whole workflow; cheaper retry after YAML or source fixes. + +**What it is:** **`validate_data(mode="revalidate", workflow_name=…)`** — creates a child `data-re-validate` workflow that re-runs **only failed partitions/levels** from a **finished** parent workflow. Not a full `mode="run"` replay. + +**Agent guidance:** + +1. Parent workflow must be **finished** with failures — use `details.progress.output.workflowName` from the run report. +2. After data or config fixes, offer revalidate before full re-setup: + +``` +validate_data(mode="revalidate", workflow_name="") +``` + +3. Repeat monitor + error-first report (same as `mode="run"`). Shared orchestrator + worker must still be up. +4. **Not the same as incremental validation** — revalidate retries **failed** work from one run; incremental skips **unchanged** partitions on subsequent scheduled runs. +5. Task-queue automatic retries (`MAX_RETRIES`, lease expiry) happen inside a single workflow — revalidate is an explicit user/agent action after the parent completes. + +See [validate_tables.md](../../validate-objects/actions/validate_tables.md) Step 5.G and [background-monitoring.md](../../validate-objects/actions/references/background-monitoring.md). + +CLI equivalent: `scai data validate revalidate `. + +--- + +## Custom normalization (Data Validation L3) + +**When the customer asks:** known benign formatting differences (case, trim, spatial WKT, Teradata PERIOD cast strings); L3 row-hash fails but values are "effectively equal"; how to whitelist normalization without `acceptedTransformations` per cell. + +**What it is:** SQL expressions applied **before** L3 row-hash and cell compare so source and target values hash/compare on a common form. Configured in DV workflow YAML — **not** DM `columnTypeMappings` (those affect extraction/load only). + +**Key fields** (workflow root, `validationConfiguration`, or per-table): + +| Field | Purpose | +|-------|---------| +| `validationCustomNormalizationRules` | **Preferred** — per `column`, `columnPattern`, or `dataType`; `sourceExpression` / `targetExpression` with `{{ col_name }}` placeholder | +| `validationCustomNormalizations` | Legacy datatype-keyed lists (still supported; granular rules win when both match) | +| `validationCustomTypes` / `validationCustomTypeRules` | L1 schema type expectations — pair with normalization when types differ but values should compare equal | + +**Agent guidance:** + +1. **Hybrid L3 requires L1** (`schemaValidation: true`) — normalization rules need L1 column metadata. +2. Use **`validationCustomNormalizationRules`** for new edits. Example — case-insensitive text: + +```yaml +validationCustomNormalizationRules: + - column: STATUS_CODE + sourceExpression: 'UPPER("{{ col_name }}")' + targetExpression: 'UPPER("{{ col_name }}")' +``` + +3. **`acceptedTransformations`** is for known source→target *value pairs*; **normalization rules** are for *expressions* applied to both sides. +4. DM **`columnTypeMappings` do not apply to DV** — do not copy migration type overrides into validation YAML expecting L3 to follow them. + +Workflow fields: [DV workflow config reference](../../setup/data-validation/references/workflow-config-reference.md#custom-normalization-l3). + +--- + +## Checksum / incremental sync — types that may not trigger re-sync + +**When the customer asks:** "I changed column X but incremental migration/validation did not re-run the partition"; checksum stayed the same; only `text`/`ntext`/`datetime`/`float`/spatial columns changed. + +**What it is:** **DM partition checksums** (and **DV incremental checksum** probes) hash a **normalized subset** of columns — not always every byte of every type. Some types are **skipped**, **rounded**, or **canonicalized** so small or lossy changes do not change the checksum. + +**Common blind spots (DM checksum):** + +| Category | Examples | Effect | +|----------|----------|--------| +| Skipped legacy LOBs | SQL Server `text`, `ntext`, `image`; Oracle LOBs, `LONG`, `XMLTYPE`, `VECTOR` | Column excluded from checksum input — changes invisible to default checksum | +| Float rounding | SQL Server float (`CONVERT(,2)`), Redshift REAL, VECTOR TME | Binary noise or tail bits may not change hash | +| Timestamp precision | High-precision `datetime2`, Redshift TIMESTAMP | Sub-nanosecond / readback truncation | +| Spatial as WKT | SQL Server / Redshift / Oracle geometry | Compared as despaced WKT; Oracle L3 WKT truncated at 4000 chars | +| Redshift `HLLSKETCH` | Redshift only | Extraction normalizes to NULL — changes invisible | +| Custom expression only | `checksumExpression: MAX(ORA_ROWSCN)` | Only that expression drives change detection — data edits elsewhere ignored | + +**DV vs DM:** DM checksum skipped columns may still appear in **DV L3 row-hash** (different pipeline). Do not tell the user "DV will catch it" without checking column selection and L3 config. + +**Agent guidance when user is confused:** + +1. Confirm **sync strategy** — watermark only sees rows above the watermark; checksum only sees partition aggregate change. +2. Identify column **data type** — if in skipped/lossy list, explain that default checksum may not detect the edit. +3. **Remediation options:** run a **full** migration/validation once; set a custom **`checksumExpression`** covering the column (DM/DV incremental checksum); switch affected columns to **watermark** if a monotonic column exists; use **DV L3** with `validationCustomNormalizationRules` when the issue is compare semantics, not sync detection. +4. For SQL Server **`text`/`ntext`/`datetime`** specifically: legacy LOBs are checksum-excluded; datetime formatting uses ODBC-style text — sub-second or timezone-only edits may not move the hash. + +DM sync field reference: [SynchronizationStrategy](../../migrate-objects/actions/data-migration/references/workflow-config-reference.md#synchronizationstrategy). diff --git a/plugin/skills/migration/data-infrastructure/references/metadata-storage-mode-reference.md b/plugin/skills/migration/data-infrastructure/references/metadata-storage-mode-reference.md new file mode 100644 index 0000000..118f6fe --- /dev/null +++ b/plugin/skills/migration/data-infrastructure/references/metadata-storage-mode-reference.md @@ -0,0 +1,64 @@ +# Metadata storage mode (Hybrid vs standard tables) + +Use this when a customer runs on a **trial / lower-tier Snowflake account**, sees bootstrap errors mentioning **`HYBRID TABLE`**, or asks why migration/validation feels **slower** or **workers seem capped** despite a healthy source. + +> **Not the same as L3 “hybrid” validation.** This topic is Snowflake **Hybrid Tables** used for orchestrator **metadata** (`TASK_QUEUE`, `TABLE_METADATA`, `PARTITION_METADATA`). It is **unrelated** to `rowValidationMode: hybrid` (L3 row-hash + cell drill-down). + +## What happens on accounts without Hybrid Tables + +The orchestrator probes Snowflake at bootstrap (or reads `SCHEMA_DEPLOYMENT_PROFILE` / env overrides). When Hybrid Tables are **unsupported**, it falls back to **standard / `TRANSIENT`** metadata tables with the same logical schema. + +| Path | Metadata DDL | Task claiming | +|------|--------------|---------------| +| **Hybrid (default on capable accounts)** | `HYBRID TABLE` + hybrid indexes | High-throughput batch pull with row locking | +| **Standard (fallback)** | `TRANSIENT` tables, no hybrid indexes | `PULL_SINGLE` loop + `MERGE` idempotency | + +Manual overrides (orchestrator / SPCS service env): + +- `SNOWFLAKE_METADATA_STORAGE_MODE=HYBRID|STANDARD|ICEBERG` — preferred three-way selector +- `SNOWFLAKE_USE_HYBRID_TABLES=1` — force hybrid when three-way mode is unset +- `SNOWFLAKE_USE_HYBRID_TABLES=0` — force standard/`TRANSIENT` (typical on trial) + +**Restart the orchestrator** after changing storage-mode env vars — resolution is cached for the process lifetime. + +In-place conversion between hybrid and standard metadata schemas is **unsupported** — match env to the existing schema or deploy a **fresh** metadata schema. + +See also: [Worker config reference](./worker-config-reference.md) (limit worker count on the standard path), [Advanced operations — rate limiting](./advanced-operations-reference.md#rate-limiting-protect-source-or-shared-resources). + +--- + +## Disclaimer for customers (consistency, performance, workers) + +When Hybrid Tables are **not** used and the framework falls back to regular/`TRANSIENT` metadata: + +### Consistency + +- **Correctness is preserved** via conditional `UPDATE`, transactional `MERGE`, and idempotent metadata writes — not via hybrid-table `SELECT … FOR UPDATE`. +- Under **high concurrent task claiming**, the standard path may show **more retries** (for example Snowflake `90232` transaction aborts) before a worker successfully leases a task. Retries are expected; do not treat them alone as data corruption. +- Metadata **`TRANSIENT`** tables have **no Time Travel** — orchestration state is rebuildable but not point-in-time recoverable like hybrid metadata. + +### Performance + +- **Task-queue throughput is lower** under contention: more round-trips per pull, more warehouse scan work without hybrid secondary indexes. +- Large backlogs of pending tasks can show a **degrading pull curve** until completed rows are purged/archived. +- This is **metadata-layer** latency — it can make the pipeline feel slow even when the **source database** and **Snowflake warehouse** have spare capacity. + +### Worker count — practical limits + +- **Do not assume unlimited horizontal scale** on the standard metadata path. Many workers × many `max_parallel_tasks` threads all compete for the same `TASK_QUEUE` rows. +- **Agent guidance:** on trial / confirmed non-hybrid metadata, **start conservatively** — for example **1–2 workers** with **`max_parallel_tasks` 2–4** — then increase gradually while watching orchestrator logs for repeated pull retries and end-to-end workflow time. +- **Source-side rate limiting** (`RATE_LIMIT`) protects the source DB but **does not** remove metadata-queue contention — both may be needed. +- If the account **supports Hybrid Tables**, prefer the hybrid metadata path (default probe) for production-scale parallelism unless the customer explicitly standardizes on trial-style deployment. + +--- + +## When to mention this to the user + +| Situation | What to say | +|-----------|-------------| +| Trial / Enterprise trial account | Fallback is automatic; set expectations on throughput and worker scaling | +| `Unsupported feature 'HYBRID TABLE'` during bootstrap | Standard path is expected; use `SNOWFLAKE_USE_HYBRID_TABLES=0` if probe is ambiguous | +| Many workers but tasks stay `pending` | Check metadata mode before adding more workers — standard path may need fewer concurrent claimers | +| Customer compares to “production Mobilize” timing | Hybrid metadata + indexes explain much of the gap on trial | + +Do **not** confuse with [rate limiting](./advanced-operations-reference.md#rate-limiting-protect-source-or-shared-resources) (source protection) or [L3 hybrid validation](../../setup/data-validation/references/workflow-config-reference.md#l3-result-codes-hybrid-mode). diff --git a/plugin/skills/migration/data-infrastructure/references/worker-config-reference.md b/plugin/skills/migration/data-infrastructure/references/worker-config-reference.md index 61543f9..01c3ee4 100644 --- a/plugin/skills/migration/data-infrastructure/references/worker-config-reference.md +++ b/plugin/skills/migration/data-infrastructure/references/worker-config-reference.md @@ -2,6 +2,8 @@ The project-default `.scai/config/dew_configuration.toml` (path relative to the SCAI project root) is generated by `scai data worker generate-config`, which pre-fills `[connections.source.]` from the project's scai source connection (referenced by name and hydrated to real credentials at `worker start`, never stored in the file). This reference documents the fields, advanced options (e.g., Redshift UNLOAD), and operational guidance. +> **Advanced options:** [Secrets management](#advanced-secrets-management) (external vaults, `$(...)` substitution), [query modifiers](#query-modifiers-toml), [custom extraction plugins](#custom-extraction-plugins), workflow-level [extraction strategies](../../migrate-objects/actions/data-migration/references/extraction-strategies-reference.md), and [advanced operations](./advanced-operations-reference.md) (rate limiting, preflight, incremental/revalidate DV). + > **Prefer SCAI CLI setup:** Use `scai data worker generate-config` and `scai data worker setup` (SPCS) or `scai data worker start --local` instead of hand-authoring TOML. Manual SPCS worker setup is only needed when the CLI path is unavailable — see official docs for egress IP allowlisting and driver host requirements on the source firewall. ## Configuration Sections @@ -110,6 +112,33 @@ extraction: externalStage: MY_DB.MY_SCHEMA.S3_EXTERNAL_STAGE ``` +## Advanced: Secrets management + +For **local or non-SPCS workers**, resolve credentials from external secret stores instead of embedding plaintext in TOML or Snowflake Secrets. + +**REST providers** — register HTTP secret backends in a separate TOML file pointed to by `SECRET_MANAGERS_CONFIG_FILE`: + +```toml +[secret_managers.providers.vault] +type = "rest" +base_url = "https://vault.example.com/v1/secret/data/dea" +``` + +Reference resolved values in connection string fields using the provider scheme (for example `vault://path/to/secret#field`). + +**Command substitution** — embed `$(...)` recipes in connection fields (for example `$( aws secretsmanager get-secret-value ... )#password`). Gated by `SECRET_MANAGERS_ALLOW_CMD_SUBSTITUTION=true` (default `false`). + +> **Inline `[secret_managers]` is ignored:** Do **not** put `[secret_managers]` blocks in `dew_configuration.toml` — the worker loader ignores unrecognized top-level sections. Use `SECRET_MANAGERS_CONFIG_FILE` or the env vars below. + +| Variable | Default | Description | +|----------|---------|-------------| +| `SECRET_MANAGERS_CONFIG_FILE` | Unset | Path to TOML with `[secret_managers.providers.*]` blocks. | +| `SECRET_MANAGERS_ALLOW_CMD_SUBSTITUTION` | `false` | Allow `$(...)` recipes in config strings. | +| `SECRET_MANAGERS_CACHE_TTL_SECONDS` | `300` | Cache resolved secrets (seconds). | +| `SECRET_MANAGERS_RESOLVE_TIMEOUT_SECONDS` | `10` | Per-resolve timeout (seconds). | + +**SPCS workers** use Snowflake Secrets for source credentials (see [worker-spcs/SKILL.md](../worker-spcs/SKILL.md) Step 3) — distinct from external secret managers above. + ## Managing Workers - Increase `max_parallel_tasks` for more parallelism on a single machine — no need to run multiple workers on the same machine. @@ -117,6 +146,10 @@ extraction: - Keep a low worker count to avoid overloading your source system. - Stop workers during peak source system usage to avoid disrupting existing operations. +> **Non-hybrid metadata (trial / standard path):** When the orchestrator uses standard/`TRANSIENT` metadata instead of Snowflake Hybrid Tables, **task-queue contention** — not source capacity — often caps useful parallelism. Prefer **fewer workers** and lower `max_parallel_tasks` on trial accounts; scale up gradually. See [Metadata storage mode reference](./metadata-storage-mode-reference.md). + +For finer-grained control than starting and stopping whole workers — for example capping only the loading tasks, or only one workflow — see [rate limiting](./advanced-operations-reference.md#rate-limiting-protect-source-or-shared-resources) (source-side; does not fix metadata-queue contention on the standard path). + ## Note: Iceberg Migrations and Workers The Worker TOML configuration **does not change** for Iceberg migrations. However, most Iceberg strategies bypass the Data Exchange Agent (Worker) entirely: diff --git a/plugin/skills/migration/data-infrastructure/teardown/SKILL.md b/plugin/skills/migration/data-infrastructure/teardown/SKILL.md index 9de45cb..eaec5e0 100644 --- a/plugin/skills/migration/data-infrastructure/teardown/SKILL.md +++ b/plugin/skills/migration/data-infrastructure/teardown/SKILL.md @@ -1,15 +1,27 @@ --- name: data-infrastructure-teardown -description: Cost-saving teardown for shared data infrastructure — verify no in-flight workflows, stop/suspend orchestrator and worker (SPCS or local, as configured), and suspend the compute pool when applicable. Invoked after every migrate_data() / validate_data() cycle and at end-of-migration. +description: Cost-saving teardown for shared data infrastructure — verify no in-flight workflows, then call data_infrastructure(mode="down") to stop/suspend orchestrator and worker (SPCS or local, as configured). The compute pool auto-suspends. Invoked after every migrate_data() / validate_data() cycle and at end-of-migration. parent_skill: data-infrastructure-setup license: Proprietary. See License-Skills for complete terms --- # Data Infrastructure Teardown -Tear down only what this project actually provisioned: **SPCS orchestrator + compute pool**, **local orchestrator process**, **SPCS DEW worker**, and/or **local worker process**. Nothing auto-resumes on the next dispatch — `migrate_data()` / `validate_data()` are pure dispatch and assume the infrastructure is already up. To run another wave after teardown, bring it back explicitly with `data_infrastructure(mode="up")` (per `../SKILL.md`). +`data_infrastructure(mode="down")` does the teardown for **both** placements in one call: -> **Why this exists:** an SPCS `DATA_MIGRATION_SERVICE` consumes compute pool seconds; a local orchestrator or local worker polling `TASK_QUEUE` wakes the warehouse on every interval — all accrue cost when idle. +- **Local** — reaps the orchestrator + worker processes this MCP session started. +- **SPCS** — when a `compute_pool` is configured for the project, it also suspends the SPCS orchestrator + service and the Data Exchange Worker. The compute pool then **auto-suspends** on its own + (`AUTO_SUSPEND_SECS`), so there is no separate pool step. Pass `drop=true` to remove the DEW service + permanently instead of suspending it. + +So teardown is normally just: **call `data_infrastructure(mode="down")`**. Everything below is the +in-flight check to do first, plus the few cases the tool cannot cover on its own. Nothing auto-resumes — +bring it back with `data_infrastructure(mode="up")` before the next wave (per `../SKILL.md`). + +> **Why this exists:** an SPCS `DATA_MIGRATION_SERVICE` consumes compute pool seconds; a local +> orchestrator or local worker polling `TASK_QUEUE` wakes the warehouse on every interval — all accrue +> cost when idle. ## When to Invoke @@ -19,45 +31,20 @@ Tear down only what this project actually provisioned: **SPCS orchestrator + com | After `validate_data()` reaches `completed` or `failed` for a wave | Caller prompts the user (default Yes) before loading this skill | | End of full migration (no more waves) | Caller invokes this skill unconditionally | -**Skip entirely** when the project never configured data infrastructure (no `compute_pool` and no local orchestrator/worker was started). +**Skip entirely** when the project never configured data infrastructure (no `compute_pool` and no local +orchestrator/worker was started). --- -## Which steps apply - -Determine from infrastructure setup and `configure()` / `.scai/settings/cloud-migration.yaml`: - -| Component | Signal | Teardown steps | -|-----------|--------|----------------| -| SPCS orchestrator | `compute_pool` configured | **Step 2a**, **Step 3** | -| Local orchestrator | No `compute_pool`; user chose local orchestrator in setup | **Step 2b** | -| SPCS DEW worker | `scai data worker setup` / `worker-spcs` completed | **Step 4a** (`scai data worker stop`; DMG0024 → no SPCS worker) | -| Local worker | `worker-local-setup` or `scai data worker start --local` | **Step 4b** | +## Step 1: Verify no in-flight workflows -Run **Step 1** always when any orchestrator/worker was used. Skip rows that do not apply — do not suspend a compute pool or SPCS service that was never provisioned. +`data_infrastructure(mode="down")` already **refuses** while a migrate/validate job it can see is running +(it reads the project's relay ledger) — so a plain `down` is safe by default. But that ledger only covers +jobs dispatched from **this project on this machine**. Before tearing down shared SPCS infrastructure that +**another machine** might be using, confirm the queue is quiet directly: -## Privilege Prerequisites - -Before running any step, ensure the active role has the following privileges. The role that ran `scai data orchestrator setup` is granted these automatically; if a different role is performing teardown (e.g. `ACCOUNTADMIN` cleaning up after the project owner), grant them explicitly: - -```sql -GRANT SELECT ON ALL TABLES IN SCHEMA SNOWCONVERT_AI.DATA_MIGRATION TO ROLE ; -GRANT MONITOR, OPERATE ON SERVICE SNOWCONVERT_AI.DATA_MIGRATION.DATA_MIGRATION_SERVICE TO ROLE ; --- If the DEW worker service is in use: -GRANT MONITOR, OPERATE ON SERVICE SNOWCONVERT_AI.DATA_MIGRATION.DATA_EXCHANGE_WORKER_SERVICE TO ROLE ; -``` - -If Step 1 or `scai data orchestrator stop` fails with an insufficient-privileges error, apply the grants above and retry from the failed step. - ---- - -## Step 1: Verify No In-Flight Workflows - -Do **not** suspend mid-job. Check the most recently observed job(s) first: - -- `job_status()` — every job must report `terminal: true`. - -Then probe the orchestrator's queue directly: +- `job_status()` — every job you know about must report `terminal: true`. +- Then probe the orchestrator's queue (SPCS, cross-machine): ```sql SELECT WORKFLOW_ID, STATUS, COUNT(*) AS N @@ -67,153 +54,101 @@ GROUP BY WORKFLOW_ID, STATUS ORDER BY WORKFLOW_ID; ``` -If any rows return, **abort teardown** and report to the user: +If any rows return, **abort teardown** and report: > Teardown skipped — workflow `` still has `` `` task(s). Wait for completion (or cancel via the troubleshooting reference) before suspending. -Otherwise continue to Step 2 (or Step 4 only if no orchestrator was ever started). +If a job is running only on this machine and you intend to stop it anyway, pass +`data_infrastructure(mode="down", force=true)`. --- -## Step 2: Stop the orchestrator - -### 2a. SPCS orchestrator — when `compute_pool` is configured +## Step 2: Tear down -**Skip 2a and Step 3** when there is no `compute_pool` (local orchestrator path). +Call the tool: -The orchestrator service is the dominant SPCS cost driver — it pins the compute pool active. Suspend it first: - -```bash -scai data orchestrator stop ``` - -Then poll status until it reports `SUSPENDED`. **Poll every 15 s for up to 3 minutes (12 polls).** If the service has not reached `SUSPENDED` after 12 polls, surface the last CLI output and stop — do not proceed to Step 3 while the orchestrator may still be running. - -```bash -scai data orchestrator status +data_infrastructure(mode="down") # suspend (resumable) — the default +data_infrastructure(mode="down", drop=true) # SPCS: also permanently drop the DEW service ``` -The CLI returns `{"status": "SUSPENDING"}` for ~60-90s before settling at `{"status": "SUSPENDED"}`. +The response reports `execution` (`local` | `cloud`) and the `orchestrator` / `worker` actions +(`stopped_local` | `suspended_spcs` | `dropped` | `stop_failed`). Relay it to the user. On SPCS the +compute pool auto-suspends shortly after the service stops — no pool call needed. -> **Why not raw SQL?** `scai data orchestrator stop` wraps `ALTER SERVICE … SUSPEND` and uses the connection / role / warehouse from the active project. Use raw `ALTER SERVICE SNOWCONVERT_AI.DATA_MIGRATION.DATA_MIGRATION_SERVICE SUSPEND` only if the CLI is unavailable. - -### 2b. Local orchestrator — when no `compute_pool` / user chose local orchestrator in setup - -**Skip 2a and Step 3.** The local orchestrator runs as a **foreground process** (`scai data orchestrator start --local`). `scai data orchestrator stop --local` is **advisory only** — it does not kill a running process. - -Tell the user: - -> Stop the local orchestrator so it stops polling Snowflake. -> - Find the terminal running `scai data orchestrator start --local` and press `Ctrl+C`. -> - If it was launched in the background, end that process (Task Manager / `pkill` as appropriate). - ---- +If the payload reports a `partial` status with `spcs_errors`, read the error to decide what it means: +- An **insufficient-privileges** failure on the `ALTER SERVICE … SUSPEND` the tool runs through scai — + grant the privileges below and re-run `data_infrastructure(mode="down")`. +- A **"service … does not exist"** failure on `worker stop` — this project has no SPCS DEW worker (Iceberg, + a local worker, or `start_worker=false` at `up`); nothing to suspend, so it is safe to ignore. -## Step 3: Suspend the compute pool (SPCS only) +### Privilege prerequisites (SPCS) -**Skip Step 3** when there is no `compute_pool`. +The tool suspends the SPCS service via `scai`, using the project's connection/role. The role that ran +`scai data orchestrator setup` holds these already; a **different** teardown role (e.g. `ACCOUNTADMIN` +cleaning up after the owner) needs them granted explicitly: ```sql -ALTER COMPUTE POOL SUSPEND; -SHOW COMPUTE POOLS LIKE ''; -``` - -`` is the value persisted by `data_infrastructure(mode="up", compute_pool=...)` — read from `.scai/settings/cloud-migration.yaml`. - -The pool's `STATE` should report `SUSPENDED` (or `STOPPING` for a few seconds, then `SUSPENDED`). - ---- - -## Step 4: Stop the Worker - -There are two worker variants and at most one is in use per project: - -### 4a. SPCS Data Exchange Worker (DEW) service — if `scai data worker setup` was used - -If the worker runs as a Snowpark Container Services service, suspend it via the CLI: - -```bash -scai data worker stop -scai data worker status +GRANT MONITOR, OPERATE ON SERVICE SNOWCONVERT_AI.DATA_MIGRATION.DATA_MIGRATION_SERVICE TO ROLE ; +-- If the DEW worker service is in use: +GRANT MONITOR, OPERATE ON SERVICE SNOWCONVERT_AI.DATA_MIGRATION.DATA_EXCHANGE_WORKER_SERVICE TO ROLE ; ``` -`stop` suspends the `DATA_EXCHANGE_WORKER_SERVICE` (default; pass `--drop` to remove permanently). `status` is the inverse of `setup`. If no DEW service exists, `scai data worker stop` returns error code `DMG0024` ("Service `SNOWCONVERT_AI.DATA_MIGRATION.DATA_EXCHANGE_WORKER_SERVICE` does not exist") — that's the signal you have no SPCS worker and should fall through to 4b. - -### 4b. Local worker process — if launched with `scai data worker start --local` - -The local worker polls the warehouse every `task_fetch_interval` seconds (see `../references/worker-config-reference.md`). Each poll wakes the warehouse and accrues credits — even with the SPCS orchestrator suspended. +### Local orchestrator started outside MCP (the tool can't reap it) -When the worker was started by **`migrate_data()` / `validate_data()`** through the MCP server, it **stops automatically when the MCP session ends**. Teardown 4b still applies when the user chose **No, keep running** on a prior wave, when the worker was started manually, or when you need to stop it before the session ends. +`data_infrastructure(mode="down")` reaps the local orchestrator/worker **this MCP session** spawned. A +local orchestrator the user launched **in their own terminal** (`scai data orchestrator start --local`) is +a foreground process the tool did not spawn and cannot kill (`scai data orchestrator stop --local` is +advisory only). Tell the user: -When the worker was started by **`migrate_data()` / `validate_data()`** through the MCP server, it is **stopped automatically when the MCP session ends** (stdio or HTTP server task exit). Teardown step 4b still applies when the user chose **No, keep running** on a prior wave, when the worker was started manually, or when you need to stop it before the session ends. - -Tell the user: - -> Stop the local worker so the warehouse can auto-suspend. -> - If you are ending the Coco/MCP session, the worker started by `migrate_data` / `validate_data` will stop on exit — no manual kill needed unless you started the worker outside MCP. -> - Otherwise find the terminal running `scai data worker start --local` and press `Ctrl+C`. -> - If the worker was launched in the background outside MCP, kill it: `pkill -f "scai data.*worker.*start"` (macOS / Linux) or end the process in Task Manager (Windows). - -Verify polling has stopped: - -```sql -SELECT QUERY_ID, USER_NAME, WAREHOUSE_NAME, START_TIME, LEFT(QUERY_TEXT, 120) AS QUERY_TEXT -FROM TABLE(INFORMATION_SCHEMA.QUERY_HISTORY_BY_USER(USER_NAME => CURRENT_USER(), RESULT_LIMIT => 20)) -WHERE QUERY_TEXT ILIKE '%TASK_QUEUE%' -ORDER BY START_TIME DESC; -``` +> Stop the local orchestrator so it stops polling Snowflake: +> - Find the terminal running `scai data orchestrator start --local` and press `Ctrl+C`. +> - If it was launched in the background, end that process (`pkill -f "scai data orchestrator.*start"`, or +> Task Manager on Windows). -Within `task_fetch_interval` seconds (default 30s) the worker's `TASK_QUEUE` queries should stop appearing. If they keep appearing, the worker process is still running. +The same applies to a local **worker** started outside MCP (`scai data worker start --local`): the +MCP-managed worker stops on session exit, but a manually-launched one needs a `Ctrl+C` / +`pkill -f "scai data.*worker.*start"`. --- -## Step 5: Cost-Hygiene Recommendations (Idempotent) +## Step 3: Cost-hygiene recommendations (idempotent, optional) -Run these once per project — they make future suspend cycles tighter. **Skip compute-pool items** when no `compute_pool` is configured. - -**Compute pool auto-suspend** — drop to ~60s so the pool unblocks even if Step 3 is skipped: +Run these once per project — they make future suspend cycles tighter. **Skip compute-pool items** when no +`compute_pool` is configured. ```sql +-- Compute pool auto-suspend — the pool unblocks quickly after the service stops: ALTER COMPUTE POOL SET AUTO_SUSPEND_SECS = 60; -``` -**Warehouse auto-suspend** — verify the warehouse used by the Snowflake connection auto-suspends quickly: - -```sql +-- Warehouse auto-suspend — verify the connection's warehouse suspends quickly: SHOW PARAMETERS LIKE 'AUTO_SUSPEND' IN WAREHOUSE ; -- If value > 60: ALTER WAREHOUSE SET AUTO_SUSPEND = 60; ``` -Both changes are persistent and safe to apply outside this teardown. +`` is the value persisted by `data_infrastructure(mode="up", compute_pool=...)` — read from +`.scai/settings/cloud-migration.yaml`. Both changes are persistent and safe to apply outside teardown. --- -## Resuming for the Next Wave +## Resuming for the next wave -Nothing auto-resumes on the next dispatch. Bring the shared infrastructure back up **once** with `data_infrastructure(mode="up")` before the next `migrate_data` / `validate_data` — it resumes the SPCS orchestrator (expect a 30–60s warm-up after suspend) or starts a persistent local orchestrator, plus the worker, depending on whether a `compute_pool` is configured. - -`data_infrastructure(mode="up")` wraps these underlying commands; run them directly only when debugging outside the tool: - -```bash -# SPCS -scai data orchestrator start -scai data worker start # SPCS DEW worker, if used -# Local -scai data orchestrator start --local -scai data worker start --local -``` +Nothing auto-resumes on the next dispatch. Bring the shared infrastructure back up **once** with +`data_infrastructure(mode="up")` before the next `migrate_data` / `validate_data` — it resumes the SPCS +orchestrator (expect a 30–60s warm-up after suspend) or starts a persistent local orchestrator, plus the +worker, depending on whether a `compute_pool` is configured. --- ## Checklist ``` -- [ ] No in-flight workflows in TASK_QUEUE (Step 1) -- [ ] SPCS orchestrator suspended (2a), or local orchestrator process stopped (2b), or N/A -- [ ] Compute pool suspended when SPCS orchestrator was used (Step 3), or N/A -- [ ] DEW worker service suspended (4a) or local worker process stopped (4b), or N/A -- [ ] Warehouse AUTO_SUSPEND <= 60s (one-time, Step 5) +- [ ] No in-flight workflows — job_status() terminal + TASK_QUEUE quiet (Step 1) +- [ ] data_infrastructure(mode="down") called; response relayed (Step 2) +- [ ] Local orchestrator/worker started outside MCP stopped by the user, or N/A +- [ ] Warehouse AUTO_SUSPEND <= 60s (one-time, Step 3), or N/A ``` Return control to the caller. + diff --git a/plugin/skills/migration/data-infrastructure/worker-spcs/SKILL.md b/plugin/skills/migration/data-infrastructure/worker-spcs/SKILL.md index 9eb2d7c..d030c77 100644 --- a/plugin/skills/migration/data-infrastructure/worker-spcs/SKILL.md +++ b/plugin/skills/migration/data-infrastructure/worker-spcs/SKILL.md @@ -84,6 +84,8 @@ Note the exact secret names — they are referenced in Step 4. **Wait for the user to confirm the secrets are created before continuing.** +> **Non-SPCS / local workers:** Snowflake Secrets above apply to SPCS container services only. For external vaults, AWS Secrets Manager REST providers, or `$(...)` command substitution in worker TOML, see [Advanced: Secrets management](../references/worker-config-reference.md#advanced-secrets-management). + **Oracle and Teradata only:** The container requires outbound network access for two destinations: the source database host and the NuGet driver download endpoint. Ask the user to provide an `EXTERNAL_ACCESS_INTEGRATION` covering both. If they do not have one, show: ```sql diff --git a/plugin/skills/migration/extensibility/TASKS.md b/plugin/skills/migration/extensibility/TASKS.md index bef8b13..817af2e 100644 --- a/plugin/skills/migration/extensibility/TASKS.md +++ b/plugin/skills/migration/extensibility/TASKS.md @@ -15,6 +15,8 @@ Drop a `SKILL.md` under either path; the plugin loads it instead of the built-in Your override SKILL.md is loaded as a normal agent skill — write it the same way you would any other skill. There is no template to subclass and no required imports. +To replace the entire `main` pipeline for one `customKind` (FiveTran, Airflow, …), discovery writes `/.scai/skills/.md` with the customer — see [Custom code units](#custom-code-units-kindcustom). That is a file, not a `/SKILL.md` directory. + To check what's currently in effect, run: ``` @@ -75,7 +77,7 @@ Tasks fall into two categories: `setup` (one-time per project) and `main` (per-o | `migrateData` | Migrates data into a deployed table (pure dispatch — requires the shared orchestrator+worker to be up). | | `validateData` | Validates migrated data against the source (pure dispatch — requires the shared orchestrator+worker to be up). | | `runTests` | Runs the scai test suite for an object. | -| `verify` | Catch-all verification for a converted object whose type has no deploy/test path of its own (Oracle PACKAGE, PACKAGE_BODY, TYPE, TYPE_BODY, SYNONYM, ...). Reached via convert's unfiltered `completed` transition, which must stay last so every type-gated route wins first. | +| `verify` | Catch-all verification for a converted object whose type has no deploy/test path of its own (Oracle PACKAGE, PACKAGE_BODY, TYPE, TYPE_BODY, SYNONYM, ...), and for procedures/functions with no source side after they deploy (SnowConvert UDF helpers). Reached via convert's unfiltered `completed` transition (must stay last) or via deploy when `source IS NULL`. | | `extractRules` | Extracts reusable migration rules from a fix. | | `applyRules` | Applies matched migration rules to an object. | | `fixCode` | Diagnoses and fixes a failing object. | @@ -89,18 +91,18 @@ Every task resolves to one **outcome**. Read-only detection (the resolver seeing | Outcome | Meaning | |---|---| | `completed` | The task finished successfully. | -| `failed` | The task could not finish; carries an `error` class (below). Routes into the fix loop, or the errored bucket for a dependency block. | +| `failed` | The task could not finish; carries an `error` class (below). Routes into the fix loop, or the errored bucket. | | `excluded` | The task was disabled for this project/object (see [How to exclude a task](#how-to-exclude-a-task)); the machine follows the task's `excluded` branch. | | `skipped` | Deferred for now (not disabled) — the task may still run later. | -| `inProgress` | An async task (`migrateData` / `validateData`) is still running. | +| `inProgress` | The task has started but has not completed. It remains the current blocking task. | **`error` classes** (required on `failed`): | Class | Use for | |---|---| | `sql` | A SQL/DDL bug the fix loop can address. | -| `dependency` | Blocked on another object not yet migrated/deployed — lands in the errored bucket, not the fix loop. | | `infra` | A transient environment failure (timeout, connection drop, cancelled run) — retry with `reset`. | +| `human` | Only a person can resolve it. Don't set this class by hand — call `transition_status(status='escalate', task=…, asks=[…])`, which sets it for you. Passing `outcome='failed'` without an error class instead sends the object into the fix loop, where no code change resolves it. A judgment you *can* make is `status='note'` (does not park; review later), not this class. | ## Per-task contracts @@ -153,6 +155,10 @@ Every entry below names the task id, what your override needs as input, and the - **Inputs:** A source language chosen in setup and the intent to migrate/validate table data (the data-infrastructure step establishes that intent). - **Done when:** Session config has `data_migration_type` (and/or `data_validation_type`) set — the data-migration-setup and data-validation-setup wizards (`progress_setup(mode="data_migration"|"data_validation")`) have run to completion. +#### `discoverExtras` +- **Inputs:** A converted project with registry populated; the list of non-built-in assets the user wants tracked. +- **Done when:** A marker file exists at `/.scai/extras_discovered` (written by the discover-extras skill once the user is done adding units). + ### `main` (per-object migration) #### `registration` @@ -173,7 +179,7 @@ Every entry below names the task id, what your override needs as input, and the #### `etlValidate` - **Inputs:** ETL test YAML present (from etlSeed or hand-authored); ETL unit deployed to Snowflake and source/Snowflake connections configured — all enforced via preconditions. -- **Done when:** Registry field `codeStatus.etlValidate` reads completed. +- **Done when:** `scai test etl-validate` stamps registry field `codeStatus.etlValidate` completed (failed runs stamp failed + error). Units skipped for a missing YAML stay pending. #### `generateTestCases` - **Inputs:** Object that needs test inputs; configured source connection. @@ -189,7 +195,7 @@ Every entry below names the task id, what your override needs as input, and the #### `captureBaseline` - **Inputs:** Object with seed data; configured source connection. -- **Done when:** Procedures/functions: the per-object YAML exists (proc seeding captures the baseline into it). BTEQ: `extensions.tasks.captureBaseline` is set — `scai test capture` uploads the baseline to the Snowflake stage and writes no local artifact, so the YAML (which `seedScript` already wrote) cannot signal capture; the agent stamps this after running capture. +- **Done when:** Procedures/functions: `VALIDATION.BASELINE_METADATA` has a row for the object's target name whose `ROW_COUNTS` sum to more than zero. BTEQ scripts have no rows in that table, so they fall through to registry field `extensions.tasks.captureBaseline`. #### `deploy` - **Inputs:** Converted SQL for the object. @@ -201,18 +207,18 @@ Every entry below names the task id, what your override needs as input, and the #### `migrateData` - **Inputs:** Deployed table; configured source connection; shared data infrastructure brought up once via data_infrastructure(mode="up"). -- **Done when:** Registry field `extensions.dataMigration` reads completed. +- **Done when:** Live cloud migration job reports the table loaded (DATA_MIGRATION.TABLE_PROGRESS). #### `validateData` - **Inputs:** Object with migrated data; shared data infrastructure brought up once via data_infrastructure(mode="up"). -- **Done when:** Registry field `extensions.dataValidation` reads completed. +- **Done when:** Live cloud validation job reports every enabled level done (DATA_VALIDATION.TABLE_PROGRESS_DETAIL). #### `runTests` - **Inputs:** Object with a captured baseline; procedures and functions are also deployed first (BTEQ scripts are not). -- **Done when:** Registry field `codeStatus.testing` reads completed. +- **Done when:** The latest run of every test case in `VALIDATION.RESULTS` passed. Procedures and functions are judged there; BTEQ scripts have no rows in that table, so they fall through to registry field `codeStatus.testing`. #### `verify` -- **Inputs:** A converted object of a type the machine routes nowhere else. +- **Inputs:** A converted object of a type the machine routes nowhere else, or a deployed procedure/function with no source counterpart. - **Done when:** Registry field `extensions.tasks.verify` reads completed. #### `extractRules` @@ -228,3 +234,79 @@ Every entry below names the task id, what your override needs as input, and the - **Done when:** Registry field `extensions.tasks.fixCode` is set. + +## Custom code units (`kind=custom`) + +Per-task overrides and custom machines are about **how** work runs. Custom code units are about **what** work runs on. The conversion engine generates a known set of object kinds (tables, views, procedures, functions, SSIS packages, …); anything outside that — orchestration tools (FiveTran, Airflow, Informatica), BI assets (SSAS cubes, Tableau extracts, dbt models), object kinds the engine doesn't generate yet (Oracle PACKAGE bodies, SQL Server triggers in some flows), or hand-maintained scripts — won't show up in the registry unless you put it there yourself. + +The registry's top-level `kind` is a closed enum with four values: `"databaseObject"`, `"script"`, `"etl"` (the three the conversion engine emits) and `"custom"` (everything else). Custom units carry an additional `customKind` discriminator on `source` and `target` — that's the free-form string the agent uses to group, filter, and route. Each custom unit carries: + +- **`kind`** — always `"custom"` for these units. The closed enum keeps registry queries and bindings simple. +- **`source.customKind` / `target.customKind`** — the free-form discriminator (`"fivetran"`, `"ssasCube"`, `"oraclePackage"`, `"airflowDag"`, …). Pick a stable name; this is what `query_registry where="source.customKind = ''"` filters on. Cannot be one of the four reserved Kind values. +- **`source.name`** — display name. +- **`source.objectType`** — *optional*. When the asset maps cleanly to a built-in `ObjectType` (e.g. an Oracle PACKAGE → `package`), set it so the unit groups with its siblings in `migration_status`. `"other"` is fine when nothing fits. +- **`files.source.path`** — *optional* path to a config file or definition, **relative to the project root** (e.g. `"source/fivetran/orders_sync/connector.py"`). Assets living outside the project must be copied under `/source/` first; an absolute or escaping path is reported in `warnings[]` because it breaks for every other checkout. +- **`dependencies.dependsOn[]`** — ids of other units (built-in or custom) this one reads/writes. `requiredBy` back-edges and `planning.topologicalRank` are derived from this by the registry on every write — never hand-write either. +- **`extensions.machine`** — name of the state machine that drives this unit, as a plain string. Only the four compiled-in machines resolve (`main`, `setup`, `data-migration-setup`, `data-validation-setup`); see the gap note below. When unset or unresolvable, a `.scai/skills/.md` skill (if present) replaces `main`; otherwise the unit walks `main`. + +### Registering custom units + +One tool, `register_units`, three shapes. Search that name if the tool is not already loaded — there is no `register_custom_unit` / `register_custom_units_from_manifest`. + +| Shape | Call | When | +|---|---|---| +| One unit | `register_units(custom_kind, name, ...)` | Walking the user through one item. | +| A list | `register_units(entries=[...])` | Manifest with many entries (CSV/JSON/YAML the user has on hand). | +| Findings | `register_units(expected_slugs=[...])` | Investigation agents already wrote `.scai/tmp/extras/findings/.json`. | + +It rejects the four reserved Kind values (`databaseObject`, `script`, `etl`, `custom`) as a `customKind`. Built-in kinds go through the regular `scai code add` / `scai code extract` paths. Per-row failures are collected into `failed[]` rather than aborting the run. + +Snake_case (`custom_kind`, `source_path`, `depends_on`, `expected_slugs`) and camelCase (`customKind`, `sourcePath`, `dependsOn`, `expectedSlugs`) both work. `dependsOn` takes either a JSON array of ids or a comma/newline-separated string. + +Every shape returns the same envelope — `registered[]` (each row has `id`, `customKind`, `name`, `machine`, `dependsOnResolved`, `dependsOnMissing`, `warnings`) plus `failed[]` / `withMissingDependencies[]` / `withWarnings[]`. Read `dependsOnMissing`: ids in it are recorded but match no unit, which is expected when the dependency is registered later and a typo otherwise. Read `warnings`: an unresolvable machine name (unit uses a `.scai/skills/.md` skill if present, otherwise `main`) or a `sourcePath` that isn't repo-relative. Don't re-query to confirm the write — `query_registry`'s default projection is `id` / `source` / `files`, so `dependencies` and `extensions` come back absent and the unit looks empty. Pass `fields=["*"]` if you do need to read them back. + +Investigation fans out; the write does not. Investigation agents write one JSON fragment each under `.scai/tmp/extras/findings/.json`. The orchestrator calls `register_units(expected_slugs=[...])`, which merges those files and registers — it does not concatenate fragments in context. Registry writes take an exclusive lock and each triggers a registry-wide graph refresh, so parallel writes from the agents themselves are slower than one batch and fragment the failure report. + +### Per-customKind skills + +Discovery writes `/.scai/skills/.md` with the customer (one cookbook per kind, reused for every object of that kind). When that file exists, those units skip `main` (register → convert → …) and run the skill as their whole workflow. Completion is the same stamp `verify` already uses: + +``` +transition_status(status='advance', task='verify', outcome='completed', where="id IN ('')") +``` + +Done when `extensions.tasks.verify` reads completed. `next_objects` / `next_task` carry `skillPath` pointing at the file. The file must be a procedure (On Entry → steps → stamp); see `setup/discover-extras/cookbook-template.md`. + +A loaded `extensions.machine` still wins when that name is compiled in. A `verify/SKILL.md` task override does not steal these units. + +### Gap: per-customKind machines are not implemented + +`Machines` loads only the four machines compiled into the server. Nothing reads `/.scai/machines/`, so a machine file written there is inert, and `extensions.machine` naming it resolves to nothing — `machine_for_unit` then uses a `.scai/skills/.md` skill if present, otherwise `main`. + +Consequences to keep in mind when extending this area: + +- Don't author a per-`customKind` machine file and report the flow as wired. The fallback is silent at resolve time; the only signals are `warnings[]` on each `registered[]` row (`withWarnings[]` on the `register_units` report) and the `Warning:` line from `update_registry(field="extensions.machine", ...)`. +- The executor kinds are `mcpTool`, `shell`, and `agent`. There is no `manual` kind and no `instructions` field — a machine using them fails to deserialize, which is a second reason a hand-written machine never takes effect. +- Making this real means loading and validating `.scai/machines/*.json` into `by_name` alongside the built-ins, and threading `project_dir` into the ~15 `Machines::load_builtin()` call sites. That's a feature, not a doc fix. + +### Invoking discovery + +The skill at `setup/discover-extras/SKILL.md` registers the units, then co-authors a `.scai/skills/.md` cookbook per kind with the customer. Load it when the user has extras to register — it is not a step in the compiled `setup` machine. + +Re-entering the skill at any time afterward is safe — it picks up where the user left off and lets them add more units. + +### Worked example: FiveTran sync depends on a table + +1. Register the FiveTran sync as a custom unit pointing at the table it reads from: + ``` + register_units( + custom_kind="fivetran", + name="orders_sync", + source_path="source/fivetran/orders_sync.yaml", + depends_on=[""], + description="Daily ingest from Shopify", + ) + ``` + Check `dependsOnMissing` on the registered row — if the table id is in there, look it up again with `query_registry` before moving on. `requiredBy` on the table and `topologicalRank` on the sync are filled in by the registry; don't touch them. +2. Discovery writes `.scai/skills/fivetran.md` with the customer (what "done" means, how one connector is migrated). Leave `machine` unset — a `fivetran-flow` machine can't load (see the gap above). +3. Run `migration_status(mode="next_objects")` — once the table the sync depends on is migrated, the FiveTran sync appears in the queue with `skillPath` pointing at that cookbook. diff --git a/plugin/skills/migration/migrate-objects/SKILL.md b/plugin/skills/migration/migrate-objects/SKILL.md index 546e1bf..4d6dd7f 100644 --- a/plugin/skills/migration/migrate-objects/SKILL.md +++ b/plugin/skills/migration/migrate-objects/SKILL.md @@ -44,12 +44,24 @@ This rule is the same for **every** task in the loop below — deploy, test, cap 1. **Do the task's work** as its skill describes. 2. **Ask what's next.** Re-pull `migration_status(mode="my_objects_summary")` (or `migration_status(mode="next_task", object_id="")` for one object). The machine advances you when it can see the work is done — a tool wrote the registry field, the object exists in Snowflake, or the expected file exists. 3. **Call `transition_status` to report or override:** - - a **failure** you can't fix — `transition_status(status='advance', task='', outcome='failed', error='')`; + - a **failure** you can't fix — `transition_status(status='advance', task='', outcome='failed', error='')`; - an outcome the system **can't observe** and you had to judge — e.g. "all tests passed" ([migrate-object/RUN_TESTS.md](migrate-object/RUN_TESTS.md)), view parity ([migrate-object/VALIDATE_VIEW.md](migrate-object/VALIDATE_VIEW.md)), ETL stabilization ([migrate-etl/SKILL.md](migrate-etl/SKILL.md)); - an **override** — `bypass` a precondition, `reset` an errored task, or `skip`. The outcome and error vocabulary is defined once in [../extensibility/TASKS.md](../extensibility/TASKS.md#outcome-vocabulary). +## Autonomous mode + +If the user asks to run the wave unattended — "autonomous", "auto-pilot", "just +migrate everything", "run objects in parallel" — load +[autonomous/SKILL.md](autonomous/SKILL.md) instead of the loop below and follow +it. That skill claims work itself and dispatches one subagent per ready task +group, up to a parallelism the user picks, escalating only when one gets stuck. + +Everything below is the interactive loop: one batch at a time, the user picks +every group and every claim. It stays the default — do not offer autonomous mode +as a menu item in 2b, and do not switch to it unless the user asks. + ## Step 2: Object Loop **IMPORTANT** Ask the user to `/compact` between work units to free up context. **IMPORTANT** @@ -77,7 +89,7 @@ Show **only** the following status lines, all derived from the cached summary re Then ask the user to pick a next action. **Only list actions you are actually offering** — never include an absent option just to acknowledge it. Build the action menu like this: -1. One numbered item per task group, labelled ` for the (s)` — e.g. `Deploy to Snowflake for the 5 tables`, `Generate test cases from source database for the 1 function`. Use `group.user_label` verbatim. +1. One numbered item per task group, labelled ` for the (s)` — e.g. `Deploy for the 5 tables`, `Generate test cases from source database for the 1 function`. Use `group.user_label` verbatim. 2. One numbered item per `blocked_groups` entry, labelled `Resolve blocked s waiting on "" (see deps)` — only if `blocked_groups` is non-empty. 3. `finishObjects` — only if `done_count > 0`. 4. `claimObjects` — only if `done_count == 0` (when `done_count > 0`, omit this entirely; the user must merge first). The exception: if the user *explicitly overrides* on a later turn ("I know, claim anyway", "skip the merge for now"), proceed to claim and flag the unmerged done objects in your reply. @@ -118,9 +130,9 @@ VERY IMPORTANT: **Wait for user input before acting.** Once the user confirms wh **User picked the errored bucket** → call `migration_status(mode="my_objects_details", group="errored")` to fetch `ErroredObject` entries and present them. **Wait for the user to pick which errored object(s) to address** before attempting any resolution; resolution depends on each `task`/`reason`. -### Resolving errored async tasks (`migrateData`, `validateData`, `runTests`) +### Resolving errored long-running tasks (`migrateData`, `validateData`, `runTests`) -When the errored task is a long-running async one and the failure is +When a long-running task fails and the failure is **transient** (Snowflake connection drop, source timeout, cancelled run), don't try to fix anything — just retry. After confirming with the user, call: diff --git a/plugin/skills/migration/migrate-objects/actions/data-migration/RUN.md b/plugin/skills/migration/migrate-objects/actions/data-migration/RUN.md index 941db1a..d7ac76c 100644 --- a/plugin/skills/migration/migrate-objects/actions/data-migration/RUN.md +++ b/plugin/skills/migration/migrate-objects/actions/data-migration/RUN.md @@ -4,7 +4,13 @@ Guide for the `migrateData` task — migrate the batch of tables the machine han ## 1. Choose the approach (once per batch, if not already set) -If the migration approach hasn't been chosen for this run, call `progress_setup(mode="data_migration")` and answer the prompts (full vs incremental, sync strategy, extraction mechanism, target table type). The choice is persisted as session defaults — skip this if it's already set. +The migration approach was chosen during setup and is persisted for all +objects. Do not re-ask or change it here. If it is unexpectedly missing, or +the user asks to change it (for example full → incremental watermark), return +the request to the main agent. The main agent must route through +[`../../../data-infrastructure/SKILL.md`](../../../data-infrastructure/SKILL.md), +which owns any setup delegation, persist the change once, and then redispatch +this task. ## 2. Generate the workflow @@ -12,7 +18,7 @@ Call `migrate_data(mode="setup", where=)`. It writes ## 3. Dispatch -Call `migrate_data(mode="run", workflow_path=)`. This is **pure dispatch** against the already-running infrastructure — there are no infra flags to pass. If the response is a `remediation` saying infrastructure is not up, **stop here — dispatch does not bring infrastructure up.** Hand back to [`../../../data-infrastructure/SKILL.md`](../../../data-infrastructure/SKILL.md): it is the single place the shared orchestrator + worker come up, and the only place the **local vs SPCS** placement is confirmed. Once it reports ready, retry this dispatch. +Call `migrate_data(mode="run", workflow_path=)`. This is **pure dispatch** against the already-running infrastructure — there are no infra flags to pass. If the response says infrastructure is not up, return the remediation to the main agent. Do not call `data_infrastructure` from this subagent; the main agent resumes the persisted setup and redispatches the task. The response carries a `monitor` block (a `job_id` and a ready-made `watch_command`). @@ -22,4 +28,6 @@ Arm the `monitor.watch_command` with the Monitor tool, or call `job_status(job_i ## When it finishes -Completion stamps the registry field `extensions.dataMigration`, which advances the machine. Present the migration summary (`SKILL.md` Step 6) and offer to tear the shared infrastructure down when the wave is done. +The machine reads live `DATA_MIGRATION.TABLE_PROGRESS` — do not stamp the registry. Present the migration summary (`SKILL.md` Step 6), and offer to tear the shared infrastructure down when the wave is done — unless you were dispatched for a single object, in which case leave it alone: `data_infrastructure(mode="down")` stops the worker every other slot is using, so that offer belongs to whoever owns the wave. + +If the job **failed**, do not re-run it and do not change Snowflake with `sql_execute`. Call `migration_status(mode="next_task")`. A failed load is `error=sql` and the machine owns the next step. A judgment you made in the converted file (meanings vs compile) is a `note` after the fix, not a live `ALTER` and not a re-dispatch of the same workflow. diff --git a/plugin/skills/migration/migrate-objects/actions/data-migration/SKILL.md b/plugin/skills/migration/migrate-objects/actions/data-migration/SKILL.md index bd2ca97..79e84e0 100644 --- a/plugin/skills/migration/migrate-objects/actions/data-migration/SKILL.md +++ b/plugin/skills/migration/migrate-objects/actions/data-migration/SKILL.md @@ -1,20 +1,20 @@ --- name: data-migration-setup -description: Setup, run, and report on cloud data migration — workflow YAML, migrate_data run, background Monitor (or poll fallback), and end-of-run summary for the user. +description: Setup, run, and report on cloud data migration — workflow YAML, migrate_data run, background Monitor, and end-of-run summary for the user. parent_skill: migration license: Proprietary. See License-Skills for complete terms --- # Data Migration Setup -One-time configuration for migrating data from a source database into Snowflake via the **scai CLI**, plus **run → monitor → report** after `migrate_data(mode="run")` (passive Monitor — always prefer it; active polling only when Monitor cannot be invoked). +One-time configuration for migrating data from a source database into Snowflake via the **scai CLI**, plus **run → monitor → report** after `migrate_data(mode="run")`. > **Always use the official tooling.** Run data migration through `migrate_data(mode="setup")` / `migrate_data(mode="run")` (backed by `scai data migrate`). **Never** suggest writing ad-hoc scripts to extract, copy, or load data outside the DMVF task pipeline — the orchestrator handles partitioning, retries, incremental sync, and load orchestration. > **Supported sources**: SQL Server, Redshift, Oracle, Teradata, PostgreSQL > **Supported targets**: Native Snowflake tables (default). **Iceberg** targets are **Redshift-only** (partial support) — see [Extraction strategies reference](./references/extraction-strategies-reference.md#iceberg-target-redshift-only--partial-support). -> **Run-only entry:** If you were routed here only to execute `migrateData` (registry task) and the workflow YAML already exists, skip Steps 1–2 and complete **Step 2a** (display the existing `workflow_path` and offer optional updates) before **Step 4**. You **must** complete **Steps 5–7** (background monitor or poll fallback, error-first migration report, teardown offer) before returning to the parent skill — even when the state machine invoked `migrate_data(mode="run")` without walking setup. +> **Run-only entry:** If you were routed here only to execute `migrateData` (registry task) and the workflow YAML already exists, skip Steps 1–2 and complete **Step 2a** (display the existing `workflow_path` and offer optional updates) before **Step 4**. Complete **Steps 5–6** (background Monitor and error-first migration report), then return the result to the parent. Per-object subagents do not manage shared infrastructure or teardown. ## Prerequisite @@ -77,10 +77,24 @@ confirmation before continuing. The migration strategy — **migration type, sync strategy, extraction mechanism, and target table type** — is chosen **once during setup** by the `dataStrategy` task (executor [`setup/data-strategy/SKILL.md`](../../../setup/data-strategy/SKILL.md)) and committed to the git main branch, so by the time you dispatch it is **already set**. Do not re-ask it here. -**Fallback only** — if the strategy is unset (e.g. a project set up before setup-phase capture): run the `data-migration-setup` wizard as a catch-up — `progress_setup(mode="data_migration")` in a loop until `completed` (idempotent; a no-op once the keys exist). Extraction is **not always ODBC** (PostgreSQL COPY; Oracle ODP.NET/DBMS_CLOUD; Redshift ODBC/UNLOAD + optional Iceberg; Teradata direct/TPT/WRITE_NOS) and each strategy has worker/infra prerequisites (worker TOML fields, `externalStage`, Oracle grants, S3/IAM, TTU) — see [Extraction strategies reference](./references/extraction-strategies-reference.md) for the per-dialect matrix. `target_table_type=iceberg` is Redshift-only. +If the user asks to change that persisted strategy, do not apply the change +inside this per-object action. Return control to the main agent, which owns the +one-time setup and must route through +[`../../../data-infrastructure/SKILL.md`](../../../data-infrastructure/SKILL.md), +which owns any setup delegation, before redispatching object work. + +If the strategy is unexpectedly unset (for example, an older project), return +that setup remediation to the main agent as well. Do not run a catch-up wizard +from an object subagent. Extraction is **not always ODBC** (PostgreSQL COPY; +Oracle ODP.NET/DBMS_CLOUD; Redshift ODBC/UNLOAD + optional Iceberg; Teradata +direct/TPT/WRITE_NOS), and each strategy has worker/infra prerequisites owned +by setup; see [Extraction strategies reference](./references/extraction-strategies-reference.md) +for the per-dialect matrix. `target_table_type=iceberg` is Redshift-only. For **Preliminary** migrations, after YAML generation (Step 2a) add `whereClauseCriteria` per table with a valid WHERE predicate; see `./references/workflow-config-reference.md`. +> **Advanced — preflight dry-run:** If the customer wants a **bounded pipeline smoke test** (one partition per table, transient `PREFLIGHT_` schema, not production targets), set `preflight: true` in the workflow YAML at Step 2a — see [Advanced operations reference](../../../data-infrastructure/references/advanced-operations-reference.md#preflight-workflows-bounded-migration-dry-run). This is **not** Preliminary row sampling to production and **not** `scai data doctor`. + ### 1.C — Confirm ``` @@ -130,6 +144,8 @@ Notes: `configure(snowflake_database=...)`), and Oracle `columnNamesToPartitionBy` (`ROWID`) when the CLI left them empty. +> **Duplicate data on re-run:** `migration_type=full` or `preliminary`, or `sync_strategy=none`, performs a **full extract and load every run**. Re-running the same workflow against a target that already has rows from a prior run **appends duplicate/extra data**. For repeatable runs, use `sync_strategy=watermark` or `checksum` (and set `primaryKeyColumns` / `watermarkColumn` / `checksumExpression` as needed — see [workflow-config-reference.md](./references/workflow-config-reference.md#synchronizationstrategy)). **Never** `TRUNCATE` or bulk-`DELETE` the target without explicit user confirmation — the table may legitimately contain pre-existing or expected rows. Before any cleanup: confirm with the user, compare source vs target row counts, and prefer switching to incremental sync for future runs. + ### Step 2a: Display, optional edits, confirm 1. Read `workflow_path`. @@ -137,7 +153,7 @@ Notes: - **Small/medium files** — show the full YAML in chat. - **Large files** — show the path, `tables:` count, `defaultTableConfiguration`, and table names; offer to show the full file or specific tables on request. - Note whether setup **reused** an existing file (`workflow_reused`) or regenerated it. -3. **Summarize:** table count, `migration_type`, sync strategy, extraction strategy, `target_table_type`, and any `partition_key_findings` from setup. +3. **Summarize:** table count, `migration_type`, sync strategy, extraction strategy, `target_table_type`, and any `partition_key_findings` and `computed_column_findings` from setup. 4. Ask verbatim: > Here is the migration workflow at ``. @@ -159,12 +175,28 @@ Notes: | Column rename/type map | `columnNameMappings`, `columnTypeMappings` | | Server-side export | `extraction.strategy`, `externalStage` + worker TOML (UNLOAD/WRITE_NOS/DBMS_CLOUD) | | Iceberg target | `target.tableType`, `target.icebergConfig`, `migrationStrategy` | + | Teradata mixed charsets / untranslatable bytes | `onUntranslatable` (`substitute` default, `fail` to stop on Error 6706); applies to ODBC, TPT, and `write_nos` — see `dmvf/docs/data-migration-orchestrator/teradata-charset-extraction.md` | For stalled or partially finished runs, see [Task model reference](./references/task-model-reference.md) and [Troubleshooting reference](./references/troubleshooting-reference.md). 6. **If the user chooses "Proceed":** skip discretionary edits unless agent-only blockers remain (step 7). 7. **Agent-only blockers** — apply without re-prompting unless you need a value from the user: - Resolve `partition_key_findings` and required `columnNamesToPartitionBy` per `edit_hints` (empty `[]` finishes the workflow without moving data; SQL Server / Redshift need an explicit PK or partition column; Oracle defaults to `ROWID`; PostgreSQL: monotonic integer PK or timestamp — avoid `ctid`). + + **Show the table's columns first.** A partition column can only be judged against + the alternatives, and the user sees your edit as a one-line diff — so in the message + **before** you write `columnNamesToPartitionBy`, give per affected table the column + you picked, why it beats the others (primary key, non-nullable, cardinality / NULL + ratio from the finding's `detail`), and the columns it was picked from. Read those + from the table's DDL under `snowflake/`, or with `query_source` against the source + catalog. + - **Narrow tables** — list every column with its type. + - **Wide tables, or several tables at once** — do not paste every column. Give the + column count and the **partition candidates** only (unique / non-nullable keys, + date and numeric columns), and offer the full list per table on request. + + A bare `suggestion` string leaves the user nothing to approve or reject on. + - Resolve `computed_column_findings` (SQL Server COMPUTED columns): exclude each listed column from the table's migration — drop it from the column list / `columnNameMappings` so its stored expression value is not migrated. The target column is redefined or recomputed post-migration. - **Preliminary type:** add `whereClauseCriteria: ""` per table or `defaultTableConfiguration` when missing. - Required **extraction strategy** fields (`extraction.strategy`, `externalStage`, worker TOML cross-refs for `unload` / `write_nos` / `dbms_cloud` / `tpt`). - **Redshift Iceberg** (`target_table_type=iceberg`): required fields from [iceberg-setup-reference.md](./references/iceberg-setup-reference.md). @@ -200,25 +232,31 @@ migrate_data(mode="run", workflow_path="") Returns immediately with `job_id` — migration is **dispatched** in the background via `scai data migrate create-workflow` against the already-running shared orchestrator + worker. -**Prerequisite:** the shared infrastructure must already be up. If you have not done so this session, run `data_infrastructure(mode="up")` once first — it brings up the orchestrator + worker (SPCS when a `compute_pool` is configured, otherwise local), runs the doctor gate, and returns the `cost_reminder` to relay. If infrastructure is not up, `migrate_data(mode="run")` returns a `remediation` pointing at `data_infrastructure(mode="up")` — bring it up and retry. +**Prerequisite:** the main agent brought the shared infrastructure up once +during setup. A per-object task never starts, repairs, or reconfigures it. If +`migrate_data(mode="run")` returns an infrastructure remediation, return that +remediation to the main agent; it resumes the persisted placement and +redispatches the object task. The `execution` field (`"local"` | `"cloud"`) and `cost_reminder` are on the `data_infrastructure(mode="up")` response — **relay the reminder to the user verbatim**. If absent, use the note that matches the `execution` field: -> **Cost note (cloud):** The SPCS orchestrator and local worker are running and shared across every dispatch this session; they can keep using Snowflake credits while idle (the worker polls the warehouse on an interval). When you are done, tear the infrastructure down — `data_infrastructure(mode="down")` for the local worker, and the teardown skill to suspend the SPCS orchestrator and compute pool. The local worker stops when this session ends; the SPCS orchestrator does not. +> **Cost note (cloud):** The SPCS orchestrator and local worker are running and shared across every dispatch this session; they can keep using Snowflake credits while idle (the worker polls the warehouse on an interval). When you are done, tear the infrastructure down with `data_infrastructure(mode="down")` — it stops the local worker **and** suspends the SPCS orchestrator (the compute pool then auto-suspends). The local worker also stops when this session ends; the SPCS orchestrator does not, so call `down` explicitly. > > **Cost note (local):** A local orchestrator and worker run for this session, shared across every dispatch, and stop when the session ends or you call `data_infrastructure(mode="down")`. Source extraction still runs SQL against your warehouse while active. Retain `workflow_path` for the report in Step 6. -After run starts, go to **Step 5** (do not busy-poll every 30–60s unless you are on the fallback path). +After run starts, go to **Step 5**. --- -## Step 5: Wait for completion (Monitor, or poll fallback) +## Step 5: Wait for completion with Monitor -Load [Background monitoring](./references/background-monitoring.md) and follow it. Summary below; the reference is authoritative for capability checks, event phases, and crash fallback. +Use Monitor for the asynchronous wait. `job_status` is for a user-requested +status update, handling a Monitor event, or producing the final report; it is +not a second polling loop. -**Status tool** (both paths): +**Status tool:** ``` job_status(job_id="") # cheap summary @@ -227,20 +265,7 @@ job_status(job_id="", details=true) # + full progress and failure repo `job_id` is `monitor.job_id` from the `migrate_data(mode="run")` response. `details` may report `details_unavailable` until `create-workflow` returns a workflow name; that is normal early in the run. -### 5.A — Choose path - -**Always prefer Monitor when it can be invoked.** Polling is never the better choice while Monitor is available — one relay poller serves every watcher, it wakes you on trouble and not only on completion, and it spends no tool call per check. - -| Condition | Path | -|-----------|------| -| Monitor tool can be invoked — **the default** | **Background** — Phases A–D in the reference | -| Monitor genuinely unavailable (absent from the tool list, or invoking it fails) | **Fallback** — active poll below | - -If you are unsure whether Monitor is available, **try it** rather than defaulting to the poll. Calling `job_status(job_id)` because the user asked for an update is not the polling path and is always fine. - -**One completion owner:** only the background Monitor path **or** the fallback poll may present Step 6 — never both. - -### 5.A.1 — Background path (always use this when Monitor is available) +### 5.A — Background monitoring 1. **Phase A — Monitor:** Start the **Monitor** tool (`persistent: true`) with `monitor.watch_command` from the run response, verbatim, from the project root. No wait for a workflow name, and no hand-built command — the cursor baked into it is what prevents replays and gaps. 2. **Phase B — Monitor fire:** Branch on the event's `phase`. `failure` / `stalled` / `relay_error` are warnings — surface them and keep watching. On `terminal`: `job_status(job_id, details=true)` once → Step 5.C if needed → **Step 6**. @@ -253,39 +278,18 @@ Tell the user once that you'll report back when the job finishes or hits trouble **If the watch goes silent,** re-arm with `job_status(job_id, monitor=true)` — it returns a fresh cursor and watch command and restores the relay's poller if it was lost. With no progress loop there is no second timer cross-checking the watch, so this is the recovery path after a compaction, a session restart, or an answer that looks stale. -### 5.A.2 — Fallback path (last resort — only when Monitor cannot be invoked) - -Poll until the job is terminal: - -- Call `job_status(job_id)` repeatedly until `terminal` is `true` (or when the user asks for an update). A natural gap between turns is enough — **do not insert your own timer**. -- On polls where you are **not** running a health check (Step 5.B), optionally share a one-line update from `summary`, or from `details.progress.output` when you pass `details=true` (e.g. `preprocessedTables`/`totalTables`, `aggregatedCounts.loadedPartitions`/`totalPartitions`). - -**Never wait with `bash sleep` (or by tailing worker logs) for migrate progress.** That burns wall-clock and skips the status tool. The only allowed wait signals are Monitor (preferred) or another `job_status` call. Short `sleep` after killing a process (1–3s) is fine; multi-tens-of-seconds sleeps to "give the workflow time" are not. - -**Stop when** `terminal` is `true`. - -Then Step 5.C → Step 6. - ### 5.B — Health monitoring (while waiting) -On the background path the relay emits a `stalled` event when counters stop moving, so you do not compute stalls yourself — warn the user and keep watching. - -On the fallback path, derive health from consecutive `job_status(job_id, details=true)` responses. Keep the previous response in memory (at least `details.progress.output.aggregatedCounts`, `preprocessedTables`, `totalTables`, and `tablePartitions`). - -**When to run:** - -- **Background path:** the relay reports stalls itself; check health when it fires an event or the user asks. -- **Fallback path:** every **2nd or 3rd** poll, or when the user asks. Skip until `details.progress.output` exists. - -**Progress key:** `loadedPartitions` from `details.progress.output.aggregatedCounts` (fallback: sum of `tablePartitions[].loadedPartitions`). Record the poll/tick time when this key last increased. +Monitor emits `stalled` and `failure` events. On either event, call +`job_status(job_id, details=true)` once to inspect the current state, surface +the warning, and keep the Monitor armed. -| Signal | Background (relay events) | Fallback (30–60s poll) | Severity | -|--------|---------------------------|-------------------------|----------| -| Stall | `stalled` event | `loadedPartitions` unchanged **≥10 minutes** | **Warning** | -| Stuck | a second `stalled` event, or one still standing when you next look | unchanged **≥20 minutes** | **Critical** | -| Partition failures | `failure` event, or `failed: true` on any event | `aggregatedCounts.failedPartitions > 0` | **Warning** (immediate) | -| Preprocessing lag | `preprocessedTables < totalTables` and running **≥30 minutes** | running **≥15 minutes** | **Warning** | -| Partial table failure | Any `tablePartitions[]` with `failedPartitions > 0` or `hasBeenPreprocessed == false` while still running | same | **Warning** | +| Signal | Severity | +|--------|----------| +| `stalled` event | **Warning** | +| Repeated unresolved `stalled` event | **Critical** | +| `failure` event, or `failed: true` on any event | **Warning** (immediate) | +| Any `tablePartitions[]` with failed partitions or incomplete preprocessing | **Warning** | When **`reports`** is present, also scan `details.reports.files.errors` — any rows mean at least one task/partition has failed even if aggregate counters look healthy. @@ -304,13 +308,13 @@ On **Warning** or **Critical**, point to [Troubleshooting Reference](./reference Fold the worst severity seen while waiting into Step 6 **Infrastructure** or **Load** only if it was never surfaced to the user. -A stall or stuck warning does **not** end monitoring — only a `terminal` event (or a fallback poll seeing `terminal: true`) does. +A stall or stuck warning does **not** end monitoring — only a `terminal` event does. When a job fails before a workflow exists, the failure text is in the job's `summary` — surface it under **Infrastructure** in `### Errors`. ### 5.C — Finished workflow with incomplete tables (anomaly) -After the terminal status (Monitor fire + confirm, or fallback poll), if `details.progress.output.isFinished == true` **and** any of: +After the terminal Monitor event and final status confirmation, if `details.progress.output.isFinished == true` **and** any of: - `preprocessedTables < totalTables` - any `tablePartitions[].hasBeenPreprocessed == false` @@ -325,6 +329,12 @@ After the terminal status (Monitor fire + confirm, or fallback poll), if `detail --- +If the job **failed**, do not re-run it and do not change Snowflake with `sql_execute`. Call `migration_status(mode="next_task")`. A failed load is `error=sql`; the machine sends you to `applyRules` → `fixCode`. Edit the converted `snowflake/` file, **deploy**, then the fix loop retries `migrateData`. + +A live `ALTER` / `DROP` / `CREATE` that is not in `snowflake/` is not a `note` — the next `deploy` overwrites it. `note` is for a judgment you already made in the file (you chose among meanings and can name the inverse). See the walker definition: meanings vs compile. Do not `note` as a substitute for the fix, and do not re-dispatch the same failed workflow. + +--- + ## Step 6: Report — data migration summary **Do not skip.** Present an **error-first** summary in chat (markdown). Build it from the final `job_status(job_id, details=true)` response (`details.progress`, `details.reports`) and classify failures — **not** a per-table results grid unless the user asks. @@ -378,7 +388,7 @@ Number fixes in the **same category order** as **Errors** (Preprocessing → Ext - [Troubleshooting Reference](./references/troubleshooting-reference.md) for worker/orchestrator/partition issues - YAML / TOML edits: `whereClauseCriteria`, partitions, `source.databaseName`, extraction strategy, worker connection fields — as required by the error category -- **Re-run only as a follow-up:** mention `migrate_data(mode="run", workflow_path=...)` in `### Suggested fixes` only when prerequisites are clear, or label it “after the steps above” — do not list re-run as the first or only fix +- **Re-run only as a follow-up:** mention `migrate_data(mode="run", workflow_path=...)` in `### Suggested fixes` only when prerequisites are clear, or label it “after the steps above” — do not list re-run as the first or only fix. Before suggesting re-run, confirm the workflow uses incremental sync (`watermark` / `checksum`) or that the user accepts a full reload; a non-incremental re-run against a populated target duplicates rows (see duplicate-data callout in Step 2). - After all tables succeed → offer [validation](../../../validate-objects/actions/validate_tables.md) for the same scope Do **not** include a per-table markdown table unless the user asks for a full audit. @@ -435,7 +445,7 @@ Omit empty category subsections. If every table completed **and Step 5.C does no - **All tables succeeded** — offer validation for this scope or continuing the wave. - **Any failures** — do **not** jump to re-run. Summarize the prerequisite actions from `### Suggested fixes`, then **ask the user** how to proceed, for example: 1. Apply fixes (YAML/TOML/config) — you or the user edits files; confirm when done - 2. Re-run the same workflow — only after prerequisites are done or the user explicitly accepts re-run without fixes (e.g. transient infra) + 2. Re-run the same workflow — only after prerequisites are done or the user explicitly accepts re-run without fixes (e.g. transient infra). **Warn:** if `sync_strategy=none` (or no `synchronization` block), re-run reloads all rows and **duplicates data** on the target; prefer adding `watermark`/`checksum` or scoped cleanup confirmed with the user — never blind `TRUNCATE`/`DELETE`. 3. Narrow scope — adjust `where` / workflow YAML and run setup + run for failed tables only 4. Investigate further — troubleshooting reference, logs, health signals from Step 5.B 5. Stop — proceed to Step 7 (teardown) without re-running @@ -446,14 +456,20 @@ Then continue to Step 7 (teardown offer) when wave data work is done for this pa ## Step 7: Offer to tear down infrastructure (cost saving) -When the wave's data work is done, offer to tear down idle infrastructure to save cost (default **Yes**), then **delegate** — do **not** re-derive what is running here: - -- **Local** orchestrator/worker this session started → `data_infrastructure(mode="down")`. -- **SPCS** orchestrator / compute pool / DEW worker (or any mixed setup) → load [`../../../data-infrastructure/teardown/SKILL.md`](../../../data-infrastructure/teardown/SKILL.md); it owns the "what's running" detection (its *Which steps apply* table) and runs only the applicable steps. +When the wave's data work is done, offer to tear down the shared infrastructure to save idle cost — a single +prompt regardless of placement (default **Yes**): -Nothing auto-resumes — dispatch is pure, so bring infrastructure back for the next wave with `data_infrastructure(mode="up")`. (A local worker started via MCP also stops when the session ends.) +> Tear down the shared data infrastructure to stop idle cost? Bring it back for the next wave with +> `data_infrastructure(mode="up")` (dispatch does not auto-resume). +> +> 1. **Yes (default)** — call `data_infrastructure(mode="down")`. +> 2. **No, keep running** — next batch soon; avoids the ~60s SPCS warm-up. -If the user picks **Yes** (or doesn't respond), run the teardown, then return to the parent skill. +On **Yes** (or no response), call `data_infrastructure(mode="down")` and relay the returned `execution` + +`orchestrator`/`worker` actions. Then load `../../../data-infrastructure/teardown/SKILL.md` **only** for the +cases the tool cannot cover on its own: the cross-machine in-flight `TASK_QUEUE` check before suspending +shared SPCS, a local orchestrator/worker the user started **outside** MCP (needs Ctrl+C / `pkill`), or a +`partial` payload reporting an SPCS privilege failure. Otherwise return to the parent skill. --- @@ -462,7 +478,7 @@ If the user picks **Yes** (or doesn't respond), run the teardown, then return to Shared infrastructure checklist is owned by `../../../data-infrastructure/SKILL.md`. Migration-specific items: ``` -- [ ] Migration strategy set (captured at setup via dataStrategy; fallback wizard only if unset) +- [ ] Migration strategy set during main-agent setup - [ ] Strategy-specific worker/infra prerequisites met per extraction-strategies-reference.md - [ ] Workflow YAML generated via migrate_data(mode="setup", ...) (or existing file reviewed at Step 2a) - [ ] User saw workflow YAML and was offered optional field updates (Step 2a) @@ -471,7 +487,7 @@ Shared infrastructure checklist is owned by `../../../data-infrastructure/SKILL. - [ ] Target database and schema exist - [ ] Iceberg prerequisites validated — if Redshift + `target_table_type=iceberg` - [ ] migrate_data(mode="run") started -- [ ] Monitor used (the default); active polling only if Monitor could not be invoked (Step 5) +- [ ] Monitor armed with the run response's watch command (Step 5) - [ ] Waited until the job reported a `terminal` event - [ ] Finished-but-incomplete anomaly checked (Step 5.C — do not report success if tables never preprocessed) - [ ] Health monitoring run while waiting when triggered (Step 5.B — stall/failure signals surfaced) @@ -485,11 +501,13 @@ Return control to the parent skill. ## Reference +- [Advanced operations reference](../../../data-infrastructure/references/advanced-operations-reference.md) — rate limiting, preflight, incremental/revalidate DV - [Background monitoring](./references/background-monitoring.md) - [Workflow Config Reference](./references/workflow-config-reference.md) - [Task Model Reference](./references/task-model-reference.md) - [Extraction Strategies Reference](./references/extraction-strategies-reference.md) - [Iceberg Setup Reference](./references/iceberg-setup-reference.md) - [Troubleshooting Reference](./references/troubleshooting-reference.md) +- [Advanced operations reference](../../../data-infrastructure/references/advanced-operations-reference.md) — rate limiting, preflight dry-run - [Data Doctor reference](../../../data-infrastructure/references/data-doctor-reference.md) - [Teardown (cost-saving suspend)](../../../data-infrastructure/teardown/SKILL.md) diff --git a/plugin/skills/migration/migrate-objects/actions/data-migration/references/task-model-reference.md b/plugin/skills/migration/migrate-objects/actions/data-migration/references/task-model-reference.md index 191a33f..8aaf726 100644 --- a/plugin/skills/migration/migrate-objects/actions/data-migration/references/task-model-reference.md +++ b/plugin/skills/migration/migrate-objects/actions/data-migration/references/task-model-reference.md @@ -51,18 +51,46 @@ Validation adds L1/L2/L3 chains with Snowpipe drain barriers when `useSnowpipeFo ## Scope grammar -Scopes are hierarchical strings used for filtering and pause/cancel operations: +Every task has a `SCOPE`: a hierarchical string of `::`-joined fragments, ordered least → most specific, describing the task's purpose and target. Scopes drive prefix queries, **rate limiting** (`RATE_LIMIT.SCOPE_PATTERN`), and the pause/resume/cancel procedures — all match `SCOPE` with SQL `LIKE`. The owning workflow is tracked separately in `TASK_QUEUE.WORKFLOW_ID` (not embedded in table/partition scopes). + +### Data migration scopes | Pattern | Meaning | |---------|---------| -| `Table[DB.SCHEMA.TABLE]::Preprocessing` | Table setup phase | -| `Table[DB.SCHEMA.TABLE]::Partition[N]::Extraction` | Partition extraction | -| `Table[DB.SCHEMA.TABLE]::Partition[N]::Loading` | Partition load | -| `preflight::::schema_drop` | Preflight cleanup | +| `Table[DB.SCHEMA.TABLE]::Preprocessing` | Table setup phase (metadata + partition strategy) | +| `Table[DB.SCHEMA.TABLE]::Partition[N]::Extraction` | Partition extraction (DEA) | +| `Table[DB.SCHEMA.TABLE]::Partition[N]::DeletionKeysExtraction` | Primary-key extraction for `trackDeletions` (distinct from `Extraction`) | +| `Table[DB.SCHEMA.TABLE]::Partition[N]::Loading` | Partition load (`COPY INTO` / Snowpipe) | +| `Table[DB.SCHEMA.TABLE]::Preprocessing::SnowpipeSetup` / `::SnowpipeTeardown` | Snowpipe pipe create / drop | +| `Table[DB.SCHEMA.TABLE]::Preprocessing::PreflightSetup` | Preflight (bounded dry-run) setup | +| `preflight::::schema_drop` | Drop transient `PREFLIGHT_` schema | +| `workflow::::transient_cleanup` | Clean up transient resources at workflow end | + +The table identifier in `Table[...]` is the **normalized source FQN** (for example `MY_DB.DBO.CUSTOMERS`). + +### Data validation scopes -Query tasks by scope prefix: +Validation tasks are prefixed with `DV::` so they never collide with migration scopes. + +| Pattern | Meaning | +|---------|---------| +| `DV::Table[ID]::Preprocessing` | Validation metadata / table prep | +| `DV::Table[ID]::SchemaValidation` | L1 schema validation | +| `DV::Table[ID]::Partition[N]::MetricsValidation` | L2 metrics validation | +| `DV::Table[ID]::Partition[N]::RowValidation` | L3 row-hash validation | +| `DV::Table[ID]::Partition[N]::CellDrilldown` | Hybrid L3 cell drill-down (may carry `Batch[k]` before the op) | +| `DV::Table[ID]::Partition[N]::WriteResults[row\|cell]` | Write results (row-hash or cell); batched as `...::Batch[k]::WriteResults[cell]` | +| `DV::Table[ID]::Evaluate[LEVEL]` | Evaluate a completed level | +| `DV::Table[ID]::ReconcilePossibleMismatches` | Post-drilldown reconcile of `POSSIBLE_MISMATCH` | +| `DV::Table[ID]::L3EarlyStopMonitor` | Periodic L3 early-stop monitor | +| `DV::Table[ID]::DetectionComplete` / `::SyncBaseline` / `::SyncFinalize` | Incremental validation bookkeeping | +| `DV::Pipe[KEY]::SnowpipeSetup\|SnowpipeTeardown\|SnowpipeDrain\|SnowpipePrepareDrain` | Snowpipe ops for validation results | +| `DV::ObjectTypeDetection::Preprocessing` | Object-type dispatch task | + +### Querying / matching by scope ```sql +-- All tasks for one table (migration): SELECT ID, NAME, STATUS, LAST_ERROR_MESSAGE FROM SNOWCONVERT_AI.DATA_MIGRATION.TASK_QUEUE WHERE WORKFLOW_ID = @@ -70,7 +98,7 @@ WHERE WORKFLOW_ID = ORDER BY ID; ``` -The table identifier in `Table[...]` is the **normalized source FQN** (for example `MY_DB.DBO.CUSTOMERS`). +The same `LIKE` matching powers rate-limit `SCOPE_PATTERN` rules (e.g. `Table[%]::Loading` caps concurrent loads) and scope-filtered queries or pause/cancel (e.g. `DV::Table[%]::Partition%::RowValidation` selects L3 tasks). See [rate limiting](../../../../data-infrastructure/references/advanced-operations-reference.md#rate-limiting-protect-source-or-shared-resources). ## Dependency model @@ -138,4 +166,4 @@ Lower number = higher priority. Fan-out / strategy tasks ≈ 1; extraction sprea - [Troubleshooting reference](./troubleshooting-reference.md) - [Workflow config reference](./workflow-config-reference.md) -- [Data Doctor reference](../../../data-infrastructure/references/data-doctor-reference.md) +- [Data Doctor reference](../../../../data-infrastructure/references/data-doctor-reference.md) diff --git a/plugin/skills/migration/migrate-objects/actions/data-migration/references/troubleshooting-reference.md b/plugin/skills/migration/migrate-objects/actions/data-migration/references/troubleshooting-reference.md index 4718cb7..ad33270 100644 --- a/plugin/skills/migration/migrate-objects/actions/data-migration/references/troubleshooting-reference.md +++ b/plugin/skills/migration/migrate-objects/actions/data-migration/references/troubleshooting-reference.md @@ -257,12 +257,41 @@ ORDER BY ID; If metadata/schema extraction completes with **zero rows** but no worker error, compare worker TOML `[connections.source.*].database` against workflow `source.databaseName` (Oracle: service name; Teradata: database name). This is the most common silent failure mode. +### Teradata Error 6701 / 5355 (mixed charsets) + +**Symptom:** Extraction task fails on Teradata with Error **6701** or **5355** when the source table has columns in different character sets (for example LATIN + KANJISJIS + GRAPHIC in one row). + +**Fix:** Ensure the workflow uses a current orchestrator build with charset-aware Teradata extraction (automatic `_TO_UNICODE` per column). If the customer still hits untranslatable-byte edge cases, set `onUntranslatable: substitute` (default) or `fail` for strict tables. See `dmvf/docs/data-migration-orchestrator/teradata-charset-extraction.md`. + +### Teradata Error 6706 (untranslatable bytes, fail mode) + +**Symptom:** Extraction fails with **6706** on a table configured with `onUntranslatable: fail`. + +**Fix:** Expected behavior — the table contains bytes that cannot map to Unicode under the chosen charset translation. Either clean/source-fix the data, use `onUntranslatable: substitute` for lossy U+FFFD replacement, or scope `whereClauseCriteria` to exclude bad rows (if acceptable). + --- ## `POSSIBLE_MISMATCH` after validation completes Hybrid L3 validation may stop early when `earlyStoppingForRowHashing` or `maxFailedRowsNumber` is reached. A workflow can finish with `POSSIBLE_MISMATCH` result codes — **do not treat as a clean pass**. Review L3 result tables and consider re-running with adjusted early-stop settings or narrower `sourceWhereClause`/`targetWhereClause` filters. +> **Data validation is read-only** — re-running a DV workflow compares source and target; it does not move or duplicate data on either side. + +--- + +## Target has more/duplicate rows after re-running a migration + +**Symptom:** Target row count exceeds source (or a prior migration run), or users report duplicate keys/rows after a second `migrate_data(mode="run")`. + +**Cause:** The workflow used **non-incremental** sync (`sync_strategy=none`, no `synchronization` block, or `migration_type=full`/`preliminary` without watermark/checksum). Each run extracts and loads **all** matching rows again — `COPY INTO` appends to the target; the orchestrator does not deduplicate on full reload. + +**Remediation (careful — do not destroy legitimate data):** + +1. **Verify:** Compare source vs target row counts (and sample keys if available). Confirm whether extra rows came from a re-run vs pre-existing target data. +2. **Confirm with the user** before any destructive action. The target may hold rows that are expected or unrelated to this migration. +3. **Do not** blindly `TRUNCATE` or bulk-`DELETE` the target. If cleanup is required, scope it (for example delete rows loaded in a specific partition/window, or dedupe by primary key) only after explicit user approval. +4. **Going forward:** Add `synchronization.strategy: watermark` or `checksum` (with `watermarkColumn`, `checksumExpression`, and `primaryKeyColumns` as needed) so subsequent runs are incremental. See [workflow-config-reference.md](./workflow-config-reference.md#synchronizationstrategy). + --- ## Workflow finished but tables incomplete @@ -327,6 +356,50 @@ ORDER BY WORKFLOW_ID; --- +## Source overloaded or too many concurrent extractions/loads + +**Symptom:** Migration or validation is slow; source DBA reports connection pressure; many extraction/load tasks run at once; customer wants to throttle without stopping workers entirely. + +**Cause:** Default parallelism (`max_parallel_tasks` per worker × number of workers) may exceed what the source can sustain. + +**Fix path (prefer in order):** + +1. **Rate limiting (advanced):** Insert rules into `DATA_MIGRATION.RATE_LIMIT` to cap concurrent tasks by scope pattern — see [Advanced operations reference](../../../../data-infrastructure/references/advanced-operations-reference.md#rate-limiting-protect-source-or-shared-resources). This is metadata SQL, not workflow YAML. +2. Lower `max_parallel_tasks` in worker TOML (per-machine parallelism). +3. Reduce worker count or pause workers during peak source hours. + +Do **not** suggest orchestrator polling-interval env vars as a throttle mechanism. + +--- + +## Tasks stay pending on trial / non-hybrid metadata accounts + +**Symptom:** Workers are running, source is healthy, but `TASK_QUEUE` rows remain `pending`; adding more workers does not help or makes it worse. + +**Cause:** Snowflake account lacks **Hybrid Table** support — orchestrator metadata fell back to **standard/`TRANSIENT`** tables. Task claiming uses a slower, contention-sensitive path; **too many workers** compete for the same queue rows. + +**Agent guidance:** + +1. Confirm metadata mode (trial account, `Unsupported feature 'HYBRID TABLE'` at bootstrap, or `SNOWFLAKE_USE_HYBRID_TABLES=0`). +2. **Reduce** worker count and `max_parallel_tasks` before adding more infrastructure — see [Metadata storage mode reference](../../../../data-infrastructure/references/metadata-storage-mode-reference.md). +3. Distinguish from source overload ([rate limiting](../../../../data-infrastructure/references/advanced-operations-reference.md#rate-limiting-protect-source-or-shared-resources)) and missing workers (affinity mismatch). + +--- + +## Incremental sync or checksum did not detect a column change + +**Symptom:** Customer edited data (especially in `text`/`ntext`/`image`, LOBs, floats, spatial, or high-precision timestamps) but the next incremental migration or incremental validation run did not re-process the partition; checksum unchanged. + +**Cause:** Built-in **partition checksums** exclude or normalize some types before hashing. A custom `checksumExpression` only reflects that SQL aggregate. **Watermark** sync ignores columns that are not the watermark. DM checksum and DV L3 row-hash use **different** pipelines — a column skipped from DM checksum may still be compared at L3. + +**Agent guidance:** + +1. Confirm sync strategy (`checksum` vs `watermark`) and whether the changed column is in the checksum input. +2. Explain using the skipped/lossy type table — see [Advanced operations reference](../../../../data-infrastructure/references/advanced-operations-reference.md#checksum--incremental-sync--types-that-may-not-trigger-re-sync). +3. Offer remediation: one-time **full** run; custom **`checksumExpression`**; switch to **watermark** if appropriate; **DV L3** + `validationCustomNormalizationRules` when the issue is compare semantics. + +--- + ## Extra / unexpected tables in the migration workflow **Symptom:** The user asked to migrate N specific tables, but the workflow YAML diff --git a/plugin/skills/migration/migrate-objects/actions/data-migration/references/workflow-config-reference.md b/plugin/skills/migration/migrate-objects/actions/data-migration/references/workflow-config-reference.md index 57604f5..1cd0e6d 100644 --- a/plugin/skills/migration/migrate-objects/actions/data-migration/references/workflow-config-reference.md +++ b/plugin/skills/migration/migrate-objects/actions/data-migration/references/workflow-config-reference.md @@ -12,9 +12,47 @@ | `affinity` | String | No | Only orchestrator and worker instances with a matching affinity will process this workflow. If the SPCS orchestrator was started with a specific affinity (visible in service logs as `Orchestrator affinity: `), the workflow **must** set the same value or it will be silently skipped. The worker's `[application].affinity` must also match. Omit from all sides for fresh setups. | | `preflight` | Boolean | No | When `true`, cap each table to one partition and run against a transient `PREFLIGHT_` schema (bounded dry-run). Default `false`. | | `preflightKeepSchema` | Boolean | No | When `preflight` is `true`, skip cleanup so the transient schema remains for manual inspection. Default `false`. | -| `cleanUpTransientResources` | `"never"` \| `"on-success"` \| `"always"` | No | Delete intermediate stage files for this workflow after it finishes (`TASK_RESULTS` and any external stages used by extraction). Default `"never"`. Underscores are accepted (`on_success`). | +| `cleanUpTransientResources` | `"never"` \| `"on-success"` \| `"always"` | No | Delete intermediate stage files for this workflow after it finishes (`TASK_RESULTS` and any external stages used by extraction). Default `"on-success"`. Underscores are accepted (`on_success`). Set `"never"` to retain stage files for debugging. When the orchestrator runs in Iceberg metadata mode, the omitted-key default may be `"always"` instead — check your deployment profile if you rely on the default. | | `intervalHandling` | `"interval"` \| `"varchar"` | No | How PostgreSQL/BigQuery mixed-family interval columns are mapped. Default `"interval"`. Can be overridden per table. | +## Preflight (bounded dry-run) + +Set top-level `preflight: true` for a **migration smoke test**: each table runs as a single partition; loads go to transient schema `PREFLIGHT_`, not the configured production target. Optional `preflightKeepSchema: true` retains the schema after the workflow for inspection. + +**Not the same as** Preliminary migration type (`whereClauseCriteria` loads to real targets) or `scai data doctor` (infra health checks). + +When a customer asks for a dry-run or pipeline test before full migration, offer preflight at Step 2a. Full agent guidance: [Advanced operations reference](../../../../data-infrastructure/references/advanced-operations-reference.md#preflight-workflows-bounded-migration-dry-run). + +## Teradata: mixed charsets and `onUntranslatable` + +**When:** Teradata extraction fails with **6701** / **5355** (mixed charsets in one row), or the customer asks how to handle **untranslatable** non-Unicode bytes during migration. + +**Default:** omit `onUntranslatable` or set `"substitute"` under `defaultTableConfiguration` — untranslatable bytes become **U+FFFD** and migration continues. + +**Strict mode:** per-table `"onUntranslatable": "fail"` when any untranslatable byte must abort the partition (Teradata **6706**). + +Applies to **`regular`**, worker-side **TPT**, and **`write_nos`** (orchestrator-built `SELECT`; no DEA TOML keys). Detail: `dmvf/docs/data-migration-orchestrator/teradata-charset-extraction.md`. + +```yaml +defaultTableConfiguration: + onUntranslatable: substitute + extraction: + strategy: regular # or write_nos + externalStage +tables: + - source: { databaseName: ecommerce, tableName: mixed_charset_orders } + target: { databaseName: TARGET_DB, schemaName: ECOMMERCE_TD, tableName: MIXED_CHARSET_ORDERS } + columnNamesToPartitionBy: [order_id] + - source: { databaseName: ecommerce, tableName: strict_audit } + target: { databaseName: TARGET_DB, schemaName: ECOMMERCE_TD, tableName: STRICT_AUDIT } + columnNamesToPartitionBy: [id] + onUntranslatable: fail +``` + +```yaml +preflight: true +preflightKeepSchema: false +``` + ## TableConfiguration | Property | Type | Required | Description | @@ -34,6 +72,7 @@ | `loading` | Object | No | Loading strategy: `warehouse` (default, `COPY INTO`) or `snowpipe`. | | `queryModifiers` | Object | No | SQL hints to reduce locking on busy source tables during extraction (see [Query modifiers](#query-modifiers)). | | `intervalHandling` | `"interval"` \| `"varchar"` | No | Per-table override of top-level `intervalHandling`. | +| `onUntranslatable` | `"substitute"` \| `"fail"` | No | **Teradata only.** How non-Unicode string columns handle untranslatable bytes during extraction. Default `"substitute"`. Set under `defaultTableConfiguration` for a workflow-wide default, or per table to override. Use `"fail"` when untranslatable non-Unicode bytes must abort the partition (Error 6706) instead of substituting U+FFFD. Detail: `dmvf/docs/data-migration-orchestrator/teradata-charset-extraction.md`. | | `executionTimeoutMinutes` | Integer | No | Wall-clock timeout in minutes for the **Analyze boundaries** DEA task only (orchestrator default is **20** when omitted). Does **not** apply to extraction or load. Use per table for large/slow boundary queries, or under `defaultTableConfiguration` to apply to all tables. | ## SourceTargetIdentifier @@ -139,10 +178,14 @@ extraction: | Strategy | Description | Best for | |----------|-------------|----------| -| `none` (default) | Full extraction every run | Small tables or unpredictable changes | -| `checksum` | Hash all column values per partition; re-extract changed partitions only | Dimension tables without a monotonic column | +| `none` (default) | Full extraction every run | One-time loads, or tables you will not re-migrate without clearing the target first | +| `checksum` | Hash all column values per partition; re-extract changed partitions only | Dimension tables without a monotonic column. **Oracle:** built-in partition checksum is supported (`STANDARD_HASH` over normalized columns); optional `checksumExpression` (for example `MAX(ORA_ROWSCN)`) overrides the default hash. Some Oracle types are excluded from the default hash — see checksum type coverage below. | | `watermark` | Track a monotonic column; sync only rows newer than the last observed value | Fact tables, event logs with a reliable `UPDATED_AT` / ID column | +> **Re-running without incremental sync:** With `strategy: none` (or no `synchronization` block), every migration run extracts and loads **all** matching rows again. Re-running the same workflow against a target that already holds data from a prior run **appends duplicate rows** (or loads more data than expected). Use `watermark` or `checksum` for repeatable incremental runs. Do **not** `TRUNCATE` or bulk-`DELETE` the target without explicit user confirmation — the table may legitimately contain pre-existing or expected rows. + +> **Checksum type coverage:** Built-in partition checksums **skip or normalize** some types (SQL Server `text`/`ntext`/`image`; Oracle LOBs/`LONG`/`XMLTYPE`/`VECTOR`; float rounding; spatial WKT; Redshift `HLLSKETCH`). Changes only in those columns may **not** change the checksum — no re-extract on the next run. Custom `checksumExpression` (for example `MAX(ORA_ROWSCN)`) only reflects what that expression measures. See [Advanced operations reference](../../../../data-infrastructure/references/advanced-operations-reference.md#checksum--incremental-sync--types-that-may-not-trigger-re-sync). + ```yaml synchronization: strategy: none diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/SKILL.md b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/SKILL.md index 06c2178..fbc81da 100644 --- a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/SKILL.md +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/SKILL.md @@ -25,6 +25,18 @@ Fix SnowConvert ETL conversion gaps through phased execution with upfront unit a - Active Snowflake connection with a warehouse - DATABASE + SCHEMA with write privileges (`CREATE TABLE`, `CREATE FUNCTION`, `CREATE PROCEDURE`) +### When the converted-output folder is not isolated + +The default contract above assumes one folder per unit. If instead you're handed a flat, whole-repository conversion output where a dbt project is referenced by more than one sibling unit's orchestration file (e.g. Informatica `SHORTCUT` mappings reused across workflows), do not edit the shared project in place: + +1. Stage a real copy of the orchestration file and every dbt project it references under `{PACKAGE_FOLDER}/Output/ETL/{unit}/`. Treat the original location read-only until sync-back. +2. If a staged dbt project's `packages.yml` has a local `path:` dependency, do **not** edit the path to account for the extra staging depth — that breaks canonical when synced back. Instead, symlink the shared-assets directory into the unit folder at the depth `packages.yml` already expects. +3. Before syncing any fix back, run the leak gate on every touched file, then diff against the original — test-environment values (schema/database names, credentials) must never reach the canonical copy: + ```bash + uv run --project {SKILL_DIR} python {SKILL_DIR}/scripts/check_sync_leaks.py {SESSION_JSON} [ ...] + ``` + Do not copy until this exits 0. + ## Persistent Files All files stored in `{UNIT}/stabilization/`: @@ -47,14 +59,18 @@ All files stored in `{UNIT}/stabilization/`: **NEVER use `bash sleep`, `bash_output`, or `cortex agent output` to wait for agents.** -Background agents deliver results via **automatic task notifications** — a new conversation turn arrives when each agent finishes. The correct flow: +You are the orchestrator. Do **not** spawn a subagent whose job is "run etl-stabilization" or "execute Phase N". Named teammates (context mappers, test-gen, fixer, apply-fixes) are the only valid spawns. + +**Claim in-flight work only from this turn's spawn tool results.** If this turn did not return live agent ids, nothing is running. Do not tell the user a subagent is executing. -1. Spawn all background agents in a **single message** (parallel tool calls) -2. **End your turn** — do not issue further tool calls or narrate while waiting -3. When a notification arrives, process that agent's result -4. When all notifications have arrived, verify output files exist and continue +How to wait, by spawn mode: -Do not attempt to call `agent_output` — it may not be available and failed attempts waste a turn. Automatic task notifications are the only reliable mechanism. +- **`run_in_background=false`** (planning mappers, and any blocking spawn): results return in **this** turn. Process them here. Do **not** end the turn to wait. +- **`run_in_background=true`**: spawn all agents in one message (parallel tool calls). Confirm each spawn returned an agent id. **Then** end the turn — no extra tool calls, no "still running" narration. A new turn arrives when each agent finishes. Process that result, then verify output files exist. + +Do not attempt to call `agent_output` — it may not be available and failed attempts waste a turn. + +**Stall:** If `STATE.md` / phase artifacts have not changed and you have no live agent ids, you are stalled. Resume from `STATE.md` (Execution Workflow) or tell the user it stalled — do not keep claiming work is in progress. ## Entry Point @@ -67,7 +83,7 @@ On every invocation: Store the flavor in session. **dbt flavor** runs the standard path below unchanged. **Scripting flavor** replaces the data-flow half of stabilization with the mapping-procedure pair (proc-test-gen → proc-fixer) while keeping the orchestration half identical — see [Scripting Flavor Routing](#scripting-flavor-routing). In the guided flow, whether a scripting unit reaches this skill is controlled upstream by the migration state machine; when it does — or on direct invocation — proceed with the scripting path. 1. Check if `{UNIT}/stabilization/tracking/STATE.md` exists 2. **If no STATE.md** → new unit → run **Planning Workflow** -3. **If STATE.md exists** → read it → run **Execution Workflow** for next pending phase +3. **If STATE.md exists** → read it → run **Execution Workflow** for next pending phase. If status is "Ready to execute" (or next-action is Execute Phase N) and Phase N has no cortex task / `start-phase` has not run, that is the post-ROADMAP stall: **start Execution Step 2 in this turn**. Do not wait for a subagent. --- @@ -117,7 +133,7 @@ cortex ctx step add -t \ "Step 3: Initialize tracking" \ "Step 4: Configure test environment" \ "Step 5: Backup and strip dead code" \ - "Step 6: Context mapping — spawn parallel agents, wait for notifications" \ + "Step 6: Context mapping — spawn parallel blocking agents, process results this turn" \ "Step 6b: Classify dbt project readiness" \ "Step 7: Create ROADMAP (7a-7c)" \ "Step 8: Create STATE.md" \ @@ -227,7 +243,7 @@ Output: `dbt-context.md` — project health, model inventory, macro inventory, s mapping and its defective blocks (block-locate / ewi-extract). The orchestration context mapper still runs for the workflow task graph. -**Spawn both agents in a single message** (parallel tool calls), then follow the **Agent Wait Protocol**: end your turn and wait for task notifications. When both notifications arrive, verify both output files exist and are non-empty before continuing. +**Spawn both agents in a single message** (parallel tool calls) with `run_in_background=false`. Process both tool results in this turn. Verify both output files exist and are non-empty before continuing. Do not end the turn to wait — blocking spawns are not background jobs. Use `team_delete` tool after verifying outputs. @@ -241,6 +257,7 @@ If `scan.json` contains dbt_projects: - **Ready**: `has_valid_config=true`, zero or low EWI count → standard dbt phase - **Needs bootstrap**: `has_valid_config=false` or `has_placeholder_config=true` → dbt phase with bootstrap sub-phase (config + macro fixes before model testing) - **Heavy EWI**: high EWI count relative to model count → dbt phase with expected baseline failures, longer fix cycle + - **Reused (pre-existing) project**: if the dbt project directory predates this unit — more than one sibling unit's orchestration file references it, or `dbt-context.md` shows it wasn't newly generated for this unit — cross-check every var default the models actually read (`sources.yml`, `dbt_project.yml` vars) against *this* unit's own source-XML identity (folder/repository/workflow name), even when `has_valid_config=true`. A previously-stabilized shared project is not a smoke-check target; wrong-but-valid-looking defaults are a silent data-correctness defect, not a compile error. These classifications inform the ROADMAP phase design in Step 7. Do NOT mark projects as `needs-user` at planning time — that decision is made by the dbt-test-gen agent after attempting test generation. @@ -258,6 +275,7 @@ Using orchestration-context.md, dbt-context.md (if dbt projects exist), and scan - **Large item**: a dbt project with >20 models. **Only dbt projects can be classified as large** — orchestration elements are always small-medium - **Phase cap**: pack up to **40-50 small-medium items per phase** OR up to **20 large items per phase** - **Do NOT mix classes in the same phase** — route small-medium and large items into separate phases so sizing stays predictable + *(This mixing rule governs phases approaching the item cap — for a phase with only a handful of items, keep them together even if one crosses a size threshold; splitting a 3-item phase into two is needless fragmentation.)* - **Batch cap**: ~10 small-medium items per batch, ~5 large items per batch - **Concurrency cap**: max 5 parallel batches per phase regardless of item size - Push each phase toward its cap rather than creating many small phases — a 48-project small-medium phase is preferable to two 24-project phases when the items share patterns @@ -331,7 +349,13 @@ Display the ROADMAP to the user. Highlight: ### Step 10: Begin Execution -After user approval, proceed to execute Phase 1 using the Execution Workflow below. +After user approval, **in the same turn** (before any wait, and without spawning an orchestrator subagent): + +1. Run Execution Workflow **Step 2** (create/verify the Phase 1 cortex task). +2. Run `track_status.py start-phase` as Step {1}.1 requires. +3. Continue Execution Step 3 for Phase 1. + +`STATE.md` saying "Ready to execute" is not evidence that work is running. If you stop after Step 8/9 without Step 2, the session is stalled at 0%. --- @@ -431,6 +455,7 @@ always has a pending task. - **Catastrophic file corruption**: Restore from `checkpoints/phase_{N}/` or `original/`. - **Snowflake connection failure**: Verify connection. Re-run `track_status.py set-test-env` if credentials changed. - **ROADMAP amendment needed**: Log via `track_status.py add-decision`, amend future phases only, document reasoning. +- **Stalled after ROADMAP / fake subagent progress**: `STATE.md` still "Ready to execute", no new artifacts, agent claiming a subagent is running. Nothing is running. Resume at Execution Step 2 in this turn. Tell the user it stalled if you cannot start. --- @@ -472,11 +497,13 @@ See [reference/examples.md](reference/examples.md). See [reference/troubleshooting.md](reference/troubleshooting.md). +`scan_unit.py` fails, or the unit folder doesn't match Prerequisites → check whether the input is a flat, multi-unit conversion output rather than an isolated per-unit folder; see "When the converted-output folder is not isolated" above. + ## Output - Fixed orchestration `.sql` file with EWI gaps resolved - Fixed dbt model files (per sub-project) - Test artifacts in `{UNIT}/stabilization/tests/` - `{UNIT}/stabilization/report.html` — self-contained HTML report aggregating all artifacts (generated during Final Validation) -- `artifacts/tracking/fix_log.md` — append-only record of every fix applied +- `artifacts/tracking/fix_log.md` — append-only record of every fix applied. Each entry carries per-instance anchors (file + stable symbol/tag anchor into the fixed tree and the `stabilization/original/` backup) plus a `Classification` — `engine-defect | conversion-improvement | intentional-decline | context-dependent`. **An `!!!RESOLVE EWI!!!` breaking wrapper (or any correctly emitted supported EWI/FDM) is `intentional-decline`, resolved manually — never log it as `engine-defect`.** See [reference/templates/fix-log-format.md](reference/templates/fix-log-format.md) and [reference/templates/batch-artifacts.md](reference/templates/batch-artifacts.md). - `artifacts/phases/phase_{N}/` — per-phase baselines, batch reports, learnings diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/dbt-fixer/SKILL.md b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/dbt-fixer/SKILL.md index 286caaa..99a1647 100644 --- a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/dbt-fixer/SKILL.md +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/dbt-fixer/SKILL.md @@ -17,7 +17,7 @@ Fix dbt projects generated by SnowConvert. Scope includes: model SQL files (stag ### Input -> **Concurrent execution note:** You may be spawned concurrently with other agents fixing different dbt projects in the same ETL unit. Each agent operates on its own project — do not access or modify files belonging to other dbt projects. +> **Concurrent execution note:** You may be spawned concurrently with other agents fixing different dbt projects in the same ETL unit. Each agent operates on its own project — do not access or modify files belonging to other dbt projects. Writing test/fix artifacts for another agent's project (even to be helpful) breaks that project's own validator lookup and causes a false missing-artifact failure at phase completion. This sub-skill expects: - `session_status.json` — from track_status.py init (contains `dbt_projects` array with project names and paths, and `source_file_path`) @@ -271,6 +271,13 @@ Follow the Main Mode workflow in full. Apply these behavioral overrides: - Process only the dbt project(s) assigned to the current phase in ROADMAP.md - Do NOT write directly to `tracking/fix-log.md` — use per-project learning file instead. The orchestrator merges these after all dbt agents complete. +Before summarizing, re-read the final state of every file you edited (`dbt_project.yml`, `sources.yml`, +`profiles.yml`) — confirm placeholder keys were actually renamed (not left alongside a new one), and +that `profile:` matches this repository's established per-project naming convention (`{project_name}`, +matching sibling projects). Do not summarize from memory of what you intended to change — the hard +`models:`/`name:` mismatch check now runs automatically at phase completion, but naming-convention +drift does not. + ### Context Management If you feel context pressure after completing a node, **stop and report partial completion** — list which nodes were fixed and which were not. Prefer stopping early over running to exhaustion. diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/dbt-test-gen/SKILL.md b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/dbt-test-gen/SKILL.md index 20d191f..66dcc12 100644 --- a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/dbt-test-gen/SKILL.md +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/dbt-test-gen/SKILL.md @@ -20,7 +20,7 @@ Tests follow the **Arrange-Act-Assert (AAA)** pattern: Arrange (dbt seeds), Act ## Scope Filtering -> **Concurrent execution note:** You may be spawned concurrently with other agents targeting different dbt projects in the same package. Each agent operates on its own project — do not access or modify files belonging to other dbt projects. +> **Concurrent execution note:** You may be spawned concurrently with other agents targeting different dbt projects in the same package. Each agent operates on its own project — do not access or modify files belonging to other dbt projects. Writing test/fix artifacts for another agent's project (even to be helpful) breaks that project's own validator lookup and causes a false missing-artifact failure at phase completion. Read ROADMAP.md for current phase (authored by stabilization with package-specific reasoning). Process only the dbt project(s) assigned to this phase. diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/orchestration-fixer/SKILL.md b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/orchestration-fixer/SKILL.md index b317864..fb96811 100644 --- a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/orchestration-fixer/SKILL.md +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/orchestration-fixer/SKILL.md @@ -118,6 +118,42 @@ After user confirms, mark all `EXECUTE DBT PROJECT` elements as `skipped` with r ``` Update status `skipped --reason disabled-in-source`. Continue to next element. + **Check downstream dependents before moving on.** Search the orchestration SQL for **every** + `AFTER` clause that lists this element's Snowflake task name ``. Snowflake never executes a + task whose required predecessor is permanently suspended, so each of those tasks (and everything + `AFTER` them) would otherwise silently stop running once deployed, with no error anywhere. + Fan-out is the common case: several downstream tasks can reference the same disabled ``. + Snowflake `AFTER` can list multiple predecessors (`AFTER T1, T2`); when rewriting a clause, + replace only `` and preserve every other name already in that list. + + **Detect a real data dependency (required recipe — do not invent a check):** + 1. From the source definition of disabled ``, collect its write targets. Informatica: the + mapping's `TARGETINSTANCE` / `TARGETLOADORDER` names and any `CONNECTOR TOINSTANCE` that + lands on a target (see `platforms/informatica/mapping-guide.md`). Other platforms: the + equivalent target/output list in the source definition. + 2. A downstream task **depends** on `` iff its converted body references any of those + concrete target names (table, view, or stream identifiers). + 3. Known limitation: names built via variables / dynamic CTAS, plus secondary effects such as + control-table row counts or session variables `` would have set, are **not** covered by + step 2. If those are the only signals, treat as a data dependency (`needs-user`) rather + than rewiring. + + - **If the downstream task does not depend on ``'s output** — for **every** downstream + task whose `AFTER` list contains ``, replace `` in that list with the set of ``'s + enabled predecessors (walk further back through any chain of disabled predecessors). If + that walk finds no enabled ancestor, mark the downstream element `needs-user` rather than + dropping the `AFTER` clause (dropping it would turn a scheduled dependent into a root task + the customer never authored). Add a one-line comment explaining the rewire and, if + applicable, cite the source `$.PrevTaskStatus` workflow variable's own description + (it documents PowerCenter's built-in skip-and-continue semantics for disabled predecessors + — check the source XML for a `WORKFLOWVARIABLE` with `DESCRIPTION` containing "not + disabled"). This is a required fix, not optional. + - **If the downstream task does depend on ``'s output** — do not rewire silently. + Mark the downstream element `needs-user`, with a reason describing the data gap (the disabled + step's output is missing and the downstream logic needs it). This is a real functional question + for the customer, not something to resolve unilaterally. + + 2. **`EXECUTE DBT PROJECT`** — mark `skipped --reason dbt-dependency`. Continue. 3. **Identify issues** — look for `!!!RESOLVE EWI!!!` markers and `--** SSC-*` comments @@ -402,6 +438,13 @@ After completing each element, if you feel context pressure (large accumulated o **Prefer stopping early with artifacts on disk over running to exhaustion.** When stopping early, send a completion message listing: elements completed (with statuses), elements NOT processed, reason: `"partial-completion: context pressure after N elements"`. +Before summarizing, re-read the final state of every file you edited (`dbt_project.yml`, `sources.yml`, +`profiles.yml`) — confirm placeholder keys were actually renamed (not left alongside a new one), and +that `profile:` matches this repository's established per-project naming convention (`{project_name}`, +matching sibling projects). Do not summarize from memory of what you intended to change — the hard +`models:`/`name:` mismatch check now runs automatically at phase completion, but naming-convention +drift does not. + ### Team Protocol When your work is complete, your agent will automatically return results to the orchestrator. diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/orchestration-test-gen/SKILL.md b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/orchestration-test-gen/SKILL.md index ea79e39..88fd778 100644 --- a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/orchestration-test-gen/SKILL.md +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/orchestration-test-gen/SKILL.md @@ -39,8 +39,8 @@ This sub-skill expects: ``` /stabilization/tests/orchestration/ - / - .sql # Isolated test (one per element) + / # Snowflake schema tests create objects in (typically public) + .sql # Isolated test (segment after last '.' in session_status name) grouped_.sql # Grouped test (shared ARRANGE for group) ... test_report.md # Report consumed by orchestration-fixer @@ -136,8 +136,12 @@ Read `ROADMAP.md` to identify the current phase number, then scope to the assign Read `session_status.json` and filter elements where `phase == `. **Only generate tests for these elements.** Elements from other phases are ignored entirely in this invocation. Skip test generation for elements that already have test files from a prior phase. Check for existing files at: -- Isolated: `/stabilization/tests/orchestration//.sql` -- Grouped: `/stabilization/tests/orchestration//grouped_.sql` +- Isolated: `/stabilization/tests/orchestration//.sql` + (`` = the Snowflake schema the test objects are created in, typically `public`; + `` = the segment after the last `.` in the element's dotted + `session_status.json` name — e.g. `s_m_last_run_date`, not the fully-qualified + `f_Warehouse_presentation.wf_bs_facts_fl_to_pl.s_m_last_run_date`) +- Grouped: `/stabilization/tests/orchestration//grouped_.sql` If a test file already exists, mark the element as `already-tested` and skip it. @@ -360,7 +364,7 @@ Categorize each failure: - **Baseline Results**: pass/fail per element, test strategy (isolated/grouped), source definition per failure, ACT vs ASSERT failure classification - **Coverage Gaps**: elements with external deps, opaque ScriptTasks, File Enumerator loops, ARRANGE:SETUP failures -5. **MANDATORY self-check before returning** — verify that `{UNIT}/stabilization/tests/orchestration//` contains at least 1 `.sql` test file and `test_report.md` exists. If ANY file is missing, generate it before returning. If you cannot write files, include full contents in your completion message so the orchestrator can write them. +5. **MANDATORY self-check before returning** — verify that `{UNIT}/stabilization/tests/orchestration//` contains at least 1 `.sql` test file and `test_report.md` exists. If ANY file is missing, generate it before returning. If you cannot write files, include full contents in your completion message so the orchestrator can write them. 6. Return to parent skill (stabilization/SKILL.md) for orchestration fixing. @@ -420,7 +424,7 @@ Do NOT include any of these in ARRANGE:SETUP. **SETUP = element-specific DDL onl ### Output -- Test files: `PACKAGE/stabilization/tests/orchestration//` +- Test files: `PACKAGE/stabilization/tests/orchestration//` - Baseline report: `PHASES_DIR/baseline_batch_{B}.md` Write artifacts **incrementally** — one element section appended at a time. The orchestrator handles partial artifacts: completed elements are already on disk and will not be re-processed on retry. @@ -458,7 +462,7 @@ Process one element at a time in the order provided: - Use `TASK_SCHEMA` as the test schema (not DATABASE.PUBLIC or any other schema) - Use **batched assertion format** (UNION ALL) for all assertions - For **clone elements**: adapt the archetype element's test file with appropriate substitutions (schema, table names, parameters) rather than generating from scratch - - Write test files to: `PACKAGE/stabilization/tests/orchestration//` + - Write test files to: `PACKAGE/stabilization/tests/orchestration//` - Follow the Main Mode workflow for ARRANGE:SETUP, ARRANGE:SEED, ACT, and ASSERT generation — applying these overrides: - Do NOT call `set-test-env` — use `TASK_SCHEMA` directly - ARRANGE:SETUP contains ONLY element-specific DDL (infrastructure is pre-created) diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/agent-prompts/dbt-context-mapper.md b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/agent-prompts/dbt-context-mapper.md index d1243d4..1916255 100644 --- a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/agent-prompts/dbt-context-mapper.md +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/agent-prompts/dbt-context-mapper.md @@ -30,6 +30,7 @@ For each project: 1. Read `dbt_project.yml` — extract: project name, profile name, vars, model-paths, seed-paths 2. Flag placeholder values (YOUR_PROJECT_NAME, YOUR_PROFILE_NAME) as bootstrap blockers 3. Check if `vars:` section defines variables used by models +4. If this project predates the current unit — more than one sibling unit's orchestration file references it, or it wasn't newly generated for this unit — cross-check every var default the models actually read (`sources.yml`, `dbt_project.yml` vars) against *this* unit's own source-XML identity (folder/repository/workflow name), even when `has_valid_config=true`. A previously-stabilized shared project is not a smoke-check target; wrong-but-valid-looking defaults are a silent data-correctness defect, not a compile error. Surface this as a bootstrap blocker with Fix Owner `needs-user` or `dbt-fixer (bootstrap)` as appropriate. ### 3. Source Definitions diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/examples.md b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/examples.md index 621b71a..935c311 100644 --- a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/examples.md +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/examples.md @@ -7,11 +7,11 @@ User says: "Fix this ETL package at /data/packages/SalesLoad" Actions: 1. No STATE.md found → run Planning Workflow 2. Scan package, initialize tracking, gather test env (user provides DATABASE with CREATE SCHEMA privileges) -3. Use the `task` tool to spawn `name="context-mapper"` with `run_in_background=false` (no team during planning) → wait for task notification → verify `orchestration-context.md` and `dbt-context.md` +3. Use the `task` tool to spawn `name="context-mapper"` with `run_in_background=false` (no team during planning) → process the tool result in this turn → verify `orchestration-context.md` and `dbt-context.md` 4. Author ROADMAP.md from template (Write tool) with 3 phases (2 orchestration + 1 final validation), each with task definitions (task table, element-to-task assignments, schema names) 5. Register phases in `artifacts/tracking/session_status.json` via `init-roadmap --phases-json`, assign elements via `assign-phases` 6. Present ROADMAP for approval → user approves -7. Execute Phase 1: +7. **Same turn:** Execution Step 2 (cortex task) + `start-phase` + Phase 1 Setup. Do not spawn an orchestrator subagent and wait. - Create schemas: `ETL_FIX_P1_B1`, `ETL_FIX_P1_B2`, `ETL_FIX_P1_B3` (batches B1.1, B1.2, B1.3) - Use `team_create` tool: team_name="etl-fix-SalesLoad-p1" - Use `task` tool to spawn 3 test-gen agents in single message: `orchestration-test-gen-B1.1`, `orchestration-test-gen-B1.2`, `orchestration-test-gen-B1.3` diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/protocols/phase-execution.md b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/protocols/phase-execution.md index eeffdfe..d4319c3 100644 --- a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/protocols/phase-execution.md +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/protocols/phase-execution.md @@ -34,6 +34,7 @@ Auto-compact may have occurred since the last action. Re-read canonical state be > **Batch ID format:** `B{P}.{M}` (e.g., `B1.1` = Phase 1 Batch 1). The `{B}` placeholder throughout this document represents the full phase-qualified batch ID. **Resume logic:** +- If `STATE.md` says "Ready to execute" and this phase has no `start-phase` / no artifacts yet → you are stalled after ROADMAP approval. Run SKILL.md Execution Step 2 in this turn. Do not wait for a subagent. - If `baseline_batch_*.md` files already exist for all tasks → skip step b (test-gen already ran), proceed to step c - If `batch_*.md` files already exist for all tasks → skip steps b-e (fixes already ran), proceed to step f - If `apply_report.md` exists → skip steps b-f, proceed to step g diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/templates/ROADMAP_TEMPLATE.md b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/templates/ROADMAP_TEMPLATE.md index 2fba6ab..2ee1ab0 100644 --- a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/templates/ROADMAP_TEMPLATE.md +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/templates/ROADMAP_TEMPLATE.md @@ -139,7 +139,7 @@ Spawn agents (MANDATORY — do NOT skip, do NOT do the work yourself): Instruction: Read {SKILL_DIR}/orchestration-test-gen/SKILL.md {end for} Max 5 agents per wave. If more, spawn first 5, wait, then remaining. -Wait for all agents: end your turn and wait for automatic task notifications. +Wait per SKILL.md Agent Wait Protocol. End the turn only if spawn tool results returned live agent ids (`run_in_background=true`). If this turn spawned nothing, do not wait. Each notification confirms one agent completed — process its result immediately. NEVER use `bash sleep`, `bash_output`, or `cortex agent output` CLI. Validate: baseline_batch_{B}.md MUST exist for EVERY batch. @@ -148,6 +148,7 @@ Validate: baseline_batch_{B}.md MUST exist for EVERY batch. After 2 retries: mark remaining elements `failed` reason `context-exhaustion`. For partial-completion recovery (agent processed some elements then died): see reference/protocols/phase-execution.md § Partial-Artifact Recovery. +> **You (the orchestrator) run this — never inside a spawned agent's prompt.** See dbt-fixer/SKILL.md "Do NOT call track_status.py directly." Update tracking (SEQUENTIAL — one call per invocation, never batched): For each element in baseline summary: uv run --project {SKILL_DIR} python {SKILL_DIR}/scripts/track_status.py \ @@ -163,10 +164,11 @@ Spawn fix agents ONLY for batches where baseline has >=1 failing element: {end for} For batches where ALL elements passed or skipped: do NOT spawn agent. Write minimal batch_{B}.md and empty learnings_batch_{B}.md yourself. -Wait for all agents via task notifications. Validate: batch_{B}.md AND learnings_batch_{B}.md MUST exist for every batch. +Wait per SKILL.md Agent Wait Protocol. Validate: batch_{B}.md AND learnings_batch_{B}.md MUST exist for every batch. Re-read session_status.json — confirm all fixed elements have terminal status. Same retry logic as Step {P}.2 (max 2, then mark failed). For partial-completion recovery: see reference/protocols/phase-execution.md § Partial-Artifact Recovery. +> **You (the orchestrator) run this — never inside a spawned agent's prompt.** See dbt-fixer/SKILL.md "Do NOT call track_status.py directly." Update tracking (SEQUENTIAL): For each element in task artifacts: uv run --project {SKILL_DIR} python {SKILL_DIR}/scripts/track_status.py \ @@ -242,6 +244,7 @@ NO test-gen agents — patterns are already proven from the archetype phase. Wait + validate: batch_{B}.md AND learnings_batch_{B}.md for every batch. Retry logic: max 2 retries per batch, then mark failed. For partial-completion recovery: see reference/protocols/phase-execution.md § Partial-Artifact Recovery. +> **You (the orchestrator) run this — never inside a spawned agent's prompt.** See dbt-fixer/SKILL.md "Do NOT call track_status.py directly." Update tracking (SEQUENTIAL): For each element in task artifacts: uv run --project {SKILL_DIR} python {SKILL_DIR}/scripts/track_status.py \ @@ -305,6 +308,11 @@ Register dbt nodes (SEQUENTIAL — one call per project): uv run --project {SKILL_DIR} python {SKILL_DIR}/scripts/track_status.py \ init-dbt {SESSION_JSON} {PROJECT_NAME} {DBT_PROJECT_PATH} {end for} +Assign dbt phase (SEQUENTIAL — one call per project): +{for each dbt_project:} + uv run --project {SKILL_DIR} python {SKILL_DIR}/scripts/track_status.py \ + assign-dbt-phase {SESSION_JSON} --phase {P} --project {PROJECT_NAME} +{end for} - [ ] **Step {P}.2: dbt-Test-Gen Wave** Resume guard: if test_report.md already exists for ALL projects @@ -320,7 +328,7 @@ Spawn even if: placeholder config, broken macros, missing sources, heavy EWI. Test generation IS the assessment — agent documents blockers in test_report.md. Early exit WITHOUT test artifacts is a protocol violation. Do NOT do this work yourself. Do NOT edit dbt files directly. -Wait for all agents: end your turn and wait for automatic task notifications. +Wait per SKILL.md Agent Wait Protocol. End the turn only if spawn tool results returned live agent ids (`run_in_background=true`). If this turn spawned nothing, do not wait. NEVER use `bash sleep`, `bash_output`, or `cortex agent output` CLI. Validate for EACH project — ALL must exist: - stabilization/tests/dbt/{PROJECT}/seeds/ — at least 1 .csv @@ -328,6 +336,7 @@ Validate for EACH project — ALL must exist: - stabilization/tests/dbt/{PROJECT}/test_report.md If ANY missing: respawn (max 2 retries), then mark nodes `failed` reason `test-gen-exhaustion`. For partial-completion recovery: see reference/protocols/phase-execution.md § Partial-Artifact Recovery. +> **You (the orchestrator) run this — never inside a spawned agent's prompt.** See dbt-fixer/SKILL.md "Do NOT call track_status.py directly." Update tracking (SEQUENTIAL): uv run ... track_status.py update-dbt {SESSION_JSON} {PROJECT} --status dbt-tested uv run ... track_status.py update-dbt-node {SESSION_JSON} {PROJECT} {NODE} --status {result} @@ -344,6 +353,7 @@ Spawn fix agents for projects with: failing tests, compilation errors, or bootst Projects where ALL nodes passed and no compilation errors: no agent needed. Write minimal dbt_learnings_{project}.md with no-fix-needed. Wait + validate: dbt_learnings_{project}.md MUST exist for every project. +> **You (the orchestrator) run this — never inside a spawned agent's prompt.** See dbt-fixer/SKILL.md "Do NOT call track_status.py directly." Update tracking (SEQUENTIAL): uv run ... track_status.py update-dbt-node per node uv run ... track_status.py update-dbt per project @@ -400,12 +410,13 @@ Spawn proc-test-gen agents (MANDATORY — UNCONDITIONAL — do NOT skip): test_schema={SCHEMA}, block FullName(s), ROADMAP_path, SESSION_JSON {end for} Max 5 agents per wave. If more, spawn first 5, wait, then remaining. -Wait for all agents: end your turn and wait for automatic task notifications. +Wait per SKILL.md Agent Wait Protocol. End the turn only if spawn tool results returned live agent ids (`run_in_background=true`). If this turn spawned nothing, do not wait. NEVER use `bash sleep`, `bash_output`, or `cortex agent output` CLI. Validate for EACH mapping proc — ALL must exist under stabilization/tests/proc/{proc_name}/: - {proc_name}.seed.sql, {proc_name}.assert.sql, test_report.md If ANY missing: respawn (max 2 retries), then mark the proc `failed` reason `test-gen-exhaustion`. For partial-completion recovery: see reference/protocols/phase-execution.md § Partial-Artifact Recovery. +> **You (the orchestrator) run this — never inside a spawned agent's prompt.** See dbt-fixer/SKILL.md "Do NOT call track_status.py directly." Update tracking (SEQUENTIAL): uv run ... track_status.py update {SESSION_JSON} {PROC} --status proc-tested @@ -421,9 +432,10 @@ Spawn fix agents ONLY for procs whose baseline did not create-clean or has faili Procs whose baseline already creates clean and passes all assertions: no agent needed. Write a minimal fix record with Outcome no-fix-needed, then set the proc's status: uv run ... track_status.py update {SESSION_JSON} {PROC} --status no-fix-needed -Wait for all agents via task notifications. Validate: a fix record exists for every proc. +Wait per SKILL.md Agent Wait Protocol. Validate: a fix record exists for every proc. Re-read session_status.json — confirm all fixed procs have terminal status. Same retry logic as Step {P}.2 (max 2, then mark failed). +> **You (the orchestrator) run this — never inside a spawned agent's prompt.** See dbt-fixer/SKILL.md "Do NOT call track_status.py directly." Update tracking (SEQUENTIAL): uv run ... track_status.py update {SESSION_JSON} {PROC} --status {status_from_artifact} diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/templates/batch-artifacts.md b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/templates/batch-artifacts.md index 21b6650..01e3c6b 100644 --- a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/templates/batch-artifacts.md +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/templates/batch-artifacts.md @@ -46,13 +46,22 @@ Each batch agent writes a per-batch learning file at `{UNIT}/stabilization/phase ## {element_name}: {one-line summary of fix} - **EWI/FDM**: {code} — {title} +- **Marker status**: NO_MARKER | MARKER_BROKEN | MARKER_PLUS_BUG | EWI_MARKED | FDM_MARKED | EWI_CRASH +- **Classification**: engine-defect | conversion-improvement | intentional-decline | context-dependent + (Intentional `!!!RESOLVE EWI!!!` declines use marker `EWI_MARKED` + classification `intentional-decline` — do not invent a separate marker code.) - **Root cause**: {what the converter did wrong or couldn't handle} - **Fix pattern**: {the SQL transformation applied} -- **Before** (abbreviated): +- **Fix instances**: (one row per file the fix touched — anchors let a consumer pull arbitrary context on demand) + +| File (repo-relative) | Anchor (fixed) | Original ref | Source XML | +|---|---|---|---| +| {models/.../foo.sql} | {block tag / macro / model + line hint} | {stabilization/original/.../foo.sql + line hint, or `new-file`} | {source.XML :: TRANSFORMATION NAME} | + +- **Before** (short inline excerpt for humans — full context recoverable via `Original ref`): \```sql {original snippet} \``` -- **After** (abbreviated): +- **After** (short inline excerpt — full context recoverable via `Anchor (fixed)`): \```sql {fixed snippet} \``` @@ -64,5 +73,7 @@ Each batch agent writes a per-batch learning file at `{UNIT}/stabilization/phase **Rules:** - Only include entries for elements where a fix was actually applied (`test-passed` or `auto-fixed-needs-review`) - `Reusable: yes` means this pattern can be applied to similar EWI/FDM codes in future phases -- Keep snippets abbreviated (relevant lines only, not full procedure bodies) +- **`Fix instances` is required**: list every file the fix touched with a stable anchor. Prefer symbol/tag anchors (`---- Start block ''`, macro name, model name) as primary and line numbers only as a hint, since line numbers drift across later phases. `Original ref` points into the `stabilization/original/` backup (or `new-file` when the fix created the artifact). This is what lets downstream consumers (e.g. bug reports) expand context instead of re-deriving it by diffing trees. +- **`Classification` is required and must be honest about intent.** Use `intentional-decline` when the converter *correctly* emitted a declined-conversion signal (the `!!!RESOLVE EWI!!!` breaking wrapper, or a supported EWI/FDM) and the fix was manual resolution — this is **not** an engine defect. Use `engine-defect` only for wrong output with no/broken signal. Use `conversion-improvement` when a better deterministic conversion is possible. Never log an intentional `!!!RESOLVE EWI!!!` decline as `engine-defect`. +- Keep the inline Before/After excerpts short (relevant lines only) — full surrounding context is reachable through the anchors, so the log stays lean and doesn't drift. - If no fixes were applied (all elements skipped or passed without changes), write an empty learnings file with a note: `No fixes applied in this batch.` diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/templates/fix-log-format.md b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/templates/fix-log-format.md index 492ea79..169f37f 100644 --- a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/templates/fix-log-format.md +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/templates/fix-log-format.md @@ -8,9 +8,9 @@ The orchestrator merges per-batch `learnings_batch_{B}.md` files into this file ### Index (rebuilt after each phase merge) -| EWI Code | Pattern Count | Phases | -|----------|--------------|--------| -| {code} | {N} | P1, P3 | +| EWI Code | Pattern Count | Instances | Classification | Phases | +|----------|--------------|-----------|----------------|--------| +| {code} | {N} | {M fix instances} | engine-defect / conversion-improvement / intentional-decline / context-dependent | P1, P3 | ### Patterns (one section per unique fix pattern) @@ -18,11 +18,18 @@ The orchestrator merges per-batch `learnings_batch_{B}.md` files into this file - **Applicability:** {which element types/contexts this applies to} - **Preconditions:** {what must be true — e.g., "element has control variables", "pure SQL element"} -- **Before:** +- **Classification:** engine-defect | conversion-improvement | intentional-decline | context-dependent — {one-line why; an `intentional-decline` (`!!!RESOLVE EWI!!!` / supported EWI/FDM) is NOT an engine bug} +- **Fix instances:** (carried from batch learnings — one row per file touched; anchors let a consumer expand context on demand) + +| Element | File (repo-relative) | Anchor (fixed) | Original ref | Marker status | Source XML | +|---|---|---|---|---|---| +| {element} | {models/.../foo.sql} | {block tag / macro / model + line hint} | {stabilization/original/.../foo.sql + line hint, or `new-file`} | {NO_MARKER / MARKER_BROKEN / MARKER_PLUS_BUG / EWI_MARKED / FDM_MARKED / EWI_CRASH} | {source.XML :: TRANSFORMATION NAME} | + +- **Before** (representative, short — full context via `Original ref`): ```sql {original code snippet} ``` -- **After:** +- **After** (representative, short — full context via `Anchor (fixed)`): ```sql {fixed code snippet} ``` @@ -34,6 +41,7 @@ The orchestrator merges per-batch `learnings_batch_{B}.md` files into this file When merging `learnings_batch_*.md` into fix_log.md: 1. Read all learnings files from the phase 2. For each learning entry: check if a pattern with the same EWI code already exists -3. If new pattern: add new section -4. If existing pattern with new variant: add as sub-pattern -5. Rebuild the Index table at the top +3. If new pattern: add new section (copy its `Classification` and all `Fix instances` rows verbatim) +4. If existing pattern with new variant: add as sub-pattern, and **append** its `Fix instances` rows to the pattern's instance table (never collapse instances away — per-instance anchors are the whole point) +5. Rebuild the Index table at the top, updating `Instances` (total fix-instance rows) and `Classification` for each code +6. If two instances of the same code disagree on `Classification`, keep them as separate rows and flag the pattern `Classification` as `mixed — see instances` rather than silently picking one diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/tools.md b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/tools.md index 4328bd3..c85378f 100644 --- a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/tools.md +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/tools.md @@ -134,7 +134,7 @@ These are built-in tools invoked through the tool-use interface, the same way yo | `snowflake_sql_execute` | Run SQL against Snowflake | SQL string, connection | | `ask_user_question` | Ask the user a question | question text, options | -**`agent_output`** — documented in CoCo guides but not reliably available. Do not attempt to call it — failed attempts waste a turn. Use **automatic task notifications** as the only mechanism for receiving background agent results (see Agent Wait Protocol in SKILL.md). +**`agent_output`** — documented in CoCo guides but not reliably available. Do not attempt to call it — failed attempts waste a turn. Use the **Agent Wait Protocol** in SKILL.md. Never tell the user a subagent is running unless this turn's spawn tool results include live agent ids. #### How to reference tools in this skill diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/troubleshooting.md b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/troubleshooting.md index 1c8d9f5..6bbf6bd 100644 --- a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/troubleshooting.md +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/reference/troubleshooting.md @@ -5,6 +5,7 @@ - Verify source definition file path is correct and readable - Verify `uv` is installed: `which uv` - If using `--platform`, verify the platform ID matches a directory under `platforms/` +- Check whether the input is a flat, multi-unit conversion output rather than an isolated per-unit folder; see SKILL.md "When the converted-output folder is not isolated" ### track_status.py errors - Verify `scan.json` exists in `{UNIT}/stabilization/planning/` diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/check_sync_leaks.py b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/check_sync_leaks.py new file mode 100644 index 0000000..70367c0 --- /dev/null +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/check_sync_leaks.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# Copyright 2026 Snowflake Inc. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Detect test-environment / credential leaks before syncing fixes back. + +Usage: + python check_sync_leaks.py [ ...] + +Exit 1 if any leak is found; exit 0 with a one-line summary otherwise. +""" +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +ETL_FIX_RE = re.compile(r"ETL_FIX_P\d+_\w+") +CREDENTIAL_KEY_RE = re.compile( + r"^\s*(password|account|authenticator|private_key_path)\s*:\s*(.*?)\s*$" +) +PLACEHOLDER_VALUE_RE = re.compile(r"^(YOUR_|your_)") +ENV_VAR_RE = re.compile(r"\{\{\s*env_var\s*\(") + + +def load_json(path: Path) -> dict: + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def _is_placeholder_credential(value: str) -> bool: + stripped = value.strip().strip("\"'") + if not stripped: + return True + if PLACEHOLDER_VALUE_RE.match(stripped): + return True + if ENV_VAR_RE.search(stripped): + return True + return False + + +def check_file(path: Path, database: str, schema: str) -> list[tuple[int, str]]: + hits: list[tuple[int, str]] = [] + text = path.read_text(encoding="utf-8", errors="replace") + is_profiles = path.name == "profiles.yml" + for i, line in enumerate(text.splitlines(), 1): + if database and database in line: + hits.append((i, database)) + if schema and schema in line: + hits.append((i, schema)) + etl_match = ETL_FIX_RE.search(line) + if etl_match: + hits.append((i, etl_match.group(0))) + if is_profiles: + cred = CREDENTIAL_KEY_RE.match(line) + if cred and not _is_placeholder_credential(cred.group(2)): + hits.append((i, cred.group(1) + ":")) + return hits + + +def main() -> None: + if len(sys.argv) < 3: + print( + "Usage: python check_sync_leaks.py [ ...]", + file=sys.stderr, + ) + sys.exit(1) + + status_path = Path(sys.argv[1]) + if not status_path.is_file(): + print(f"Error: '{status_path}' not found", file=sys.stderr) + sys.exit(1) + + session = load_json(status_path) + test_env = session.get("test_environment") or {} + database = test_env.get("database") or "" + schema = test_env.get("schema") or "" + + files = [Path(p) for p in sys.argv[2:]] + any_hits = False + for path in files: + if not path.is_file(): + print(f"Error: '{path}' not found", file=sys.stderr) + sys.exit(1) + for line_no, pattern in check_file(path, database, schema): + print(f"{path}:{line_no}: matched '{pattern}'") + any_hits = True + + if any_hits: + sys.exit(1) + + print(f"no leaks found in {len(files)} files") + + +if __name__ == "__main__": + main() diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/path_resolver.py b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/path_resolver.py index e4d755e..7088e02 100644 --- a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/path_resolver.py +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/path_resolver.py @@ -38,10 +38,35 @@ def phases_dir(code_unit_dir: str | Path) -> Path: def phase_dir(code_unit_dir: str | Path, phase_num: int) -> Path: - """Return a specific phase directory.""" + """Return the canonical phase directory (hyphenated `phase-{N}`).""" return phases_dir(code_unit_dir) / f"phase-{phase_num}" +def phase_dir_candidates(code_unit_dir: str | Path, phase_num: int) -> tuple[Path, ...]: + """Hyphen (`phase-N`) is canonical; underscore (`phase_N`) is still read.""" + root = phases_dir(code_unit_dir) + candidates = (root / f"phase-{phase_num}", root / f"phase_{phase_num}") + unique: list[Path] = [] + seen: set[Path] = set() + for path in candidates: + key = path.resolve() if path.exists() else path + if key in seen: + continue + seen.add(key) + unique.append(path) + return tuple(unique) + + +def resolve_phase_dir(code_unit_dir: str | Path, phase_num: int) -> Path: + """Existing phase dir, preferring a populated `phase-N` then `phase_N`.""" + candidates = phase_dir_candidates(code_unit_dir, phase_num) + existing = [path for path in candidates if path.is_dir()] + if not existing: + return candidates[0] + populated = [path for path in existing if any(path.iterdir())] + return populated[0] if populated else existing[0] + + def original_backup_dir(code_unit_dir: str | Path) -> Path: """Return the original backup directory.""" return stabilization_root(code_unit_dir) / "original" diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/scan_unit.py b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/scan_unit.py index da4c131..07c38c8 100644 --- a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/scan_unit.py +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/scan_unit.py @@ -81,7 +81,14 @@ def find_orchestration_file(unit_path: Path) -> Path | None: return None -PLACEHOLDER_VALUES = {"YOUR_PROJECT_NAME", "YOUR_PROFILE_NAME", "your_project_name", "your_profile_name"} +PLACEHOLDER_VALUES = { + "YOUR_PROJECT_NAME", + "YOUR_PROFILE_NAME", + "your_project_name", + "your_profile_name", + "YOUR_SCHEMA", + "YOUR_DB", +} def assess_dbt_health(project_path: Path) -> dict: @@ -119,6 +126,14 @@ def assess_dbt_health(project_path: Path) -> dict: health["has_placeholder_config"] = True health["health_issues"].append(f"Placeholder '{placeholder}' in dbt_project.yml") + sources_yml = project_path / "models" / "sources.yml" + if sources_yml.is_file(): + sources_content = sources_yml.read_text(encoding="utf-8", errors="replace") + for placeholder in PLACEHOLDER_VALUES: + if placeholder in sources_content: + health["has_placeholder_config"] = True + health["health_issues"].append(f"Placeholder '{placeholder}' in sources.yml") + models_dir = project_path / "models" if models_dir.is_dir(): model_files = list(models_dir.rglob("*.sql")) diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/__init__.py b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/fixtures/session_status_sanitized.json b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/fixtures/session_status_sanitized.json new file mode 100644 index 0000000..827b75b --- /dev/null +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/fixtures/session_status_sanitized.json @@ -0,0 +1,23 @@ +# Sanitized session_status.json derived from wf_op_ar_accomprice (credentials/schema names stripped). +# Used as a shape reference; tests build working copies via helpers.base_session(). +{ + "migration_object": "wf_op_ar_accomprice", + "migration_object_path": "/sanitized/unit", + "orchestration_file": null, + "source_file_path": "/sanitized/source.xml", + "platform_id": "informatica", + "elements": [ + {"name": "el_fixed_a", "statement": "public.wf_op_ar_accomprice", "status": "fixed", "phase": 1, "test_strategy": "isolated"}, + {"name": "el_fixed_b", "statement": "public.wf_op_ar_accomprice", "status": "fixed", "phase": 1, "test_strategy": "isolated"}, + {"name": "el_needs_user", "statement": "public.wf_op_ar_accomprice", "status": "needs-user", "phase": 1, "test_strategy": "grouped:inf0058_dg1"} + ], + "dbt_projects": [{"name": "m_op_ar_accomprice", "path": "m_op_ar_accomprice", "status": "pending"}], + "dbt_nodes": [], + "test_environment": {"database": "TEST_DB", "schema": "TEST_SCHEMA"}, + "roadmap": { + "phases": [ + {"phase": 1, "name": "Orchestration", "scope": "orchestration", "status": "in_progress", "goal": "fix orch", "decisions": [], "parallel_safe": false, "depends_on": []}, + {"phase": 2, "name": "dbt", "scope": "dbt", "status": "pending", "goal": "fix dbt", "decisions": [], "parallel_safe": false, "depends_on": [1]} + ] + } +} diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/helpers.py b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/helpers.py new file mode 100644 index 0000000..4346cd9 --- /dev/null +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/helpers.py @@ -0,0 +1,127 @@ +"""Shared helpers for etl-stabilization script tests.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +SCRIPTS_DIR = Path(__file__).resolve().parent.parent +TESTS_DIR = Path(__file__).resolve().parent +FIXTURES_DIR = TESTS_DIR / "fixtures" + + +def run_script(script_name: str, *args: str, check: bool = False) -> subprocess.CompletedProcess: + env = os.environ.copy() + env["PYTHONPATH"] = str(SCRIPTS_DIR) + os.pathsep + env.get("PYTHONPATH", "") + return subprocess.run( + [sys.executable, str(SCRIPTS_DIR / script_name), *args], + capture_output=True, + text=True, + env=env, + check=check, + ) + + +def run_track_status(*args: str) -> subprocess.CompletedProcess: + return run_script("track_status.py", *args) + + +def write_json(path: Path, data: dict) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + return path + + +def load_json(path: Path) -> dict: + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def minimal_scan(*, source_file_path: str | None = "/sanitized/source.xml") -> dict: + return { + "unit_name": "wf_op_ar_accomprice", + "unit_path": "/sanitized/unit", + "orchestration_file": None, + "source_file_path": source_file_path, + "platform_id": "informatica", + "statements": [ + { + "name": "public.wf_op_ar_accomprice", + "elements": [ + {"name": "el_fixed_a", "issues": []}, + {"name": "el_fixed_b", "issues": []}, + {"name": "el_needs_user", "issues": []}, + ], + } + ], + "dbt_projects": [{"name": "m_op_ar_accomprice", "path": "m_op_ar_accomprice"}], + } + + +def base_session(*, unit_path: str = "/sanitized/unit") -> dict: + """Sanitized session shaped like wf_op_ar_accomprice (credentials stripped).""" + return { + "migration_object": "wf_op_ar_accomprice", + "migration_object_path": unit_path, + "orchestration_file": None, + "orchestration_file_hash": None, + "source_file_path": "/sanitized/source.xml", + "platform_id": "informatica", + "started_at": "2026-08-18T00:00:00+00:00", + "last_updated": "2026-08-18T00:00:00+00:00", + "elements": [ + { + "name": "el_fixed_a", + "statement": "public.wf_op_ar_accomprice", + "status": "fixed", + "phase": 1, + "test_strategy": "isolated", + }, + { + "name": "el_fixed_b", + "statement": "public.wf_op_ar_accomprice", + "status": "fixed", + "phase": 1, + "test_strategy": "isolated", + }, + { + "name": "el_needs_user", + "statement": "public.wf_op_ar_accomprice", + "status": "needs-user", + "phase": 1, + "test_strategy": "grouped:inf0058_dg1", + "reason": "stage path needs-user", + }, + ], + "dbt_projects": [ + {"name": "m_op_ar_accomprice", "path": "m_op_ar_accomprice", "status": "pending", "nodes": []}, + ], + "dbt_nodes": [], + "test_environment": {"database": "TEST_DB", "schema": "TEST_SCHEMA"}, + "roadmap": { + "phases": [ + { + "phase": 1, + "name": "Orchestration", + "goal": "fix orch", + "scope": "orchestration", + "status": "in_progress", + "decisions": [], + "parallel_safe": False, + "depends_on": [], + }, + { + "phase": 2, + "name": "dbt", + "goal": "fix dbt", + "scope": "dbt", + "status": "pending", + "decisions": [], + "parallel_safe": False, + "depends_on": [1], + }, + ] + }, + } diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/test_check_sync_leaks.py b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/test_check_sync_leaks.py new file mode 100644 index 0000000..83f25ed --- /dev/null +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/test_check_sync_leaks.py @@ -0,0 +1,77 @@ +"""Tests for check_sync_leaks.py (PR-4).""" +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from helpers import base_session, run_script, write_json + +PROFILES_WITH_LEAKS = """\ +m_op_ar_accomprice: + target: dev + outputs: + dev: + type: snowflake + account: preprod_joel + user: "{{ env_var('SNOWFLAKE_USER', 'cortex_code') }}" + password: "{{ env_var('SNOWFLAKE_PASSWORD', '') }}" + database: TEST_DB + schema: ETL_FIX_P2_ARACR + warehouse: COMPUTE_WH + role: SYSADMIN + threads: 1 +""" + +PROFILES_CLEAN = """\ +m_op_ar_accomprice: + target: dev + outputs: + dev: + type: snowflake + account: YOUR_ACCOUNT + user: "{{ env_var('SNOWFLAKE_USER') }}" + password: "{{ env_var('SNOWFLAKE_PASSWORD') }}" + database: "{{ env_var('SNOWFLAKE_DATABASE') }}" + schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}" +""" + + +class TestCheckSyncLeaks(unittest.TestCase): + def test_profiles_yml_with_credentials_and_etl_fix_schema_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + status_path = Path(tmp) / "session_status.json" + write_json(status_path, base_session()) + profiles = Path(tmp) / "profiles.yml" + profiles.write_text(PROFILES_WITH_LEAKS, encoding="utf-8") + + result = run_script("check_sync_leaks.py", str(status_path), str(profiles)) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("profiles.yml", result.stdout) + self.assertRegex(result.stdout, r"profiles\.yml:\d+: matched ") + self.assertTrue( + "account" in result.stdout or "preprod_joel" in result.stdout, + result.stdout, + ) + self.assertIn("ETL_FIX_P2_ARACR", result.stdout) + + def test_clean_files_exit_zero(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + status_path = Path(tmp) / "session_status.json" + session = base_session() + session["test_environment"] = {"database": "TEST_DB", "schema": "TEST_SCHEMA"} + write_json(status_path, session) + profiles = Path(tmp) / "profiles.yml" + profiles.write_text(PROFILES_CLEAN, encoding="utf-8") + model = Path(tmp) / "model.sql" + model.write_text("SELECT 1 AS id;\n", encoding="utf-8") + + result = run_script( + "check_sync_leaks.py", str(status_path), str(profiles), str(model) + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("no leaks found in 2 files", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/test_scan_unit.py b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/test_scan_unit.py new file mode 100644 index 0000000..9b9b875 --- /dev/null +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/test_scan_unit.py @@ -0,0 +1,75 @@ +"""Tests for scan_unit.py placeholder detection in sources.yml (PR-3).""" +from __future__ import annotations + +import sys +import tempfile +import unittest +from pathlib import Path + +from helpers import SCRIPTS_DIR + +sys.path.insert(0, str(SCRIPTS_DIR)) +from scan_unit import PLACEHOLDER_VALUES, assess_dbt_health # noqa: E402 + + +def _write_project(root: Path, *, dbt_project: str, sources: str | None) -> Path: + (root / "models").mkdir(parents=True) + (root / "dbt_project.yml").write_text(dbt_project, encoding="utf-8") + if sources is not None: + (root / "models" / "sources.yml").write_text(sources, encoding="utf-8") + return root + + +class TestAssessDbtHealthSourcesPlaceholders(unittest.TestCase): + def test_sources_yml_your_schema_your_db_sets_has_placeholder_config(self) -> None: + self.assertIn("YOUR_SCHEMA", PLACEHOLDER_VALUES) + self.assertIn("YOUR_DB", PLACEHOLDER_VALUES) + + with tempfile.TemporaryDirectory() as tmp: + project = _write_project( + Path(tmp) / "m_last_run_date", + dbt_project=( + "name: m_last_run_date\n" + "profile: m_last_run_date\n" + "version: '1.0.0'\n" + ), + sources=( + "version: 2\n" + "sources:\n" + " - name: raw\n" + " schema: \"{{ var('m_last_run_date_schema', 'YOUR_SCHEMA') }}\"\n" + " database: \"{{ var('m_last_run_date_database', 'YOUR_DB') }}\"\n" + " tables:\n" + " - name: BATCH_INSTANCE\n" + ), + ) + health = assess_dbt_health(project) + self.assertTrue(health["has_placeholder_config"]) + issues = " ".join(health["health_issues"]) + self.assertIn("sources.yml", issues) + self.assertTrue( + "YOUR_SCHEMA" in issues or "YOUR_DB" in issues, + health["health_issues"], + ) + + def test_clean_sources_yml_does_not_flag_placeholder_config(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + project = _write_project( + Path(tmp) / "m_clean", + dbt_project="name: m_clean\nprofile: m_clean\nversion: '1.0.0'\n", + sources=( + "version: 2\n" + "sources:\n" + " - name: raw\n" + " schema: ANALYTICS\n" + " database: PROD_DB\n" + " tables:\n" + " - name: T\n" + ), + ) + health = assess_dbt_health(project) + self.assertFalse(health["has_placeholder_config"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/test_track_status.py b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/test_track_status.py new file mode 100644 index 0000000..abe1e90 --- /dev/null +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/tests/test_track_status.py @@ -0,0 +1,606 @@ +"""Acceptance tests for track_status.py (PR-1, PR-2, PR-3, PR-5).""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from helpers import ( + SCRIPTS_DIR, + base_session, + load_json, + minimal_scan, + run_track_status, + write_json, +) + + +class TestPr1InitSourceFilePath(unittest.TestCase): + def test_init_copies_source_file_path_from_scan(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) + scan_path = unit / "planning" / "scan.json" + write_json(scan_path, minimal_scan(source_file_path="/sanitized/source.xml")) + + result = run_track_status("init", str(scan_path)) + self.assertEqual(result.returncode, 0, result.stderr) + + session = load_json(unit / "tracking" / "session_status.json") + self.assertEqual(session.get("source_file_path"), "/sanitized/source.xml") + + +class TestPr1UsageStartPhase(unittest.TestCase): + def test_bare_invocation_usage_includes_start_phase(self) -> None: + result = run_track_status() + self.assertNotEqual(result.returncode, 0) + combined = result.stdout + result.stderr + self.assertIn("start-phase", combined) + self.assertIn("session_status.json", combined) + + +class TestPr1ValidatePhaseBreakdown(unittest.TestCase): + def test_validate_phase_prints_needs_user_distinctly(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + status_path = Path(tmp) / "session_status.json" + write_json(status_path, base_session()) + + result = run_track_status("validate-phase", str(status_path), "1") + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertIn("Needs-user: 1", result.stdout) + self.assertIn("Fixed/passed: 2", result.stdout) + self.assertIn("PASS:", result.stdout) + + +class TestPr1ListTestFiles(unittest.TestCase): + def test_list_test_files_uses_tests_orchestration_not_phase_dir(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) + stmt = "public.wf_op_ar_accomprice" + orch_tests = unit / "stabilization" / "tests" / "orchestration" / stmt + orch_tests.mkdir(parents=True) + (orch_tests / "el_fixed_a.sql").write_text("SELECT 1;\n", encoding="utf-8") + (orch_tests / "grouped_inf0058_dg1.sql").write_text("SELECT 1;\n", encoding="utf-8") + # decoy in the old (wrong) phase-dir location + decoy = unit / "stabilization" / "phases" / "phase-1" / stmt + decoy.mkdir(parents=True) + (decoy / "el_fixed_a.sql").write_text("SELECT decoy;\n", encoding="utf-8") + + session = base_session(unit_path=str(unit)) + status_path = Path(tmp) / "session_status.json" + write_json(status_path, session) + + result = run_track_status( + "validate-phase", str(status_path), "1", "--list-test-files" + ) + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertIn("tests/orchestration/", result.stdout) + self.assertNotIn("phases/phase-1/", result.stdout) + self.assertGreaterEqual(result.stdout.count("el_fixed_a.sql"), 1) + + def test_list_test_files_finds_schema_short_name_layout(self) -> None: + """Real agent output: tests/orchestration//.sql.""" + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) + public = unit / "stabilization" / "tests" / "orchestration" / "public" + public.mkdir(parents=True) + files = [ + "s_m_last_run_date.sql", + "wk_pl_acr_booking_fact.sql", + "s_m_fl_tmp_bookings_services_upd.sql", + "s_GEN_PARAMETER_FILE.sql", + "s_m_pl_acr_booking_fact_trans_point_upd.sql", + "wf_bs_facts_fl_to_pl.sql", + ] + for name in files: + (public / name).write_text("SELECT 1;\n", encoding="utf-8") + + session = base_session(unit_path=str(unit)) + session["elements"] = [ + { + "name": "public.wf_bs_facts_fl_to_pl", + "statement": "public.wf_bs_facts_fl_to_pl", + "status": "fixed", + "phase": 1, + "test_strategy": "isolated", + }, + { + "name": "f_Warehouse_presentation.wf_bs_facts_fl_to_pl.s_m_pl_acr_booking_fact_trans_point_upd", + "statement": "public.f_Warehouse_presentation_wf_bs_facts_fl_to_pl_s_m_pl_acr_booking_fact_trans_point_upd", + "status": "fixed", + "phase": 1, + "test_strategy": "isolated", + }, + { + "name": "f_Warehouse_presentation.wf_bs_facts_fl_to_pl.s_GEN_PARAMETER_FILE", + "statement": "public.f_Warehouse_presentation_wf_bs_facts_fl_to_pl_s_GEN_PARAMETER_FILE", + "status": "fixed", + "phase": 1, + "test_strategy": "isolated", + }, + { + "name": "f_Warehouse_presentation.wf_bs_facts_fl_to_pl.s_m_fl_tmp_bookings_services_upd", + "statement": "public.f_Warehouse_presentation_wf_bs_facts_fl_to_pl_s_m_fl_tmp_bookings_services_upd", + "status": "skipped", + "phase": 1, + "test_strategy": "isolated", + }, + { + "name": "f_Warehouse_presentation.wf_bs_facts_fl_to_pl.wk_pl_acr_booking_fact", + "statement": "public.f_Warehouse_presentation_wf_bs_facts_fl_to_pl_wk_pl_acr_booking_fact", + "status": "fixed", + "phase": 1, + "test_strategy": "isolated", + }, + { + "name": "f_Warehouse_presentation.wf_bs_facts_fl_to_pl.s_m_last_run_date", + "statement": "public.f_Warehouse_presentation_wf_bs_facts_fl_to_pl_s_m_last_run_date", + "status": "fixed", + "phase": 1, + "test_strategy": "isolated", + }, + ] + status_path = Path(tmp) / "session_status.json" + write_json(status_path, session) + + result = run_track_status( + "validate-phase", str(status_path), "1", "--list-test-files" + ) + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertIn("Test files (6):", result.stdout) + for name in files: + self.assertIn(name, result.stdout) + self.assertNotIn("(not found)", result.stdout) + # Old dotted-statement / fully-qualified-name construction must not be used + self.assertNotIn( + "public.f_Warehouse_presentation_wf_bs_facts_fl_to_pl_s_m_last_run_date/", + result.stdout, + ) + + def test_list_test_files_reports_all_short_name_collisions(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) + orch = unit / "stabilization" / "tests" / "orchestration" + (orch / "public").mkdir(parents=True) + (orch / "other_schema").mkdir(parents=True) + (orch / "public" / "s_m_last_run_date.sql").write_text("SELECT 1;\n", encoding="utf-8") + (orch / "other_schema" / "s_m_last_run_date.sql").write_text("SELECT 2;\n", encoding="utf-8") + + session = base_session(unit_path=str(unit)) + session["elements"] = [ + { + "name": "f_Warehouse_presentation.wf_a.s_m_last_run_date", + "statement": "public.wf_a_s_m_last_run_date", + "status": "fixed", + "phase": 1, + "test_strategy": "isolated", + }, + { + "name": "f_Warehouse_presentation.wf_b.s_m_last_run_date", + "statement": "public.wf_b_s_m_last_run_date", + "status": "fixed", + "phase": 1, + "test_strategy": "isolated", + }, + ] + status_path = Path(tmp) / "session_status.json" + write_json(status_path, session) + + result = run_track_status( + "validate-phase", str(status_path), "1", "--list-test-files" + ) + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertIn("2 candidates", result.stdout) + self.assertIn("(verify)", result.stdout) + self.assertIn("public/s_m_last_run_date.sql", result.stdout.replace("\\", "/")) + self.assertIn("other_schema/s_m_last_run_date.sql", result.stdout.replace("\\", "/")) + + def test_list_test_files_reports_missing_short_name(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) + (unit / "stabilization" / "tests" / "orchestration").mkdir(parents=True) + session = base_session(unit_path=str(unit)) + session["elements"] = [ + { + "name": "f_Warehouse_presentation.wf.s_missing", + "statement": "public.wf_s_missing", + "status": "fixed", + "phase": 1, + "test_strategy": "isolated", + }, + ] + status_path = Path(tmp) / "session_status.json" + write_json(status_path, session) + + result = run_track_status( + "validate-phase", str(status_path), "1", "--list-test-files" + ) + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertIn("s_missing.sql: (not found)", result.stdout) + + +class TestPr1UnverifiedDoesNotBlockComplete(unittest.TestCase): + def test_unverified_dbt_node_does_not_block_complete_phase(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) + project = "m_op_ar_accomprice" + tests_root = unit / "stabilization" / "tests" / "dbt" / project + (tests_root / "seeds").mkdir(parents=True) + (tests_root / "tests").mkdir(parents=True) + (tests_root / "seeds" / "seed.csv").write_text("id\n1\n", encoding="utf-8") + (tests_root / "tests" / "test_x.sql").write_text("SELECT 1;\n", encoding="utf-8") + (tests_root / "test_report.md").write_text("# report\n", encoding="utf-8") + learnings = unit / "stabilization" / "phases" / "phase-2" + learnings.mkdir(parents=True) + (learnings / f"dbt_learnings_{project}.md").write_text("# learnings\n", encoding="utf-8") + + session = base_session(unit_path=str(unit)) + session["dbt_nodes"] = [ + { + "name": "stg_raw", + "path": "models/staging/stg_raw.sql", + "status": "unverified", + "project": project, + "phase": 2, + } + ] + status_path = Path(tmp) / "session_status.json" + write_json(status_path, session) + + result = run_track_status("complete-phase", str(status_path), "2") + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertIn("marked as completed", result.stdout) + + +class TestPr2AssignDbtPhase(unittest.TestCase): + def test_assign_dbt_phase_sets_phase_on_matching_nodes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + session = base_session() + session["dbt_nodes"] = [ + {"name": "n1", "path": "models/n1.sql", "status": "pending", "project": "m_op_ar_accomprice", "phase": None}, + {"name": "n2", "path": "models/n2.sql", "status": "pending", "project": "m_op_ar_accomprice", "phase": None}, + {"name": "other", "path": "models/o.sql", "status": "pending", "project": "m_other", "phase": None}, + ] + status_path = Path(tmp) / "session_status.json" + write_json(status_path, session) + + result = run_track_status( + "assign-dbt-phase", str(status_path), "--phase", "2", "--project", "m_op_ar_accomprice" + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("Assigned 2 dbt nodes", result.stdout) + + saved = load_json(status_path) + by_name = {n["name"]: n for n in saved["dbt_nodes"]} + self.assertEqual(by_name["n1"]["phase"], 2) + self.assertEqual(by_name["n2"]["phase"], 2) + self.assertIsNone(by_name["other"]["phase"]) + + def test_assign_dbt_phase_listed_in_usage(self) -> None: + result = run_track_status() + combined = result.stdout + result.stderr + self.assertIn("assign-dbt-phase", combined) + + def test_validate_phase_dbt_scope_does_not_require_elements(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + session = base_session() + session["elements"] = [] # dbt-scope phases have zero orchestration elements + session["dbt_nodes"] = [ + {"name": "n1", "path": "models/n1.sql", "status": "fixed", "project": "m_op_ar_accomprice", "phase": 2}, + {"name": "n2", "path": "models/n2.sql", "status": "unverified", "project": "m_op_ar_accomprice", "phase": 2}, + ] + status_path = Path(tmp) / "session_status.json" + write_json(status_path, session) + + result = run_track_status("validate-phase", str(status_path), "2") + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertNotIn("no elements assigned", result.stderr) + self.assertIn("Validation (dbt)", result.stdout) + self.assertIn("PASS: All dbt nodes resolved.", result.stdout) + + def test_validate_phase_infers_dbt_from_assigned_nodes_when_scope_is_missing(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + session = base_session() + session["elements"] = [] + session["roadmap"]["phases"][1].pop("scope") + session["dbt_nodes"] = [ + { + "name": "n1", + "path": "models/n1.sql", + "status": "fixed", + "project": "m_op_ar_accomprice", + "phase": 2, + } + ] + status_path = Path(tmp) / "session_status.json" + write_json(status_path, session) + + result = run_track_status("validate-phase", str(status_path), "2") + + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertNotIn("no elements assigned", result.stderr) + self.assertIn("Validation (dbt)", result.stdout) + self.assertIn("PASS: All dbt nodes resolved.", result.stdout) + + def test_update_dbt_node_syncs_top_level_dbt_nodes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + session = base_session() + session["dbt_projects"] = [ + { + "name": "m_op_ar_accomprice", + "path": "m_op_ar_accomprice", + "status": "pending", + "nodes": [{"name": "n1", "path": "models/n1.sql", "status": "pending"}], + } + ] + session["dbt_nodes"] = [ + {"name": "n1", "path": "models/n1.sql", "status": "pending", "project": "m_op_ar_accomprice", "phase": 2}, + ] + status_path = Path(tmp) / "session_status.json" + write_json(status_path, session) + + result = run_track_status( + "update-dbt-node", str(status_path), "m_op_ar_accomprice", "n1", + "--status", "unverified", "--reason", "no warehouse", + ) + self.assertEqual(result.returncode, 0, result.stderr) + saved = load_json(status_path) + self.assertEqual(saved["dbt_nodes"][0]["status"], "unverified") + self.assertEqual(saved["dbt_projects"][0]["nodes"][0]["status"], "unverified") + + +class TestPr5ConcurrentUpdates(unittest.TestCase): + def test_concurrent_update_dbt_node_both_survive(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + session = base_session() + session["dbt_projects"] = [ + { + "name": "m_op_ar_accomprice", + "path": "m_op_ar_accomprice", + "status": "pending", + "nodes": [ + {"name": "node_a", "path": "models/a.sql", "status": "pending"}, + {"name": "node_b", "path": "models/b.sql", "status": "pending"}, + ], + } + ] + session["dbt_nodes"] = [ + {"name": "node_a", "path": "models/a.sql", "status": "pending", "project": "m_op_ar_accomprice", "phase": 2}, + {"name": "node_b", "path": "models/b.sql", "status": "pending", "project": "m_op_ar_accomprice", "phase": 2}, + ] + status_path = Path(tmp) / "session_status.json" + write_json(status_path, session) + + env = os.environ.copy() + env["PYTHONPATH"] = str(SCRIPTS_DIR) + os.pathsep + env.get("PYTHONPATH", "") + + def update(node: str, status: str) -> subprocess.CompletedProcess: + return subprocess.run( + [ + sys.executable, str(SCRIPTS_DIR / "track_status.py"), + "update-dbt-node", str(status_path), "m_op_ar_accomprice", node, + "--status", status, + ], + capture_output=True, text=True, env=env, + ) + + with ThreadPoolExecutor(max_workers=2) as pool: + futs = [ + pool.submit(update, "node_a", "fixed"), + pool.submit(update, "node_b", "unverified"), + ] + results = [f.result() for f in futs] + + for r in results: + self.assertEqual(r.returncode, 0, r.stderr) + + saved = load_json(status_path) + by_name = {n["name"]: n["status"] for n in saved["dbt_nodes"]} + self.assertEqual(by_name["node_a"], "fixed") + self.assertEqual(by_name["node_b"], "unverified") + nested = {n["name"]: n["status"] for n in saved["dbt_projects"][0]["nodes"]} + self.assertEqual(nested["node_a"], "fixed") + self.assertEqual(nested["node_b"], "unverified") + + +def _write_dbt_complete_phase_artifacts(unit: Path, project: str) -> None: + tests_root = unit / "stabilization" / "tests" / "dbt" / project + (tests_root / "seeds").mkdir(parents=True) + (tests_root / "tests").mkdir(parents=True) + (tests_root / "seeds" / "seed.csv").write_text("id\n1\n", encoding="utf-8") + (tests_root / "tests" / "test_x.sql").write_text("SELECT 1;\n", encoding="utf-8") + (tests_root / "test_report.md").write_text("# report\n", encoding="utf-8") + learnings = unit / "stabilization" / "phases" / "phase-2" + learnings.mkdir(parents=True) + (learnings / f"dbt_learnings_{project}.md").write_text("# learnings\n", encoding="utf-8") + + +def _dbt_session_for_project(unit: Path, project: str) -> dict: + session = base_session(unit_path=str(unit)) + session["dbt_projects"] = [ + {"name": project, "path": project, "status": "pending", "nodes": []} + ] + session["dbt_nodes"] = [ + { + "name": "stg_raw", + "path": "models/staging/stg_raw.sql", + "status": "unverified", + "project": project, + "phase": 2, + } + ] + return session + + +class TestPr3DbtProjectYmlSanity(unittest.TestCase): + def test_complete_phase_fails_when_models_key_is_placeholder(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) + project = "foo" + _write_dbt_complete_phase_artifacts(unit, project) + (unit / project).mkdir(parents=True) + (unit / project / "dbt_project.yml").write_text( + "name: foo\n" + "profile: snowflake_test\n" + "models:\n" + " YOUR_PROJECT_NAME:\n" + " staging:\n" + " +materialized: view\n", + encoding="utf-8", + ) + status_path = Path(tmp) / "session_status.json" + write_json(status_path, _dbt_session_for_project(unit, project)) + + result = run_track_status("complete-phase", str(status_path), "2") + self.assertNotEqual(result.returncode, 0) + combined = result.stderr + result.stdout + self.assertIn("dbt_project.yml", combined) + self.assertIn("YOUR_PROJECT_NAME", combined) + self.assertIn("name: 'foo'", combined) + + def test_complete_phase_passes_when_models_key_matches_name(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) + project = "foo" + _write_dbt_complete_phase_artifacts(unit, project) + (unit / project).mkdir(parents=True) + (unit / project / "dbt_project.yml").write_text( + "name: foo\n" + "profile: foo\n" + "models:\n" + " foo:\n" + " staging:\n" + " +materialized: view\n" + " intermediate:\n" + " +materialized: ephemeral\n" + " marts:\n" + " +materialized: incremental\n", + encoding="utf-8", + ) + status_path = Path(tmp) / "session_status.json" + write_json(status_path, _dbt_session_for_project(unit, project)) + + result = run_track_status("complete-phase", str(status_path), "2") + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertIn("marked as completed", result.stdout) + + def test_complete_phase_infers_dbt_from_assigned_nodes_when_scope_is_missing(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) + project = "foo" + _write_dbt_complete_phase_artifacts(unit, project) + session = _dbt_session_for_project(unit, project) + session["elements"] = [] + session["roadmap"]["phases"][1].pop("scope") + status_path = Path(tmp) / "session_status.json" + write_json(status_path, session) + + result = run_track_status("complete-phase", str(status_path), "2") + + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertIn("marked as completed", result.stdout) + + def test_complete_phase_skips_check_when_dbt_project_yml_missing(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) + project = "foo" + _write_dbt_complete_phase_artifacts(unit, project) + (unit / project).mkdir(parents=True) + status_path = Path(tmp) / "session_status.json" + write_json(status_path, _dbt_session_for_project(unit, project)) + + result = run_track_status("complete-phase", str(status_path), "2") + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertNotIn("models: key", result.stderr + result.stdout) + + def test_validate_phase_fails_on_models_key_mismatch(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) + project = "foo" + (unit / project).mkdir(parents=True) + (unit / project / "dbt_project.yml").write_text( + "name: foo\nmodels:\n YOUR_PROJECT_NAME:\n +materialized: view\n", + encoding="utf-8", + ) + status_path = Path(tmp) / "session_status.json" + write_json(status_path, _dbt_session_for_project(unit, project)) + + result = run_track_status("validate-phase", str(status_path), "2") + self.assertNotEqual(result.returncode, 0) + self.assertIn("YOUR_PROJECT_NAME", result.stderr + result.stdout) + + +class TestCompletePhaseArtifactLayout(unittest.TestCase): + def test_complete_phase_dbt_finds_learnings_under_underscore_phase_dir(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) + project = "foo" + tests_root = unit / "stabilization" / "tests" / "dbt" / project + (tests_root / "seeds").mkdir(parents=True) + (tests_root / "tests").mkdir(parents=True) + (tests_root / "seeds" / "seed.csv").write_text("id\n1\n", encoding="utf-8") + (tests_root / "tests" / "test_x.sql").write_text("SELECT 1;\n", encoding="utf-8") + (tests_root / "test_report.md").write_text("# report\n", encoding="utf-8") + underscore = unit / "stabilization" / "phases" / "phase_2" + underscore.mkdir(parents=True) + (underscore / f"dbt_learnings_{project}.md").write_text("# learnings\n", encoding="utf-8") + + status_path = Path(tmp) / "session_status.json" + write_json(status_path, _dbt_session_for_project(unit, project)) + + result = run_track_status("complete-phase", str(status_path), "2") + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertIn("marked as completed", result.stdout) + + def test_complete_phase_orch_accepts_fix_log_instead_of_batch_files(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) + tracking = unit / "stabilization" / "tracking" + tracking.mkdir(parents=True) + (tracking / "fix-log.md").write_text("# fixes applied in main session\n", encoding="utf-8") + status_path = Path(tmp) / "session_status.json" + write_json(status_path, base_session(unit_path=str(unit))) + + result = run_track_status("complete-phase", str(status_path), "1") + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertNotIn("baseline_batch_", result.stderr) + self.assertIn("marked as completed", result.stdout) + + def test_complete_phase_orch_finds_batch_files_under_underscore_phase_dir(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) + phase = unit / "stabilization" / "phases" / "phase_1" + phase.mkdir(parents=True) + (phase / "baseline_batch_1.md").write_text("# baseline\n", encoding="utf-8") + (phase / "batch_1.md").write_text("# batch\n", encoding="utf-8") + (phase / "learnings_batch_1.md").write_text("# learnings\n", encoding="utf-8") + (phase / "apply_report.md").write_text("# apply\n", encoding="utf-8") + status_path = Path(tmp) / "session_status.json" + write_json(status_path, base_session(unit_path=str(unit))) + + result = run_track_status("complete-phase", str(status_path), "1") + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertIn("marked as completed", result.stdout) + + def test_complete_phase_orch_still_requires_terminal_elements(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + unit = Path(tmp) + tracking = unit / "stabilization" / "tracking" + tracking.mkdir(parents=True) + (tracking / "fix-log.md").write_text("# fixes\n", encoding="utf-8") + session = base_session(unit_path=str(unit)) + session["elements"][0]["status"] = "pending" + status_path = Path(tmp) / "session_status.json" + write_json(status_path, session) + + result = run_track_status("complete-phase", str(status_path), "1") + self.assertNotEqual(result.returncode, 0) + self.assertIn("not in terminal status", result.stderr + result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/track_status.py b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/track_status.py index ba811ef..b9ecabd 100644 --- a/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/track_status.py +++ b/plugin/skills/migration/migrate-objects/actions/etl-stabilization/scripts/track_status.py @@ -22,6 +22,7 @@ python track_status.py set-test-env python track_status.py init-roadmap --phases-json '' python track_status.py init-roadmap --phases-file + python track_status.py start-phase python track_status.py assign-phases --phase --elements --strategy python track_status.py assign-phases --phase --elements-file --strategy python track_status.py batch-assign-phases --assignments-file @@ -34,10 +35,13 @@ python track_status.py add-decision --phase --decision python track_status.py update-dbt --status [--reason ] python track_status.py init-dbt + python track_status.py assign-dbt-phase --phase --project python track_status.py update-dbt-node --status [--reason ] """ from __future__ import annotations +import contextlib +import fcntl import hashlib import json import os @@ -47,7 +51,14 @@ from datetime import datetime, timezone from pathlib import Path -from path_resolver import phase_dir, report_path, stabilization_root, tests_dir +from path_resolver import ( + fix_log_path, + phase_dir_candidates, + report_path, + resolve_phase_dir, + stabilization_root, + tests_dir, +) VALID_STATUSES = {"pending", "in_progress", "fixed", "skipped", "needs-user", "failed", "no-fix-needed", "orch-tested", "proc-tested", "test-passed", "test-failed", "auto-fixed-needs-review"} @@ -108,6 +119,17 @@ def save_json(path: Path, data: dict) -> None: raise +@contextlib.contextmanager +def _locked(status_path: Path): + lock_path = status_path.with_suffix(status_path.suffix + ".lock") + with open(lock_path, "w") as lf: + fcntl.flock(lf, fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(lf, fcntl.LOCK_UN) + + def now_iso() -> str: return datetime.now(timezone.utc).isoformat() @@ -182,7 +204,7 @@ def cmd_init(scan_path: Path) -> None: if orch_path.is_file(): orch_hash = _file_hash(orch_path) - source_file_path = scan.get("source_definition_path") + source_file_path = scan.get("source_file_path") platform_id = scan.get("platform_id") if platform_id is None and "platform_id" not in scan: # Backward compat: old scan results created before multi-platform support @@ -220,44 +242,45 @@ def cmd_update(status_path: Path, element_name: str, status: str, reason: str | print(f"Error: invalid status '{status}'. Valid: {', '.join(sorted(VALID_STATUSES))}", file=sys.stderr) sys.exit(1) - session = load_json(status_path) - found = None - for el in session["elements"]: - if el["name"] == element_name: - found = el - break - - if found is None: - print(f"Error: element '{element_name}' not found", file=sys.stderr) - sys.exit(1) + with _locked(status_path): + session = load_json(status_path) + found = None + for el in session["elements"]: + if el["name"] == element_name: + found = el + break - current = found["status"] - allowed = VALID_TRANSITIONS.get(current) - if allowed and status not in allowed: - print(f"Warning: unusual transition '{current}' -> '{status}' for '{element_name}'", file=sys.stderr) + if found is None: + print(f"Error: element '{element_name}' not found", file=sys.stderr) + sys.exit(1) - # Validate needs-user requires reason - if status == "needs-user" and reason is None: - print("Error: 'needs-user' status requires --reason documenting prior fix attempts", file=sys.stderr) - sys.exit(1) + current = found["status"] + allowed = VALID_TRANSITIONS.get(current) + if allowed and status not in allowed: + print(f"Warning: unusual transition '{current}' -> '{status}' for '{element_name}'", file=sys.stderr) - # Validate skipped requires valid reason - if status == "skipped": - if reason is None: - print("Error: 'skipped' status requires --reason. Valid: " + ", ".join(sorted(VALID_SKIP_REASONS)), file=sys.stderr) - sys.exit(1) - if reason not in VALID_SKIP_REASONS: - print(f"Error: invalid skip reason '{reason}'. Valid skip reasons: " + ", ".join(sorted(VALID_SKIP_REASONS)), file=sys.stderr) + # Validate needs-user requires reason + if status == "needs-user" and reason is None: + print("Error: 'needs-user' status requires --reason documenting prior fix attempts", file=sys.stderr) sys.exit(1) - found["status"] = status - if reason is not None: - found["reason"] = reason - elif "reason" in found and status not in ("skipped", "failed", "needs-user", "test-failed", "orch-tested"): - del found["reason"] + # Validate skipped requires valid reason + if status == "skipped": + if reason is None: + print("Error: 'skipped' status requires --reason. Valid: " + ", ".join(sorted(VALID_SKIP_REASONS)), file=sys.stderr) + sys.exit(1) + if reason not in VALID_SKIP_REASONS: + print(f"Error: invalid skip reason '{reason}'. Valid skip reasons: " + ", ".join(sorted(VALID_SKIP_REASONS)), file=sys.stderr) + sys.exit(1) - session["last_updated"] = now_iso() - save_json(status_path, session) + found["status"] = status + if reason is not None: + found["reason"] = reason + elif "reason" in found and status not in ("skipped", "failed", "needs-user", "test-failed", "orch-tested"): + del found["reason"] + + session["last_updated"] = now_iso() + save_json(status_path, session) print(f"Updated '{element_name}' -> {status}") @@ -315,29 +338,37 @@ def cmd_init_roadmap(status_path: Path, phases_json: str) -> None: # complete-phase # --------------------------------------------------------------------------- +def _phase_has_rglob(pkg_path: str, phase_num: int, pattern: str) -> bool: + return any( + path.is_dir() and any(path.rglob(pattern)) + for path in phase_dir_candidates(pkg_path, phase_num) + ) + + def _validate_orch_artifacts(session: dict, phase_num: int, pkg_path: str) -> list[str]: """Return a list of validation errors for an orchestration phase. - Checks: - - baseline_batch_*.md exists in the phase dir - - batch_*.md exists - - learnings_batch_*.md exists - - apply_report.md exists - - All elements assigned to the phase have terminal status + Batch-agent filenames (`baseline_batch_*.md`, `batch_*.md`, + `learnings_batch_*.md`, `apply_report.md`) are one valid layout, not a + required one: a main-session run writes `tracking/fix-log.md` instead. + Phase dirs may be `phase-{N}` or `phase_{N}`. """ errors: list[str] = [] if pkg_path: - phase_directory = phase_dir(pkg_path, phase_num) - # Search recursively: artifacts may be in task subdirectories - if not any(phase_directory.rglob("baseline_batch_*.md")): - errors.append(f"Missing artifact: baseline_batch_*.md not found under {phase_directory}") - if not any(phase_directory.rglob("batch_*.md")): - errors.append(f"Missing artifact: batch_*.md not found under {phase_directory}") - if not any(phase_directory.rglob("learnings_batch_*.md")): - errors.append(f"Missing artifact: learnings_batch_*.md not found under {phase_directory}") - if not any(phase_directory.rglob("apply_report.md")): - errors.append(f"Missing artifact: apply_report.md not found under {phase_directory}") + has_batch_layout = ( + _phase_has_rglob(pkg_path, phase_num, "baseline_batch_*.md") + or _phase_has_rglob(pkg_path, phase_num, "batch_*.md") + or _phase_has_rglob(pkg_path, phase_num, "learnings_batch_*.md") + or _phase_has_rglob(pkg_path, phase_num, "apply_report.md") + ) + has_fix_log = fix_log_path(pkg_path).is_file() + if not has_batch_layout and not has_fix_log: + phase_directory = resolve_phase_dir(pkg_path, phase_num) + errors.append( + f"Missing phase artifacts: no batch reports under {phase_directory} " + f"and {fix_log_path(pkg_path)} not found" + ) else: errors.append("Missing artifact: migration_object_path not set in session; cannot locate phase artifacts") @@ -352,6 +383,111 @@ def _validate_orch_artifacts(session: dict, phase_num: int, pkg_path: str) -> li return errors +def _dbt_project_dir(session: dict, pkg_path: str, project: str) -> Path | None: + """Resolve a dbt project directory from session `dbt_projects[].path`.""" + rel = None + for proj in session.get("dbt_projects", []): + if proj.get("name") == project: + rel = proj.get("path") + break + if not rel: + rel = project + path = Path(rel) + if path.is_absolute(): + return path + if pkg_path: + return Path(pkg_path) / rel + return None + + +def _top_level_yaml_scalar(text: str, key: str) -> str | None: + prefix = f"{key}:" + for raw in text.splitlines(): + line = raw.split("#", 1)[0].rstrip() + if not line.strip(): + continue + indent = len(line) - len(line.lstrip()) + if indent != 0: + continue + content = line.strip() + if content.startswith(prefix): + value = content[len(prefix):].strip().strip("'\"") + return value or None + return None + + +def _first_level_yaml_mapping_keys(text: str, section: str) -> list[str]: + keys: list[str] = [] + in_section = False + child_indent: int | None = None + for raw in text.splitlines(): + line = raw.split("#", 1)[0].rstrip() + if not line.strip(): + continue + indent = len(line) - len(line.lstrip()) + content = line.strip() + if indent == 0: + in_section = content == f"{section}:" or content.startswith(f"{section}:") + child_indent = None + continue + if not in_section: + continue + if child_indent is None: + child_indent = indent + if indent < child_indent: + in_section = False + continue + if indent != child_indent: + continue + if ":" not in content: + continue + key = content.split(":", 1)[0].strip().strip("'\"") + if not key or key.startswith("+"): + continue + keys.append(key) + return keys + + +def _check_dbt_project_yml_sanity(project_path: Path) -> list[str]: + """Structural sanity checks independent of placeholder-scanning (scan_unit.py's job). + + Catches a specific, deterministic class of incomplete-bootstrap bug: the `models:` + block's key must match the project's own `name:` — if not, +materialized config + for the real project silently never applies to any model. + """ + errors: list[str] = [] + dbt_project_yml = project_path / "dbt_project.yml" + if not dbt_project_yml.is_file(): + return errors + try: + text = dbt_project_yml.read_text(encoding="utf-8") + except OSError: + return errors + name = _top_level_yaml_scalar(text, "name") + models_keys = _first_level_yaml_mapping_keys(text, "models") + if name and models_keys and name not in models_keys: + errors.append( + f"{dbt_project_yml}: models: key(s) {models_keys} do not match name: '{name}' " + f"— placeholder rename incomplete (expected a 'models: {name}:' block)" + ) + return errors + + +def _dbt_project_yml_sanity_errors(session: dict, phase_num: int, pkg_path: str) -> list[str]: + dbt_nodes = session.get("dbt_nodes", []) + projects = sorted({ + node.get("project") + for node in dbt_nodes + if node.get("phase") == phase_num and node.get("project") + }) + errors: list[str] = [] + for project in projects: + project_dir = _dbt_project_dir(session, pkg_path, project) + if project_dir: + errors.extend(_check_dbt_project_yml_sanity(project_dir)) + return errors + + def _validate_dbt_artifacts(session: dict, phase_num: int, pkg_path: str) -> list[str]: """Return a list of validation errors for a dbt phase. @@ -361,6 +497,7 @@ def _validate_dbt_artifacts(session: dict, phase_num: int, pkg_path: str) -> lis - stabilization/tests/dbt/{PROJECT}/test_report.md exists - dbt_learnings_{project}.md exists in phase dir - All dbt nodes assigned to this phase have terminal status + - dbt_project.yml `models:` top-level key matches `name:` (incomplete bootstrap) """ errors: list[str] = [] @@ -377,8 +514,15 @@ def _validate_dbt_artifacts(session: dict, phase_num: int, pkg_path: str) -> lis seeds_directory = stab_tests_dir / "dbt" / project / "seeds" tests_directory = stab_tests_dir / "dbt" / project / "tests" report_file = stab_tests_dir / "dbt" / project / "test_report.md" - phase_directory = phase_dir(pkg_path, phase_num) - learnings_file = phase_directory / f"dbt_learnings_{project}.md" + learnings_name = f"dbt_learnings_{project}.md" + learnings_file = next( + ( + candidate / learnings_name + for candidate in phase_dir_candidates(pkg_path, phase_num) + if (candidate / learnings_name).is_file() + ), + resolve_phase_dir(pkg_path, phase_num) / learnings_name, + ) if not seeds_directory.is_dir() or not any(seeds_directory.glob("*.csv")): errors.append(f"Missing dbt artifact: {seeds_directory} has no .csv seed files") @@ -386,12 +530,15 @@ def _validate_dbt_artifacts(session: dict, phase_num: int, pkg_path: str) -> lis errors.append(f"Missing dbt artifact: {tests_directory} has no .sql test files") if not report_file.is_file(): errors.append(f"Missing dbt artifact: {report_file} not found") - if not learnings_file.is_file(): + if not learnings_file.is_file() and not fix_log_path(pkg_path).is_file(): errors.append(f"Missing dbt artifact: {learnings_file} not found") + project_dir = _dbt_project_dir(session, pkg_path, project) + if project_dir: + errors.extend(_check_dbt_project_yml_sanity(project_dir)) else: errors.append("Missing artifact: migration_object_path not set in session; cannot locate dbt test artifacts") - node_terminal_statuses = TERMINAL_STATUSES | {"passing"} + node_terminal_statuses = TERMINAL_STATUSES | {"passing", "unverified"} non_terminal_nodes = [ node["name"] for node in dbt_nodes @@ -440,6 +587,19 @@ def _validate_dataflow_proc_artifacts(session: dict, phase_num: int, pkg_path: s return errors +def _phase_scope(session: dict, phase_num: int) -> str: + phases = session.get("roadmap", {}).get("phases", []) + phase_data = next((p for p in phases if p["phase"] == phase_num), None) + explicit_scope = (phase_data or {}).get("scope") + if explicit_scope: + return explicit_scope + + if any(node.get("phase") == phase_num for node in session.get("dbt_nodes", [])): + return "dbt" + + return "orchestration" + + def _validate_phase_artifacts(session: dict, phase_num: int) -> list[str]: """Return validation errors for a phase based on its scope. @@ -449,9 +609,7 @@ def _validate_phase_artifacts(session: dict, phase_num: int) -> list[str]: - dataflow-proc → _validate_dataflow_proc_artifacts - final-validation → checks all elements terminal + artifacts/report.html """ - phases = session.get("roadmap", {}).get("phases", []) - phase_data = next((p for p in phases if p["phase"] == phase_num), None) - scope = (phase_data or {}).get("scope", "orchestration") + scope = _phase_scope(session, phase_num) pkg_path = session.get("migration_object_path", "") if scope == "dbt": @@ -611,19 +769,20 @@ def _generate_state_md(session: dict, current_phase: int, phase_status: str, nex def cmd_update_state(status_path: Path, current_phase: int, phase_status: str, next_action: str) -> None: - session = load_json(status_path) - if "roadmap" not in session: - print("Error: no roadmap in session. Run init-roadmap first.", file=sys.stderr) - sys.exit(1) - session["last_updated"] = now_iso() - if "planning_completed_at" not in session: - session["planning_completed_at"] = now_iso() - save_json(status_path, session) + with _locked(status_path): + session = load_json(status_path) + if "roadmap" not in session: + print("Error: no roadmap in session. Run init-roadmap first.", file=sys.stderr) + sys.exit(1) + session["last_updated"] = now_iso() + if "planning_completed_at" not in session: + session["planning_completed_at"] = now_iso() + save_json(status_path, session) - state_md = _generate_state_md(session, current_phase, phase_status, next_action) - state_dir = status_path.parent - state_path = state_dir / "STATE.md" - state_path.write_text(state_md, encoding="utf-8") + state_md = _generate_state_md(session, current_phase, phase_status, next_action) + state_dir = status_path.parent + state_path = state_dir / "STATE.md" + state_path.write_text(state_md, encoding="utf-8") print(f"STATE.md generated: {state_path}") @@ -694,24 +853,47 @@ def cmd_add_decision(status_path: Path, phase_num: int, decision: str) -> None: # --------------------------------------------------------------------------- def cmd_assign_phases(status_path: Path, phase: int, element_names: list[str], strategy: str) -> None: - session = load_json(status_path) - updated = 0 - for el in session["elements"]: - if el["name"] in element_names: - el["phase"] = phase - el["test_strategy"] = strategy - updated += 1 - - if updated == 0: - print("Warning: no elements matched the provided names", file=sys.stderr) - sys.exit(1) + with _locked(status_path): + session = load_json(status_path) + updated = 0 + for el in session["elements"]: + if el["name"] in element_names: + el["phase"] = phase + el["test_strategy"] = strategy + updated += 1 - session["last_updated"] = now_iso() - save_json(status_path, session) + if updated == 0: + print("Warning: no elements matched the provided names", file=sys.stderr) + sys.exit(1) + + session["last_updated"] = now_iso() + save_json(status_path, session) print(f"Assigned {updated} elements to phase {phase} with strategy '{strategy}'") +# --------------------------------------------------------------------------- +# assign-dbt-phase +# --------------------------------------------------------------------------- + +def cmd_assign_dbt_phase(status_path: Path, phase: int, project: str) -> None: + with _locked(status_path): + session = load_json(status_path) + updated = 0 + for node in session.get("dbt_nodes", []): + if node.get("project") == project: + node["phase"] = phase + updated += 1 + + if updated == 0: + print(f"Warning: no dbt_nodes found for project '{project}'", file=sys.stderr) + sys.exit(1) + + session["last_updated"] = now_iso() + save_json(status_path, session) + print(f"Assigned {updated} dbt nodes in project '{project}' to phase {phase}") + + # --------------------------------------------------------------------------- # batch-assign-phases # --------------------------------------------------------------------------- @@ -734,31 +916,32 @@ def cmd_batch_assign_phases(status_path: Path, assignments_json: str) -> None: print(f"Error: assignment[{i}] missing required key '{key}'", file=sys.stderr) sys.exit(1) - session = load_json(status_path) - - roadmap_phases = {p["phase"] for p in session.get("roadmap", {}).get("phases", [])} - if roadmap_phases: - for i, entry in enumerate(assignments): - if entry["phase"] not in roadmap_phases: - print(f"Warning: assignment[{i}] references phase {entry['phase']} which does not exist in roadmap (available: {sorted(roadmap_phases)})", file=sys.stderr) - - total_updated = 0 - for entry in assignments: - phase = entry["phase"] - element_set = set(entry["elements"]) - strategy = entry["strategy"] - for el in session["elements"]: - if el["name"] in element_set: - el["phase"] = phase - el["test_strategy"] = strategy - total_updated += 1 - - if total_updated == 0: - print("Warning: no elements matched any provided names", file=sys.stderr) - sys.exit(1) + with _locked(status_path): + session = load_json(status_path) + + roadmap_phases = {p["phase"] for p in session.get("roadmap", {}).get("phases", [])} + if roadmap_phases: + for i, entry in enumerate(assignments): + if entry["phase"] not in roadmap_phases: + print(f"Warning: assignment[{i}] references phase {entry['phase']} which does not exist in roadmap (available: {sorted(roadmap_phases)})", file=sys.stderr) + + total_updated = 0 + for entry in assignments: + phase = entry["phase"] + element_set = set(entry["elements"]) + strategy = entry["strategy"] + for el in session["elements"]: + if el["name"] in element_set: + el["phase"] = phase + el["test_strategy"] = strategy + total_updated += 1 + + if total_updated == 0: + print("Warning: no elements matched any provided names", file=sys.stderr) + sys.exit(1) - session["last_updated"] = now_iso() - save_json(status_path, session) + session["last_updated"] = now_iso() + save_json(status_path, session) print(f"Batch-assigned {total_updated} elements across {len(assignments)} phases") @@ -770,6 +953,12 @@ def cmd_batch_assign_phases(status_path: Path, assignments_json: str) -> None: def cmd_validate_phase(status_path: Path, phase_num: int, list_test_files: bool = False) -> None: session = load_json(status_path) + scope = _phase_scope(session, phase_num) + + if scope == "dbt": + _validate_dbt_phase(session, status_path, phase_num, list_test_files) + return + phase_els = [e for e in session["elements"] if e.get("phase") == phase_num] if not phase_els: print(f"Error: no elements assigned to phase {phase_num}", file=sys.stderr) @@ -789,6 +978,7 @@ def cmd_validate_phase(status_path: Path, phase_num: int, list_test_files: bool status_counts[s] = status_counts.get(s, 0) + 1 fixed = status_counts.get("fixed", 0) + status_counts.get("test-passed", 0) + status_counts.get("no-fix-needed", 0) + needs_user = status_counts.get("needs-user", 0) + status_counts.get("auto-fixed-needs-review", 0) failed = status_counts.get("failed", 0) + status_counts.get("test-failed", 0) pending = status_counts.get("pending", 0) + status_counts.get("in_progress", 0) skipped = status_counts.get("skipped", 0) @@ -797,30 +987,38 @@ def cmd_validate_phase(status_path: Path, phase_num: int, list_test_files: bool print(f"Orchestration file: {line_count} lines, {ewi_count} EWI markers (file-wide, not phase-scoped)") print(f"Elements: {len(phase_els)} total") print(f" Fixed/passed: {fixed}") + print(f" Needs-user: {needs_user}") print(f" Failed: {failed}") print(f" Pending: {pending}") print(f" Skipped: {skipped}") if list_test_files: pkg_path = session.get("migration_object_path", "") - test_directory = phase_dir(pkg_path, phase_num) if pkg_path else None + test_directory = (tests_dir(pkg_path) / "orchestration") if pkg_path else None seen: set[str] = set() found: list[str] = [] for el in phase_els: strategy = el.get("test_strategy") or "" - stmt = el.get("statement", "") if strategy.startswith("grouped:"): group_name = strategy.split(":", 1)[1] - fname = f"grouped_{group_name}.sql" + short_name = f"grouped_{group_name}" else: - fname = f"{el['name']}.sql" + short_name = el["name"].rsplit(".", 1)[-1] + fname = f"{short_name}.sql" if fname in seen: continue seen.add(fname) if test_directory: - candidate = test_directory / stmt / fname - if candidate.is_file(): - found.append(str(candidate)) + matches = sorted(test_directory.rglob(fname)) + if not matches: + found.append(f"{fname}: (not found)") + elif len(matches) == 1: + found.append(str(matches[0])) + else: + found.append( + f"{fname}: {len(matches)} candidates — " + f"{', '.join(str(m) for m in matches)} (verify)" + ) print(f"\nTest files ({len(found)}):") for f in found: print(f" {f}") @@ -838,6 +1036,64 @@ def cmd_validate_phase(status_path: Path, phase_num: int, list_test_files: bool sys.exit(1) +def _validate_dbt_phase(session: dict, status_path: Path, phase_num: int, list_test_files: bool) -> None: + pkg_path = session.get("migration_object_path", "") + dbt_nodes = [n for n in session.get("dbt_nodes", []) if n.get("phase") == phase_num] + if not dbt_nodes: + print(f"Error: no dbt nodes assigned to phase {phase_num}", file=sys.stderr) + sys.exit(1) + + status_counts: dict[str, int] = {} + for node in dbt_nodes: + s = node["status"] + status_counts[s] = status_counts.get(s, 0) + 1 + + fixed = ( + status_counts.get("fixed", 0) + + status_counts.get("test-passed", 0) + + status_counts.get("no-fix-needed", 0) + + status_counts.get("passing", 0) + + status_counts.get("unverified", 0) + ) + needs_user = status_counts.get("needs-user", 0) + status_counts.get("auto-fixed-needs-review", 0) + failed = status_counts.get("failed", 0) + status_counts.get("test-failed", 0) + pending = status_counts.get("pending", 0) + status_counts.get("in_progress", 0) + + print(f"=== Phase {phase_num} Validation (dbt) ===") + print(f"dbt nodes: {len(dbt_nodes)} total") + print(f" Fixed/passed: {fixed}") + print(f" Needs-user: {needs_user}") + print(f" Failed: {failed}") + print(f" Pending: {pending}") + + if list_test_files: + projects = sorted({n.get("project") for n in dbt_nodes if n.get("project")}) + stab_tests_dir = tests_dir(pkg_path) if pkg_path else None + for project in projects: + if stab_tests_dir: + project_tests = stab_tests_dir / "dbt" / project / "tests" + for f in sorted(project_tests.glob("*.sql")) if project_tests.is_dir() else []: + print(f" {f}") + + yml_errors = _dbt_project_yml_sanity_errors(session, phase_num, pkg_path) if pkg_path else [] + for err in yml_errors: + print(f"Error: {err}", file=sys.stderr) + + ok = failed == 0 and pending == 0 and not yml_errors + if ok: + print("PASS: All dbt nodes resolved.") + else: + reasons = [] + if pending > 0: + reasons.append(f"{pending} nodes still pending") + if failed > 0: + reasons.append(f"{failed} nodes failed") + if yml_errors: + reasons.append(f"{len(yml_errors)} dbt_project.yml sanity error(s)") + print(f"FAIL: {'; '.join(reasons)}") + sys.exit(1) + + # --------------------------------------------------------------------------- # update-dbt # --------------------------------------------------------------------------- @@ -847,27 +1103,28 @@ def cmd_update_dbt(status_path: Path, project_name: str, status: str, reason: st print(f"Error: invalid dbt status '{status}'. Valid: {', '.join(sorted(VALID_DBT_STATUSES))}", file=sys.stderr) sys.exit(1) - session = load_json(status_path) - dbt_projects = session.get("dbt_projects", []) + with _locked(status_path): + session = load_json(status_path) + dbt_projects = session.get("dbt_projects", []) - found = None - for proj in dbt_projects: - if proj["name"] == project_name: - found = proj - break + found = None + for proj in dbt_projects: + if proj["name"] == project_name: + found = proj + break - if found is None: - print(f"Error: dbt project '{project_name}' not found", file=sys.stderr) - sys.exit(1) + if found is None: + print(f"Error: dbt project '{project_name}' not found", file=sys.stderr) + sys.exit(1) - found["status"] = status - if reason is not None: - found["reason"] = reason - elif "reason" in found and status not in ("dbt-failed", "skipped"): - del found["reason"] + found["status"] = status + if reason is not None: + found["reason"] = reason + elif "reason" in found and status not in ("dbt-failed", "skipped"): + del found["reason"] - session["last_updated"] = now_iso() - save_json(status_path, session) + session["last_updated"] = now_iso() + save_json(status_path, session) print(f"Updated dbt project '{project_name}' -> {status}") @@ -876,46 +1133,47 @@ def cmd_update_dbt(status_path: Path, project_name: str, status: str, reason: st # --------------------------------------------------------------------------- def cmd_init_dbt(status_path: Path, project_name: str, dbt_project_path: Path) -> None: - session = load_json(status_path) - dbt_projects = session.get("dbt_projects", []) - - found = None - for proj in dbt_projects: - if proj["name"] == project_name: - found = proj - break - - if found is None: - print(f"Error: dbt project '{project_name}' not found in session", file=sys.stderr) - sys.exit(1) + with _locked(status_path): + session = load_json(status_path) + dbt_projects = session.get("dbt_projects", []) + + found = None + for proj in dbt_projects: + if proj["name"] == project_name: + found = proj + break + + if found is None: + print(f"Error: dbt project '{project_name}' not found in session", file=sys.stderr) + sys.exit(1) - models_dir = dbt_project_path / "models" - nodes: list[dict] = [] - if models_dir.is_dir(): - for sql_file in sorted(models_dir.rglob("*.sql")): - rel = str(sql_file.relative_to(dbt_project_path)) - node_name = sql_file.stem - nodes.append({ - "name": node_name, - "path": rel, - "status": "pending", + models_dir = dbt_project_path / "models" + nodes: list[dict] = [] + if models_dir.is_dir(): + for sql_file in sorted(models_dir.rglob("*.sql")): + rel = str(sql_file.relative_to(dbt_project_path)) + node_name = sql_file.stem + nodes.append({ + "name": node_name, + "path": rel, + "status": "pending", + }) + + found["nodes"] = nodes + # Also populate top-level dbt_nodes for validation queries + top_nodes = session.setdefault("dbt_nodes", []) + # Remove existing nodes for this project (idempotent) + top_nodes[:] = [n for n in top_nodes if n.get("project") != project_name] + for node in nodes: + top_nodes.append({ + "name": node["name"], + "path": node["path"], + "status": node["status"], + "project": project_name, + "phase": None, }) - - found["nodes"] = nodes - # Also populate top-level dbt_nodes for validation queries - top_nodes = session.setdefault("dbt_nodes", []) - # Remove existing nodes for this project (idempotent) - top_nodes[:] = [n for n in top_nodes if n.get("project") != project_name] - for node in nodes: - top_nodes.append({ - "name": node["name"], - "path": node["path"], - "status": node["status"], - "project": project_name, - "phase": None, - }) - session["last_updated"] = now_iso() - save_json(status_path, session) + session["last_updated"] = now_iso() + save_json(status_path, session) print(f"Initialized {len(nodes)} dbt nodes for project '{project_name}'") @@ -928,34 +1186,44 @@ def cmd_update_dbt_node(status_path: Path, project_name: str, node_name: str, st print(f"Error: invalid dbt node status '{status}'. Valid: {', '.join(sorted(VALID_DBT_NODE_STATUSES))}", file=sys.stderr) sys.exit(1) - session = load_json(status_path) - proj = None - for p in session.get("dbt_projects", []): - if p["name"] == project_name: - proj = p - break - if proj is None: - print(f"Error: dbt project '{project_name}' not found", file=sys.stderr) - sys.exit(1) - - nodes = proj.get("nodes", []) - found = None - for node in nodes: - if node["name"] == node_name: - found = node - break - if found is None: - print(f"Error: node '{node_name}' not found in project '{project_name}'", file=sys.stderr) - sys.exit(1) + with _locked(status_path): + session = load_json(status_path) + proj = None + for p in session.get("dbt_projects", []): + if p["name"] == project_name: + proj = p + break + if proj is None: + print(f"Error: dbt project '{project_name}' not found", file=sys.stderr) + sys.exit(1) - found["status"] = status - if reason is not None: - found["reason"] = reason - elif "reason" in found and status not in ("failed", "skipped", "unverified"): - del found["reason"] + nodes = proj.get("nodes", []) + found = None + for node in nodes: + if node["name"] == node_name: + found = node + break + if found is None: + print(f"Error: node '{node_name}' not found in project '{project_name}'", file=sys.stderr) + sys.exit(1) - session["last_updated"] = now_iso() - save_json(status_path, session) + found["status"] = status + if reason is not None: + found["reason"] = reason + elif "reason" in found and status not in ("failed", "skipped", "unverified"): + del found["reason"] + + for top_node in session.get("dbt_nodes", []): + if top_node.get("project") == project_name and top_node.get("name") == node_name: + top_node["status"] = status + if reason is not None: + top_node["reason"] = reason + elif "reason" in top_node and status not in ("failed", "skipped", "unverified"): + del top_node["reason"] + break + + session["last_updated"] = now_iso() + save_json(status_path, session) print(f"Updated node '{node_name}' in '{project_name}' -> {status}") @@ -1098,6 +1366,35 @@ def main() -> None: cmd_assign_phases(status_path, phase, element_names, strategy) + elif command == "assign-dbt-phase": + if len(sys.argv) < 3: + print("Usage: python track_status.py assign-dbt-phase --phase --project ", file=sys.stderr) + sys.exit(1) + status_path = Path(sys.argv[2]) + if not status_path.is_file(): + print(f"Error: '{status_path}' not found", file=sys.stderr) + sys.exit(1) + + phase = None + project = None + i = 3 + while i < len(sys.argv): + if sys.argv[i] == "--phase" and i + 1 < len(sys.argv): + phase = parse_int(sys.argv[i + 1], "phase") + i += 2 + elif sys.argv[i] == "--project" and i + 1 < len(sys.argv): + project = sys.argv[i + 1] + i += 2 + else: + print(f"Error: unexpected argument '{sys.argv[i]}'", file=sys.stderr) + sys.exit(1) + + if phase is None or project is None: + print("Error: --phase and --project are required", file=sys.stderr) + sys.exit(1) + + cmd_assign_dbt_phase(status_path, phase, project) + elif command == "batch-assign-phases": if len(sys.argv) < 3: print("Usage: python track_status.py batch-assign-phases --assignments-file | --assignments-json ''", file=sys.stderr) diff --git a/plugin/skills/migration/migrate-objects/actions/finish_objects.md b/plugin/skills/migration/migrate-objects/actions/finish_objects.md index c2bfcf3..a9239f1 100644 --- a/plugin/skills/migration/migrate-objects/actions/finish_objects.md +++ b/plugin/skills/migration/migrate-objects/actions/finish_objects.md @@ -22,9 +22,6 @@ The response includes a `git_activity` array of human-readable strings describin > - Pushed commit abc1234f to origin/main with 3 files (registry/obj-1.json, snowflake/proc_1.sql, snowflake/proc_1_test.sql) > - Rebased branch 'migrate-alice' onto origin/main -> - Unblocked 2 object(s) that depended on the finished objects - - On error (e.g. merge conflict), attempt to resolve it: 1. Read each conflicted file and decide the correct resolution (accept incoming, keep current, or merge both sides). @@ -41,8 +38,6 @@ The handler stamps each object done (`isDone`) and closes Snowflake claims. No g ### After success (both modes) -After the handler succeeds, it automatically scans for objects that were blocked with `error="dependency"` and depend on any of the just-finished objects. Their error stamps are cleared so the next `migration_status` walk picks them up as ready. If the response includes `"woke_dependents": N` (N > 0), tell the user: - -> Also unblocked **N** object(s) that were waiting on these dependencies. They'll appear in your next status check. +The finish payload is `{status, action, actual_size}` plus git extras (`merge_commit`, `git_activity`, …) or `git_disabled`. It does **not** include `my_objects_summary` — pull claim status with `migration_status(mode="my_objects_summary")` if you need it. -Return to [../SKILL.md](../SKILL.md). +Dependents waiting on these objects are offered again on the next walk — finish does not stamp or clear a dependency wait. Return to [../SKILL.md](../SKILL.md). diff --git a/plugin/skills/migration/migrate-objects/autonomous/SKILL.md b/plugin/skills/migration/migrate-objects/autonomous/SKILL.md new file mode 100644 index 0000000..1fc36cf --- /dev/null +++ b/plugin/skills/migration/migrate-objects/autonomous/SKILL.md @@ -0,0 +1,504 @@ +--- +name: migrate-objects-auto +description: Autonomous version of migrate-objects — dispatches one subagent per ready task group, refills as they finish, and only stops to ask when a subagent is stuck. Triggers: autonomous migration, run unattended, migrate objects automatically, auto-pilot the wave, migrate everything in parallel, swarm the objects. +parent_skill: migrate-objects +license: Proprietary. See License-Skills for complete terms +--- + +# Migrate Objects — Autonomous + +## On Entry **IMPORTANT DO NOT SKIP** + +Tell the user: + +> **Autonomous mode.** I'll work the wave by giving each object its own subagent, +> which walks it the whole way — convert, deploy, tests, fixes — and I'll refill as +> they finish. I'll interrupt you only when an agent is stuck, when a dependency +> needs a human decision, and before any data moves. Say "stop" at any point and +> I'll let in-flight work land. + +## Step 0: Preflight + +1. `configure(project_dir=, subagent_mode=true, snowflake_connection=)`. Read the appended migration-status block. Bring + the shared data infrastructure up: `data_infrastructure(mode="up")`. Relay its + `cost_reminder`. `status="not_ready"` is not a green light — a first local bring-up + runs every schema migration and can need more than one call, so follow its `remediation` + and call `up` again until it reports `ready`. Once means one *successful* bring-up for + the wave, not one call. +2. If `configure()` reports setup is not finished, read + [../../setup/SKILL.md](../../setup/SKILL.md) then come back here. + +`subagent_mode` attributes writes per agent. Set it once here; it lasts the +life of the server and cannot be turned off. Yours is `0000`: park a looping +object, reset a remediated transient failure, and relay non-terminal guidance. +Acknowledge unreviewed notes only in Step 3 (`review`), never in the dispatch +loop. Object-level outcomes stay parked for a person in an interactive session. +`deploy`, `migrate_data`, and `validate_data` are not binding-checked — keep +each agent on its own object. + +## Step 1: Ask how many subagents to run at once + +Offer 2 / 4 / 6, recommend **4**. State the +trade-off: more slots finish the wave faster, burn proportionally more tokens, +and produce more escalations competing for their attention. Call it `N`. Never +exceed it, and never quietly raise it. + +**Wait for the user's response — do not dispatch until they answer.** + +## Step 2: The loop + +The unit of dispatch is an **object**: one subagent takes one object and walks it +through as many tasks as it takes, then stops. You hold no migration state — +[the server does](#where-the-state-lives). Track three things per in-flight +object so you never put two agents on one object and so a wake can continue the +same conversation: the **object id**, the minted `agentId`, and the Cortex +`resume` id (`task` returns it as `agentId` — a UUID, not the four-hex mint). + +### 2a. Read the board + +``` +migration_status(mode="my_objects_board") +migration_status(mode="escalations") +``` + +`my_objects_board` is the only status shape you read for dispatch. Each row is +one object: `objectId`, `name`, `type`, `agentId`, `bucket` +(`ready` | `blocked` | `done` | `escalated` | `errored`). Overlay your own +in-flight set — an object you have a live child on is in-flight even if the +board says `ready`. + +Do not call `my_objects_summary`, `my_objects_details`, `next_task`, or +`task_views`. Those name the current task and why it is waiting; they are for +interactive sessions and for the child walking the object. + +`escalations` is the human queue. Read `escalations` (open asks) and `answered` +(`guidance` + `agentId`). Do not pass `details=true`. Do not read `notes[].asks` +/ `choice` or `overrideAcceptedCases` — default payloads omit those bodies. +`unreviewedCount` is a tally for Step 3, not a reason to stop. + +Re-read both every time a child **returns**. + +### 2b. Top off the object pool + +A slot is occupied only by a **live Cortex child**. `waiting` on a relay +job, `stuck` / `escalated` parks, leftover claims from a dead session, and +`completed` objects do not occupy one. `free_slots` is `N` minus the live +child count. When it is greater than zero, pull: + +``` +migration_status(mode="next_objects", limit=) +``` + +The board lists only claimed objects. Unclaimed work is invisible there — +`next_objects` is the only way to see it. Do not skip this pull because a +leftover looks blocked on a sibling still in flight: the picker returns what +can start, or comes back empty. Do not wait for the flight to empty. + +`next_objects` also returns `leftover_claims`: open claims this run minted +whose objects are **hidden** from `objects` (the picker will not double-book a +live walker). Each entry is `{object_id, name, agentId}`. **Before** waiting, +first-send every leftover whose `agentId` is **not** a live child and whose +object is not `done` — same minted `agentId`, no `resume`, no fresh mint. +Skip leftovers you still have a live child for. Count those first-sends +against `free_slots`, then spawn from `objects` for whatever slots remain. + +Each `objects` entry has `object_id`, `name`, `display_name`, `type`, and `agentId`. Hand the +ids to the subagents you dispatch — **each subagent claims its own object**. You never +call `transition_status(status="begin")` yourself: the agent that does the work owns +the claim. A spawn that dies *before* `begin` leaves the object unclaimed (2b will +offer it again). A child that dies *after* `begin` still binds that minted +`agentId` — first-send that id again from `leftover_claims` (or from 2d when +the child returns `completed` without `done`). + +`agentId` is minted here, one per object, and this and `answered` in 2e are the only +places you can get one. Copy it verbatim into that object's **first** dispatch +prompt: the server refuses an id it did not issue, so an id you compose yourself +fails the agent's first call. Keep the triple — object id, minted `agentId`, +Cortex `resume` id — in the same bookkeeping line. A later send to the same +object reuses **the same minted `agentId`** (the claim is bound to it) and +**resumes** the Cortex conversation; a fresh mint would be refused as an +attempt to take a claim it does not hold. + +If you have lost the minted pairing, the refusal tells you: `begin` under the +wrong id comes back naming the object and the agent that holds its claim, and +that name is the minted id to send with. + +> **Claim narrowly.** [../actions/claim_objects.md](../actions/claim_objects.md) +> forbids auto-claiming because a claim hides an object from every teammate's +> picker. Autonomous mode claims without asking, so keep the batch small: only the +> free slots, only ids from this turn's `next_objects`, never a category `where` +> predicate. The user authorized `N` slots, not the whole wave. + +### 2c. Dispatch + +**One object per subagent, and the agent is always +[`general-task`](../../../../agents/general-task.md).** It walks its object through +every task the machine offers — convert, deploy, tests, fixes — and stops when the +object is done or needs a human. You do not route by task, and there is no +per-task agent to choose. + +Up to `N` in flight. First send and later send are different `task` calls. + +**First send** — you have no Cortex `resume` id for this object. Spawn in a +single turn, one prompt each, and always pass `description` (a short object +label). Do not pass `resume` or `fork_conversation_history`. + +``` +Migrate this object end-to-end, following your agent definition. + +objectId: +agentId: +projectDir: +pluginDir: +snowflakeConnection: +snowflakeDatabase: +guidance: +``` + +`task` returns an `agentId` (UUID). Store it as this object's `resume` id. That +is the conversation. A later `task(resume=…)` returns a **different** UUID — +that one is only the wait handle for `agent_output`. Do not overwrite the +stored `resume` id with it. + +One imperative line, then values. The line matters: four bare `key: value` pairs +read as context rather than a request, and an agent handed only context asks +what you want — which nobody is there to answer. Say what to do, once, and leave +what to do it with to the definition. + +Those values are the rest of the first prompt. `pluginDir` is among them because +`executor.skill` values are relative to the plugin's `skills/migration/` directory and +the subagent cannot locate that on its own; `agentId` is there for the same kind of +reason — it is minted by the server and handed to *you*, so the agent has no way to +obtain it, and every write it makes is refused without one. `snowflakeConnection` +on the first-send is Cortex `sql_execute` `-c` and may be a **reader** account. +MCP writes use the parent `configure(snowflake_connection=…)` connection, which +can differ. Without `snowflakeConnection` the child omits `-c` +and hits Cortex's default account; without `snowflakeDatabase` it qualifies SQL +with the source catalog name. +Do not tell the child to load `snowflake-migration:migration` — that is the +interactive router; its contract is [`general-task`](../../../../agents/general-task.md). +The agent definition is the contract — it already carries the loop, the fix-loop +thresholds, the escalation test, and the return schema — and anything the prompt +adds competes with it instead of replacing it. A prompt that invents a retry limit, +names a field the tools do not return, or specifies its own return shape leaves the +agent choosing between two sets of rules; a prompt that tells it not to escalate +converts a decision that needed a human into a silent guess. + +**Later send** — you already have a `resume` id. The child finished a turn +(`waiting` / `partial` / `stuck`) and this conversation still has the walk. +Call `task` with `resume` set to that stored UUID, `description` set, and a +prompt that is **only the new line**. Do not repeat `objectId` / `projectDir` / +`pluginDir` / `snowflakeConnection` / `snowflakeDatabase` / the migrate-this-object line — they are +already in that conversation. Do not pass `fork_conversation_history`. + +``` +relay_wake: +``` + +or, after a person answered: + +``` +guidance: +``` + +or, when the board says `ready` again after a `partial` / remediated `reset`: + +``` +Continue this object from the machine's next task. +``` + +Store the UUID this `task` call returned as the wait handle. Do not wait yet — +finish every other first send and wake for this turn, then wait once in 2d. +Keep resuming the stored id. + +If you have lost the `resume` id, this is a first send: full prompt, no +`resume`. That is the killed-session path, not a wake. + +**One minted `agentId` per object, never shared across two live agents.** Two +agents holding one id can write to each other's objects and the binding cannot +tell them apart — the whole guarantee collapses to the one it replaced. +Resuming the same conversation is the sequential reuse; it is the only reuse +there is. + +**Never dispatch two agents for one object.** Because an agent walks the whole +pipeline, an object it is mid-walk on will keep resolving to a *new* task each time +you re-read the board. Track in-flight by **object id**, not by task: the object is +the unit of work and the only safe key. Two agents on one object race on the same +files and the same registry entries. A `resume` call is not a second agent. + +**Check `answered` before every send.** An answered escalation has already +un-parked its object, so it comes back looking like ordinary work with no sign that +a person decided anything. If `migration_status(mode="escalations")` lists the +object under `answered`, the next send is a later send when you have its +`resume` id (prompt is the `guidance:` line) and a first send when you do not +— take that entry's minted `agentId` either way. An answered object is claimed +by nobody, so a first send claims it like any other work. Dropping the guidance +sends an agent to make a decision that was already made for it. + +`N` is the only dispatch limit you control. Do not inspect why a sibling is +`blocked` or which task it is on. + +### 2d. Refill + +This session is headless: ending the turn exits the process and kills every +child. Keep it alive with **one wait per turn**, and only after this turn has +filled every free slot. + +**Top off, then wait.** Re-read the board (2a) and pull `next_objects` (2b) +before every wait — after a child returns, after a wake, after empty +`job_status` wakes. First-send every `leftover_claims` entry that is not a +live child (same minted `agentId`), then spawn every other first send and +every pending wake in this turn (background `task` calls). Then wait once. +Do not `agent_output(wait=true)` on one live child while leftovers or free +slots remain. A live Cortex child on one object does not block filling the +other slots. Neither does a `waiting` object or an escalation. + +Wait once: +- If a live object-walker remains: `agent_output(wait=true)` on the UUID + **that child's most recent `task` call returned** (the wait handle, which + equals the stored `resume` id only on a first send). Do not wait on a + nested test-writer or verifier UUID. Do not call it to peek at a running + child's tools or transcript. +- If no live object-walker remains **and** 2b returned nothing (or + `free_slots` is 0): `job_status(wait=true, confirm=true, agent_id="0000", + cursor=)` instead of ending the turn. Without `confirm=true` the + tool returns a reminder and does not block — that is not a wake. Empty + `wakes` means go back to 2a/2b, not wait again with idle slots. Do not + call `job_status(wait=true)` while 2b would still return objects. + +Arm **one** Monitor for the wave: `orchestrator_watch.watch_command` from +`configure` or `job_status(monitor=true)`, `persistent: true`. Each line is +an instruction. Do not open the job, call `job_status` on it, or read the +relay log. + +A wake looks like `wake up 86e1 and ask it to check new event from relay. …`. +That is a later send: `task(resume=)` with +the `relay_wake:` line. Do not spawn a new `general-task` for it. That later +send is the live child; it occupies a slot only while it is live. Do not +inspect the event. + +When `agent_output` returns, read only these keys from its result JSON: +`objectId`, `tasksCompleted`, `result`, `reopenedCodeUnits`. Ignore `Recent Output`, `task`, +`failed`, `blocked`, `evidence`, `asks`, and `notes`. Then 2a and 2b +**before** you send a wake or wait again. + +Report one line per return (`dbo.Customers: done, 4 tasks`) and the running +tally. + +| `result` | What you do | +|---|---| +| `completed` | Re-read the board. `done` → drop the `resume` id and the minted `agentId`. Anything else → the child died holding the claim. First send again with the **same** minted `agentId` (no `resume`, no fresh mint from 2b). | +| `partial` | Re-read the board. `blocked` → leave it. `ready` → later send, same minted `agentId` (the machine routed recovery). `escalated` → 2e. | +| `stuck` | 2e. Do not send again until a person answers. | +| `waiting` | Keep the `resume` id; drop the slot. The object is on a relay job — its own migrate/validate, or a dependency the machine registered. Later send only on a wake for that minted `agentId`. | +| `reopened` | Keep the waiter's `resume` id; drop the slot. First-send each id in `reopenedCodeUnits` that is not already in flight (2b mint, or the same minted `agentId` if that code unit already has one). Do not later-send the waiter until those walks finish. This is not a 2e interrupt. | + +Do not send again for a `partial` whose board bucket is still `blocked`. + +Reset only from evidence you already have — infrastructure now reports ready, +or a shared job reached terminal — never from the child's `failed` payload: + +``` +transition_status(status="reset", task="", where="id = ''", + agent_id="0000") +``` + +Then a later send of the same object **with the same minted `agentId`**, once. +Record the remediation or clearing evidence in your report. If nothing changed, +leave an existing escalation open; if the object is not parked yet, park it with +the exact failure and the repair needed before a rerun. + +Deterministic row mismatches, schema drift, invalid identifiers or compilation +errors in generated SQL, and a tool invocation that fails the same way on repeat are +not transient infrastructure failures. Park them rather than resetting or +sending the unchanged task again. `reset` clears both the stamp and escalation +marker, so using it before remediation erases the durable record of a failure the +next run will reproduce. + +### 2e. Escalate + +Escalation is the only reason to interrupt the user. An escalated object does +not occupy a slot — keep dispatching up to `N` live children while they read. + +**Read the queue, don't rely on what came back.** A subagent returning `stuck` is +one source; the authoritative one is: + +``` +migration_status(mode="escalations") +``` + +Check it on every board read (2a), not only when an agent returns. It is +project-wide, so it also surfaces escalations raised by another person's agents, +and answers recorded in an earlier session — including one you were killed in the +middle of. + +Present each **open** row: object, task, `causeClass` if set, and the numbered +choices from `asks`. That is the only time you read inside an object. Batch +them. Do not open `notes` or `overrideAcceptedCases`. + +**Record the decision with one call — the user's decision, never your own.** An +escalation exists because the agent that raised it judged the choice not to be an +agent's to make, and `answer` records *a person's* choice: the row names who +decided. Do not settle one by reading the escalating agent's skill and writing +guidance back yourself. If the queue cannot be answered without the user, the run +terminates with it open — 2f allows that. + +The call closes the escalation *and* un-parks the object: + +``` +transition_status(status="answer", where="id IN (...)", task="", + resolution="guidance|decompose|needs_repair", + reason="", agent_id="0000") +``` + +`0000` is yours here, on a remediated `reset`, on `escalate`, and on `review`. It is the id that +says a person decided — the row records which agent recorded the answer, so +answering under a subagent's id would attribute the user's decision to the agent that +asked. + +| The user says | `resolution` | Then you | +|---|---|---| +| Try again, here's how | `guidance` | Later send with their words as the `guidance:` line. `reason` is required. | +| It's too big — split it | `decompose` | Decompose per [LONG_PROCEDURE.md](../migrate-object/references/LONG_PROCEDURE.md), then later send. | +| Leave it for a human | `needs_repair` | Nothing. It stays parked but leaves the open queue, so it stops being re-offered every cycle. | +| Apply an object-level outcome | — | Leave it parked. Putting an object out of scope or accepting its current state belongs to the interactive migration flow. | + +`skip` remains a human-interactive out-of-scope action. It is not an autonomous +`answer` resolution; leave that choice parked for the interactive session. + +**Do not send again without answering first.** `error="human"` has no failure +transition, so the object stays parked until the stamp is cleared: `next_task` keeps +resolving `needsHuman`, and the agent you sent reads that and returns `stuck` +having done nothing. `guidance` and `decompose` are the two resolutions that un-park; +the response reports `unparked` so you can tell it happened. + +One thing parked here is not a decision to make: a transient failure whose condition +has demonstrably cleared. The remediated `reset` in 2d un-parks it without recording +that anybody chose anything. Every other way out of the queue is a person answering. + +A missing dependency (`reason: "missing"`) needs the register / stub / out-of-scope +menu from [../SKILL.md](../SKILL.md) Step 2c. Register and stub are work you can +dispatch. Keep choices that determine the object's scope or unmet requirements +parked for the interactive migration flow. + +Not every escalation is a failure. An agent that stops on a design decision — no +Snowflake equivalent, a definition missing from the source, two readings that +differ in row count — has done the right thing and has nothing to show you but the +ambiguity. Those arrive with no `causeClass`. + +**Stop the loop** is still an option at any point: let in-flight agents land, then +Step 3. + +### 2f. Termination + +The loop ends when all four hold: + +- nothing is in flight (no live child, and no object `waiting` on a relay + job — that job is still the wave, even though it does not occupy a slot), +- the board has no `ready` objects (`escalated` / `blocked` do not block + termination, and waiting for them to empty would hang the run on an + unanswered question), +- `next_objects` is empty, +- every remaining object is `escalated`, `blocked`, `done`, `errored`, or + out of scope. + +A minted `agentId` whose child is gone and whose object is not `done` is +none of those: first-send that id (2d `completed`, or `leftover_claims` in +2b). Do not treat `next_objects.objects` being empty as the wave being empty +while `leftover_claims` is non-empty or you still hold those triples. +Credential / OAuth expiry is not a human question — do not escalate it; +the run cannot continue until the owner refreshes auth. + +A `ready` object with nothing in flight is none of those: dispatch it or park +it, but never report around it. An object that returns `ready` again with an +empty `tasksCompleted` must not be dispatched again until the board changes. +If there is no concrete remediation or clearing evidence, park it: + +``` +transition_status(status="escalate", task="", + asks=[""], + cause="", + reason="", where="id = ''", + agent_id="0000") +``` + +It leaves `ready` because it is parked, which meets the fourth condition, and the +user gets a question they can answer instead of an object no board read would have +shown them. + +Then run Step 3. If the board still has `escalated` rows or `openCount` is +non-zero at that point, say so in the report — the run finished, the wave did +not. + +## Where the state lives + +You keep no ledger. Every question about the run has an authoritative answer +somewhere else, and re-reading beats remembering: + +| Question | Source | +|---|---| +| What claimed work is ready? | `my_objects_board` → `bucket=ready` | +| What unclaimed object can take a free slot? | `next_objects` | +| What is parked on a person? | `my_objects_board` → `bucket=escalated`, and `escalations` for the asks | +| What is claimed, by whom? | `my_objects_board` → `agentId` | +| Who to wake for a relay event? | The wake line. Later send: `resume` that object's stored Cortex id. Do not read the job. | +| Is an object done? | `bucket=done` — the machine closes a verified terminal (`isDone`) | +| What is waiting on a human, and what did they decide? | `escalations` → `escalations` and `answered` | +| Unreviewed judgments (count only, mid-loop) | `escalations` → `unreviewedCount` | + +Do not call `next_task`, `my_objects_details`, or `task_views`. You do not +need where an object is in its pipeline or why it failed. + +One thing the server cannot tell you, so it stays in your reply text: which **object +ids** you have agents on right now, the minted `agentId` each was dispatched with, +and the Cortex `resume` id for that conversation. That is dispatcher bookkeeping, +not migration state, and safe to lose — a re-read of the board plus the errored +bucket rebuilds the objects, and a killed run's agents are gone anyway so their +ids retire with them. Track object ids rather than tasks: an agent walks its +object across several tasks, so a task-keyed ledger would show the same object as +new work each time it advanced. + +Escalations survive the session — read them, don't remember them. + +## Step 3: Report + +Call `migration_status()` and report as [../SKILL.md](../SKILL.md) Step 3 does, +plus what autonomous mode adds: objects finished without intervention, objects +escalated (open asks, as in 2e), unreviewed notes as **object + task** only +(do not pass `details=true`; do not quote SQL or choice text), then +`transition_status(status="review", …)` in one batch with `agent_id="0000"`. +Also: subagents dispatched, remediated tasks reset, and outcomes stamped by +an agent because the machine could not observe them. + +A stage count is not a done count. Take the finished number from the +`migration_status()` you already called — `objects_done` — and leave `stage_totals` +out of it: an object can be deployed, data-migrated, and still not finished, so +adding stage counts together counts one object several times. `objects_done` is +project-wide, which is what a wave total should be; `doneCount` on +`my_objects_board` counts only what you hold a claim on. + +Build every line the board can confirm from the board. A returning agent's `result` +is a claim, not a finding: report what you cannot confirm as what that agent said, +attributed to it, and never let one agent's `completed` become a report line of its +own. Nothing counts remediated resets for you — take those from the board and +what you recorded when you reset, and if you cannot tell how many there were, +say what you saw instead of a number. + +Then tear down: `data_infrastructure(mode="down")`, or +[../../data-infrastructure/teardown/SKILL.md](../../data-infrastructure/teardown/SKILL.md) +when the project configured a `compute_pool` and data work ran. + +## Resuming an interrupted run + +A killed session loses your dispatch bookkeeping and nothing else. Claims live in +Snowflake, task stamps in the registry, merges in git, escalations and their +answers in Snowflake. Re-enter this skill: Step 0, then 2a, and the board shows +exactly where the wave stands — including objects whose subagent died mid-task, +which resolve back to that task as pending (`reclaimed_from_other_sessions` names +the ones taken back from the dead session). Nothing needs manual cleanup. + +Read `migration_status(mode="escalations")` before dispatching anything. A question +you asked before the session died is still open, and an answer the user gave is +still waiting to be acted on — sending those objects again without answering them +first just parks them again. The Cortex `resume` ids died with the session, so +every object is a first send. diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/CAPTURE.md b/plugin/skills/migration/migrate-objects/baseline-capture/CAPTURE.md index 555d169..5c046ac 100644 --- a/plugin/skills/migration/migrate-objects/baseline-capture/CAPTURE.md +++ b/plugin/skills/migration/migrate-objects/baseline-capture/CAPTURE.md @@ -6,7 +6,7 @@ After creating the test YAML files, capture baselines from the source database a ## Step 1: Capture Baselines from Source Database -Baselines upload to the Snowflake stage `@.VALIDATION.BASELINES`; no copy is kept on the user's laptop (customer data residency). +Baselines upload to the Snowflake stage `@.VALIDATION.BASELINES` — the database named by `testing_results_database` in `.scai/settings/test_config.yaml`; no copy is kept on the user's laptop (customer data residency). ```bash scai test capture \ @@ -25,26 +25,26 @@ scai test capture \ List the stage, filtering server-side to just this object's baselines: ```bash -snow stage list-files @.VALIDATION.BASELINES \ +snow stage list-files @.VALIDATION.BASELINES \ --pattern ".*\..*" \ -c ``` ## Step 3 (BTEQ scripts only): mark capture complete -For BTEQ scripts the baseline is uploaded to the stage and nothing is written to the test YAML, so the state machine cannot infer capture from the file — stamp the task explicitly: +For BTEQ scripts the baseline is uploaded to the stage and `VALIDATION.BASELINE_METADATA` is not written, so the state machine cannot infer capture from Snowflake — stamp the task explicitly: ``` transition_status status=advance task=captureBaseline --where "id = ''" ``` -Procedures/functions skip this — their `captureBaseline` completes from the per-object test YAML. +Procedures/functions skip this — their `captureBaseline` completes once `VALIDATION.BASELINE_METADATA` has a row for the object whose `ROW_COUNTS` sum to more than zero. ## CHECKPOINT Confirm: - [ ] Baselines captured for `` from source database -- [ ] Baselines visible on Snowflake stage `@.VALIDATION.BASELINES` +- [ ] Baselines visible on Snowflake stage `@.VALIDATION.BASELINES` - [ ] At least 15-25 test cases for this object ## Next Steps diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/SWARM.md b/plugin/skills/migration/migrate-objects/baseline-capture/SWARM.md index 1791d6b..9b59fee 100644 --- a/plugin/skills/migration/migrate-objects/baseline-capture/SWARM.md +++ b/plugin/skills/migration/migrate-objects/baseline-capture/SWARM.md @@ -115,36 +115,41 @@ Use the `files.source.path` from Step 2. Internalize: | Complexity | Signals | Agents to spawn | |---|---|---| | **Simple** | 1–3 params, straightforward logic | 3 (one of each type) | -| **Complex** | 4+ params, multiple branches, table lookups, OUT params | 6 (two of each type — see A/B split in each agent file) | +| **Complex** | 4+ params, multiple branches, table lookups, OUT params | 6 (two of each type — pass `split: A` and `split: B`) | ### 5.3 — Spawn agents in parallel -Use the Task tool. Each agent reads its own instruction file; do **not** paste instructions inline. +Use the Task tool. Spawn each as its `subagent_type` with a facts-only +prompt — the agent definition is the contract. Do not paste instructions +inline and do not tell it to read a path. -| Agent | Instruction file | Needs source DB | Focus | +| Agent | `subagent_type` | Needs source DB | Focus | |---|---|---|---| -| **Data-Driven** (most important) | `agents/data_driven.md` | Yes (or testbed CSVs as fallback) | Real parameter values from actual data | -| **Edge Cases & Boundaries** | `agents/edge_cases.md` | No | NULLs, zeros, type limits, overflow | -| **Business Logic** | `agents/business_logic.md` | No | Branch coverage from source SQL analysis | - -Spawn prompt for each agent: +| **Data-Driven** (most important) | [`data_driven`](../../../../agents/data_driven.md) | Yes (or testbed CSVs as fallback) | Real parameter values from actual data | +| **Edge Cases & Boundaries** | [`edge_cases`](../../../../agents/edge_cases.md) | No | NULLs, zeros, type limits, overflow | +| **Business Logic** | [`business_logic`](../../../../agents/business_logic.md) | No | Branch coverage from source SQL analysis | ``` -Read the instructions at /agents/.md -then produce test_cases for . - -Object signature: -Source code: -Referenced tables: # data-driven only -Source connection name: # data-driven only -Project directory: +Produce test_cases for this object, following your agent definition. + +object_name: +signature: +source_code: +project_dir: +referenced_tables: # data_driven only +source_connection: # data_driven only +split: A|B # complex only ``` -For **complex** objects spawn 2 agents per type — the agent files describe the A/B split. +For **complex** objects spawn 2 of each type, one with `split: A` and +one with `split: B`. ### 5.4 — Collect, dedupe, target 15–25 rows -Each agent writes its rows to `/.scai/tmp/_.yml` (also prints them to stdout as a backup). After all agents complete: +Each agent writes its rows to +`/.scai/tmp/_.yml` +(or `_*_a.yml` / `_*_b.yml` when `split` was set; also prints them to +stdout as a backup). After all agents complete: 1. Read each tmp file. Fall back to stdout parsing if a tmp file is missing. 2. Concatenate `test_cases:` lists. Drop exact duplicates. diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/agents/business_logic.md b/plugin/skills/migration/migrate-objects/baseline-capture/agents/business_logic.md deleted file mode 100644 index a20bc34..0000000 --- a/plugin/skills/migration/migrate-objects/baseline-capture/agents/business_logic.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -name: business-logic-test-agent -description: Produces `test_cases:` rows for an existing step-based YAML stub ensuring code path coverage — IF/ELSE branches, CASE WHEN conditions, happy paths, error paths. No source DB access — uses synthetic values derived from source SQL analysis. Triggers: business logic tests, code coverage tests, branch coverage tests. -parent_skill: baseline-capture ---- - -# Agent: Business Logic & Code Path Coverage - -You produce **`test_cases:` rows** for `` that exercise every code path in the source SQL. - -> You are NOT writing a YAML file. The stub YAML already exists (created by `scai test seed`). Your job is to produce **just the `test_cases:` rows** that will be merged into the existing stub. -> -> See [`../../references/step-based-yaml.md` → Placeholders and `test_cases`](../../references/step-based-yaml.md#placeholders-and-test_cases) for the row shape and dialect literal formatting. - -## Inputs - -- **Object signature**: `` -- **Source code**: `` -- **Project directory**: `` - -## Instructions - -Analyze the source code and produce rows that: - -- Exercise each `IF` / `ELSEIF` / `ELSE` branch. -- Cover each `CASE WHEN` arm. -- Hit the happy path with typical values. -- Trigger early-return conditions. -- Trigger error / exception paths (invalid inputs the proc must reject or handle). - -**Do not query the source database.** Generate rows purely from code analysis. For parameter values that depend on data (e.g. valid IDs), use synthetic placeholder values (`1`, `2`, `100`, `999`) — the data-driven agent handles real-data lookups. - -## Output - -Write your rows to: `/.scai/tmp/_business_logic.yml` - -The file must contain only valid YAML starting with `test_cases:`. Also print the rows to stdout as a backup. - -```yaml -test_cases: - - [1, 100.00] # happy path - main IF branch - - [1, 1500.00] # high-value branch - CASE WHEN amount > 1000 - - [-1, 10.00] # error path - negative ID - - [null, 10.00] # NULL guard - COALESCE branch -``` - -Each row is a JSON-ish array of literals matching the proc's parameter order. Add a trailing `# ...` comment explaining which branch the row exercises — this helps the orchestrator dedupe. - -## When the orchestrator splits business logic into A/B - -For complex objects, two business-logic agents may be spawned: - -- **Agent 3A** — focus on happy paths and main branches. -- **Agent 3B** — focus on error paths, exceptions, and edge conditions found in the source SQL. - -Each writes its own tmp file (`_business_logic_a.yml` vs `_b.yml`); the orchestrator merges them. diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/SKILL.md b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/SKILL.md index fe9b0e4..ae93956 100644 --- a/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/SKILL.md +++ b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/SKILL.md @@ -61,7 +61,7 @@ Tell the user: ## Steps -1. Ensure the MCP session is configured: if you were spawned as a sub-agent, call `configure(project_dir=)` first (session state is not inherited). +1. Ensure the MCP session is configured: if you were spawned as a sub-agent, call `configure(project_dir=)` first (you may be the first to touch the server, and there is no working-directory fallback). 2. **Mine phase.** Run the driver: ``` @@ -139,11 +139,43 @@ Runs after the enrich pass and **before compile** (`mine → enrich → compile uv run --project {SKILL_DIR} python {SKILL_DIR}/scripts/run_pipeline.py enrich --project-dir {PROJECT_DIR} ``` + Between *assemble* and *propose*, the driver runs the **critic station** (the deterministic + backbone ① + citation gate ③, SNOW-3717841). On the first pass over a given envelope it runs the + backbone and stops with `stop_kind: "critique"`, having written + `testbed/enrich/critic/critique-request.json` (the entries that passed the backbone, with the spans + to judge). **You then run the critic prompts** — `prompts/critics/joins_critic.md` for the structural + arrays, `prompts/critics/spec_critic.md` for the value arrays — and write your verdict to + `testbed/enrich/critic/verdict.json` per `prompts/critics/verdict-contract.md`. Re-invoke the driver: + with a fresh verdict present (its `envelope_sig` matching the assembled envelope) the driver applies + the citation gate and, on all-ACCEPT, proceeds to `propose-enrichments`. + 2. **On exit 0:** the driver wrote `testbed/enrich/enrichment-view.json` and the workload is ready — proceed to compile. 3. **On non-zero exit:** read `testbed/enrich/enrichment-report.json` and act on `stop_kind`: - - `reject` — a fragment was malformed. Re-run the prompt named by `prompt_type`, overwrite its fragment file, and re-invoke step 1. Bounded by the per-prompt-type budget. - - `iterate` — `validate` is not ready. For each `blocking` issue whose `remediation.enrichment_fixable` is true, re-run the prompt for its `remediation.enrichment_type` (using the `hint`), overwrite the fragment, re-invoke. Bounded by the global iteration budget. - - `documented-stop` — a structural gap (e.g. an FK cycle, surfaced at propose-time as `TBD0015`) or CAS-retry exhaustion. Surface the `detail` to the user and stop; do not re-prompt the same edge. + - `reject` — a fragment was malformed OR the critic backbone/gate rejected an entry (`reason: + critic_backbone` with `citations`, or `reason: critic_reject`). Re-run the prompt named by + `prompt_type` (`critic` for a critic rejection), overwrite its fragment, re-invoke step 1. Bounded + by the per-prompt-type budget. + - `critique` — the backbone passed; the envelope awaits your semantic judgment. Run the two critic + prompts against `critique-request.json`, write `verdict.json` (copy `envelope_sig` verbatim), then + re-invoke step 1. A `REJECT` needs a verifiable mode-`a` citation; when you can't cite a + contradicting span, use `REVISE` (mode `b`). Re-authoring a fragment on REVISE changes the envelope + and re-runs the backbone on the new version. + - `iterate` — either `validate` is not ready, or the citation gate downgraded a critic `REJECT` to + a bounded `REVISE`. When `validate` is not ready, for each `blocking` issue whose + `remediation.enrichment_fixable` is true, re-run the prompt for its `remediation.enrichment_type` + (using the `hint`), overwrite the fragment, re-invoke. When the stop instead carries `reason: + critic_revise` / `prompt_type: critic`, the payload is not the `blocking`/`remediation` shape but + `entries[]` — each with `kind`, the offending `table`/`column(s)`, and the critic's `feedback` — + so re-author the named fragment(s) per that `feedback` and re-invoke; re-authoring changes the + envelope and re-runs the backbone. The two `iterate` causes are bounded by two independent + budgets: a not-ready `validate` iterate draws on the global iteration budget + (`--max-iterations`, exhausting as `reason: iteration_budget`), while a `critic_revise` iterate + draws on the `critic` bucket of the per-prompt-type reject budget + (`--max-rejections-per-type`, exhausting as `reason: critic_budget`). + - `documented-stop` — a structural gap (FK cycle → `TBD0015`), CAS-retry exhaustion, a **malformed + `verdict.json`** (`reason: critic_verdict_malformed`), or a **value cycle** (`reason: value_cycle` + — the same value was rejected and re-proposed). Surface the `detail`/`citations` and stop; a human + resolves it. The critic never auto-applies an un-critiqued or cyclically-failing entry. - `budget-exhausted` — the retry/iterate budget is spent. Surface the report and stop; a human fixes the fragment and re-runs with `--reset-budget`. 4. **Only after enrich exit 0**, run compile (enrich MUST precede compile). Enrich's terminal `validate ready` is the readiness precondition the downstream `generate` gate checks; the standalone `validate` step above remains a cheap idempotent re-check. @@ -157,4 +189,6 @@ Note: `fk_cycle` and `parent_missing_key` gaps are **advisory** (never blocking) - `testbed/compile/clusters-view.json` — the compile deliverable: cluster summary (written by the driver). - `testbed/generate/summary-view.json` — the generate deliverable and **task completion predicate**: row counts + CSV/manifest paths (written by the driver). - Generated CSVs + provenance `manifest.json` — the `generate` deliverable. Each table's CSV lands under its own object's `/testbed/` folder; the `manifest.json` (beside `state.bin`, at the view's `manifest_path`) lists them as project-root-relative `csv_path` entries. The view's `out_path` is that project root — not a CSV directory. No output dir is passed to `scai testbed generate`. +- `testbed/enrich/critic/critique-request.json` — the backbone-pass entries + spans the critic judges (written by the driver at the `critique` stop). +- `testbed/enrich/critic/verdict.json` — the agent-authored critic verdict (`ACCEPT`/`REVISE`/`REJECT` per entry); consumed by the citation gate on re-invocation. - `.scai/testbed/run.json` — derived progress ledger (mine/validate/compile/generate status, PENDING records). diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/prompts/critics/joins_critic.md b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/prompts/critics/joins_critic.md new file mode 100644 index 0000000..fb976e3 --- /dev/null +++ b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/prompts/critics/joins_critic.md @@ -0,0 +1,32 @@ +# joins_critic — structural enrichment critic + +Scope: `fk_chains`, `correlated_groups`, `temporal_alignment`, `anti_join_tables` (workload-level; +FK chains span objects). The backbone already confirmed existence, non-self-join, range, and structural +backing. Judge only the semantic residue below, then emit your verdict per **verdict-contract.md**. + +## Per-type residue + +### fk_chains — is this the semantically correct parent? +The backbone confirmed the child cell has a mined `fk`/`join_edge` (see `anchors.mined_edges`). Compare +the proposed `parent_table` to the parent the mined edge names. +- Mined edge names a **different** parent than proposed → **REJECT**, mode `a`, citation + `fk_parent_mismatch` (`{child_table, child_col, proposed_parent_table}`). Example: `REP_ID` joins + `SALES_REP` in the source but the chain proposes `REGION`. +- Direction/cardinality implausible given the PKs (child→parent points at a non-key) → **REVISE**, mode `b`. +- Proposed parent matches the mined edge → **ACCEPT**. + +### correlated_groups — real correlation or coincidence? +The backbone confirmed the group maps to a `branch_predicate` gap. Ask whether the source jointly implies +the tuple (the columns co-vary in the same predicate/branch), or the pairing is coincidental. +- No joint support in the source → **REVISE**, mode `b`. +- Jointly implied → **ACCEPT**. + +### temporal_alignment — does the source imply `column_deb <= column_fin`? +There is no span to hard-gate against. +- No ordering/predicate in the source that implies the inequality → **REVISE**, mode `b`. +- Source implies the ordering → **ACCEPT**. + +### anti_join_tables — is there a real anti-join in the source? +Look for `NOT EXISTS` / `LEFT JOIN ... IS NULL` semantics on the cited fk_column. +- No real anti-join present → **REVISE**, mode `b`. +- Present → **ACCEPT**. diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/prompts/critics/spec_critic.md b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/prompts/critics/spec_critic.md new file mode 100644 index 0000000..73013d1 --- /dev/null +++ b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/prompts/critics/spec_critic.md @@ -0,0 +1,28 @@ +# spec_critic — value enrichment critic + +Scope: `inferred_enum`, `branch_values`, `must_include`, `null_fraction_override` (per-object; per +column). The backbone already verbatim-checked literals against `source_evidence`, applied the +declared-enum ACCEPT-and-skip, and range-checked fractions. Judge only the semantic residue below, then +emit your verdict per **verdict-contract.md**. + +## Per-type residue + +### inferred_enum — is the domain complete and genuinely enum-like? +The backbone already rejected invented literals and skipped declared enums. +- The domain **misses** a discriminating branch literal that appears in the source → **REVISE**, mode `b`. +- A listed literal **positively contradicts** a declared domain (rare) → **REJECT**, mode `a`, citation + `declared_enum`. +- Complete and enum-like → **ACCEPT**. + +### branch_values — did it capture ALL discriminating branch literals? +- A shallow subset (source branches on more literals than listed) → **REVISE**, mode `b`. +- Complete → **ACCEPT**. + +### must_include — are these values the source actually requires? +- A value that **positively contradicts** a source predicate/domain → **REJECT**, mode `a` (cite the span). +- Unsupported-but-not-contradictory (source doesn't require it) → **REVISE**, mode `b`. +- Required by the source → **ACCEPT**. + +### null_fraction_override — does the source justify overriding nullability? +- No justification in the source (procs don't treat the column as optional) → **REVISE**, mode `b`. +- Justified → **ACCEPT**. diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/prompts/critics/verdict-contract.md b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/prompts/critics/verdict-contract.md new file mode 100644 index 0000000..7ea23e2 --- /dev/null +++ b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/prompts/critics/verdict-contract.md @@ -0,0 +1,59 @@ +# Critic verdict contract (shared by joins_critic and spec_critic) + +You are a **critic**. Your one objective, stated explicitly: decide whether each machine-proposed +enrichment is **grounded in the source**. You are not the schema checker or the readiness checker — +those already ran. You judge only the **semantic residue** the deterministic backbone could not. + +## Inputs +- `testbed/enrich/critic/critique-request.json` — the entries that passed the backbone. Each entry: + `{kind, container, identity, status ("pass"|"flag"), citation, anchors}`. `anchors` carries the spans + and questions to judge against (e.g. `mined_edges`, `grounded_in`, a yes/no `question`). +- The per-object testbed JSON under `artifacts/**` and the mine view + `testbed/mine/unsolved-view.json` — your grounding evidence. + +## Grounded-conservatism (the governing rule) +Inspect exhaustively. **ACCEPT** if nothing in the source contradicts or under-supports the entry. +Only fail a criterion when you can point at a specific, source-grounded defect. Do not REVISE by reflex. + +## Reason before the verdict +For each non-ACCEPT entry, first write one line per criterion — `PASS`/`FAIL` + a one-sentence cited +evidence (name the span: a column, a predicate, a mined edge) — **then** the verdict label. No numeric +scores. + +## The two modes (the asymmetry — read carefully) +- **mode `a` — positive contradiction.** You found a concrete span that *contradicts* the entry. Only a + mode-`a` verdict may be a **REJECT**, and it MUST carry a `citation` the harness can re-resolve. The + harness re-verifies your citation and **drops your REJECT** if it doesn't hold — so cite only real + contradictions. +- **mode `b` — absence of support.** Nothing in the source supports the entry, but no span contradicts + it. You **cannot** hard-reject a negative. If you cannot cite a contradicting span, you MUST use + **REVISE** (mode `b`), never REJECT. + +## Verdict labels (fixed order): `ACCEPT` | `REVISE` | `REJECT` + +## Citation types for a mode-`a` REJECT +- `invented_literal` — `{ "type": "invented_literal", "literal": "" }` +- `declared_enum` — `{ "type": "declared_enum", "literal": "" }` (contradicts a declared domain) +- `fk_parent_mismatch` — `{ "type": "fk_parent_mismatch", "child_table", "child_col", "proposed_parent_table" }` + +## Self-consistency +Judge each entry three times and take the majority. A 1-1-1 split resolves to the **more conservative** +verdict (prefer REVISE over ACCEPT) — but never fabricate a REJECT to break a tie. + +## Output — write `testbed/enrich/critic/verdict.json` +Copy `envelope_sig` verbatim from `critique-request.json`. List every non-ACCEPT entry (ACCEPT entries +may be omitted; an empty `verdicts` array means "all accepted"). + +```json +{ + "envelope_sig": "", + "verdicts": [ + { "kind": "", "table": "", "column": "", + "verdict": "REVISE", "mode": "b", + "feedback": "[type — T.C] → → " } + ] +} +``` +Use `"columns": ["A","B"]` instead of `"column"` for multi-column structural entries. Feedback is +specific and localized — never "go deeper." On re-critique, re-verify against the source, not "did the +fragment change." diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/assembler.py b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/assembler.py index e0aaf89..0986f1f 100644 --- a/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/assembler.py +++ b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/assembler.py @@ -36,15 +36,18 @@ def load_fragments(fragments_dir: str) -> list[dict]: fragments: list[dict] = [] for p in sorted(root.glob("*.json")): try: - frag = json.loads(p.read_text()) + frag = json.loads(p.read_text(encoding="utf-8")) except json.JSONDecodeError as e: # A malformed fragment is a re-promptable reject, not a station crash: name the offending # fragment (via prompt_type) so run_enrich emits stop_kind "reject" and the agent knows # which prompt to re-run — mirroring assemble()'s flag_for_llm handling. raise AssemblyError(p.stem, f"fragment '{p.name}' is not valid JSON: {e}") from e - except OSError as e: - # Unreadable (removed between glob and read, permissions): same reject class, not a - # traceback out of the station — but named honestly as an I/O failure, not bad JSON. + except (OSError, UnicodeDecodeError) as e: + # Unreadable (removed between glob and read, permissions, or a fragment written mid- + # codepoint): same reject class, not a traceback out of the station — but named honestly + # as an I/O failure, not bad JSON. UnicodeDecodeError is a ValueError, so it escapes + # both arms above unless named here; the explicit utf-8 read is what can raise it, + # instead of the locale default silently decoding a fragment to mojibake. raise AssemblyError(p.stem, f"fragment '{p.name}' could not be read: {e}") from e if not isinstance(frag, dict): # Valid JSON but not an object (a bare array/scalar) can't be merged; reject it here so diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_backbone.py b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_backbone.py new file mode 100644 index 0000000..0f162f7 --- /dev/null +++ b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_backbone.py @@ -0,0 +1,414 @@ +"""Deterministic backbone (layer ①) for the enrichment critics. + +Runs cheap, 100%-reproducible checks per enrichment entry and classifies each into: + fail -> hard REJECT (guaranteed-true citation); skips the LLM residue. + accept_skip -> boundary rule (declared-enum wins / declared-FK duplicate); no residue needed. + flag -> semantically-unsupported-but-in-range; routed to the residue as a mode-(b) question. + pass -> existence/grounding OK; residue judges the semantic bit. +This module makes ZERO model calls. It holds the value + structural entry checks, the run_backbone +dispatch over an envelope, and build_critique_request (the pass/flag entries handed to the residue). +""" +from __future__ import annotations + +import enum +from dataclasses import dataclass, field + +from critic_index import ObjectIndex, canon_owner, canon_table +from critic_normalize import literal_grounded_normalized, normalize + + +class CheckStatus(enum.StrEnum): + # One source of truth for the backbone verdict a check carries; compared across module + # boundaries (run_pipeline reads FAIL, build_critique_request reads PASS/FLAG), so a bare + # literal here would let a typo silently route a check to the wrong gate branch. + FAIL = "fail" + ACCEPT_SKIP = "accept_skip" + FLAG = "flag" + PASS = "pass" + + +@dataclass +class EntryCheck: + kind: str + container: str + identity: dict + status: CheckStatus + citation: str = "" + anchors: dict = field(default_factory=dict) + + +def _cell(entry: dict) -> tuple: + return entry.get("table"), entry.get("column") + + +def _in_unit_range(v: object) -> bool: + return isinstance(v, (int, float)) and not isinstance(v, bool) and 0.0 <= v <= 1.0 + + +def _wire_array(value: object) -> list | None: + # None for a value the wire contract declares as an array but that arrived as something else. A bare + # string is the case that matters: truthy and iterable, so an unguarded loop grades its characters as + # literals and "GOLD" is accepted as four of them. Falsy stays [] so an absent field reads as absent. + if not value: + return [] + return value if isinstance(value, list) else None + + +def _fail(name: str, ident: dict, msg: str) -> list[EntryCheck]: + # Each structural array is its own kind and container, so the FAIL shape lives in exactly one place. + return [EntryCheck(name, name, ident, CheckStatus.FAIL, msg)] + + +def _resolved_key(index: ObjectIndex, table: str | None) -> str: + # Identity key for a table name: its owner when the name resolves to exactly one, else the name as + # written. Two names are then equal only when they provably denote the same table -- which is what a + # guaranteed-true FAIL needs. canon_table would instead collapse SALES.ORDERS and STAGING.ORDERS. + owners = index.table_owners(table) + return owners[0] if len(owners) == 1 else canon_owner(table) + + +def _table_matches(index: ObjectIndex, a: str | None, b: str | None) -> bool: + # Deliberately looser than _resolved_key equality: a declared FK's referenced table is written at + # inconsistent qualification depth across artifacts, so require owner equality only when both names + # resolve uniquely. Used solely where a match SKIPS review, never where it hard-blocks. + ka, kb = index.table_owners(a), index.table_owners(b) + if len(ka) == 1 and len(kb) == 1: + return ka[0] == kb[0] + return canon_table(a) == canon_table(b) + + +def _ambiguity_check(kind: str, container: str, ident: dict, index: ObjectIndex, + *tables) -> list[EntryCheck]: + # An unresolvable bare name yields no column facts, so every existence/type check below it would + # read as a guaranteed-true FAIL on a column that does exist in each candidate. The ambiguity + # itself is deterministic, so hand it to the residue with the candidate owners attached instead. + unresolved = [t for t in dict.fromkeys(tables) if index.ambiguous_table(t)] + if not unresolved: + return [] + return [EntryCheck(kind, container, ident, CheckStatus.FLAG, + anchors={"owners": sorted({o for t in unresolved for o in index.table_owners(t)}), + "question": "the same bare table name is declared in more than one " + "schema; does this entry mean one of them?"})] + + +@dataclass(frozen=True) +class _ValueCell: + """The (table, column) context each value-field check needs, once the entry cleared the guards. + + Bundled rather than threaded through every helper as six parameters, and built exactly once per + entry: `cell_corpus` re-walks constraints and re-derefs `source_evidence` on each call, and + `normalize` would otherwise re-fold the whole corpus once per literal. + """ + container: str + table: str | None + column: str | None + index: ObjectIndex + corpus_normalized: list[str] + grounded_sample: list[str] + + @classmethod + def build(cls, container: str, table: str | None, column: str | None, + index: ObjectIndex) -> _ValueCell: + corpus = index.cell_corpus(table, column) + return cls(container, table, column, index, [normalize(c) for c in corpus], corpus[:5]) + + @property + def ident(self) -> dict: + # A fresh dict per check: EntryCheck.identity is handed to the residue and the gate, so a + # shared one would let a later mutation rewrite an already-emitted check's identity. + return {"table": self.table, "column": self.column} + + def literal_ident(self, literal: object) -> dict: + # The FAIL shape the value-cycle memo reads: `literal` present <=> a value was cited. + return {"table": self.table, "column": self.column, "literal": literal} + + +def _nfo_range_fail(nfo, container: str, ident: dict) -> list[EntryCheck]: + # The range is provable whichever table the name resolves to, so it must survive the ambiguity + # guard in check_value_entry rather than be softened to a question the residue has to re-derive. + if nfo is None or _in_unit_range(nfo): + return [] + return [EntryCheck("null_fraction_override", container, ident, CheckStatus.FAIL, + f"null_fraction_override {nfo!r} for {ident['table']}.{ident['column']} " + f"is outside [0,1]")] + + +def _declared_enum_check(cell: _ValueCell, lit) -> EntryCheck: + # The declared domain wins over an inferred one only for literals it actually contains. + # An out-of-domain literal is the same guaranteed-true contradiction the gate adjudicates + # as CitationType.DECLARED_ENUM, and ACCEPT_SKIP is forwarded to neither the residue nor + # the gate -- so if this doesn't FAIL, nothing downstream ever checks it. + if cell.index.literal_in_declared_domain(cell.table, cell.column, lit): + return EntryCheck("inferred_enum", cell.container, cell.ident, CheckStatus.ACCEPT_SKIP) + return EntryCheck("inferred_enum", cell.container, cell.literal_ident(lit), CheckStatus.FAIL, + f"inferred_enum literal {lit!r} for {cell.table}.{cell.column} is outside the " + f"declared enum domain {cell.index.declared_enum_domain(cell.table, cell.column)}") + + +def _grounded_enum_check(cell: _ValueCell, lit) -> EntryCheck: + if literal_grounded_normalized(lit, cell.corpus_normalized): + return EntryCheck("inferred_enum", cell.container, cell.ident, CheckStatus.PASS, + anchors={"grounded_in": cell.grounded_sample}) + return EntryCheck("inferred_enum", cell.container, cell.literal_ident(lit), CheckStatus.FAIL, + f"inferred_enum literal {lit!r} for {cell.table}.{cell.column} " + f"is not present in any source_evidence span") + + +def _check_inferred_enum(cell: _ValueCell, raw) -> list[EntryCheck]: + literals = _wire_array(raw) + if literals is None: + return [EntryCheck("inferred_enum", cell.container, cell.ident, CheckStatus.FLAG, + anchors={"value": raw, + "question": "inferred_enum is not an array; is it intended?"})] + if cell.index.is_declared_enum(cell.table, cell.column): + return [_declared_enum_check(cell, lit) for lit in literals] + return [_grounded_enum_check(cell, lit) for lit in literals] + + +def _check_must_include(cell: _ValueCell, raw) -> list[EntryCheck]: + values = _wire_array(raw) + out: list[EntryCheck] = [] + if values is None: + out.append(EntryCheck("must_include", cell.container, cell.ident, CheckStatus.FLAG, + anchors={"value": raw, + "question": "must_include is not an array; is it intended?"})) + values = [] + for val in values: + if not isinstance(val, str) or not val.strip(): + # Nothing to ground against, so neither verdict is provable: a PASS here would carry a + # grounded_in anchor asserting evidence that was never matched, steering the residue to ACCEPT. + out.append(EntryCheck("must_include", cell.container, cell.ident, CheckStatus.FLAG, + anchors={"value": val, + "question": "must_include value is not a non-empty string; " + "is it intended?"})) + elif not literal_grounded_normalized(val, cell.corpus_normalized): + out.append(EntryCheck("must_include", cell.container, cell.literal_ident(val), + CheckStatus.FAIL, + f"must_include value {val!r} for {cell.table}.{cell.column} " + f"is not present in any source_evidence span")) + else: + out.append(EntryCheck("must_include", cell.container, cell.ident, CheckStatus.PASS, + anchors={"grounded_in": cell.grounded_sample})) + return out + + +def _check_null_fraction_override(cell: _ValueCell, nfo) -> list[EntryCheck]: + # Only the in-range case is graded here; an out-of-range value already FAILed in _nfo_range_fail, + # which runs ahead of the ambiguity guard because its verdict needs no table resolution. + if nfo is None or not _in_unit_range(nfo): + return [] + if nfo > 0 and cell.index.is_pk_or_notnull(cell.table, cell.column): + return [EntryCheck("null_fraction_override", cell.container, cell.ident, CheckStatus.FAIL, + f"null_fraction_override {nfo!r} cannot introduce nulls into " + f"{cell.table}.{cell.column}, which is a primary key or declared NOT NULL")] + return [EntryCheck("null_fraction_override", cell.container, cell.ident, CheckStatus.FLAG, + anchors={"fraction": nfo, + "question": "does the source justify overriding nullability?"})] + + +def check_value_entry(entry: dict, container: str, index: ObjectIndex) -> list[EntryCheck]: + # Existence/ambiguity guards first, then one independent per-field check composed in wire order. + # Each field's rules live in its own helper: they share only the cell they are graded against, so + # inlining them here bought nothing but one function past the length and complexity budget. + table, col = _cell(entry) + nfo = entry.get("null_fraction_override") + # Each guard builds its own identity dict for the same reason _ValueCell.ident does: two emitted + # checks sharing one object would let a later mutation rewrite an already-emitted identity. + range_fail = _nfo_range_fail(nfo, container, {"table": table, "column": col}) + ambiguous = _ambiguity_check("column", container, {"table": table, "column": col}, index, table) + if ambiguous: + return range_fail + ambiguous + if not index.has_column(table, col): + return range_fail + [EntryCheck("column", container, {"table": table, "column": col}, + CheckStatus.FAIL, f"column {table}.{col} does not exist")] + cell = _ValueCell.build(container, table, col, index) + return (range_fail + + _check_inferred_enum(cell, entry.get("inferred_enum")) + + _check_must_include(cell, entry.get("must_include")) + + _check_null_fraction_override(cell, nfo)) + + +def _both_cells_distinct(index: ObjectIndex, t1, c1, t2, c2) -> bool: + return not (_resolved_key(index, t1) == _resolved_key(index, t2) and str(c1).upper() == str(c2).upper()) + + +def check_fk_chain(entry: dict, index: ObjectIndex) -> list[EntryCheck]: + ct, cc = entry.get("child_table"), entry.get("child_col") + pt, pc = entry.get("parent_table"), entry.get("parent_col") + ident = {"child_table": ct, "child_col": cc, "parent_table": pt, "parent_col": pc} + + def fail(msg: str) -> list[EntryCheck]: + return _fail("fk_chains", ident, msg) + + ambiguous = _ambiguity_check("fk_chains", "fk_chains", ident, index, ct, pt) + if ambiguous: + return ambiguous + if not index.has_column(ct, cc): + return fail(f"fk_chains child column {ct}.{cc} does not exist") + if not index.has_column(pt, pc): + return fail(f"fk_chains parent column {pt}.{pc} does not exist") + if not _both_cells_distinct(index, ct, cc, pt, pc): + return fail("fk_chains is a self-join on identical columns") + dfk = index.declared_fk(ct, cc) + if dfk and _table_matches(index, dfk[0], pt) and str(dfk[1] or "").upper() == str(pc).upper(): + return [EntryCheck("fk_chains", "fk_chains", ident, CheckStatus.ACCEPT_SKIP)] + if not index.has_kind_for_cell(ct, cc, ("fk", "join_edge")): + return fail(f"fk_chains for {ct}.{cc} maps to no unsolved fk/join_edge constraint (spurious)") + mined = [c.get("detail") for c in index.cell_constraints(ct, cc) if c.get("kind") in ("fk", "join_edge")] + return [EntryCheck("fk_chains", "fk_chains", ident, CheckStatus.PASS, + anchors={"child": f"{ct}.{cc}", "proposed_parent": f"{pt}.{pc}", "mined_edges": mined})] + + +def _group_table_names(tables: list) -> list: + # Wire shape is [{table, columns}] (CorrelatedGroupTableEntry). A bare string is tolerated so a + # hand-built entry degrades to a name instead of pushing an unhashable dict through the guards. + return [t.get("table") if isinstance(t, dict) else t for t in tables] + + +def _split_tuple_col(tc, tables: list) -> list[tuple]: + # "TABLE.COLUMN" -> (table, col); bare "COLUMN" -> pair with every table in the group. + if isinstance(tc, str) and "." in tc: + t, c = tc.rsplit(".", 1) + return [(t, c)] + return [(t, tc) for t in tables] + + +def check_correlated_group(entry: dict, index: ObjectIndex) -> list[EntryCheck]: + names = _group_table_names(entry.get("tables", []) or []) + tuple_cols = entry.get("tuple_columns", []) or [] + ident = {"tables": entry.get("tables", []) or [], "tuple_columns": tuple_cols, + "group_id": entry.get("group_id")} + + def fail(msg: str) -> list[EntryCheck]: + return _fail("correlated_groups", ident, msg) + + # Two names are one table only when they resolve to the same owner, so this survives resolution and + # stays a hard FAIL even for an ambiguous name. Comparing bare names instead would reject + # SALES.ORDERS + STAGING.ORDERS -- a legitimate cross-schema group. + if len({_resolved_key(index, n) for n in names}) < 2: + return fail("correlated_groups needs at least 2 distinct tables") + ambiguous = _ambiguity_check("correlated_groups", "correlated_groups", ident, index, *names) + if ambiguous: + return ambiguous + cells = [pair for tc in tuple_cols for pair in _split_tuple_col(tc, names)] + for t, c in cells: + if not index.has_column(t, c): + return fail(f"correlated_groups column {t}.{c} does not exist") + if not any(index.has_kind_for_cell(t, c, ("branch_predicate",)) for t, c in cells): + return fail("correlated_groups maps to no unsolved branch_predicate gap (spurious)") + return [EntryCheck("correlated_groups", "correlated_groups", ident, CheckStatus.PASS, + anchors={"cells": [f"{t}.{c}" for t, c in cells], + "question": "is the correlation real, or a coincidence?"})] + + +def check_temporal(entry: dict, index: ObjectIndex) -> list[EntryCheck]: + # One table, two of its date columns: column_deb / column_fin = the span's start ("début") and end + # ("fin"). The names are wire-contract-locked (TemporalAlignmentEntry), not descriptive labels. + t = entry.get("table") + deb, fin = entry.get("column_deb"), entry.get("column_fin") + ident = {"table": t, "column_deb": deb, "column_fin": fin} + + def fail(msg: str) -> list[EntryCheck]: + return _fail("temporal_alignment", ident, msg) + + # Provable whichever table the name resolves to, so it survives the ambiguity guard. + same_column = [] if str(deb).upper() != str(fin).upper() else fail( + "temporal_alignment references the same column twice") + ambiguous = _ambiguity_check("temporal_alignment", "temporal_alignment", ident, index, t) + if ambiguous: + return same_column + ambiguous + if same_column: + return same_column + for col in (deb, fin): + if not index.has_column(t, col): + return fail(f"temporal_alignment column {t}.{col} does not exist") + if not index.is_date(t, col): + return fail(f"temporal_alignment column {t}.{col} is not a date/timestamp") + return [EntryCheck("temporal_alignment", "temporal_alignment", ident, CheckStatus.FLAG, + anchors={"question": "does the source imply column_deb <= column_fin?"})] + + +def check_anti_join(entry: dict, index: ObjectIndex) -> list[EntryCheck]: + t, fkc = entry.get("table"), entry.get("fk_column") + frac = entry.get("anti_join_fraction") + ident = {"table": t, "fk_column": fkc, "anti_join_fraction": frac} + + def fail(msg: str) -> list[EntryCheck]: + return _fail("anti_join_tables", ident, msg) + + # The range check needs no table resolution, so it stays a hard FAIL even for an ambiguous name. + if not _in_unit_range(frac): + return fail(f"anti_join_tables anti_join_fraction {frac!r} is outside [0,1]") + ambiguous = _ambiguity_check("anti_join_tables", "anti_join_tables", ident, index, t) + if ambiguous: + return ambiguous + if not index.has_column(t, fkc): + return fail(f"anti_join_tables column {t}.{fkc} does not exist") + return [EntryCheck("anti_join_tables", "anti_join_tables", ident, CheckStatus.FLAG, + anchors={"anti_join_fraction": frac, + "question": "does the source contain a real NOT EXISTS / LEFT JOIN ... IS NULL?"})] + + +_STRUCTURAL_CHECKERS = { + "fk_chains": check_fk_chain, + "correlated_groups": check_correlated_group, + "temporal_alignment": check_temporal, + "anti_join_tables": check_anti_join, +} + +_VALUE_CONTAINERS = ("column_enrichments", "branch_values") + + +def reads_column_facts(envelope: dict) -> bool: + """True when grading this envelope would actually read the per-object column facts. + + Derived from the two registries above rather than restated as its own literal list, so registering a + new checker cannot leave a caller's answer stale. The partition is exactly those registries: every + array with a checker reads facts (each one consults `has_column` before it asserts anything, and the + value containers additionally read the grounding corpus), while every array without one falls to + `_uncovered`, which inspects only the entry's own keys and can only ever FLAG. + + Emptiness is what makes this answerable at all: `run_backbone` iterates `envelope.get(array, []) or + []`, so an absent or empty array contributes no entries and therefore reads nothing -- the same + truthiness test used here. Callers use this to tell "the facts are missing and something needs them" + from "the facts are missing and nothing was going to read them", which are different situations: only + the first can manufacture a verdict out of an empty fact base. + """ + return any(envelope.get(array) for array in (*_VALUE_CONTAINERS, *_STRUCTURAL_CHECKERS)) + + +def _uncovered(array: str, entry: dict) -> list[EntryCheck]: + # An enrichment type this backbone predates has no deterministic verdict available, but skipping it + # is the one outcome that must not happen: run_backbone would iterate neither it nor the residue, so + # the entry would be applied having been checked by nobody. Hand it to the residue as a question. + return [EntryCheck(array, array, {"container": array}, CheckStatus.FLAG, + anchors={"fields": sorted(k for k, v in entry.items() if v is not None), + "question": "this enrichment type has no deterministic checker in the " + "backbone; does the entry hold up on inspection?"})] + + +def run_backbone(envelope: dict, index: ObjectIndex) -> list[EntryCheck]: + checks: list[EntryCheck] = [] + for container in _VALUE_CONTAINERS: + for entry in envelope.get(container, []) or []: + checks.extend(check_value_entry(entry, container, index)) + for array, checker in _STRUCTURAL_CHECKERS.items(): + for entry in envelope.get(array, []) or []: + checks.extend(checker(entry, index)) + covered = set(_VALUE_CONTAINERS) | set(_STRUCTURAL_CHECKERS) + for array, entries in envelope.items(): + if array in covered or not isinstance(entries, list): + continue + for entry in entries: + if isinstance(entry, dict): + checks.extend(_uncovered(array, entry)) + return checks + + +def build_critique_request(checks: list[EntryCheck]) -> dict: + entries = [ + {"kind": c.kind, "container": c.container, "identity": c.identity, + "status": c.status, "citation": c.citation, "anchors": c.anchors} + for c in checks if c.status in (CheckStatus.PASS, CheckStatus.FLAG) + ] + return {"schema_version": 1, "entries": entries} diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_benchmark.py b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_benchmark.py new file mode 100644 index 0000000..b8c56ab --- /dev/null +++ b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_benchmark.py @@ -0,0 +1,95 @@ +"""Benchmark harness for the enrichment critics — offline-deterministic (CI-green) by default. + +Scores a frozen seeded-fault set two ways: + residue=False -> backbone-only ablation (the baseline; isolates the residue's marginal lift). + residue=True -> backbone + replayed critic verdicts (the canned residue double from cases.json). +The live path (--run-live) is the seam a future live-residue runner (not run in CI) plugs into; it +makes model calls. One diffable JSON report: per-type coverage + per-layer catch counts + false-REJECT count. +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from critic_index import load_index # noqa: E402 +from critic_backbone import CheckStatus, run_backbone # noqa: E402 +from critic_gate import GateAction, parse_verdict, apply_gate # noqa: E402 + + +def _never(_t, _c, _v): + return False + + +def _outcome(case, index, *, residue, run_live): + envelope = case["envelope"] + if any(c.status == CheckStatus.FAIL for c in run_backbone(envelope, index)): + return "reject", "backbone" + if not residue: + return "accept", "none" + if run_live: + raise NotImplementedError("live residue path requires an agent runner (not run in CI)") + verdict = parse_verdict(json.dumps(case.get("verdict") or {"verdicts": []})) + if not verdict.ok: + return "escalate", "residue" + decision = apply_gate(verdict, index, _never) + # "accept"/"reject"/"escalate" below are the benchmark's own outcome vocabulary, compared against + # cases.json `expected_action` -- deliberately not GateAction, which names the gate's decisions. + if decision.action == GateAction.ACCEPT_ALL: + return "accept", "none" + return decision.action, "residue" + + +def score(cases_dir: str | Path, *, residue: bool = True, run_live: bool = False) -> dict: + root = Path(cases_dir) + view = json.loads((root / "unsolved-view.json").read_text(encoding="utf-8")) + index = load_index(str(root / "artifacts"), view) + cases = json.loads((root / "cases.json").read_text(encoding="utf-8")) + + per_type: dict = {} + layers = {"backbone": 0, "residue": 0} + false_rejects = 0 + results = [] + for case in cases: + outcome, layer = _outcome(case, index, residue=residue, run_live=run_live) + caught = outcome != "accept" + pt = per_type.setdefault(case["type"], {"seeded_total": 0, "seeded_caught": 0, + "clean_total": 0, "clean_ok": 0}) + if case["bucket"] == "seeded": + pt["seeded_total"] += 1 + if caught: + pt["seeded_caught"] += 1 + if layer in layers: + layers[layer] += 1 + else: + pt["clean_total"] += 1 + if caught: + false_rejects += 1 + else: + pt["clean_ok"] += 1 + results.append({"id": case["id"], "type": case["type"], "bucket": case["bucket"], + "layer": case.get("layer", "na"), "expected": case["expected_action"], + "outcome": outcome, "ok": outcome == case["expected_action"]}) + return {"residue": residue, "total": len(cases), "per_type": per_type, + "layers": layers, "false_rejects": false_rejects, "cases": results} + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(prog="critic_benchmark") + ap.add_argument("--set", required=True, help="fixture dir with artifacts/, unsolved-view.json, cases.json") + ap.add_argument("--no-residue", action="store_true", help="backbone-only ablation baseline") + ap.add_argument("--run-live", action="store_true", help="author residue live (a future live-residue runner; not CI)") + ap.add_argument("--out", default=None, help="write the JSON report here") + a = ap.parse_args(argv) + report = score(a.set, residue=not a.no_residue, run_live=a.run_live) + text = json.dumps(report, indent=2) + if a.out: + Path(a.out).write_text(text, encoding="utf-8") + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_gate.py b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_gate.py new file mode 100644 index 0000000..ee18559 --- /dev/null +++ b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_gate.py @@ -0,0 +1,214 @@ +"""Citation gate (layer ③) for the enrichment critics — asymmetric by design. + +A positive-contradiction (mode a) is hard-gated on a harness-verifiable span: the gate re-resolves +the cited fact and confirms the contradiction, so a fabricated critique is dropped toward ACCEPT. +An absence-of-support (mode b) is NOT hard-gatable (no span verifies a negative), so it becomes a +bounded REVISE that escalates to a human at budget-exhaustion. A REJECT that is not a verifiable +mode-a contradiction is coerced to a bounded REVISE — the gate never hard-blocks on an unverifiable +claim. This module makes ZERO model calls. +""" +from __future__ import annotations + +import enum +import json +import re +from collections.abc import Callable +from dataclasses import dataclass, field + +from critic_index import ObjectIndex, canon_table +from critic_normalize import normalize +from manifest import classify_rejections + +_IDENT = re.compile(r"[A-Za-z_][A-Za-z0-9_$]*") + + +class CitationType(enum.StrEnum): + # The mode-a citation contract shared with prompts/critics/verdict-contract.md; one source of + # truth so a typo in a comparison (or an emitted label) can't silently downgrade a REJECT. + INVENTED_LITERAL = "invented_literal" + DECLARED_ENUM = "declared_enum" + FK_PARENT_MISMATCH = "fk_parent_mismatch" + + +@dataclass +class VerdictEntry: + kind: str + table: str | None + columns: list + verdict: str + mode: str | None + citation: dict + feedback: str + + +@dataclass +class Verdict: + ok: bool + error: str = "" + envelope_sig: str | None = None + verdicts: list = field(default_factory=list) + + +def parse_verdict(text: object) -> Verdict: + try: + doc = json.loads(text) + except (json.JSONDecodeError, TypeError, RecursionError) as exc: + # RecursionError: a deeply nested array is malformed input, not a harness bug, and the caller's + # contract is a Verdict carrying the reason -- not a traceback out of the enrich phase. + return Verdict(False, f"verdict is not valid JSON: {exc}") + if not isinstance(doc, dict): + return Verdict(False, "verdict is not a JSON object") + raw = doc.get("verdicts") + if not isinstance(raw, list): + return Verdict(False, "verdict 'verdicts' is missing or not a list") + entries: list = [] + for index, entry in enumerate(raw): + if not isinstance(entry, dict) or "verdict" not in entry: + return Verdict(False, f"verdict entry {index} is malformed (needs a 'verdict' field)") + cols = entry.get("columns") + if not isinstance(cols, list): + cols = [entry["column"]] if entry.get("column") is not None else [] + entries.append(VerdictEntry( + kind=entry.get("kind", ""), + table=entry.get("table"), + columns=cols, + verdict=str(entry.get("verdict", "")).strip().upper(), + mode=(str(entry["mode"]).strip().lower() if entry.get("mode") is not None else None), + citation=entry.get("citation") if isinstance(entry.get("citation"), dict) else {}, + feedback=str(entry.get("feedback", "")), + )) + return Verdict(True, "", doc.get("envelope_sig"), entries) + + +class GateAction(enum.StrEnum): + # The gate's outcome vocabulary. run_pipeline branches on these and _SEVERITY orders them, so a + # bare literal here would silently route a decision to the wrong arm (or drop out of _SEVERITY). + DROP = "drop" # fabricated critique: discard it, the enrichment survives + REVISE = "revise" # unverifiable claim: bounded re-prompt of the fragment + REJECT = "reject" # verified mode-a contradiction: hard block + ESCALATE = "escalate" # a human must look (value cycle / unknown verdict label) + ACCEPT_ALL = "accept_all" + + +@dataclass +class GateDecision: + action: GateAction + report: dict + message: str + record_values: list = field(default_factory=list) + + +_SEVERITY = {GateAction.ESCALATE: 3, GateAction.REJECT: 2, GateAction.REVISE: 1} + + +def _revise_detail(e: VerdictEntry, reason: str) -> dict: + return {"kind": e.kind, "table": e.table, "columns": e.columns, + "mode": "b", "reason": reason, "feedback": e.feedback} + + +def _adjudicate_reject(e: VerdictEntry, index: ObjectIndex) -> tuple[GateAction, dict]: + # Only a harness-verifiable mode-a contradiction can hard-gate; everything else -> bounded REVISE. + if e.mode != "a" or not e.citation: + return GateAction.REVISE, _revise_detail(e, "reject_without_verifiable_mode_a_citation") + # Normalized like `verdict` and `mode`: an off-case label is a producer slip, and leaving it raw would + # silently demote the only verifiable REJECT classes to unverifiable_citation_type. + ctype = str(e.citation.get("type") or "").strip().lower() + table = e.table + col = e.columns[0] if e.columns else None + lit = e.citation.get("literal") + if ctype in (CitationType.INVENTED_LITERAL, CitationType.DECLARED_ENUM): + # No literal cited: nothing to verify either way. Without this, an empty literal is vacuously + # "grounded" (DROP, with an audit line claiming a match) or vacuously out-of-domain (hard REJECT). + # Emptiness is judged after normalize, so a numeric or boolean literal still counts as cited. + if not normalize(lit): + return GateAction.REVISE, _revise_detail(e, "citation_missing_literal") + if ctype == CitationType.INVENTED_LITERAL: + if index.literal_grounded_in_cell(table, col, lit): + return GateAction.DROP, {"kind": e.kind, "table": table, "column": col, "literal": lit, + "why": "cited literal is grounded in source"} + return GateAction.REJECT, {"kind": e.kind, "table": table, "column": col, "literal": lit, + "citation": CitationType.INVENTED_LITERAL, "feedback": e.feedback} + if ctype == CitationType.DECLARED_ENUM: + # Both the domain test and the membership test are the backbone's, so the two layers cannot + # disagree on whether this column has a declared domain or on what is in it. + if index.is_declared_enum(table, col) and not index.literal_in_declared_domain(table, col, lit): + return GateAction.REJECT, {"kind": e.kind, "table": table, "column": col, "literal": lit, + "citation": CitationType.DECLARED_ENUM, "feedback": e.feedback} + return GateAction.DROP, {"kind": e.kind, "table": table, "column": col, "literal": lit, + "why": "literal is within the declared domain / no domain to contradict"} + if ctype == CitationType.FK_PARENT_MISMATCH: + ct, cc = e.citation.get("child_table"), e.citation.get("child_col") + proposed = e.citation.get("proposed_parent_table") + parent = canon_table(proposed).casefold() + if not parent: # no parent named -> nothing to verify; fail-safe to bounded REVISE, never drop + return GateAction.REVISE, _revise_detail(e, "fk_citation_missing_proposed_parent") + mined = [normalize(c.get("detail")) for c in index.cell_constraints(ct, cc) + if c.get("kind") in ("fk", "join_edge")] + if not mined: + return GateAction.REVISE, _revise_detail(e, "fk_no_mined_edge_to_verify") + # Identifier-boundary match, not substring: a degenerate name canonicalizes to "", which is a + # substring of every edge and would DROP the critique unconditionally. The child's own name is + # deliberately still eligible -- a self-referencing FK is legitimate, and excluding it would turn + # one into a hard REJECT, which this gate may only ever do on a verified contradiction. + if any(parent in _IDENT.findall(m) for m in mined): + return GateAction.DROP, {"kind": e.kind, "child": f"{ct}.{cc}", "proposed_parent": proposed, + "why": "proposed parent matches a mined edge"} + return GateAction.REJECT, {"kind": e.kind, "child": f"{ct}.{cc}", "proposed_parent": proposed, + "citation": CitationType.FK_PARENT_MISMATCH, "mined_edges": mined, + "feedback": e.feedback} + return GateAction.REVISE, _revise_detail(e, "unverifiable_citation_type") + + +def apply_gate(verdict: Verdict, index: ObjectIndex, + previously_rejected: Callable[[str | None, str | None, object], bool]) -> GateDecision: + """Adjudicate a parsed critic verdict into one gate action for the whole envelope. + + Each non-ACCEPT entry is adjudicated independently, then the most severe outcome wins + (escalate > reject > revise); DROPs are recorded but never escalate. `previously_rejected` + answers whether a (table, column, literal) has already been rejected in an earlier round, which + turns a repeat REJECT into an escalate rather than another loop. Its signature is the one + manifest.classify_rejections declares -- the literal is whatever the citation carried (a number + or a bool is still a cited value), and one declared shape is what lets the same predicate serve + this path and the backbone-stop. Returns ACCEPT_ALL when nothing is actionable. `record_values` + is the caller's to persist -- the gate holds no state. + """ + actions: list = [] # [(GateAction, detail)] + dropped: list = [] + record_values: list = [] + for e in verdict.verdicts: + if e.verdict == "ACCEPT": + continue + if e.verdict == "REJECT": + kind, detail = _adjudicate_reject(e, index) + elif e.verdict == "REVISE": + kind, detail = GateAction.REVISE, _revise_detail(e, "absence_of_support") + else: + kind, detail = GateAction.ESCALATE, {"kind": e.kind, "reason": "unknown_verdict_label", + "label": e.verdict, "feedback": e.feedback} + if kind == GateAction.DROP: + dropped.append(detail) + continue + if kind == GateAction.REJECT: + # Same helper the backbone-stop consumes, over this path's own data shape: an fk + # mismatch carries no `literal`, so it records nothing and cannot read as a repeat. + cycle, recorded = classify_rejections( + [(detail.get("table"), detail.get("column"), detail.get("literal"))], + previously_rejected) + record_values.extend(recorded) + if cycle: + actions.append((GateAction.ESCALATE, {**detail, "reason": "value_cycle"})) + continue + actions.append((GateAction.REJECT, detail)) + else: + actions.append((kind, detail)) + + if not actions: + return GateDecision(GateAction.ACCEPT_ALL, + {"reason": "critic_accept", "dropped": dropped}, + "critic verdict: all entries accepted", record_values) + # Picked from `actions` rather than by searching _SEVERITY for the key holding the max value, which + # only resolves to the right action while every severity is distinct. + action = max(actions, key=lambda a: _SEVERITY[a[0]])[0] # GateAction, so f-strings emit its value + report = {"reason": f"critic_{action}", "entries": [d for _, d in actions], "dropped": dropped} + message = f"critic {action}: {len(actions)} actionable verdict(s); see enrichment-report.json" + return GateDecision(action, report, message, record_values) diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_index.py b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_index.py new file mode 100644 index 0000000..fbe242c --- /dev/null +++ b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_index.py @@ -0,0 +1,281 @@ +"""Read-only index over per-object testbed JSONs + the list-unsolved view. + +The backbone dispatches on facts, not on the opaque state.bin: column existence/type from the +per-object `columns[]`, the list-unsolved constraints (per kind and per cell), and a grounding +corpus per (table,column) built from constraint `detail` + deref'd `source_evidence`. Both column +facts and constraint cells are keyed on the resolved qualified owner and reached through a bare-name +alias: bare keying alone would let two schemas' same-named tables overwrite each other's columns and +share one constraint bucket, and the critic would then validate enrichments against another table's +facts or ground a literal in another table's evidence. list-unsolved rows are sometimes +schema-qualified, sometimes bare, and fk/enum/check rows carry table=None with the owner in `object`, +so resolution is guarded by `names_compatible`: a row may only be attributed to a table whose name it +under- or over-qualifies, never to one whose schema it contradicts. A row that cannot be attributed +keeps the name it was written with (qualified) or the bare name, and is recorded in `unattributed` +when a differently-schema'd table would otherwise have absorbed it. + +Cell lookups are then deliberately path-dependent, because the two readers fail in opposite +directions. `cell_corpus` (grounding) is permissive and merges the unattributable bare bucket into +every candidate owner: a row it cannot see makes a golden literal false-REJECT, and a backbone FAIL +is a contractually guaranteed-true hard reject. `cell_constraints` / `has_kind_for_cell` (structural) +is strict: there an EXTRA row from a table the entry never named manufactures a hard reject, while a +missing one only routes the entry to the residue. A bare name that maps to more than one owner is +`ambiguous_table` and yields no column facts at all -- the backbone's FAILs are contractually +guaranteed-true, so an unresolvable name must route to the residue rather than assert anything. + +An artifact's own name is read as `object ?? table`, mirroring TestbedArtifact.ResolvedName: procedural +artifacts name themselves in `object`, table artifacts in `table`. A TABLE artifact with neither is +recorded in `skipped` rather than indexed, and two artifacts claiming one owner are recorded in +`ambiguities` -- either would otherwise make a table's columns vanish with nothing in the report. +""" +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path + +from critic_normalize import deref_pointer, literal_grounded, normalize + + +def canon_table(name: str | None) -> str: + return str(name or "").split(".")[-1].upper() + + +def canon_owner(name: str | None) -> str: + return str(name or "").strip().upper() + + +def names_compatible(a: str | None, b: str | None) -> bool: + """True when two object names can denote the same table: every part they both spell agrees. + + Compared right-to-left over the shared depth, so `ORDERS` and `DB.SALES.ORDERS` are both + compatible with the artifact `SALES.ORDERS` (one under-qualifies, one over-qualifies the same + table) while `STAGING.ORDERS` is not (it names a schema the artifact contradicts). This is the + test `_resolve`'s bare-alias fallback lacks: that fallback answers "which captured table shares + this leaf name", which for a row that already named its own schema is a different question. + """ + pa = [p for p in canon_owner(a).split(".") if p] + pb = [p for p in canon_owner(b).split(".") if p] + n = min(len(pa), len(pb)) + return n > 0 and pa[-n:] == pb[-n:] + + +@dataclass +class ObjectIndex: + objects: dict = field(default_factory=dict) # object name (as written) -> testbed json + tables: dict = field(default_factory=dict) # canon owner (qualified) -> {col upper: colfacts} + _bare: dict = field(default_factory=dict) # canon bare table -> [canon owner, ...] + _by_kind: dict = field(default_factory=dict) # kind -> [constraint] + _by_cell: dict = field(default_factory=dict) # (canon table, col upper) -> [constraint] + skipped: list = field(default_factory=list) # testbed JSONs dropped as unreadable/corrupt + ambiguities: list = field(default_factory=list) # bare table names owned by >1 qualified object + unattributed: list = field(default_factory=list) # qualified rows naming a table no artifact covers + + def _resolve(self, table: str | None) -> list[str]: + # Exact qualified match first, then the bare alias -- so a 3-part name still finds a 2-part + # artifact, and a bare list-unsolved row still finds its uniquely-named table. + owner = canon_owner(table) + if owner in self.tables: + return [owner] + return list(self._bare.get(canon_table(table), [])) + + def table_owners(self, table: str | None) -> list[str]: + return self._resolve(table) + + def ambiguous_table(self, table: str | None) -> bool: + return len(self._resolve(table)) > 1 + + def has_column(self, table: str | None, col: str | None) -> bool: + return self.col_facts(table, col) is not None + + def col_facts(self, table: str | None, col: str | None) -> dict | None: + owners = self._resolve(table) + if len(owners) != 1: # unknown table, or a bare name several schemas answer to + return None + return self.tables[owners[0]].get(str(col).upper()) + + def is_date(self, table: str | None, col: str | None) -> bool: + c = self.col_facts(table, col) + return bool(c and c.get("is_date")) + + def is_declared_enum(self, table: str | None, col: str | None) -> bool: + c = self.col_facts(table, col) + return bool(c and c.get("enum_domain") and c.get("semantic_class") == "enum") + + def declared_enum_domain(self, table: str | None, col: str | None) -> list[object]: + # list[object], not list[str]: the domain is copied out of the artifact verbatim, so a + # numeric or boolean member arrives as it was mined. Every reader goes through normalize. + return list((self.col_facts(table, col) or {}).get("enum_domain") or []) + + def literal_in_declared_domain(self, table: str | None, col: str | None, + literal: object) -> bool: + # The backbone's declared-enum fast path and the gate's CitationType.DECLARED_ENUM arm + # adjudicate the same fact from opposite directions; sharing one predicate is what keeps them + # from disagreeing, which would let the gate DROP a literal the backbone hard-REJECTed. + return normalize(literal) in {normalize(x) for x in self.declared_enum_domain(table, col)} + + def declared_fk(self, table: str | None, col: str | None) -> tuple | None: + c = self.col_facts(table, col) + if c and c.get("foreign_key") and c.get("foreign_key_referenced_table"): + return (c["foreign_key_referenced_table"], c.get("foreign_key_referenced_column")) + return None + + def is_pk_or_notnull(self, table: str | None, col: str | None) -> bool: + c = self.col_facts(table, col) + return bool(c and (c.get("primary_key") or c.get("nullable") is False)) + + def _cell_keys(self, table: str | None, col: str | None, *, permissive: bool) -> list[tuple]: + # Two buckets are always in scope: the row's resolved owner (when the name resolves to exactly + # one table AND that table does not contradict the schema the name spells), and the name exactly + # as written -- which is where load_index parks a row it could not attribute, so a lookup naming + # the same table still reaches its own evidence. + col_u = str(col).upper() + owners = self._resolve(table) + keys: list = [] + if len(owners) == 1 and names_compatible(table, owners[0]): + keys.append((owners[0], col_u)) + if canon_owner(table) and (canon_owner(table), col_u) not in keys: + keys.append((canon_owner(table), col_u)) + if not permissive: + return keys + # The permissive tail: the bare-name bucket (rows whose owner resolved to zero or several + # tables), plus every candidate owner of an ambiguous bare name. Reserved for the GROUNDING + # corpus, where a missing row makes a golden literal false-REJECT -- a guaranteed-true hard + # reject asserted on evidence the reader simply could not see. The structural path deliberately + # does NOT take this tail: there the asymmetry inverts (see cell_constraints). + for candidate in [(canon_table(table), col_u)] + [(o, col_u) for o in owners]: + if candidate not in keys: + keys.append(candidate) + return keys + + def _cell_rows(self, table: str | None, col: str | None, *, permissive: bool) -> list[dict]: + rows: list = [] + for key in self._cell_keys(table, col, permissive=permissive): + rows.extend(self._by_cell.get(key, [])) + return rows + + def cell_corpus(self, table: str | None, col: str | None) -> list[str]: + corpus: list = [] + for c in self._cell_rows(table, col, permissive=True): + if c.get("detail"): + corpus.append(c["detail"]) + # deref target = THIS row's own object json (the TABLE for fk/enum/check, the proc for edges/predicates) + owner = self.objects.get(c.get("object")) + corpus.extend(deref_pointer(c.get("source_evidence"), owner or {})) + return corpus + + def literal_grounded_in_cell(self, table: str | None, col: str | None, literal: object) -> bool: + return literal_grounded(literal, self.cell_corpus(table, col)) + + def cell_constraints(self, table: str | None, col: str | None) -> list[dict]: + # Strict, unlike cell_corpus, because the failure asymmetry runs the other way on this path. + # These rows answer "does a mined constraint back this structural claim", and an EXTRA row from + # a table the entry never named manufactures a hard reject: critic_gate's fk_parent_mismatch arm + # REVISEs (bounded, re-promptable) when `mined` is empty but hard-REJECTs when it is non-empty + # and the proposed parent is absent from it -- so borrowing another schema's edge converts a + # bounded revise into a guaranteed-true-shaped REJECT the entry cannot answer. A MISSING row + # only routes the entry to the residue, which is the recoverable direction. (Nothing is lost by + # dropping the bare bucket here: a name that resolves to zero or several tables is stopped + # upstream by the backbone's own has_column / _ambiguity_check guards before it reaches a kind + # test, so the bare bucket has no reachable reader on this path.) + return self._cell_rows(table, col, permissive=False) + + def has_kind_for_cell(self, table: str | None, col: str | None, kinds: tuple) -> bool: + return any(c.get("kind") in kinds for c in self.cell_constraints(table, col)) + + +def _iter_testbed_jsons(artifacts_path: str, skipped: list[str]): + root = Path(artifacts_path) + if not root.is_dir(): + return + for p in sorted(root.rglob("*.testbed.json")): + try: + doc = json.loads(p.read_text(encoding="utf-8")) + # read_text raises UnicodeDecodeError on a file that died mid-codepoint, which is a ValueError + # and so escapes OSError/JSONDecodeError entirely -- one truncated artifact would abort the + # whole index build. The explicit utf-8 is what makes that guarantee hold: on the locale + # default a non-UTF-8 host (cp1252 leaves only 5 byte values undefined) decodes an accented + # artifact to mojibake without raising at all, so nothing lands in `skipped` and a genuine + # golden literal instead false-REJECTs as absent from every source_evidence span. + except (OSError, ValueError) as exc: + # A corrupt/unreadable artifact silently vanishing from the index makes the backbone + # hard-REJECT that table's columns as "does not exist"; record it so the drop is visible. + skipped.append(f"{p}: {exc}") + continue + if isinstance(doc, dict): + yield p, doc + + +def load_index(artifacts_path: str | Path, unsolved_view: dict | None) -> ObjectIndex: + idx = ObjectIndex() + for path, doc in _iter_testbed_jsons(str(artifacts_path), idx.skipped): + # Mirrors TestbedArtifact.ResolvedName (object ?? table): procedural artifacts name themselves in + # `object`, table artifacts in `table`. Reading only `object` collapsed every artifact of the + # latter shape onto one nameless key, and the last one read silently won. + name = doc.get("object") or doc.get("table") + if name: + idx.objects[name] = doc + if doc.get("object_type") == "TABLE": + owner = canon_owner(name) + if not owner: + skipped_reason = "TABLE artifact names itself in neither 'object' nor 'table'" + idx.skipped.append(f"{path}: {skipped_reason}") + continue + cols = {str(c.get("name")).upper(): c for c in doc.get("columns", []) if isinstance(c, dict)} + if owner in idx.tables and idx.tables[owner] != cols: + # Two artifacts claiming one owner: the second overwrites the first, so the critic would + # validate against half a table with nothing in the report to explain it. + idx.ambiguities.append(f"table {owner} is described by more than one artifact with " + f"differing columns; the last one read wins") + idx.tables[owner] = cols + aliases = idx._bare.setdefault(canon_table(name), []) + if owner not in aliases: # a re-read of the same owner is not a collision + aliases.append(owner) + for bare, owners in idx._bare.items(): + if len(owners) > 1: + idx.ambiguities.append(f"bare table name {bare} is owned by {', '.join(sorted(owners))}") + by_kind = (unsolved_view or {}).get("by_kind", {}) + for kind, block in by_kind.items(): + for c in block.get("constraints", []): + if not isinstance(c, dict): + continue + idx._by_kind.setdefault(kind, []).append(c) + owner = c.get("table") or c.get("object") + # Key on the resolved owner, not the bare name: bare keying put SALES.ORDERS.STATUS and + # STAGING.ORDERS.STATUS in one bucket, so one schema's grounding corpus contained the + # other's evidence and the backbone PASSed literals invented for the table it was checking. + # `_resolve` is fully populated by now (the artifact loop and the collision scan above have + # both run). + # + # Resolution alone is not enough, because `_resolve` falls back to the bare alias: with only + # SALES.ORDERS captured, a row that explicitly says STAGING.ORDERS resolves to + # ['SALES.ORDERS'] -- length 1, so the "cannot be attributed" escape below never fires -- + # and STAGING's evidence lands in SALES' bucket, re-creating the exact leak this keying + # exists to close (one schema grounding the other's literals, and one schema's mined fk + # backing the other's fk_chain as a hard reject). `names_compatible` is the missing guard: it + # accepts a row that under- or over-qualifies the same table and rejects one that names a + # contradicting schema. A name that resolves to zero or several tables genuinely cannot be + # attributed, so it stays on the bare key and `_cell_keys`' permissive tail merges it into + # every candidate owner for grounding. + owners = idx._resolve(owner) + if len(owners) == 1 and names_compatible(owner, owners[0]): + key_owner = owners[0] + elif owners and "." in str(owner or ""): + # Qualified, and the bare alias points at a table whose schema this name contradicts -- + # the mis-attribution above. Keep the row on its own name so it is reachable from a + # lookup naming the same table, and from nowhere else. Narrow on `owners` deliberately: + # when NOTHING is captured under that leaf name there is no other schema to absorb the + # row, so it keeps the bare key and a bare lookup still reaches it. + key_owner = canon_owner(owner) + # Silence here is what made this invisible: `ambiguities` stays empty (a qualified name + # is not ambiguous) and `ambiguous_table` is False, so nothing in the report explained + # why the row's evidence stopped counting anywhere. + note = (f"constraint rows name {key_owner}, which has no TABLE artifact; the only " + f"captured table sharing that name is {', '.join(sorted(owners))}, so those " + f"rows are attributed to neither") + if note not in idx.unattributed: + idx.unattributed.append(note) + else: + key_owner = canon_table(owner) + cell = (key_owner, str(c.get("column")).upper()) + idx._by_cell.setdefault(cell, []).append(c) + return idx diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_normalize.py b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_normalize.py new file mode 100644 index 0000000..76fed62 --- /dev/null +++ b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/critic_normalize.py @@ -0,0 +1,119 @@ +"""Verbatim-groundedness primitives for the enrichment critics. + +The backbone asks "does this literal appear in the source?" against a corpus built from +constraint `detail` strings and deref'd `source_evidence` spans. Normalization must be +lenient enough that a correctly-cased golden literal never false-REJECTs, strict enough +that an invented literal ('Z' in a STATUS enum) is not spuriously matched. +""" +from __future__ import annotations + +import re + +_PTR = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*)\[(\d+)\]$") + +# Where one literal ends and the next begins, applied as negative lookarounds rather than \b: \b is +# defined against the *pattern's* own edge characters, so a literal like '-1' would lose its left +# boundary, while a lookaround on an explicit class is keyed on the corpus text and holds for any +# literal shape. +# +# Deliberately `\w` and NOT critic_gate's _IDENT class ([A-Za-z_][A-Za-z0-9_$]*), on two counts: +# * No `$`. _IDENT's `$` is a SQL-identifier *continuation* character, correct where that class +# tokenizes a mined FK edge into identifiers. Here the corpus is prose-and-SQL constraint detail, +# where `$` is overwhelmingly a currency *prefix*: with `$` as a boundary char the correctly-mined +# 'min balance $100' stops grounding the literal '100' -- a false-REJECT of a golden literal, the +# one failure this module's docstring forbids outright, and worse than a stray false positive +# because a backbone FAIL is a contractually-guaranteed-true hard reject. The trade is one observed +# positive against a hypothetical `$`-in-identifier false positive: no `$`-containing identifier +# appears anywhere in the corpora this actually sees (constraint `detail` strings plus deref'd +# `source_evidence` string leaves, across the critics benchmark set and every mine-phase fixture). +# * Unicode, not an ASCII class. Snowflake permits non-ASCII quoted identifiers, so an ASCII-only +# boundary leaves 'GOLD' grounding in "GOLDN-with-enye" -- the exact aliasing this check exists to +# stop, just spelled outside [A-Za-z]. `\w` on a str pattern is Unicode-aware by default. +_BOUND = r"\w" + +# Numeric literals ground on VALUE, not on spelling -- see literal_grounded_normalized. Corpus numbers +# carry the same `\w` boundary as strings, so on word adjacency the value path is no more permissive than +# the string path and the widening adds only scale-equivalence: '50' grounds in neither 'ITEM_50' nor +# 'ITEM50'. The '-' and '.' in the lookbehind are a SIGN/SCALE guard on the corpus side, not an adjacency +# guard: they stop a digit run being read as a whole number when it is really the magnitude of a negative +# ('-5' yields no number, so the literal '5.0' does not ground in 'min temp -5') or the tail of a longer +# dotted run ('1.2.3' yields nothing rather than the fragment '2.3'). They do NOT withhold a hyphen +# adjacency from the overall answer, because '-' is not in \w and the STRING path above already grounds +# there first -- measured, literal_grounded('1', ['ABC-1']) is True, under the same boundary rule that +# lets the whole SKU 'ABC-1' ground. +_CORPUS_NUM = re.compile(r"(? str: + t = "" if s is None else str(s) + t = t.strip().casefold() + if len(t) >= 2 and t[0] == t[-1] and t[0] in "'\"": + t = t[1:-1].strip() + return " ".join(t.split()) + + +def string_leaves(node: object) -> list[str]: + out: list[str] = [] + if isinstance(node, str): + out.append(node) + elif isinstance(node, dict): + for v in node.values(): + out.extend(string_leaves(v)) + elif isinstance(node, list): + for v in node: + out.extend(string_leaves(v)) + return out + + +def deref_pointer(pointer: object, obj: object) -> list[str]: + if not isinstance(pointer, str) or not isinstance(obj, dict): + return [] + m = _PTR.match(pointer.strip()) + if not m: + return [] + array, idx = m.group(1), int(m.group(2)) + seq = obj.get(array) + if not isinstance(seq, list) or idx >= len(seq): + return [] + return string_leaves(seq[idx]) + + +def literal_grounded(literal: object, corpus: list) -> bool: + return literal_grounded_normalized(literal, [normalize(c) for c in corpus]) + + +def _numeric_value(t: str) -> float | None: + # None for anything not a plain number, so a non-numeric literal never enters the value comparison. + return float(t) if _NUMERIC_LITERAL.match(t) else None + + +def literal_grounded_normalized(literal: object, normalized_corpus: list[str]) -> bool: + # Takes an already-normalized corpus so a caller checking M literals against one corpus + # normalizes it once instead of M times. + n = normalize(literal) + if not n: + return True # empty literal: nothing to ground, must not false-REJECT + # Boundary match, not substring. A bare `n in c` grounds any literal that merely happens to sit + # inside a longer word, and the corpus always contains the column's own name and type: 'A' would + # be "grounded" by "STATUS IN ('I','P')" via STATUS, and the docstring's own counter-example 'Z' + # by BRONZE. Single-char CHAR(1) enum codes -- the canonical inferred_enum case, whose accept + # example is ["G","S","B"] -- are exactly the values that alias most, so the backbone would PASS + # an invented literal to the residue carrying a grounded_in anchor asserting evidence that was + # never matched. Boundaries are _BOUND only, so a literal stays matchable when it is delimited by + # quotes, commas, pipes, operators or a currency '$', and a hyphenated SKU like 'ABC-1' still + # matches whole (tokenizing the corpus into SQL literals instead would split it at the hyphen). + pat = re.compile(f"(? str: return datetime.now(timezone.utc).isoformat() +def classify_rejections(cells: Iterable[tuple], + previously_rejected: Callable[[str | None, str | None, object], bool], + ) -> tuple[bool, list[tuple]]: + """The single home of the "a re-proposed rejected value is a cycle" rule. + + `cells` is (table, column, literal) triples read out of whatever shape the caller holds -- the + backbone's EntryCheck.identity or the gate's adjudicated REJECT detail. Returns + (is_cycle, recorded): whether any cited literal was already rejected in an earlier round, and the + triples the caller must persist with record_rejected_value. + + Two things live here rather than at each call site, which is why they cannot drift apart: + + * A triple whose literal is None cites no VALUE (a structural or out-of-range FAIL), so it + neither votes on the cycle nor enters the memo. Recording one would key the memo on None and + make every later non-literal FAIL on that cell read as a repeat; letting one vote would + escalate a first-ever structural reject to a human. Its crash-loop bound is the per-type + reject budget instead. + * The cycle question is answered against the memo as it stands BEFORE anything here is + persisted, so a round can never trip on its own recording. The (is_cycle, recorded) return + shape is what holds that ordering: a caller asks once and persists afterwards, because + `recorded` is handed back rather than written here. + + Stateless on purpose: the gate holds no ledger, so the memo predicate arrives as a callable and + persisting `recorded` stays the caller's step. + """ + recorded = [(t, c, lit) for t, c, lit in cells if lit is not None] + return any(previously_rejected(t, c, lit) for t, c, lit in recorded), recorded + + @dataclass class Manifest: schema_version: int = SCHEMA_VERSION @@ -50,7 +83,7 @@ def load(cls, project_dir: str) -> "Manifest": # (OSError), so guard the whole triple — the same set run_pipeline's predicate # readers guard. try: - raw = json.loads(p.read_text()) + raw = json.loads(p.read_text(encoding="utf-8")) except (json.JSONDecodeError, UnicodeDecodeError, OSError): return cls() if not isinstance(raw, dict): @@ -110,13 +143,62 @@ def iteration_budget_exhausted(self, cap: int) -> bool: def reset_enrich_budget(self) -> None: # Explicit fresh-human-run override only; never implicit, or a crash-loop would reset and bound nothing. + # Clearing rejected_values_by_cell is the point, not a leak: a value_cycle stop would otherwise be + # unrecoverable, since the human's corrected re-proposal keeps tripping the same memo. self.enrichment = {"fragments": 0, "rejections": 0, "retries": 0, "iterations": 0, "ready": False, "rejections_by_type": {}} + _CELL_SEP = "\x1f" + + @classmethod + def _cell_key(cls, table: str | None, column: str | None) -> str: + # A qualified name keys on its own owner. Keying every name on the bare table put SALES.ORDERS.TIER + # and STAGING.ORDERS.TIER in one bucket, so a first-ever literal on one read as a re-proposal of the + # one rejected on the other, and escalated a healthy run to a human. + name = str(table or "") + owner = canon_owner(name) if "." in name else canon_table(name) + return f"{owner}{cls._CELL_SEP}{str(column).upper()}" + + def _buckets_for(self, table: str | None, column: str | None): + # Lookup is deliberately more permissive than the key: a bare name could denote any owner, so it + # also matches what an earlier round recorded qualified. A qualified name matches only itself. + memo = self.enrichment.get("rejected_values_by_cell", {}) + exact = self._cell_key(table, column) + if exact in memo: + yield memo[exact] + if "." in str(table or ""): + return + want = f"{canon_table(table)}{self._CELL_SEP}{str(column).upper()}" + for key, values in memo.items(): + owner, _, col = key.partition(self._CELL_SEP) + if key != exact and f"{canon_table(owner)}{self._CELL_SEP}{col}" == want: + yield values + + def record_rejected_value(self, table: str | None, column: str | None, value: object) -> None: + memo = self.enrichment.setdefault("rejected_values_by_cell", {}) + bucket = memo.setdefault(self._cell_key(table, column), []) + # normalize() folds case and strips matching quotes, matching how the gate grounds literals, + # so a re-proposed 'GOLD'/gold/GOLD is recognized as the same already-rejected value. + s = normalize(value) + if s not in bucket: + bucket.append(s) + + def value_previously_rejected(self, table: str | None, column: str | None, + value: object) -> bool: + s = normalize(value) + return any(s in bucket for bucket in self._buckets_for(table, column)) + def save(self, project_dir: str) -> None: self.updated_at = _now() p = self.path(project_dir) p.parent.mkdir(parents=True, exist_ok=True) tmp = p.with_suffix(".json.tmp") - tmp.write_text(json.dumps(asdict(self), indent=2)) + # Explicit utf-8 both ways (load reads it back the same). This write cannot corrupt today + # and the pin is not what stops it: json.dumps escapes non-ASCII (ensure_ascii defaults to + # True), so these bytes are pure ASCII and identical under every ASCII-superset locale + # codec. What the pin buys is that the ledger's encoding is declared here rather than + # inherited, so the round trip stays well-defined the day a writer emits the artifacts' + # object/column names raw -- a mis-decoded rejected-value memo silently stops recognizing a + # re-proposed literal. + tmp.write_text(json.dumps(asdict(self), indent=2), encoding="utf-8") os.replace(tmp, p) # atomic on POSIX diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/run_pipeline.py b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/run_pipeline.py index cd8be25..7566b20 100644 --- a/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/run_pipeline.py +++ b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/run_pipeline.py @@ -11,15 +11,19 @@ import json import os import re +import hashlib import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from subcommands import TestbedCli, default_runner, enumerate_artifacts # noqa: E402 from recovery import RecoveryClass, classify # noqa: E402 -from manifest import Manifest # noqa: E402 +from manifest import Manifest, classify_rejections # noqa: E402 from assembler import ( # noqa: E402 load_fragments, assemble, structural_coverage_warnings, value_conflict_warnings, AssemblyError) +from critic_index import load_index # noqa: E402 +from critic_backbone import run_backbone, build_critique_request, reads_column_facts, CheckStatus # noqa: E402 +from critic_gate import parse_verdict, apply_gate, GateAction # noqa: E402 STATE_REL = ".scai/testbed/state.bin" # written by init (opaque; driver never touches) VIEW_REL = "testbed/mine/unsolved-view.json" # non-hidden: it's the state-machine completion predicate @@ -29,6 +33,9 @@ ENRICH_VIEW_REL = "testbed/enrich/enrichment-view.json" # completion predicate: present <=> ready ENRICH_REPORT_REL = "testbed/enrich/enrichment-report.json" # not-ready / reject diagnostics FRAGMENTS_DIR_REL = "testbed/enrich/fragments" +CRITIQUE_REQUEST_REL = "testbed/enrich/critic/critique-request.json" +CRITIC_VERDICT_REL = "testbed/enrich/critic/verdict.json" +CRITIC_PROMPT_TYPE = "critic" # the max_rejections_per_type bucket critic re-author rounds draw down QUARANTINE_REL = ".scai/testbed/quarantine" MAX_QUARANTINE_ATTEMPTS = 20 # PENDING key for a malformed whole-workload entry that carries no identity to attribute it to. @@ -53,7 +60,11 @@ def _write_json(path: Path, obj: dict) -> None: # then os.replace (atomic on POSIX). path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(path.name + ".tmp") - tmp.write_text(json.dumps(obj, indent=2)) + # Explicit utf-8, matching every reader of these views. Not a live crash fix: json.dumps + # escapes non-ASCII, so this payload is pure ASCII and writes identically under any + # ASCII-superset locale codec. The pin declares the views' encoding at the call site instead of + # inheriting the host's, which is what the readers below (and the guard test) rely on. + tmp.write_text(json.dumps(obj, indent=2), encoding="utf-8") os.replace(tmp, path) @@ -374,7 +385,7 @@ def _readiness_gate(project_dir: str, ignore_readiness: bool) -> tuple[bool, str if not readiness_path.exists(): return False, "generate blocked: run validate first", False try: - readiness = json.loads(readiness_path.read_text()) + readiness = json.loads(readiness_path.read_text(encoding="utf-8")) except (json.JSONDecodeError, UnicodeDecodeError, OSError): # Predicate files are atomic-written, but a crash mid-write (or a hand-edit) can still # truncate one — mid-codepoint (UnicodeDecodeError), malformed (JSONDecodeError), or racing @@ -424,10 +435,14 @@ def _read_json(project_dir, rel): if not p.exists(): return None try: - return json.loads(p.read_text()) - except (json.JSONDecodeError, OSError): + return json.loads(p.read_text(encoding="utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError, OSError): # Mirror _readiness_gate: a truncated/unreadable predicate (crash mid-write, permissions, - # partial write) degrades to None rather than raising straight out of the driver. + # partial write) degrades to None rather than raising straight out of the driver -- + # including the mid-codepoint truncation, which raises UnicodeDecodeError (a ValueError, + # so it escapes OSError). The explicit utf-8 is what makes the degrade reachable at all: + # on the locale default a non-UTF-8 host decodes an accented view to mojibake without + # raising, so nothing degrades and the driver reads names that were never written. return None @@ -462,6 +477,7 @@ def _commit_enrich_ready(project_dir, manifest, report, applied, warnings, note) {"schema_version": 1, "ready": True, "counts": report.get("counts", {}), "applied": applied, "warnings": warnings, "enrichment": manifest.enrichment}) _clear_stale_report(project_dir) + _clear_critic_state(project_dir) manifest.save(project_dir) return 0, note @@ -550,12 +566,176 @@ def _enrich_handle_reject(project_dir, manifest, env, warnings, *, cap, cli, env warnings, f"enrich stopped [{env.error.code}]: {env.error.message}") -def run_enrich(project_dir, cli, *, max_rejections_per_type=3, max_iterations=3, +def _envelope_sig(envelope) -> str: + # Ties a verdict fragment to the exact envelope it critiqued: if the agent re-authors any fragment, + # the sig changes and the stale verdict is ignored (Invocation A re-runs on the new envelope). + return hashlib.sha256(json.dumps(envelope, sort_keys=True).encode()).hexdigest()[:16] + + +def _clear_critic_state(project_dir) -> None: + for rel in (CRITIC_VERDICT_REL, CRITIQUE_REQUEST_REL): + p = Path(project_dir) / rel + if p.exists(): + p.unlink() + + +def _critic_backbone_stop(project_dir, manifest, envelope, index, warnings, sig, *, cap): + # Backbone policy, for an envelope with no fresh verdict. A FAIL never reaches the critic: a + # re-proposed rejected value escalates, otherwise it spends the reject budget. An all-clear writes + # the critique request and stops for the critic's second invocation. + checks = run_backbone(envelope, index) + fails = [c for c in checks if c.status == CheckStatus.FAIL] + if fails: + # Same helper the gate's REJECT arm consumes, so "repeat rejection -> escalate as a cycle, + # else record it" is decided in one place for both shapes of rejection. + cycle, recorded = classify_rejections( + ((f.identity.get("table"), f.identity.get("column"), f.identity.get("literal")) + for f in fails), + manifest.value_previously_rejected) + for table, column, value in recorded: + manifest.record_rejected_value(table, column, value) + citations = [{"identity": f.identity, "citation": f.citation} for f in fails] + if cycle: + return _enrich_stop(project_dir, manifest, "documented-stop", + {"reason": "value_cycle", "citations": citations}, warnings, + "critic backbone: a previously-rejected value was re-proposed; escalating for human review") + manifest.record_rejection(CRITIC_PROMPT_TYPE) + if manifest.rejection_budget_exhausted(CRITIC_PROMPT_TYPE, cap): + return _enrich_stop(project_dir, manifest, "budget-exhausted", + {"reason": "critic_budget", "prompt_type": CRITIC_PROMPT_TYPE, + "citations": citations}, warnings, + "critic reject budget exhausted; fix fragments and re-run with --reset-budget") + return _enrich_stop(project_dir, manifest, "reject", + {"reason": "critic_backbone", "prompt_type": CRITIC_PROMPT_TYPE, + "citations": citations}, warnings, + "critic backbone rejected an entry; re-prompt the fragment; see enrichment-report.json") + request = build_critique_request(checks) + request["envelope_sig"] = sig + _write_json(Path(project_dir) / CRITIQUE_REQUEST_REL, request) + return _enrich_stop(project_dir, manifest, "critique", + {"reason": "await_critic", "critique_request": CRITIQUE_REQUEST_REL, + "entries": len(request["entries"])}, warnings, + "critic backbone passed; run the critic prompt, write verdict.json, then re-run enrich") + + +def _critic_gate_stop(project_dir, manifest, verdict, index, warnings, *, cap): + # Gate policy, for a verdict matching the current envelope. Returns None only on accept_all; + # reject/revise spend the reject budget, and every other action is a documented stop. + decision = apply_gate(verdict, index, manifest.value_previously_rejected) + for table, column, value in decision.record_values: + manifest.record_rejected_value(table, column, value) + if decision.action == GateAction.ACCEPT_ALL: + return None + if decision.action in (GateAction.REJECT, GateAction.REVISE): + manifest.record_rejection(CRITIC_PROMPT_TYPE) + if manifest.rejection_budget_exhausted(CRITIC_PROMPT_TYPE, cap): + # decision.report carries "reason": f"critic_{action}", so override it AFTER the spread: + # a budget-exhausted stop must report critic_budget (as the backbone-reject arm does), + # not the underlying critic_revise/critic_reject action. + return _enrich_stop(project_dir, manifest, "budget-exhausted", + {**decision.report, "prompt_type": CRITIC_PROMPT_TYPE, "reason": "critic_budget"}, + warnings, "critic revise/reject budget exhausted; fix fragments and re-run with --reset-budget") + stop_kind = "reject" if decision.action == GateAction.REJECT else "iterate" + return _enrich_stop(project_dir, manifest, stop_kind, + {"prompt_type": CRITIC_PROMPT_TYPE, **decision.report}, warnings, decision.message) + return _enrich_stop(project_dir, manifest, "documented-stop", + {"prompt_type": CRITIC_PROMPT_TYPE, **decision.report}, warnings, decision.message) + + +def _run_critic_station(project_dir, artifacts_path, manifest, envelope, unsolved, warnings, *, cap): + # Two-invocation bridge. Returns None to proceed to propose, else a terminal (rc, msg) stop. + sig = _envelope_sig(envelope) + verdict_path = Path(project_dir) / CRITIC_VERDICT_REL + verdict = None + if verdict_path.exists(): + # A present verdict.json that can't be read or parsed is malformed, not "stale/absent": + # surface it rather than silently re-critiquing over a truncated/corrupt handoff (never + # auto-apply an un-critiqued envelope). An absent file leaves verdict None -> backbone below. + def _malformed(detail): + return _enrich_stop(project_dir, manifest, "documented-stop", + {"reason": "critic_verdict_malformed", "detail": detail}, warnings, + "critic verdict.json malformed; a human must review " + "(never auto-apply an un-critiqued envelope)") + try: + raw = verdict_path.read_text(encoding="utf-8") + # read_text raises UnicodeDecodeError on a handoff written mid-codepoint, which is a ValueError + # and so escapes OSError -- crashing the run instead of reaching the documented stop above. The + # explicit utf-8 is load-bearing for that: on the locale default a non-UTF-8 host mis-decodes + # the handoff to mojibake without raising, so a real critique parses as different text. + except (OSError, ValueError) as exc: + return _malformed(f"verdict.json unreadable: {exc}") + verdict = parse_verdict(raw) + if not verdict.ok: + return _malformed(verdict.error) + + # A re-authored fragment changes the sig, so a stale-but-valid verdict falls through to a fresh + # backbone run. Build the read-only facts index once here and share it across both branches -- before + # the accept-all fast path, or a run whose only diagnostics are index warnings reports none of them. + fresh = verdict is not None and verdict.envelope_sig == sig + index = load_index(artifacts_path, unsolved) + warnings.extend(f"critic index skipped unreadable artifact {s}" for s in index.skipped) + warnings.extend( + f"critic index: {a}; entries naming it bare cannot be resolved to one table and are routed " + "to the critic instead of checked deterministically" for a in index.ambiguities) + # A qualified row the index could not attribute to any captured table. Reported for the same reason + # `ambiguities` is: its evidence stops counting toward every other schema's cells, and without this + # line nothing explains why (a qualified name is not ambiguous, so `ambiguities` stays empty). + warnings.extend(f"critic index: {u}" for u in index.unattributed) + # Both index-health diagnostics are recorded before the fast path below, for the same reason the + # index is built there: a run whose only diagnostics are these would otherwise report none of them. + no_root = not Path(artifacts_path).is_dir() + if no_root: + warnings.append(f"critic artifacts root {artifacts_path} is not a directory; column-existence " + "checks have no facts to read") + elif not index.tables: + # Warning, not a stop: a purely procedural envelope over a procedural-only artifact root is + # legitimate, so hard-stopping here would be a false positive. It is still worth naming, since + # it is the reason every column entry rejects rather than any property of the entries. + warnings.append(f"critic index resolved zero TABLE artifacts under {artifacts_path}; " + "column-existence checks cannot pass and every column entry will be rejected") + if fresh and not verdict.verdicts: + return None + + # An absent root plus at least one entry that would be graded against the column facts. Both halves + # are load-bearing, and the finding is only the conjunction: the harm is a manufactured verdict, not a + # missing directory. With no facts every column reads as non-existent, so a fact-reading entry draws a + # guaranteed-true-shaped FAIL worded identically to a genuinely missing column, and the gate would + # adjudicate its citations against nothing. + # + # `reads_column_facts` is what keeps this from over-reaching. An envelope of only uncovered arrays + # reads no facts at all -- the backbone's `_uncovered` inspects the entry's own keys and FLAGs -- so it + # is graded identically with or without a root, and stopping it would contradict the reason the + # present-but-tableless root above is only a warning: a purely procedural envelope with nothing to look + # up is a legitimate steady state, and it stays one when the lookup target is absent rather than merely + # empty. Such an envelope keeps its critique handoff, and the warning recorded above is what tells the + # operator the root was missing. + # + # A documented-stop rather than a reject because a misconfigured root is not an entry defect: it spends + # no reject budget and writes no rejected-value memo, so fixing the flag is enough to re-run without + # --reset-budget. (Absent is a misconfiguration, never a steady state: the mine phase creates the + # default root, and --artifacts-path is per-phase, so `mine --artifacts-path /elsewhere` plus a plain + # `enrich` lands here.) It sits below the accept-all fast path deliberately -- an already-accepted + # verdict consumes no facts either, so stopping it there would be a false positive for the same reason. + if no_root and reads_column_facts(envelope): + return _enrich_stop(project_dir, manifest, "documented-stop", + {"reason": "critic_no_artifacts", "artifacts_path": str(artifacts_path)}, + warnings, + f"critic artifacts root {artifacts_path} is not a directory; pass " + "--artifacts-path (it is not inherited from the mine phase)") + + if not fresh: + return _critic_backbone_stop(project_dir, manifest, envelope, index, warnings, sig, cap=cap) + + return _critic_gate_stop(project_dir, manifest, verdict, index, warnings, cap=cap) + + +def run_enrich(project_dir, cli, *, artifacts_path=None, max_rejections_per_type=3, max_iterations=3, reset_budget=False) -> tuple[int, str]: # Enrich assembles the LLM prompt fragments into one envelope, proposes it to the CLI, then # validates readiness. Idempotent: the enrichment view is the completion predicate. state.bin # stays opaque — the driver reads only fragments + the machine-readable envelopes. project_dir = str(project_dir) + artifacts_path = artifacts_path or str(Path(project_dir) / "artifacts") if _present(project_dir, ENRICH_VIEW_REL): return 0, "enrich already complete (enrichment view present)" @@ -578,6 +758,11 @@ def run_enrich(project_dir, cli, *, max_rejections_per_type=3, max_iterations=3, report={"reason": "assembly_rejected", "prompt_type": e.prompt_type, "detail": e.message}, warnings=warnings, msg=f"enrich rejected: {e.message}") + gate_stop = _run_critic_station(project_dir, artifacts_path, manifest, envelope, unsolved, warnings, + cap=max_rejections_per_type) + if gate_stop is not None: + return gate_stop + env = cli.propose_enrichments(project_dir, json.dumps(envelope, sort_keys=True)) if not env.success: return _enrich_handle_reject(project_dir, manifest, env, warnings, @@ -666,6 +851,8 @@ def main(argv=None) -> int: enr = sub.add_parser("enrich", help="run the enrichment orchestration phase") enr.add_argument("--project-dir", required=True) enr.add_argument("--scai", default=None, help="override the scai binary") + enr.add_argument("--artifacts-path", default=None, + help="per-object testbed JSON root the critic reads (default: /artifacts)") enr.add_argument("--reset-budget", action="store_true", help="clear the retry/iterate budget (fresh human run)") enr.add_argument("--max-rejections-per-type", type=int, default=3, help="re-prompt reject cap before stopping. v1 buckets every malformed-field reject " @@ -683,7 +870,7 @@ def main(argv=None) -> int: rc, msg = run_generate(args.project_dir, cli, args.rows, args.seed, args.ignore_readiness) elif args.phase == "enrich": - rc, msg = run_enrich(args.project_dir, cli, + rc, msg = run_enrich(args.project_dir, cli, artifacts_path=args.artifacts_path, max_rejections_per_type=args.max_rejections_per_type, max_iterations=args.max_iterations, reset_budget=args.reset_budget) else: diff --git a/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/subcommands.py b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/subcommands.py index 5c96e90..ebd3283 100644 --- a/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/subcommands.py +++ b/plugin/skills/migration/migrate-objects/baseline-capture/testbed-generator/scripts/subcommands.py @@ -52,16 +52,34 @@ def default_runner(scai_bin: str | None = None, def _run(argv: list[str], cwd: str | None = None, stdin: str | None = None) -> subprocess.CompletedProcess: - # The two failures that strike before scai can emit an envelope — an - # unlaunchable binary and a hang past the bound — become a non-zero - # CompletedProcess so _envelope reports EXEC instead of raising or - # blocking the whole skill forever. + # The failures that strike before scai can emit a parseable envelope — an + # unlaunchable binary, a hang past the bound, and I/O that is not valid UTF-8 — + # become a non-zero CompletedProcess so _envelope reports EXEC instead of + # raising or blocking the whole skill forever. try: + # encoding= is not optional here. text=True alone decodes stdout/stderr with + # locale.getencoding(), and this call is the main ingress for mined identities: scai + # writes its --json envelope as UTF-8 (Program.cs pins Console.OutputEncoding) and that + # envelope (list-unsolved, propose-enrichments) carries object/column names verbatim. + # On a cp1252 host — only five undefined byte values — those bytes decode to mojibake + # without raising, so a golden literal false-REJECTs as absent from its own + # source_evidence span. Pin the codec instead of inheriting the shell's. The same + # keyword pins the stdin encode too, which is contract rather than repair: today's only + # caller hands propose-enrichments json.dumps output, already escaped to pure ASCII by + # ensure_ascii, so no locale codec can fail on it. return subprocess.run( - [program, *argv], input=stdin, capture_output=True, text=True, cwd=cwd, timeout=timeout) + [program, *argv], input=stdin, capture_output=True, text=True, + encoding="utf-8", cwd=cwd, timeout=timeout) except subprocess.TimeoutExpired: return subprocess.CompletedProcess( [program, *argv], 124, "", f"scai timed out after {timeout}s: {' '.join(argv)}") + except UnicodeError as exc: + # Pinning the codec converts a silent mis-decode into a raise; keep that raise inside + # the seam. UnicodeDecodeError/UnicodeEncodeError subclass ValueError, not OSError, so + # without this arm they escape as a traceback out of the driver. Loud EXEC beats both + # mojibake (indistinguishable from an object genuinely named that) and a crash. + return subprocess.CompletedProcess( + [program, *argv], 125, "", f"scai I/O was not valid UTF-8: {exc}") except OSError as exc: return subprocess.CompletedProcess( [program, *argv], 127, "", f"could not execute scai ({program}): {exc}") @@ -177,7 +195,12 @@ def enumerate_artifacts(artifacts_path: str) -> list[ArtifactRef]: def _artifact_ref(path: Path, stem: str) -> ArtifactRef: try: - doc = json.loads(path.read_text()) + # The engine writes these artifacts as UTF-8. Decoding with the platform locale instead + # (cp1252 on a Windows runner) raises UnicodeDecodeError, which is a ValueError and so is + # caught below as if the file were unreadable — silently collapsing a non-ASCII identity + # to the bare stem. The stem then never matches an `object` the CLI echoes, so a real + # procedure becomes an unresolvable PENDING. Pin the encoding to the one it was written in. + doc = json.loads(path.read_text(encoding="utf-8")) except (OSError, ValueError): return ArtifactRef(stem, stem, "") if not isinstance(doc, dict): diff --git a/plugin/skills/migration/migrate-objects/migrate-etl/etl-validate/SKILL.md b/plugin/skills/migration/migrate-objects/migrate-etl/etl-validate/SKILL.md index a66d48f..eec6840 100644 --- a/plugin/skills/migration/migrate-objects/migrate-etl/etl-validate/SKILL.md +++ b/plugin/skills/migration/migrate-objects/migrate-etl/etl-validate/SKILL.md @@ -1,6 +1,6 @@ --- name: etl-validate -description: Run scai test etl-validate to compare live SSIS/Informatica package output against the converted Snowflake output, and record the result on the registry entry. +description: Run scai test etl-validate to compare live SSIS/Informatica package output against the converted Snowflake output. The CLI stamps codeStatus.etlValidate — do not invent advance outcomes. parent_skill: migrate-etl license: Proprietary. See License-Skills for complete terms --- @@ -51,21 +51,22 @@ If the session uses named-connection overrides, append: The command streams per-package results. Watch for the summary line reporting the failed-unit count. -## Step 3: Record Result +**The CLI owns the registry stamp.** On each unit it actually ran, `scai test etl-validate` writes `codeStatus.etlValidate`: -**All packages passed (failed units = 0):** +| CLI outcome | Registry stamp | +|---|---| +| Unit passed | `{ status: "completed", updatedAt }` | +| Unit failed | `{ status: "failed", updatedAt, error: "comparison" }` | +| Skipped (no ETL test YAML) | *no stamp* — field stays pending | -``` -transition_status(status="advance", task="etlValidate", outcome="completed", where="id = '{ETL_ID}'") -``` +## Step 3: Confirm stamp — do not invent advance -**One or more packages failed (failed units > 0):** +**Do not** call `transition_status(status="advance", …)` to invent a green or red outcome from narration. Re-read the unit via `query_registry` / `migration_status` and confirm `codeStatus.etlValidate` matches the CLI summary. -``` -transition_status(status="advance", task="etlValidate", outcome="failed", error="comparison", where="id = '{ETL_ID}'") -``` +- All packages passed → registry should already read `completed`; the ETL flow is terminal. +- One or more packages failed → registry should read `failed` with `error: "comparison"`. Surface the failed package names and row-level differences from the CLI output so the user can investigate the conversion gap. -Then surface the failed package names and the row-level differences to the user so they can investigate the conversion gap. +If the CLI exited non-zero but the stamp is missing (registry write failed), say so explicitly and do **not** stamp green yourself — ask the user to re-run or investigate the registry write error. ## Step 4: Exclusion diff --git a/plugin/skills/migration/migrate-objects/migrate-object/DEPLOY.md b/plugin/skills/migration/migrate-objects/migrate-object/DEPLOY.md index d1d2e1a..7f7e37e 100644 --- a/plugin/skills/migration/migrate-objects/migrate-object/DEPLOY.md +++ b/plugin/skills/migration/migrate-objects/migrate-object/DEPLOY.md @@ -15,7 +15,7 @@ The tool uses the connection and database from `configure` and deploys via `scai Converted files may contain issues that need manual fixes before deployment: 1. `USE DATABASE ` at the top referencing the **source** database — remove it. -2. Fully qualify the object name with the **target** database: `..`. +2. Keep the converted qualifier (`TaskTracker.dbo.X` or `schema.X`). `deploy` already passes `-d` and rewrites it onto the configured database. Do **not** write that database name into the file — the next run's `-d` will differ, and disk will disagree with Snowflake. 3. **Variable binding in LANGUAGE SQL procedures:** Parameters and variables inside SQL statements (SELECT, INSERT, WHERE, etc.) must use `:param_name` syntax. SnowConvert often omits the colon prefix — always verify. 4. **Preserve original comments:** When editing or rewriting converted SQL, always retain the original source code comments (synopsis, metadata, author, archive, change log, etc.). These comments document provenance and authorship — do not strip them during conversion or fixes. @@ -23,16 +23,16 @@ Converted files may contain issues that need manual fixes before deployment: | Error | Cause | Fix | |-------|-------|-----| -| Syntax error | Invalid SQL | Fix the code, redeploy | +| Syntax error | Invalid SQL | A pre-deploy check above, if one matches — otherwise `transition_status(outcome='failed', error='sql')` and let the fix loop take it | | Unknown function `` | Missing dependency | Deploy that function first | | Object does not exist | Missing table/view | Deploy or check schema | | Schema does not exist | Missing schema | `CREATE SCHEMA IF NOT EXISTS ` | -| `Error [XXXXXXX]: ...` | Planner error | Try to fix the SQL and redeploy; if the error persists, do **not** use `sql_execute` — call `transition_status(outcome='failed', error='dependency')` and surface the error to the user | +| `Error [XXXXXXX]: ...` | Planner error | Try to fix the SQL and redeploy; if the error persists, do **not** use `sql_execute` — leave the task unstamped if a known dependency is not ready (the walk will show `blockedOn`), or `error='sql'` / escalate if a person must register or stub a missing object | 1. **Read the error message** — identify the line/issue. 2. **Look for EWI comments** — SnowConvert comments (`--** SSC-`) near the error indicate unconverted constructs. -3. **Fix the code** — make the minimal change to resolve the error. -4. **Redeploy** — repeat until deployment succeeds. +3. **Fix it only when it is one of the pre-deploy checks above** — those are known, bounded edits. Make the minimal change and redeploy. +4. **Otherwise stamp `outcome='failed', error='sql'`** — `applyRules` → `fixCode` owns converted-SQL defects, carries the diagnosis guidance, and turns the fix into a rule other objects reuse. Rewriting the object here skips all three, and a redeploy loop on a real conversion defect churns the file instead of fixing it. ## File-update rule @@ -46,6 +46,7 @@ Workflow: Re-pull `migration_status(mode="my_objects_summary")` (or `next_task` with the object's `object_id`). The machine advances you once it sees the deployment — `cloudStatus.deployment` -from the `deploy` tool, or the object existing in Snowflake. If deployment **failed** and you -cannot fix it, call `transition_status(status='advance', task='deploy', outcome='failed')` with -the appropriate `error` code. See [Advancing and reporting](../SKILL.md#advancing-and-reporting). +from the `deploy` tool, or the object existing in Snowflake. If deployment **failed** with +anything the pre-deploy checks do not cover, call +`transition_status(status='advance', task='deploy', outcome='failed')` with the appropriate +`error` code. See [Advancing and reporting](../SKILL.md#advancing-and-reporting). diff --git a/plugin/skills/migration/migrate-objects/migrate-object/DIAGNOSE_FIX.md b/plugin/skills/migration/migrate-objects/migrate-object/DIAGNOSE_FIX.md index 44c0bcb..d9a64e3 100644 --- a/plugin/skills/migration/migrate-objects/migrate-object/DIAGNOSE_FIX.md +++ b/plugin/skills/migration/migrate-objects/migrate-object/DIAGNOSE_FIX.md @@ -1,314 +1,134 @@ # Diagnose & Fix -Analyze test failures and fix them. Uses error history from previous iterations, searches for relevant rules by error text, spawns investigation agents in parallel, then applies fixes with confidence-gated review. +`fixCode` for one object. Read the failures, name a root cause, apply a +faithful fix or stop. The machine retries the failed task when you stamp +`completed` — do not jump to [SKILL.md](SKILL.md) by hand. -## Step 1: Accumulate Iteration Context +You diagnose. Do not spawn a review agent, and do not spawn investigators +because a later step is titled "swarm". Read `VALIDATION.LATEST`, the source +SQL, and the converted SQL yourself. The one +[`test_case_verifier`](../../../../agents/test_case_verifier.md) is spawned +from `runTests` after you stamp, not from here. -Before investigating, gather context from all previous iterations of this fix loop. This prevents retrying approaches that already failed. +## Step 1: Iteration context -**Build the iteration log** by recalling from the current session: - -- **Iteration number** — which pass through the deploy-test-fix loop is this? -- **Previous errors** — what errors/failures were seen in earlier iterations? -- **Previous fix attempts** — what code changes were made, and what was the outcome? -- **Approaches to avoid** — any fix strategies that were tried and failed? - -Structure this as: +If this is the first pass, skip. Otherwise recall what you already tried +and do not repeat an unchanged approach. ``` Iteration: - Previous attempts: -- Iteration 1: → -- Iteration 2: → -... - +- Iteration 1: → Approaches to avoid: -- -``` - -**If this is iteration 1**, skip this step — there is no prior context. - -## Step 2: Get Current Failure Context - -Read local test results for the failing object: - -```bash -cat /test-results/results.json -``` - -Filter entries where `code_unit_name` matches `` and `status` is `FAIL` or `ERROR`. - -Gather from each failing entry: -- `params_hash` and `params` — input parameters -- `error` — error message (if ERROR status) -- `differences` — human-readable diff descriptions -- `in_memory_diff.cell_diffs` — exact cell-level differences (row, column, baseline value, actual value) -- `in_memory_diff.row_counts` — baseline vs actual row counts -- `in_memory_diff.summary_stats` — per-column aggregate mismatches - -## Step 2.5: Detect Special Cases - -Before the general rule search and investigation swarm, check whether this failure matches a **known error pattern**, involves **dynamic SQL**, or falls under a **documented troubleshooting scenario**. These checks can short-circuit or enrich the diagnosis. - -### Check for Known Errors - -Scan the error messages and diff descriptions from Step 2 against the patterns in [references/KNOWN_ERRORS.md](references/KNOWN_ERRORS.md). - -**If a match is found:** Apply the documented fix directly — skip the investigation swarm and go straight to Step 6 (Apply Fix). - -Known patterns include: -- Error 002232 (invalid virtual column expression) — inline UDF logic -- Mixed-quote PIVOT column identifiers — fix quoting -- All-rows-different due to column ordering, case, or formatting — reorder/alias/cast -- Decimal/rounding and timestamp precision differences — normalize or flag - -### Check for Dynamic SQL Issues - -If the error message or the converted SQL file contains any of these patterns: -- `EXECUTE IMMEDIATE` -- `IDENTIFIER(` -- `sp_executesql` -- `EXEC(` -- `EXEC @` - -Then the failure likely involves dynamic SQL conversion. Read the canonical dynamic SQL conversion reference at [../../rule-engine/resolving-ewis/reference/SSC-EWI-0030.md](../../rule-engine/resolving-ewis/reference/SSC-EWI-0030.md) and pass the relevant conversion patterns into the investigation agents (Step 4) as additional context. This reference covers: -- Variable transformation and identifier quoting -- `sp_executesql` to `EXECUTE IMMEDIATE ... USING` conversion -- System catalog mappings (e.g., `sys.tables` to `INFORMATION_SCHEMA`) -- Temp table transformation in dynamic context -- Handling of commented-out dynamic SQL (SnowConvert `!!!RESOLVE EWI!!!` markers) - -### Check for Test YAML Shape Mismatch - -Before assuming the failure is a SQL bug, ask: **does the test YAML's `steps:` block match the actual shape of this procedure?** `scai test seed` and the swarm both produce a default-shape YAML (one CALL step, one return-value comparison). Many procedures don't fit that default and need a different step structure. - -Scan the failure context for any of these smells: - -| Smell | Likely YAML-shape issue | Recipe | -|---|---|---| -| Error mentions multiple result sets, "got N result sets, expected M", or first-RS-only comparison while source proc emits several `SELECT` statements | Default `validate: true` only compares the first RS | [`EDIT_TEST_YAML.md` → Multi-result-set validation](EDIT_TEST_YAML.md#multi-result-set-validation) | -| Proc has OUT / INOUT params and the diff is on a column that looks like a param name, or "expected non-null, got null" on what should be an output | YAML doesn't emit Output Parameter Comparison step | [`EDIT_TEST_YAML.md` → OUT / INOUT parameter comparison](EDIT_TEST_YAML.md#out--inout-parameter-comparison) | -| Proc is DML (INSERT / UPDATE / DELETE / MERGE) and result shows "no rows captured" or empty diff | Delta capture not active or no post-condition SELECT in `steps:` | [`EDIT_TEST_YAML.md` → Side-effect-only DML](EDIT_TEST_YAML.md#side-effect-only-dml) | -| Proc populates a temp table or persistent table and the diff is on that table not being read | YAML missing Table Read Step after the CALL | [`EDIT_TEST_YAML.md` → Table-read assertion](EDIT_TEST_YAML.md#table-read-assertion) | -| Redshift proc returns a refcursor and the diff is "RESULT_SCAN returned no rows" / "cursor not found" | YAML missing Cursor Read Step | [`EDIT_TEST_YAML.md` → Cursor-read step](EDIT_TEST_YAML.md#cursor-read-step) | -| Snowflake column name is `"ANONYMOUS BLOCK"` while baseline expects the Redshift param name | Anonymous-block alias missing in `validate:` list | [`EDIT_TEST_YAML.md` → Per-dialect gotchas: Redshift scalar INOUT](EDIT_TEST_YAML.md#redshift-scalar-inout--anonymous-block--column-aliasing) | -| Teradata proc fails with `Error 5315: does not have SELECT/INSERT access` on the source side | Cross-database GRANT steps missing | [`EDIT_TEST_YAML.md` → Per-dialect gotchas: Teradata cross-database GRANTs](EDIT_TEST_YAML.md#teradata-cross-database-grants-when-using-clone-isolation) | - -**If a match is found:** load [`EDIT_TEST_YAML.md`](EDIT_TEST_YAML.md), apply the matching recipe to the failing YAML under `//test/` (glob `*.yml` — `files.artifacts.path` from `query_registry`, verbatim; synthetic seeding may write multiple `.0.yml`, `.1.yml` files), then re-run `scai test capture --where "source.canonicalName ILIKE '%%'"` to refresh the baseline. Re-run `scai test validate` to confirm — go directly to [SKILL.md](SKILL.md) Step 4 (retest), skipping the investigation swarm. This is **not a SQL fix**, so there is nothing to redeploy. - -**If unsure whether the YAML shape is at fault:** prefer to continue with the investigation swarm. Agent 3 (Output Analysis) will surface the same smells with more context, and you can come back to this recipe after seeing its output. False positives here cost a YAML edit that might not be needed; false negatives cost an extra swarm iteration. - -### Check Troubleshooting Reference - -If none of the above matched, also consult [../references/troubleshooting.md](../references/troubleshooting.md) for common test failure scenarios (wrong schema prefix, missing base data, connection issues, etc.) that may explain the failure without needing the full swarm. - ---- - -## Step 3: Search Rules by Error Text - -Before spawning the investigation swarm, check if a known rule already addresses this error. This can short-circuit the diagnosis entirely. - -Collect the primary error messages from Step 2 (the `error` field from ERROR entries, or the first `differences` entry from FAIL entries). Use `find_similar_rules` with the error text as the query: - -Use the `find_similar_rules` tool with `query` set to `""`. - -**If a matching rule is found:** - -| Rule's `replacement_mode` | Action | -|---------------------------|--------| -| `regex` | Apply `replacement_find` / `replacement_replace` mechanically to the SQL file. Skip to Step 6 (Apply Fix). | -| `ai` | Read the rule's `ai_context` and `examples`. Pass them into the investigation agents (Step 4) as additional context to guide diagnosis. | - -After applying a matched rule (regex mode), record the application: use the `record_rule_application` tool with `rule_id` set to the matched rule's ID, `outcome` set to `"applied"`, and `code_unit_name` set to ``. - -**If the rule's name starts with `[AVOID]`:** This is a negative rule (anti-pattern). Note the `ai_context` — it describes an approach that was tried before and failed. Add it to the "Approaches to avoid" list from Step 1. - -**If no matching rule is found**, proceed to Step 4. - -## Step 4: Spawn Investigation Swarm - -Launch 3 agents **in parallel** to investigate different causes. Each agent receives the iteration context from Step 1 so it can avoid repeating failed approaches. - -### Agent 1: Code Comparison - -``` -Investigate test failures for by comparing SOURCE vs TARGET code. - -Source (truth): -Snowflake target: - -Compare: -1. Parameter handling - same names, types, defaults? -2. Business logic - IF/CASE branches match? -3. Table/view references - correct schema prefixes? -4. Function calls - source functions converted correctly? -5. Return values - same columns, same order? - -Look for SnowConvert EWI comments (--** SSC-) indicating conversion issues. - -Failing tests context: - - -Iteration context (if iteration > 1): - - -Matching rules context (if any from Step 3): - - -IMPORTANT: Do NOT suggest approaches listed under "Approaches to avoid". - -Output: List specific code differences that could cause the failures. -``` - -### Agent 2: Data Investigation - -``` -Investigate test failures for by checking UNDERLYING DATA. - -Referenced tables/views in the code: - - -For each table, check: -1. Does it exist in Snowflake with correct schema prefix? -2. Row counts match between source baseline and Snowflake? -3. Any data type differences that could affect results? - -Failing tests context: - - -Iteration context (if iteration > 1): - - -IMPORTANT: Do NOT suggest approaches listed under "Approaches to avoid". - -Run queries to verify data exists: -- SELECT COUNT(*) FROM .
-- Sample rows if needed - -Output: List any data issues that could cause the failures. +- ``` -### Agent 3: Output Analysis - -``` -Investigate test failures for by analyzing TEST OUTPUT DIFFERENCES. - -Failing test details: - - -Analyze: -1. Row count differences - missing rows? extra rows? -2. Column value differences - which columns differ? by how much? -3. Data type/format differences - precision, date formats, case? -4. Pattern across failures - same issue in all, or different issues? - -Iteration context (if iteration > 1): - - -IMPORTANT: Do NOT suggest approaches listed under "Approaches to avoid". - -Output: -- What specifically differs (columns, values, row counts) -- Pattern analysis (is it the same root cause across all failures?) -- Likely root cause category (logic bug, precision issue, data issue, etc.) -``` - -## Step 5: Synthesize & Identify Root Cause - -After agents complete, combine findings: - -| Finding | Root Cause | Action | -|---------|------------|--------| -| Code logic differs | Conversion bug | Fix the code | -| Missing schema prefix | Wrong table reference | Add prefix | -| Function not converted | T-SQL function used | Replace with Snowflake equivalent | -| Data missing | Table not synced | Sync data or check schema | -| Precision differs | Type mismatch | Add explicit CAST | - -For common issues and their solutions, also consult [../references/troubleshooting.md](../references/troubleshooting.md). - -## Step 6: Apply Fix - -1. **Open the Snowflake SQL file:** - ```bash - find /snowflake -iname "**" -type f - ``` - -2. **Make the minimal change** to fix the root cause. **Preserve all original source code comments** (synopsis, metadata, author, archive, change log, examples) — do not strip them during fixes or rewrites. Don't stub or delete logic to silence an error (no `NULL`/empty `… WHERE FALSE` bodies), and don't reuse another object or invent a new one for a missing reference — comment the original out with a `-- NEEDS-USER:` note instead. - - Also verify these common conversion issues before redeploying: - - `USE DATABASE ` at the top referencing the source database — remove it. - - Object name must be fully qualified with the target database: `..`. - - Variable binding in LANGUAGE SQL procedures: parameters/variables in SQL statements must use `:param_name` syntax (SnowConvert often omits the colon). - - Edit the file in `snowflake/` — do NOT deploy directly without updating the file. - - If the fix was informed by a rule from Step 3, record the application: use the `record_rule_application` tool with `rule_id` set to the rule's ID and `outcome` set to `"applied"`. - -3. **Redeploy** → Return to [SKILL.md](SKILL.md) Step 3 - -4. **Retest** → Return to [SKILL.md](SKILL.md) Step 4 - -### Common Fixes - -Query the rule engine for known fix patterns: `search_rules(description="")` - -## Step 7: Fix Review Agent - -After making the fix, spawn a review agent to verify the change: +## Step 2: Failure context ``` -Review the code fix for . - -Original issue: - - -File changed: - -Changes made: - - -Source (truth): -Snowflake target (fixed): - -Iteration: of fix loop - -Checklist: -1. Does the fix address the specific root cause identified? -2. Are there any unrelated changes that should be reverted? -3. Is the SQL syntactically valid (no unclosed parens, missing semicolons)? -4. Are all schema references correct for the target environment? -5. Does the fixed code match the source logic for this specific area? -6. Are there any other instances of the same issue in the file that should also be fixed? -7. Could this fix introduce any new issues? -8. Are there any tests hardcoded? -9. Has this same approach been tried before and failed? (Check iteration context) - -Output ONE of: -- HIGH_CONFIDENCE: Fix directly addresses root cause, syntactically correct, no regressions expected. Ready to deploy. -- LOW_CONFIDENCE: Fix is plausible but uncertain (e.g., edge cases unclear, partial fix, or similar approach partially failed before). List specific concerns. Deploy but flag for user review. -- NEEDS_CHANGES: List what needs to be fixed before deploying. +query_registry(where="id = ''", fields="id,source,files,target") ``` -If review agent returns **NEEDS_CHANGES**, address the feedback and re-run the review. **Max 5 review iterations** — if the review agent still returns NEEDS_CHANGES after 5 rounds, present the current state to the user and ask for guidance. - -**HIGH_CONFIDENCE** → Proceed to deploy (return to [SKILL.md](SKILL.md) Step 3). - -**LOW_CONFIDENCE** → Present the fix and the reviewer's concerns to the user: - -> The review agent flagged this fix as **low confidence**: -> - Concerns: `` -> - Changes: `` -> -> Deploy anyway, or adjust the fix first? - -If user approves → deploy. If user wants changes → revise and re-run review. - -**NEEDS_CHANGES** → Address the feedback and re-run the review. - -## After Review - -1. Redeploy → Return to [SKILL.md](SKILL.md) Step 3 -2. Retest → Return to [SKILL.md](SKILL.md) Step 4 -3. If still failing → repeat from Step 1 -4. If all pass → done +Hold `files.source.path`, `files.converted.path`, `files.artifacts.path`. +Read `.VALIDATION.LATEST` (attach names that +database; it is not the migration target). There is no +`/test-results/results.json`. For each `FAIL` / `ERROR` on +this object, take `params_hash`, `parameters`, `error_message`, and +`differences`. Then read the source SQL and the converted SQL. A deploy +error is the context when this loop entered from a compile failure. + +## Step 3: Known shortcuts + +Check these before inventing a new theory. A match is a fix (or a stop), +not a swarm. + +**Documented conversion / load errors** + +- Error 002232 (invalid virtual column) — inline the UDF. +- Mixed-quote PIVOT identifiers — fix quoting. +- All-rows-different from column order, case, or formatting — reorder / + alias / cast when that still means what the source means. +- Decimal / rounding / timestamp precision — normalize only when that + still means what the source means. A wall-clock expression (`GETDATE`, + `CURRENT_TIMESTAMP`, age-from-today) is not a normalize: go to Step 5. +- Snowflake 001187 (`COPY INTO` refused on CHECK constraints) — delete + the `CONSTRAINT … CHECK` clauses from the converted `snowflake/` file, + then deploy. Do not `ALTER TABLE … DROP CONSTRAINT` on the live table. + +**Dynamic SQL** in the error or the converted file (`EXECUTE IMMEDIATE`, +`IDENTIFIER(`, `sp_executesql`, `EXEC(` / `EXEC @`): read +[../../rule-engine/resolving-ewis/reference/SSC-EWI-0030.md](../../rule-engine/resolving-ewis/reference/SSC-EWI-0030.md) +and use those patterns in the edit. + +**YAML `steps:` do not match the proc** — this is not a SQL bug. Load +[`EDIT_TEST_YAML.md`](EDIT_TEST_YAML.md) when the failure smells like: + +| Smell | Recipe | +|---|---| +| Multiple result sets / "got N, expected M" | Multi-result-set validation | +| OUT / INOUT param column null or missing | OUT / INOUT parameter comparison | +| DML proc, "no rows captured" | Side-effect-only DML | +| Proc writes a table the YAML never reads | Table-read assertion | +| Redshift refcursor / `RESULT_SCAN` empty | Cursor-read step | +| `"ANONYMOUS BLOCK"` vs Redshift param name | Redshift scalar INOUT aliasing | +| Teradata 5315 on the source side | Teradata cross-database GRANTs | + +Apply the recipe under `//test/` (glob +`*.yml`; `files.artifacts.path` verbatim). Recapture with +`scai test capture --where "source.canonicalName ILIKE '%%'"`, +then stamp `fixCode` completed so the machine retries `runTests`. Nothing +to redeploy. + +**Rules.** `find_similar_rules` with the primary error or first diff. +`regex` → apply `replacement_find` / `replacement_replace` and +`record_rule_application`. `ai` → use `ai_context` as diagnosis, not as +an accept. A name starting `[AVOID]` is an approach not to repeat. + +## Step 4: Name the root cause + +| Finding | Action | +|---|---| +| Converted logic ≠ source | Edit converted SQL | +| Missing schema prefix | Add the prefix | +| Source function left as T-SQL | Snowflake equivalent | +| YAML `steps:` mismatch | Step 3 recipe, recapture | +| Precision / collation / padding, and a CAST or `RTRIM` still means what the source means | Edit converted SQL, then **note** (you chose among meanings) | +| Diffs are clock drift and the source (or a view it reads) computes from now | Step 5 — do not edit | +| Dialect error-code / SQLSTATE mismatch (`error_mismatch`) while both sides error | Do not delete the YAML case. Stamp and retry; a second verifier is allowed after a later code change. | +| Missing object, and `next_task` on that id is not terminal | Do not stamp. The walk derives the wait from that object's status. | +| You cannot name a cause after reading the files and any matching rule | Escalate with the diffs and the readings you cannot choose between | + +Override-accept is last. "The baseline moved by a day" is not enough if +CAST, a prefix, YAML shape, or a collation/`RTRIM` would make the case +pass. + +A missing reference is an escalation, not a `-- NEEDS-USER:` comment and +not a stub (`NULL` / `WHERE FALSE`) or a borrowed object. + +## Step 5: Wall-clock — stop, do not overlay here + +When the only remaining failures are clock drift from an expression that +must stay (`GETDATE`, `CURRENT_TIMESTAMP`, `CURRENT_DATE`, `DATEDIFF` +against now, age-from-today), a code change would lie. Do not edit. Do +not call `override_accept_case` yourself. Do not spawn a verifier. + +Stamp `fixCode` completed. The machine retries `runTests`; that guide +hands every remaining hash to one +[`test_case_verifier`](../../../../agents/test_case_verifier.md). +Rejected hashes come back as `error="sql"`. + +## Step 6: Apply a code fix + +Edit `files.converted.path` only. Minimal change. Keep source comments. +Do not strip `EXECUTE AS` or the SnowConvert `COMMENT` provenance block +unless the error named them. + +Before deploy, also drop a leading `USE DATABASE `. Do not +rewrite the database qualifier to the SNAP / configure name — `deploy -d` +does that. + +If the edit chose among meanings (CHECK drop so COPY can run, `RTRIM` for +trailing-space collation), **note** per +[general-task.md](../../../../agents/general-task.md) §4, then stamp +`fixCode` completed. The machine's `retryEntry` is the redeploy / retest. +Do not spawn a reviewer for the note. diff --git a/plugin/skills/migration/migrate-objects/migrate-object/RUN_TESTS.md b/plugin/skills/migration/migrate-objects/migrate-object/RUN_TESTS.md index e05c077..e4bdd23 100644 --- a/plugin/skills/migration/migrate-objects/migrate-object/RUN_TESTS.md +++ b/plugin/skills/migration/migrate-objects/migrate-object/RUN_TESTS.md @@ -6,13 +6,24 @@ Guide for the `runTests` task. The machine invokes this after deployment succeed The `scai test validate` tool compares Snowflake output against captured baselines. Baselines were captured during the prep phase — this step only re-runs validation. -Read the results file: +`scai test validate` writes each case into +`.VALIDATION.RESULTS`. Read the latest run from +`VALIDATION.LATEST` (attach names `metadata_database` — same catalog as +ORCHESTRATION, not the migration target). There is no +`/test-results/results.json`. ``` -/test-results/results.json +sql_execute: +SELECT params_hash, status, parameters, error_message, differences, + baseline_rows, actual_rows, match_type +FROM .VALIDATION.LATEST +WHERE UPPER(procedure_name) IN ( + UPPER('..'), + UPPER('.') + ) ``` -Each entry has `code_unit_name`, `status`, `match_type`, `error`, and `differences`. Focus on entries where `status` is not `PASS`. BTEQ result rows carry `metadata.kind: "bteq"` and compare file I/O + table deltas rather than a return value. +Each row has `procedure_name`, `status`, `match_type`, `error_message`, and `differences`. Focus on rows where `status` is not `PASS`. BTEQ result rows carry `metadata.kind: "bteq"` and compare file I/O + table deltas rather than a return value. ## Test statuses @@ -44,23 +55,28 @@ Scan the `error` field from failing test entries for these patterns: ### If a dependency failure is detected 1. **Identify the missing object** from the error message (e.g., `Unknown function: dbo.HelperFunc`). -2. **Look it up in the registry** with `query_registry`: +2. **Find it in the registry** with `query_registry` — you need its `id`: ``` query_registry( where="source.canonicalName ILIKE '%%'", - fields="id,source,codeStatus,cloudStatus,extensions" + fields="id,source" ) ``` -3. **Route based on the returned fields:** +3. **Ask the resolver where that object stands**, rather than reading a status field yourself: + ``` + migration_status(mode="next_task", object_ids=[""]) + ``` + It consults whichever source is authoritative for each task and object type — a test-results query, a Snowflake probe, a registry field — and which one that is differs per task and changes. A field you read yourself is not the verdict. +4. **Route on what it says:** -| Registry result | Action | +| Resolver result | Action | |---|---| -| No row returned | **Skip this object.** Report: "Blocked on `` — not in registry." | -| Row exists but `cloudStatus.deployment.status != "completed"` | **Skip this object.** Report: "Blocked on `` — not yet deployed." | -| Deployed but `codeStatus.testing.status != "completed"` (procs/funcs) or `extensions.dataValidation.status` is `failed`/`error` (tables) | **Skip this object.** Report: "Blocked on `` — deployed but failing tests/validation." | -| Otherwise | Not a dependency failure — proceed to fix the code. | +| `query_registry` returned no row | **Skip this object.** Report: "Blocked on `` — not in registry." | +| `errored`, or `blocked: true` | **Skip this object.** Report: "Blocked on `` — " plus its error, or the reason on its `blockedOn` entry. | +| `nextTask` is not null | **Skip this object.** Report: "Blocked on `` — still at ``." It has not finished its own walk, so it is not what you should be testing against yet. | +| `nextTask: null`, not errored, not blocked | The dependency is done. If the FAIL is still a defect **in that code unit** (wrong column type, bad view body, missing CAST on *its* SQL), do **not** patch this procedure around it. Spawn one foreground [`task-invalidate`](../../../../agents/task-invalidate.md) (`subagent_type="task-invalidate"`) with that code unit's `id`, the resume task (`validateView` on a view, `runTests` on a procedure), and why — plus this waiter's `objectId` and `runTests` so its verification is also reopened. Then **return** `result: "reopened"` with `reopenedCodeUnits`. Do not stay in the fix loop. If the dependency's SQL is faithful and the FAIL is in *this* code unit, this is not a dependency failure — proceed to fix the code. | -If blocked on a dependency, call `transition_status(status='advance', task='runTests', outcome='failed', error='dependency')`. Name the specific dependency in your user-facing reply so the user knows what to migrate. The state machine will land this in the errored bucket without entering the fix loop — the SQL isn't broken, the precondition is. +If blocked on a dependency, **do not stamp this task**. Name the specific dependency in your user-facing reply so the user knows what to migrate. The next walk derives `blocked` / `blockedOn` from that dependency's live status. ### If not a dependency failure @@ -69,4 +85,6 @@ Proceed — the machine will route to the fix loop. ## After tests complete - **All pass:** call `transition_status(status='advance', task='runTests', outcome='completed')`. -- **Any fail (not dependency):** call `transition_status(status='advance', task='runTests', outcome='failed', error='sql')` — the machine routes to rule application and the fix loop. Use `error='sql'` only when the test failure is caused by a SQL/DDL bug the fix loop can address; for transient infrastructure issues (timeouts, connection drops) use `error='infra'` instead. +- **Any fail (not dependency):** call `transition_status(status='advance', task='runTests', outcome='failed', error='sql')` — the machine routes to rule application and the fix loop. Try to make the cases pass there before any overlay. Use `error='sql'` only when the test failure is caused by a SQL/DDL bug the fix loop can address; for transient infrastructure issues (timeouts, connection drops) use `error='infra'` instead. +- **Still failing after the fix loop, and a code change would be illogical** (source wall-clock expression; freezing "now" would lie): spawn **one** foreground [`test_case_verifier`](../../../../agents/test_case_verifier.md) (`subagent_type="test_case_verifier"`) with fresh context for **all** remaining hashes on this object. Hand `objectId`, `projectDir`, the `params_hash` list, and the RESULTS diffs — not a verdict. That child reads each case and decides. Do **not** call `configure(subagent_mode=true)` for this — that latch is the autonomous orchestrator's, before dispatching walkers. Call it yourself only if the project set `require_independent_override_accept: false`. See [general-task.md](../../../../agents/general-task.md) §4. Do not stamp `runTests` completed and do not delete the YAML case. The oracle joins the overlay on `(procedure, params_hash, target)`; RESULTS still shows FAIL. The call is also a `note` (same unreviewed queue). If that child **rejects** and you then change the converted SQL, a later FAIL on the same hash gets a **new** verifier — the first child judged the pre-fix SQL. A second override-accept is allowed. +- **Never delete a YAML test case** (overflow, short-input, leftover RESULTS FAIL). Keep the row; overlay or fix SQL. diff --git a/plugin/skills/migration/migrate-objects/migrate-object/SKILL.md b/plugin/skills/migration/migrate-objects/migrate-object/SKILL.md index 1b1e033..cb61f0d 100644 --- a/plugin/skills/migration/migrate-objects/migrate-object/SKILL.md +++ b/plugin/skills/migration/migrate-objects/migrate-object/SKILL.md @@ -16,12 +16,13 @@ Each object goes through a pipeline managed by the state machine: 1. **Claim** — `claimObject` reserves the object for this user. 2. **Checkout** — `checkoutBranch` creates or switches to a git branch. 3. **Convert** — `convert` runs SnowConvert to produce initial Snowflake SQL. -4. **Test prep** — procedures/functions: `createTests` + `captureBaseline` generate test YAML and capture source-side baselines. BTEQ scripts: `seedScript` (binding values + import fixtures resolved from the shell script that runs it via `scai test seed --bindings-from`, hand-filled otherwise — see [../baseline-capture/seed-script/SKILL.md](../baseline-capture/seed-script/SKILL.md)) then `captureBaseline`. +4. **Test prep** — procedures/functions **with a source side**: `createTests` + `captureBaseline` generate test YAML and capture source-side baselines. Procedures/functions **with no source** (UDF helpers) skip this and go straight to deploy. BTEQ scripts: `seedScript` (binding values + import fixtures resolved from the shell script that runs it via `scai test seed --bindings-from`, hand-filled otherwise — see [../baseline-capture/seed-script/SKILL.md](../baseline-capture/seed-script/SKILL.md)) then `captureBaseline`. 5. **Deploy** — `deploy` pushes the SQL to Snowflake. See [DEPLOY.md](DEPLOY.md). 6. **Validate** — depending on object type: - Tables: `migrateData` → `validateData` - Views: `validateView` - - Procedures/functions: `runTests`. See [RUN_TESTS.md](RUN_TESTS.md). + - Procedures/functions with a source side: `runTests`. See [RUN_TESTS.md](RUN_TESTS.md). + - Procedures/functions with no source: `verify`. See [VERIFY.md](VERIFY.md). - BTEQ scripts: `runTests` (deploy is skipped — the converted script is run by the test). See [RUN_TESTS.md](RUN_TESTS.md). 7. **Fix loop** — if deployment or validation fails, the machine enters a cycle: - `applyRules` — apply known migration rules from the rule engine. @@ -51,7 +52,5 @@ Do NOT iterate blindly. Escalate to the user when: |-----------|---------| | Same error persists | Same primary error for 3 consecutive iterations | | Errors churning | Errors keep changing but never resolve after 5 total iterations | -| Review loop | Fix review returns NEEDS_CHANGES twice for the same root cause | -| Low confidence repeated | Review returns LOW_CONFIDENCE on 2 consecutive iterations | On escalation, present iteration history and offer: provide guidance, decompose and retry, skip, mark as needs human repair, or mark done. diff --git a/plugin/skills/migration/migrate-objects/migrate-object/VALIDATE_VIEW.md b/plugin/skills/migration/migrate-objects/migrate-object/VALIDATE_VIEW.md index deb619c..770a312 100644 --- a/plugin/skills/migration/migrate-objects/migrate-object/VALIDATE_VIEW.md +++ b/plugin/skills/migration/migrate-objects/migrate-object/VALIDATE_VIEW.md @@ -7,7 +7,11 @@ Compare a deployed view between source and Snowflake to verify the migration pro ## Inputs - `.` — the view to validate -- All connections and database are configured via MCP (`configure()`) +- `` — attach `snowflake_connection:` / first-prompt + `snowflakeConnection`. Pass it as `sql_execute` `connection` (`-c`). +- `` — the Snowflake **target** from `configure` attach + (`snowflake_database:`) / the walker's first prompt. Not `source.database` + and not the workload catalog name. ## Step 1: Row Count Comparison @@ -17,11 +21,19 @@ query_source("SELECT COUNT(*) FROM .") Compare with: -```sql -SELECT COUNT(*) FROM .. +``` +sql_execute( + connection="", + sql="SELECT COUNT(*) FROM ..", + description="Count rows in the deployed target view" +) ``` -on Snowflake (via the configured connection). +`query_source` is the source database. `sql_execute` is Cortex Snowflake and +does not inherit `configure` — pass `connection` from attach +`snowflake_connection:` / the first-prompt `snowflakeConnection`, and qualify +with ``. Do not fall back to shell, `snow sql`, Python, or +an unqualified query. ## Step 2: Spot Check @@ -31,8 +43,12 @@ query_source("SELECT TOP 10 * FROM .") Compare with: -```sql -SELECT * FROM .. LIMIT 10 +``` +sql_execute( + connection="", + sql="SELECT * FROM .. LIMIT 10", + description="Sample rows from the deployed target view" +) ``` Check for: @@ -57,12 +73,12 @@ Before reporting fail, identify *why*. The classification drives how the parent | Cause | Signal | `error` code | |---|---|---| | The view's DDL is wrong (compile error, column missing, type mismatch, bad expression) | Snowflake-side query errors, or values diverge despite full data | `sql` | -| Snowflake side has 0 rows (or far fewer) because base tables haven't had data migrated yet | Source has rows; Snowflake-side count is 0 or anomalously low; the view's `dependencies` in the registry list tables whose `extensions.dataMigration.status` is not `completed` | `dependency` | -| Validation couldn't run (connection drop, query timeout, permissions) | Tool-level error, not a row-count divergence | `infra` | +| Snowflake side has 0 rows (or far fewer) because base tables haven't had data migrated yet | Source has rows; Snowflake-side count is 0 or anomalously low; the view's `dependencies` in the registry list tables whose live `migrateData` job is not completed | *(do not stamp)* | +| Validation couldn't run (connection drop, query timeout, permissions) | Tool-level error, not a row-count divergence; do not substitute a shell query | `infra` | When the parent calls `transition_status(... outcome="failed", error=)`: - `error="sql"` routes to the rule engine + fix loop (`applyRules` → `fixCode`). Use ONLY when the SQL needs editing. -- `error="dependency"` lands the task in the errored bucket without entering the fix loop. In your user-facing reply, name the specific tables that need data migration first so the user knows what to fix. +- Do not stamp a dependency wait. The walk already surfaces those tables on `blockedOn`; name them in your reply and leave the task unstamped. - `error="infra"` lands in the errored bucket; the user retries. If validation passes, return success and the parent will call `transition_status(... outcome="completed")` with no `error`. diff --git a/plugin/skills/migration/migrate-objects/migrate-object/VERIFY.md b/plugin/skills/migration/migrate-objects/migrate-object/VERIFY.md index a403adb..cd4691a 100644 --- a/plugin/skills/migration/migrate-objects/migrate-object/VERIFY.md +++ b/plugin/skills/migration/migrate-objects/migrate-object/VERIFY.md @@ -1,6 +1,11 @@ # Verify -Guide for the `verify` task. The machine routes an object here after conversion when its type has no deploy/test path of its own — Oracle `PACKAGE`, `PACKAGE_BODY`, `TYPE`, `TYPE_BODY`, `SYNONYM`, and any other type the machine doesn't route onward. There is no `scai` command for this step: **verify the object by whatever means the object allows**, then record the verdict. +Guide for the `verify` task. The machine routes an object here in two cases: + +- After conversion, when its type has no deploy/test path of its own — Oracle `PACKAGE`, `PACKAGE_BODY`, `TYPE`, `TYPE_BODY`, `SYNONYM`, and any other type the machine doesn't route onward. +- After **deploy**, when a procedure or function has **no source side** (SnowConvert UDF helpers under `snowflake/UDF Helpers/`). Those objects are Snowflake-only: there is no source to seed or baseline against, so `createTests` / `runTests` will never pass. + +There is no `scai` command for this step: **verify the object by whatever means the object allows**, then record the verdict. ## Step 1: Read the conversion output @@ -14,10 +19,11 @@ Open the object's source and converted files (`files.source.path` and `files.con Use as much evidence as the object gives you. In rough order of strength: -1. **It produced deployable DDL** → deploy it and confirm it compiles in Snowflake. -2. **Its logic moved into referencing units** → list the referencing code units (`dependencies` on the registry entry, or `query_registry` for units that reference this name) and confirm each one converted, and that the packaged logic is present in them. Their own `runTests` results are the real proof the logic survived. -3. **Nothing was emitted and nothing references it** → confirm that: no referencing unit, no remaining references to the name in the converted SQL. -4. **Unresolved EWIs on the converted output** → treat as not verified; load [DIAGNOSE_FIX.md](DIAGNOSE_FIX.md). +1. **It is already deployed** (source-less helper) → confirm the object exists in Snowflake (`SHOW FUNCTIONS` / `SHOW PROCEDURES` or `DESC`) and is callable with a few smoke inputs. Do not try to seed, capture a source baseline, or author a test YAML. +2. **It produced deployable DDL** → deploy it and confirm it compiles in Snowflake. +3. **Its logic moved into referencing units** → list the referencing code units (`dependencies` on the registry entry, or `query_registry` for units that reference this name) and confirm each one converted, and that the packaged logic is present in them. Their own `runTests` results are the real proof the logic survived. +4. **Nothing was emitted and nothing references it** → confirm that: no referencing unit, no remaining references to the name in the converted SQL. +5. **Unresolved EWIs on the converted output** → treat as not verified; load [DIAGNOSE_FIX.md](DIAGNOSE_FIX.md). Report what you checked and what you concluded. Do not claim more than the evidence supports — "the package body was inlined into 3 procedures, all 3 pass their tests" is a verification; "conversion reported success" is not. diff --git a/plugin/skills/migration/migrate-objects/references/BTEQ_TEST_YAML.md b/plugin/skills/migration/migrate-objects/references/BTEQ_TEST_YAML.md index d720850..c6e2ce8 100644 --- a/plugin/skills/migration/migrate-objects/references/BTEQ_TEST_YAML.md +++ b/plugin/skills/migration/migrate-objects/references/BTEQ_TEST_YAML.md @@ -58,7 +58,7 @@ A binding used by a single script stays in its per-object YAML; one used by 2+ i | `files.writes[]` | `scriptMetadata.IO` `direction:write` — declared target name | ## Baseline / results -`scai test capture` writes `baseline_type:"script"` baselines (`exit_code`, `stderr`, `success`, `table_deltas`, `script_io`); `{ eval }` recipes are evaluated at capture and their resolved values pinned into the baseline. `scai test validate` re-runs `snowflake/BTEQ/.sql` and compares (reuse targets take the capture-pinned value); results in `test-results/results.json` with `metadata.kind:"bteq"`. +`scai test capture` writes `baseline_type:"script"` baselines (`exit_code`, `stderr`, `success`, `table_deltas`, `script_io`); `{ eval }` recipes are evaluated at capture and their resolved values pinned into the baseline. `scai test validate` re-runs `snowflake/BTEQ/.sql` and compares (reuse targets take the capture-pinned value); results land in `.VALIDATION.RESULTS` with `metadata.kind:"bteq"`. ## Prerequisites - The `bteq` binary (Teradata Tools & Utilities) must be on PATH on the capture host - capture runs the source `.btq` through it. diff --git a/plugin/skills/migration/migrate-objects/references/collaboration-model.md b/plugin/skills/migration/migrate-objects/references/collaboration-model.md index 9eb684b..f6ab1bb 100644 --- a/plugin/skills/migration/migrate-objects/references/collaboration-model.md +++ b/plugin/skills/migration/migrate-objects/references/collaboration-model.md @@ -14,9 +14,8 @@ How the migration plugin manages git for multi-user concurrent migrations. 1. Files for the finished objects are committed on top of `origin/main` using pure git plumbing (the working tree stays on your branch). 2. The commit is pushed to the remote. If a teammate pushed in between, the plugin re-fetches, checks for overlapping files, rebuilds, and retries (up to 3 times). 3. Your branch is rebased onto the updated `main`, pulling in teammates' merged work. -4. Dependency-blocked objects that depended on the just-finished objects are unblocked. -All of these steps are reported in the response's `git_activity` array. +All of these steps are reported in the response's `git_activity` array. Dependents waiting on the finished objects are offered again on the next walk — that is derived, not a finish-time stamp clear. ## What Happens During `migration_status` diff --git a/plugin/skills/migration/migrate-objects/references/overrides.md b/plugin/skills/migration/migrate-objects/references/overrides.md index 6fb5aa1..9ec0f51 100644 --- a/plugin/skills/migration/migrate-objects/references/overrides.md +++ b/plugin/skills/migration/migrate-objects/references/overrides.md @@ -124,17 +124,14 @@ This improves the framework rather than accumulating overrides. ## Pre-Override Investigation Checklist -Before adding any override, check local results and Snowflake: - -```bash -# 1. Read detailed cell-level diffs -cat /test-results/results.json -``` +Before adding any override, read the latest case from Snowflake +(`.VALIDATION.LATEST` — not the migration target): ```sql --- 2. Get difference details from Snowflake -SELECT differences FROM VALIDATION.LATEST -WHERE code_unit_name = 'RPT.Name' AND params_hash = 'abc12345'; +-- Difference details for one case +SELECT differences, error_message, parameters, status +FROM .VALIDATION.LATEST +WHERE UPPER(procedure_name) = UPPER('RPT.Name') AND params_hash = 'abc12345'; -- 3. Run the procedure to see actual output CALL .Name(param1 => value1, param2 => value2); diff --git a/plugin/skills/migration/migrate-objects/references/sql-queries.md b/plugin/skills/migration/migrate-objects/references/sql-queries.md index 0b81798..cb21780 100644 --- a/plugin/skills/migration/migrate-objects/references/sql-queries.md +++ b/plugin/skills/migration/migrate-objects/references/sql-queries.md @@ -2,6 +2,11 @@ Queries against the `VALIDATION` schema created by `scai test validate --create-schema`. +The schema lives in the SnowConvert metadata database — attach +`metadata_database:` / `.scai/config/plugin.yml` (`SNOWCONVERT_AI` unless +overridden), not the migration target. Qualify every query with that name. +The examples below leave the database off only as a shorthand. + ## Validation Results ```sql @@ -9,7 +14,7 @@ Queries against the `VALIDATION` schema created by `scai test validate --create- SELECT * FROM VALIDATION.SUMMARY ORDER BY pass_rate DESC; -- Latest result per test case -SELECT * FROM VALIDATION.LATEST ORDER BY code_unit_name; +SELECT * FROM VALIDATION.LATEST ORDER BY procedure_name; -- Failures with details SELECT * FROM VALIDATION.FAILURES; @@ -19,30 +24,32 @@ SELECT * FROM VALIDATION.FAILURES; ```sql -- Failures for a specific code unit -SELECT code_unit_name, params_hash, status, baseline_rows, actual_rows, error_message +SELECT procedure_name, params_hash, status, baseline_rows, actual_rows, error_message FROM VALIDATION.LATEST -WHERE code_unit_name = '' +WHERE UPPER(procedure_name) = UPPER('') AND status IN ('FAIL', 'ERROR'); --- All results history for a code unit -SELECT code_unit_name, params_hash, status, match_type, error_message, executed_at +-- All results history for a code unit (VARIANT `data`; prefer LATEST) +SELECT data:procedure::VARCHAR, data:params_hash::VARCHAR, + data:status::VARCHAR, data:match_type::VARCHAR, + data:error::VARCHAR, data:executed_at::TIMESTAMP_TZ FROM VALIDATION.RESULTS -WHERE code_unit_name = '' -ORDER BY executed_at DESC; +WHERE UPPER(data:procedure::VARCHAR) = UPPER('') +ORDER BY data:executed_at::TIMESTAMP_TZ DESC; ``` -## Local Results (Preferred) - -The agent should prefer reading local results from `test-results/results.json` for detailed cell-level diffs that are not stored in Snowflake: +## Source of truth -```bash -cat /test-results/results.json -``` +`scai test validate` writes each case into `VALIDATION.RESULTS`. `VALIDATION.LATEST` +is the newest run per `(procedure_name, test_name, params_hash)`. Qualify with +`metadata_database` from attach — not the migration target. There is no +`/test-results/results.json`. -Each entry contains: -- `code_unit_name`, `params_hash`, `status`, `match_type`, `error` +Each LATEST row has: +- `procedure_name`, `params_hash`, `status`, `match_type`, `error_message` +- `parameters` — the case inputs - `differences` — human-readable diff descriptions -- `in_memory_diff` — structured cell-level diffs with `row_counts`, `summary_stats`, `cell_diffs` +- `baseline_rows`, `actual_rows` — counts (not the row payloads) ## Available Views diff --git a/plugin/skills/migration/migrate-objects/references/testing-framework-perms.md b/plugin/skills/migration/migrate-objects/references/testing-framework-perms.md index 8341c68..8a42c13 100644 --- a/plugin/skills/migration/migrate-objects/references/testing-framework-perms.md +++ b/plugin/skills/migration/migrate-objects/references/testing-framework-perms.md @@ -8,7 +8,7 @@ What Snowflake and source-DB privileges the testing framework needs, and what to When the user opts into testing (answers Q1 in `migrate-objects/SKILL.md` Step 2), `configure(recheck=true)` triggers two probes that may write to Snowflake: -1. **`scai test validate --create-schema`** — installs the `VALIDATION` schema in the configured database. Creates a stage (`@VALIDATION.BASELINES`), a results table (`VALIDATION.RESULTS`), supporting views (`SUMMARY`, `LATEST`, `FAILURES`), and the validation stored procedures (`VALIDATE_SINGLE`, `VALIDATE_BATCH`). Idempotent on the scai side — re-runs against an already-deployed schema are no-ops. +1. **`scai test validate --create-schema`** — installs the `VALIDATION` schema in the database named by `testing_results_database` in `.scai/settings/test_config.yaml` (the SnowConvert metadata database, not the migration target; projects predating that setting still have it in the target). Creates a stage (`@VALIDATION.BASELINES`), a results table (`VALIDATION.RESULTS`), supporting views (`SUMMARY`, `LATEST`, `FAILURES`), and the validation stored procedures (`VALIDATE_SINGLE`, `VALIDATE_BATCH`). Idempotent on the scai side — re-runs against an already-deployed schema are no-ops. 2. **`SHOW GRANTS TO ROLE CURRENT_ROLE()`** — read-only check that the active Snowflake role has `CREATE DATABASE on ACCOUNT`, needed for clone-based test isolation during `scai test validate`. Later, during the deploy-test-fix loop, the framework also: diff --git a/plugin/skills/migration/migrate-objects/references/troubleshooting.md b/plugin/skills/migration/migrate-objects/references/troubleshooting.md index 92ddd23..6ff1f8e 100644 --- a/plugin/skills/migration/migrate-objects/references/troubleshooting.md +++ b/plugin/skills/migration/migrate-objects/references/troubleshooting.md @@ -174,7 +174,7 @@ Apply this to every PIVOT column reference (SELECT list, `NVL`, `CASE`, aliases, 1. **Find the extra/missing rows:** ```sql SELECT differences FROM VALIDATION.LATEST - WHERE code_unit_name = 'RPT.Name' AND params_hash = 'abc123'; + WHERE UPPER(procedure_name) = UPPER('RPT.Name') AND params_hash = 'abc123'; ``` 2. **Check for filter differences:** @@ -229,6 +229,15 @@ Apply this to every PIVOT column reference (SELECT list, `NVL`, `CASE`, aliases, ls -la ~/.ssh/rsa_key.p8 ``` +4. **Entra ID / OIDC:** `externalbrowser` is SAML SSO, not OIDC. Use + `authenticator = "oauth_authorization_code"` with `user`, client id/secret, + both HTTPS endpoints, `oauth_scope`, and a **fixed** loopback + `oauth_redirect_uri` registered exactly in Entra. See + `../connection/snowflake-connection/SKILL.md` and + `Snowflake.SnowConvertDesktop/Snowflake.SnowConvert.Cli/docs/entra-oidc-oauth.md`. + Headless/CI cannot complete this flow — switch to PAT or key-pair. Data + validation and test generation do not support Authorization Code (`CNX0037`). + ## Test Runner Errors **Symptoms:** `scai test capture` or `scai test validate` fails. diff --git a/plugin/skills/migration/migrate-objects/rule-engine/apply/BATCH.md b/plugin/skills/migration/migrate-objects/rule-engine/apply/BATCH.md index 847f01c..b9b5e8b 100644 --- a/plugin/skills/migration/migrate-objects/rule-engine/apply/BATCH.md +++ b/plugin/skills/migration/migrate-objects/rule-engine/apply/BATCH.md @@ -62,5 +62,5 @@ After each successful application: > scai test validate -c \ > --where "source.canonicalName IN ()" > ``` -> Read `/test-results/results.json` to confirm; the state machine picks up testing status when the agent calls `transition_status(status='advance', task='runTests', outcome=...)` per object. +> Confirm in `.VALIDATION.LATEST`; the state machine picks up testing status from that table when the agent calls `transition_status(status='advance', task='runTests', outcome=...)` per object. > - **For any failures, run the full diagnose/fix loop per object** → [../../migrate-object/SKILL.md](../../migrate-object/SKILL.md) diff --git a/plugin/skills/migration/register-code-units/extract-code-units/SKILL.md b/plugin/skills/migration/register-code-units/extract-code-units/SKILL.md index df0b1b5..a3ed0bf 100644 --- a/plugin/skills/migration/register-code-units/extract-code-units/SKILL.md +++ b/plugin/skills/migration/register-code-units/extract-code-units/SKILL.md @@ -45,6 +45,9 @@ Ask the user via `ask_user_question` (`multiSelect = false`): Allow combining options (e.g. specific types within a specific schema). +If the source server hosts several databases and the user wants more than the connection's +own, also ask **which databases** — see the multi-database flow in Step 2. + > **Name matching:** `-n` / `--name` is an **exact** case-insensitive match unless the > pattern contains `*`. Do **not** pass a bare prefix expecting substring match — > that used to pull sibling objects (e.g. `FOO` matching `FOO_SWTCH`). For partial @@ -61,12 +64,39 @@ Build the `scai code extract` command based on user selections: scai code extract -s --json # Add flags based on user choices: +# -d, --database extract from a specific database on the server, +# overriding the connection's own database # --schema filter by schema # -t TYPE1,TYPE2 filter by object type # -n "Name" exact name (case-insensitive); use * for wildcards # --driver-path path to driver .nupkg (Oracle only, first use) ``` +**Multiple databases on one server (SqlServer, AzureSynapse, Redshift, Postgresql, Teradata).** +A source connection points at one database, but its server may host several. To migrate more +than one, extract each with `-d ` — the run is repointed at that database without a separate +connection. Do **not** re-run without `-d`; that only re-pulls the connection's own database. +(Oracle and BigQuery don't bind a swappable database in the connection, so `-d` is rejected there — +use the connection's own database / schemas.) + +1. **List the databases** on the connection's server with `query_source` (dialect-specific): + - **SqlServer / AzureSynapse:** `SELECT name FROM sys.databases WHERE database_id > 4` (excludes the + `master`/`tempdb`/`model`/`msdb` system databases). + - **Redshift / Postgresql:** `SELECT datname FROM pg_database WHERE datistemplate = false`. + - **Teradata:** `SELECT DatabaseName FROM DBC.DatabasesV WHERE DBKind = 'D'`. +2. **Ask which to extract** via `ask_user_question` (`multiSelect = true`), defaulting to the + connection's own database. +3. **Loop the extraction**, once per selected database: + +```bash +scai code extract -s --database Sales --json +scai code extract -s --database Inventory --json +``` + +Output nests each under `source//…`, and the Code Unit Registry accumulates every +database (each run re-scans the whole `source/` tree), so all coexist in one project. Omit +`--database` when the connection's own database is the only target. + **Driver note (Oracle / Teradata):** `configure()` seeds the driver cache, so `--driver-path` is normally not needed here. If the cache was missed for any reason, pass `--driver-path ` on first use; SCAI persists the path machine-wide and reuses it across projects. **Examples:** @@ -74,6 +104,9 @@ scai code extract -s --json # All objects scai code extract -s --json +# All objects from a specific database on the server +scai code extract -s --database Sales --json + # Only tables and views in the dbo schema scai code extract -s --schema dbo -t TABLE,VIEW --json @@ -151,7 +184,7 @@ Confirm with user: ## On Completion -After the CHECKPOINT passes, tell the user. Fill placeholders from the JSON envelope returned by `scai code extract --json` (`catalog.{discovered,extracted,failed}`, `byType`, `failures[]`, `executionTimeSeconds`). +After the CHECKPOINT passes, tell the user. Fill placeholders from the JSON envelope returned by `scai code extract --json` (`catalog.{discovered,extracted,failed}`, `byType`, `failures[]`, `executionTimeSeconds`). When you extracted several databases, sum the counts across each run's envelope and report per-database. > **Extraction complete.** `/` objects extracted in ``, broken down by type (filled from `byType`). Files saved under `source/`. > *If `failed > 0`:* `` failed. Most common error: ``. Full list in the reports. diff --git a/plugin/skills/migration/sas/INTEGRATION.md b/plugin/skills/migration/sas/INTEGRATION.md new file mode 100644 index 0000000..f608686 --- /dev/null +++ b/plugin/skills/migration/sas/INTEGRATION.md @@ -0,0 +1,62 @@ +# SAS domain — integration wiring + +How the SAS parallel track connects to the AIM migration plugin, what to verify, and what not to change. + +## Wiring table + +| Component | Location | Role | +|-----------|----------|------| +| **Router** | `plugin/skills/migration/sas/SKILL.md` | Classifies SAS intent; loads one child skill | +| **Skill-match (router)** | `plugin/skills/migration/SKILL.md` | SAS section under the migration `` — natural-language SAS prompts route here | +| **Structural test** | `ai/crates/mcp-server/tests/sas_skill_integration_check.rs` | Pins required files; asserts SAS is **present** in the migration `` and that `plugin/commands/` stays absent | +| **Root packaging test** | `ai/crates/mcp-server/tests/plugin_skill_root_check.rs` | Ensures single top-level skill root | + +Discovery is **natural-language routed**. The SAS track is registered in the migration `` in `plugin/skills/migration/SKILL.md`, so SAS prompts route here the same way the other migration skills do. There are **no dedicated SAS slash commands** (`plugin/commands/` was removed in SNOW-3973489). Per SNOW-3915306, PMs promoted the Preview track out of the earlier explicit-invoke-only scope; `sas_skill_integration_check` now guards that SAS stays in skill-match and that the commands tree stays gone. + +## Do not touch (unless a separate design revisits AIM integration) + +| Area | Constraint | +|------|------------| +| Migration `` | SAS entries live here (natural-language routed). Edit only `plugin/skills/migration/SKILL.md` (`sas_skill_integration_check` enforces the SAS paths stay listed) | +| `plugin/.cortex-plugin/plugin.json` `skills` array | Must remain `["skills/migration"]` only — no second top-level SAS root | +| `ai/crates/mcp-server/data/machines/*.json` | Do not add SAS tasks to MCP state machines | +| `plugin_skill_root_check` expected set | `plugin/skills/` must contain only `migration/` | +| SnowConvert dialect registry | SAS is not a SnowConvert source dialect in v1 | + +Adding SAS to AIM claims, waves, `OBJECT_CLAIMS`, or the migration `` would require a separate product design — not this parallel track. + +## Verification commands + +Run from the repo root: + +```bash +cd ai/crates/mcp-server && SKIP_DASHBOARD_BUILD=1 cargo test --test sas_skill_integration_check +cd ai/crates/mcp-server && SKIP_DASHBOARD_BUILD=1 cargo test --test plugin_skill_root_check +``` + +`sas_skill_integration_check` asserts: + +- Router + three child `SKILL.md`s + validate sub-skill exist +- `README.md` and `INTEGRATION.md` exist +- The SAS skill files resolve on disk +- SAS is **present** in the migration `` (natural-language routed) +- `plugin/commands/` does not exist + +## Manual invocation checklist + +The SAS track is reached by **natural-language routing** through the migration ``. There are no dedicated SAS slash commands. + +### Should route to SAS + +| Prompt | Expected behavior | +|--------|-------------------| +| "assess this SAS portfolio" | Routes to `sas/assess-sas-migration/SKILL.md` | +| "convert MyJob.sas to Snowflake" | Routes to `sas/convert-sas-to-snowflake/SKILL.md` | +| "load these `.sas7bdat` files from a stage" | Routes to `sas/migrate-sas7bdat-to-snowflake/SKILL.md` | +| "migrate SAS" (programs vs datasets ambiguous) | Routes to the `sas/SKILL.md` router, which asks: programs (`.sas`) vs datasets (`.sas7bdat`) | + +### Should NOT route to SAS + +| Prompt | Expected behavior | +|--------|-------------------| +| "assess my SQL Server migration" | AIM migration workflow (SnowConvert / registry) — unaffected | diff --git a/plugin/skills/migration/sas/README.md b/plugin/skills/migration/sas/README.md new file mode 100644 index 0000000..0eabbb7 --- /dev/null +++ b/plugin/skills/migration/sas/README.md @@ -0,0 +1,55 @@ +# SAS → Snowflake (parallel domain) + +> **Status: Preview.** This capability ships as a Preview and is discoverable by the migration router — natural-language SAS requests (e.g. "assess this SAS portfolio", "convert MyJob.sas to Snowflake") route here via the migration ``. It has no dedicated slash commands. + +This folder is the **SAS parallel track** inside the AIM migration plugin. It hosts vendored skills for assessing SAS portfolios, converting `.sas` programs to Snowflake, and bulk-loading `.sas7bdat` datasets from a stage. + +Unlike the main AIM workflow, this track does **not** use SnowConvert, the object registry, MCP state machines, or wave claims. The agent routes SAS-specific requests here and follows the child skill end-to-end without requiring `configure` or an AIM project. + +## Skills + +| Skill | Path | Purpose | +|-------|------|---------| +| **sas** (router) | `SKILL.md` | Classifies intent and loads exactly one child skill | +| **assess-sas-migration** | `assess-sas-migration/SKILL.md` | Portfolio analysis — complexity, volume, dependencies, wave plan, LOE | +| **convert-sas-to-snowflake** | `convert-sas-to-snowflake/SKILL.md` | Convert `.sas` programs (DATA steps, PROC, macros) to Snowflake SQL / stored procedures | +| **validate-sas-conversion** | `convert-sas-to-snowflake/validate-sas-conversion/SKILL.md` | Validate an existing SAS conversion (sub-skill of convert) | +| **migrate-sas7bdat-to-snowflake** | `migrate-sas7bdat-to-snowflake/SKILL.md` | Bulk-load `.sas7bdat` files from a Snowflake stage into tables | + +## Recommended flow + +``` +Assess → Convert +``` + +1. **Assess** produces portfolio-level analysis (`assessment.json`, reports, wave plan). +2. **Convert** reuses `assessment.json` when present to prioritize and inform conversion. + +**Migrate-data** (`.sas7bdat` load) is **independent** — it does not require Assess or Convert and can run on its own when the user only needs dataset ingestion from a stage. + +## Provenance + +Vendored from an internal Snowflake SAS-to-Snowflake skill repository on **2026-08-10**. The three primary skill trees (`assess-sas-migration/`, `convert-sas-to-snowflake/`, `migrate-sas7bdat-to-snowflake/`) were copied into this nested domain under `plugin/skills/migration/sas/`. + +Upstream `.snowflake/cortex/plans/` and generated test output were **not** vendored. + +## Non-goals + +This domain deliberately does **not**: + +- Register SAS as a SnowConvert source dialect +- Add SAS tasks to MCP state machines or `OBJECT_CLAIMS` +- Require an AIM project, `configure`, or `migration_status` before starting +- Add a second top-level skill root in `plugin.json` (packaging stays `skills/migration` only) + +See [INTEGRATION.md](./INTEGRATION.md) for wiring details and verification steps. + +## How to extend + +To add a new SAS capability: + +1. Create `sas//SKILL.md` (+ references, scripts, assets as needed). +2. Add a routing row to `sas/SKILL.md` intent table. +3. Add a skill-match bullet under the SAS section in `plugin/skills/migration/SKILL.md`. +4. Update the required path list in `ai/crates/mcp-server/tests/sas_skill_integration_check.rs`. +5. Update this README and `INTEGRATION.md`. diff --git a/plugin/skills/migration/sas/SKILL.md b/plugin/skills/migration/sas/SKILL.md new file mode 100644 index 0000000..fededd7 --- /dev/null +++ b/plugin/skills/migration/sas/SKILL.md @@ -0,0 +1,36 @@ +--- +name: sas +description: Preview. SAS to Snowflake parallel migration track — assess portfolios, convert .sas programs, or load .sas7bdat from a stage. Not SnowConvert / AIM registry. Triggers: SAS, sas migration, convert SAS, assess SAS, sas7bdat, PROC SQL, DATA step, load SAS datasets. +parent_skill: migration +license: Proprietary. See License-Skills for complete terms +--- + +# SAS → Snowflake (parallel track) + +Tell the user: +> **SAS migration track (Preview)** — assessment, code conversion, or `.sas7bdat` load. This path does not use SnowConvert or the AIM object registry. + +Do **not** call `configure` or `migration_status` as a prerequisite. + +## Intent routing + +| User intent | Load | +|-------------|------| +| Assess / size / complexity / volume / LOE / waves / readiness | `./assess-sas-migration/SKILL.md` | +| Convert / translate / migrate SAS **programs** (DATA step, PROC, macros) | `./convert-sas-to-snowflake/SKILL.md` | +| Load / ingest **`.sas7bdat`** from a Snowflake stage | `./migrate-sas7bdat-to-snowflake/SKILL.md` | +| Validate an **existing** SAS conversion | `./convert-sas-to-snowflake/validate-sas-conversion/SKILL.md` | + +If the request is only "migrate SAS" and does not distinguish code vs datasets, ask: + +> Do you want to (1) convert SAS **programs** (`.sas`), or (2) load SAS **datasets** (`.sas7bdat`) from a stage? + +**Wait for the user's response — do not proceed until they choose.** + +Then load exactly one child skill above and follow it end-to-end. + +## Notes + +- Recommended code flow: Assess → Convert (Convert reuses `assessment.json` when present). +- `.sas7bdat` load is independent of Assess/Convert. +- Snowflake credit-using steps stay gated by the child skill's consent rules. diff --git a/plugin/skills/migration/sas/assess-sas-migration/SKILL.md b/plugin/skills/migration/sas/assess-sas-migration/SKILL.md new file mode 100644 index 0000000..64ef8fe --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/SKILL.md @@ -0,0 +1,284 @@ +--- +name: assess-sas-migration +parent_skill: sas +description: "Preview. Assess SAS migration complexity and volume for Snowflake. Produces portfolio analysis with tier distribution, complexity heatmap, dependency DAG, a migration wave plan, and (on request) a separate effort/staffing plan. Triggers: assess SAS, SAS assessment, migration complexity, SAS analysis, SAS dependency diagram, migration waves, migration phases, migration sizing, LOE estimate SAS, SAS volume analysis, SAS migration readiness." +license: Proprietary. See License-Skills for complete terms +--- + +# Assess SAS Migration + +> © Snowflake Inc. This skill and its contents are the proprietary intellectual property of Snowflake Inc. + +Analysis-only tool for assessing SAS-to-Snowflake migration complexity, volume, and sequencing. + +**Purpose:** Pre-conversion assessment that informs the `convert-sas-to-snowflake` skill with portfolio-level insights, per-file tier classification, and a migration wave plan. + +**Output:** A merged assessment report (`assessment_complete.md`) covering executive +summary, CLI quantitative detail, dependency DAG, complexity analysis, a migration wave plan, +open questions & dependencies, and recommendations — backed by the CLI's JSON/Markdown/Mermaid +artifacts. On request, a separate `effort_staffing_plan.md` adds effort + staffing and embeds the +same wave plan. + +--- + +## Architecture + +``` +User provides SAS files + │ + ▼ +┌─────────────────────────────┐ +│ Python CLI (assess_sas.py) │ ← Standalone, no Snowflake needed +│ - Parse .sas files │ +│ - Score complexity │ +│ - Classify tiers │ +│ - Build dependency graph │ +└──────────────┬──────────────┘ + │ Produces: + ▼ + assessment.json + assessment_report.md + assessment_report.html + dependency_dag.mmd + │ + ▼ +┌─────────────────────────────┐ +│ This Skill (CoCo layer) │ ← Adds judgment + qualitative analysis +│ - Complexity analysis │ (reads .sas source directly) +│ - Migration wave plan │ +│ - Effort estimation (opt-in)│ +│ - Staffing model (opt-in) │ +│ - Executive summary │ +└─────────────────────────────┘ +``` + +**Two analysis layers:** +1. **Python CLI (default, quantitative):** parses, scores, classifies, builds the DAG. Fast and deterministic. +2. **CoCo-native (additive, qualitative):** the skill reads the `.sas` files directly to explain *areas of complexity* — themes, construct hotspots, and risks — on top of the CLI numbers. If the CLI cannot run (no Python, or user opts out), this layer runs **standalone** using the heuristics in `references/complexity-analysis.md`. + +--- + +## Workflow + +### Step 1: Gather Input + +Ask user for: +1. SAS source path (file, directory, or already-generated `assessment.json`) +2. Output location + +**Default path:** Run the Python CLI for quantitative metrics, then add the CoCo-native +complexity layer (Step 4) on top. + +Branch on what's available: +- **`assessment.json` already exists** → skip the CLI, present it (Step 3), then add CoCo analysis (Step 4). +- **SAS files provided + Python available** → run the CLI (Step 2), present (Step 3), add CoCo analysis (Step 4). +- **Python unavailable or user opts out** → skip Steps 2-3; run Step 4 in **standalone mode** (CoCo reads `.sas` directly and produces both distributions and the complexity narrative). + +### Step 2: Run Assessment Tool + +Execute the standalone Python CLI: + +```bash +cd /tool +python assess_sas.py --output +``` + +**Expected output:** +- `/assessment.json` — structured metrics +- `/assessment_report.md` — human-readable report +- `/assessment_report.html` — self-contained, SCAI-themed HTML report (KPIs, tier mix, complexity/volume charts, dependency DAG, per-file detail). Open in a browser; share as the visual deliverable. +- `/dependency_dag.mmd` — Mermaid dependency graph + +Use `--format html` to emit only the HTML report, or `--format all` (default) for every artifact. + +If the tool fails, check: +- Python >= 3.8 available +- Source path contains `.sas` files +- Output directory is writable + +### Step 3: Present Assessment Results + +Read `assessment.json` and present: + +> **Two independent axes** (see `../references/block-tiering-spec.md`): +> **Translation tier** (Tier 1/2/3) is *how* each block migrates (SQL / stored proc / notebook) +> and is computed identically to the `convert-sas-to-snowflake` skill — a file is Tier 3 if it +> has any Tier-3 block, else Tier 2 if any Tier-2 block, else Tier 1. **Complexity** +> (LOW/MEDIUM/HIGH) is a separate volume/effort score used only for sizing and wave planning. A +> file can be Tier 1 yet HIGH complexity, or Tier 3 yet LOW complexity. Do not conflate them. + +1. **Executive Summary** (4 KPIs): + - Total files / Total lines + - Tier distribution (% Tier 1 / 2 / 3) + - Complexity split (% LOW / MEDIUM / HIGH) + +2. **Complexity x Volume Heatmap** (3x3 matrix) + +3. **Tier Distribution** with approach descriptions + +4. **Top Complex Files** (highest scoring, most likely to need manual review) + +5. **Dependency DAG** (render Mermaid or describe topology) + +### Step 4: CoCo-Native Complexity Analysis (additive) + +After the CLI metrics are presented, **read the `.sas` source files directly** to add a +qualitative complexity narrative. This is the layer that explains *why* files are complex and +*what* will need attention — the numbers alone don't tell that story. + +**Load `references/complexity-analysis.md`** for the construct catalog and heuristics. Keep +these aligned with the CLI so the narrative does not contradict `assessment.json`. + +Produce a **portfolio-level summary** (not a per-file or per-block dump): + +1. **Areas of Complexity (themes):** aggregate construct findings across files — report the + theme, count of files affected, and migration implication (e.g. "Statistical modeling in 4 files → PySpark"). +2. **Construct hotspots:** short table of the most impactful construct categories (HASH, + CALL EXECUTE, stat PROCs, multi-OUTPUT, INTNX/INTCK) with file counts and target tier. +3. **Top complex files:** 5-10 highest-complexity files, one line each on the dominant driver. + Rank by CLI `overall_score` when available; otherwise by Tier 3/2 construct density × volume. +4. **Boilerplate note:** flag DI Studio / DataFlow scaffolding as high-volume / low-complexity. +5. **Cross-cutting risks:** external DB dependencies, dynamic %INCLUDE, statistical modeling, + circular cross-file dependencies. + +Use the **Complexity Analysis Section** in `templates/assessment-report.md` for formatting. + +**Standalone mode (CLI not run):** If the Python tool was not run (no Python available, or +the user is working purely through CoCo), generate this entire analysis from direct source +reading — including the Executive Summary tier/complexity distributions derived from the +heuristics. Label the report **"CoCo-native estimate (CLI not run)"** so consumers know the +counts are LLM-derived, not tool-computed. + +### Step 5: Migration Wave Planning + +**Load `references/wave-planning.md`.** + +Always runs — the wave plan is core to the assessment report (Part 5). Break the portfolio into +**dependency-aware blocks of scripts** and sequence them into **migration waves**. NO effort, +staffing, or durations at this step. + +Produce: +- A wave table (Wave · Scripts · Tier mix · Rationale · Entry gate) +- The pilot wave (Wave 0) script selection with rationale +- Qualitative wave gates (no calendar time) + +Keep dependency clusters intact, isolate externally-blocked scripts into a later gated wave, and +sequence the rest Tier 1 → Tier 2 → Tier 3. This wave plan is generated **once** and reused +verbatim in both Part 5 of the report and (if requested) the effort & staffing file. + +### Step 5b: Effort & Staffing (opt-in — separate deliverable) + +Run **only if** the user opted in at the Step 4 stopping point. **Load `references/sizing-model.md`.** + +Apply per-file effort based on: +- Tier: Tier 1 = base effort, Tier 2 = moderate, Tier 3 = highest +- Confidence: LOW confidence = multiplier +- Volume: HIGH volume files take longer + +Produce (for the separate `effort_staffing_plan.md`, NOT the main report): +- Total estimated effort (hours) and effort by tier +- Suggested team composition (staffing) +- The **same** Migration Wave Plan from Step 5, embedded verbatim + +This becomes the `effort_staffing_plan.md` file written in Step 8. + +### Step 6: Open Questions & Dependencies to Clarify + +Identify everything the customer must answer or provide before/early in the migration, with a +focus on **source availability and access gaps** — anything that would block conversion, +compilation, or validation if unresolved. Derive these from: +- The DAG's external inputs (`assessment.json` external refs / `dependency_dag.mmd`) +- External DB LIBNAMEs (Oracle/DB2/Teradata) and their source tables +- `%INCLUDE` references whose source is not in scope +- Control/reference tables of unknown provenance (static seed vs runtime-built) +- Unknown source schemas, row counts, and refresh cadence (affects orchestration) + +Use the **Part 6** format in `templates/assessment-report.md`: an Open Questions table +(question · why it matters · what it blocks · owner), a Missing/Unverified Inputs table, and a +Resolution Priority (P1 blocks pilot wave / P2 blocks later waves / P3 clarification). If nothing +is outstanding, state "No open dependencies — all sources available in scope." + +### Step 7: Recommendations + +Based on assessment data, provide: + +1. **Migration approach recommendation:** + - If >80% Tier 1: "Highly automatable — bulk conversion in early waves" + - If >20% Tier 3: "High complexity — pilot with complex scripts first, defer the rest to a later wave" + - If many cross-file dependencies: "Wave-based migration by dependency cluster" + +2. **Risk areas:** + - Files with LOW confidence + - External database dependencies + - Statistical PROCs requiring PySpark + +3. **Wave recommendation:** + - Reference the Migration Wave Plan from Step 5 (Wave 0 pilot → independent Tier 1 → procedural + Tier 2 → complex/external Tier 3). Do not restate effort or staffing here. + +4. **Handoff to conversion skill:** + - "Ask to convert the SAS programs to Snowflake to begin conversion" (routes to `convert-sas-to-snowflake`) + - The conversion skill can consume `assessment.json` at Step 3 to skip re-classification + +### Step 8: Assemble Final Merged Report + +Write the complete assessment to `/assessment_complete.md` using the **merged +multi-part format** in `templates/assessment-report.md`: + +1. Part 1 — Executive Summary +2. Part 2 — CLI Quantitative Detail (block types, top functions, per-file table) +3. Part 3 — Dependency DAG (topology + fan-in/fan-out + external deps) +4. Part 4 — Complexity Analysis (themes, hotspots, top complex files, risks) +5. Part 5 — Migration Wave Plan (dependency-aware blocks + wave sequence; NO effort/staffing) +6. Part 6 — Open Questions & Dependencies to Clarify +7. Part 7 — Recommendations & Next Steps + +This single file is the primary deliverable. The CLI's raw artifacts (`assessment.json`, +`assessment_report.md`, `assessment_report.html`, `dependency_dag.mmd`) remain alongside it as +supporting detail — `assessment_report.html` is the shareable, SCAI-styled visual report. Point the +user to it as the browser-friendly counterpart to `assessment_complete.md`. + +**Effort & staffing file (opt-in):** If the user opted into effort & staffing at Step 4, ALSO +write `/effort_staffing_plan.md` using `templates/effort-staffing-report.md` — effort +by tier, staffing, and the **same** Migration Wave Plan embedded verbatim from Part 5. If the user +did not opt in, skip this file; the wave plan still appears in Part 5 of the main report. + +**Standalone mode:** When the CLI was not run, generate Parts 1-3 from direct source reading +and title the report "CoCo-native estimate (CLI not run)". + +--- + +## Stopping Points + +- ✋ Step 1: Confirm source path and output location +- ✋ Step 3: Present CLI assessment results — confirm before deeper analysis +- ✋ Step 4: Present complexity analysis — the Migration Wave Plan is always produced; ask whether to ALSO produce the separate effort & staffing file (`effort_staffing_plan.md`) +- ✋ Step 7: Present recommendations +- ✋ Step 8: Confirm before writing the final merged report (and the effort & staffing file if opted in) + +--- + +## Integration with convert-sas-to-snowflake + +The assessment output (`assessment.json`) can be consumed by the conversion skill: +- **Auto-discovered — no manual co-location needed.** At Step 3 the conversion skill searches common + locations (its own output dir, the SAS source dir, `assessment_output/`, and a shallow glob) for + `assessment.json` and consumes the most recent readable match. If none is found it proceeds + silently with fresh classification (no prompt). +- At Step 3 (Classify Each Block): pre-computed tier classifications are available +- At Step 4 (Present Block Analysis): dependency graph already built +- **Baseline reconciliation:** conversion compares its own per-file block counts and tiers against + the assessment and flags any mismatch (a signal the assessment is stale or a bug) — informational, + never blocking. Both skills share `../references/block-tiering-spec.md`, so a current assessment + should match exactly. +- Reduces conversion startup time for large portfolios + +--- + +## Python Tool Location + +The standalone CLI lives at: `assess-sas-migration/tool/assess_sas.py` + +**No Snowflake connection required.** The tool uses Python standard library only (no pip install needed). + +See `tool/README.md` for CLI usage, threshold tuning, and architecture details. diff --git a/plugin/skills/migration/sas/assess-sas-migration/references/complexity-analysis.md b/plugin/skills/migration/sas/assess-sas-migration/references/complexity-analysis.md new file mode 100644 index 0000000..f9f223d --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/references/complexity-analysis.md @@ -0,0 +1,82 @@ +# Complexity Analysis (CoCo-Native) + +Heuristics for the LLM to read `.sas` source directly and produce a **portfolio-level** +complexity narrative. This layer is qualitative — it explains *why* files are complex and +*what* will need attention, complementing the quantitative metrics from the Python CLI. + +Keep these heuristics aligned with the CLI (`scorer.py`, `classifier.py`) so the narrative +does not contradict the numbers. When the CLI has already run, use `assessment.json` for the +counts and use this reference only to add the qualitative "areas of complexity" interpretation. + +--- + +## Construct Catalog + +Scan each file for the constructs below. Group findings into **themes** at the portfolio +level (e.g. "Statistical modeling appears in 4 files") rather than listing every occurrence. + +### Tier 3 signals (no SQL equivalent — PySpark/SCOS) + +| Construct | What to grep for | Why it's complex | +|-----------|------------------|------------------| +| HASH objects | `declare hash`, `hash(` | No SQL equivalent; in-memory key lookup | +| Dynamic code gen | `call execute` | Runtime-generated SAS; needs procedural rewrite | +| Statistical PROCs | `proc reg`, `proc glm`, `proc logistic`, `proc cluster`, `proc factor`, `proc phreg`, `proc lifetest`, `proc mixed`, `proc genmod`, `proc nlmixed` | Requires ML/stats libraries, not SQL | +| Stateful DO loops | `do until` / `do while` + `symput` or many `call ` | External state mutation across iterations | + +### Tier 2 signals (procedural — Stored Procedure) + +| Construct | What to grep for | Why it's complex | +|-----------|------------------|------------------| +| RETAIN w/ reset | `retain ` + `first.` + (`= 0` or `= .`) | Running totals with conditional reset | +| FIRST./LAST. + multi-OUTPUT | `first.`/`last.` + 2+ `output ` | BY-group splitting into multiple datasets | +| Multiple OUTPUT datasets | 2+ `output ` (not `output;`) | One DATA step writing several tables | +| Complex branching | >5 `if `/`when ` **in a DATA step** (not PROC SQL CASE WHEN) | Dense procedural conditional logic | +| Sequential DML | 3+ of `delete `/`insert `/`update ` | Multi-statement procedural mutation | + +### Tier 1 advanced (SQL-translatable, but non-trivial) + +`retain `, `array `, `merge `, `first.`/`last.`, `%do `, `%if `, `infile `, `ods ` — +translatable to window functions / CTEs but worth calling out as complexity drivers. + +### Confidence reducers (raise risk even within Tier 1) + +| Signal | What to grep for | Effect | +|--------|------------------|--------| +| Nested/heavy macros | >2 `%macro` | LOW confidence | +| Dynamic %INCLUDE | `%include` + `&` | LOW confidence (resolved at runtime) | +| External DB engines | `oracle`, `teradata`, `db2`, `sqlsvr`, `odbc`, `oledb` | LOW confidence; passthrough/function mapping | +| Date interval funcs | `intck`, `intnx` | MEDIUM confidence; SAS vs Snowflake alignment differs | +| NOTSORTED BY | `notsorted` | MEDIUM confidence; ordering assumptions | + +--- + +## Boilerplate Discount + +DI Studio / DataFlow-generated files contain large volumes of scaffolding. Do NOT treat these +as genuine complexity. Indicators (3+ present ⇒ file is boilerplate-heavy): +`etls_`, `sas data integration studio`, `%macro rcset`, `perfinit`, `log4sas`, `armsubsys`. + +Boilerplate macro names to ignore when counting business logic: `rcset`, `rcsetds`, +`etls_startperformancestats`, `etls_setdebug`, `etls_recordcount`, `etls_endperformancestats`, +`etls_recordtable`, `etls_getrecordcount`, `etls_jobstatus`, `etls_logerror`. + +When a file is boilerplate-heavy, describe it as "high line count, low business complexity" +so the narrative matches the CLI's discounted score. + +--- + +## Producing the Portfolio Narrative + +1. **Identify themes**: aggregate construct findings across all files. Report the *theme*, + the *count of files* affected, and the *migration implication* — not a per-file dump. +2. **Top complex files**: name the 5-10 highest-complexity files (use CLI `overall_score` when + available; otherwise rank by Tier 3/2 construct density × volume). One line each on the + dominant complexity driver. +3. **Construct hotspots**: a short table of the most impactful construct categories present, + with file counts and the recommended target (SQL / SP / PySpark). +4. **Cross-cutting risks**: external DB dependencies, dynamic %INCLUDE, statistical modeling, + and any circular cross-file dependencies — these gate the migration approach. + +Keep it concise. This is a summary, not a per-block audit. If the user needs block-level +detail, that is the job of the `convert-sas-to-snowflake` skill. diff --git a/plugin/skills/migration/sas/assess-sas-migration/references/sizing-model.md b/plugin/skills/migration/sas/assess-sas-migration/references/sizing-model.md new file mode 100644 index 0000000..694f972 --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/references/sizing-model.md @@ -0,0 +1,72 @@ +# Sizing Model + +## Per-File Effort Estimation + +### Base Effort by Tier + +| Tier | Min Hours | Typical Hours | Max Hours | Description | +|------|-----------|---------------|-----------|-------------| +| Tier 1 (SQL) | 0.5 | 1.5 | 4 | Direct SQL translation — CTAS, CTEs, window functions | +| Tier 2 (Stored Proc) | 2 | 5 | 12 | Procedural logic — cursors, state management, multi-step | +| Tier 3 (PySpark/SCOS) | 4 | 10 | 20 | Complex patterns — HASH, CALL EXECUTE, statistical modeling | + +### Effort Multipliers + +| Factor | Multiplier | When Applied | +|--------|-----------|--------------| +| LOW confidence | 1.5x | File has nested macros, dynamic %INCLUDE, or external LIBNAME | +| HIGH volume (>1000 lines) | 1.3x | Large files take proportionally longer to review | +| External DB dependency | 1.2x | Oracle/DB2/SQL Server passthrough requires function mapping | +| Boilerplate (DI Studio) | 0.7x | Scaffolding code is mostly deletable | +| Cross-file dependency | 0.9x per dependent | Files in same cluster share context (batch discount) | + +### Formula + +``` +file_effort = base_hours[tier] × confidence_multiplier × volume_multiplier × external_multiplier × boilerplate_multiplier +``` + +Where `base_hours[tier]` uses the "Typical Hours" column by default. + +### Portfolio-Level Effort + +``` +total_effort = SUM(file_effort for each file) +overhead_factor = 1.15 # integration testing, orchestration setup, documentation +total_with_overhead = total_effort × overhead_factor +``` + +--- + +## Staffing Model + +### Team Composition by Portfolio Size + +| Portfolio Size | SE (Snowflake) | Developer | QA | Duration | +|---------------|----------------|-----------|-----|----------| +| Small (≤20 files) | 0.25 FTE | 1 FTE | 0.25 FTE | 2-4 weeks | +| Medium (21-100 files) | 0.5 FTE | 2-3 FTE | 0.5 FTE | 6-12 weeks | +| Large (100+ files) | 0.5 FTE | 3-5 FTE | 1 FTE | 12-24 weeks | + +### Timeline Formula + +``` +calendar_weeks = total_effort_hours / (developer_count × 30 hours/week × 0.8 utilization) +``` + +The 0.8 utilization accounts for meetings, context switching, and review cycles. + +> **Phasing/sequencing lives in `references/wave-planning.md`.** This file covers effort and +> staffing only. To map effort onto the migration timeline, apply the wave plan from +> `wave-planning.md` — both deliverables share the same waves. + +--- + +## Calibration Notes + +These estimates are starting points derived from 50+ SAS migration engagements. Adjust based on: + +1. **Pilot results**: After the pilot wave (see `wave-planning.md`), compare actual hours vs estimated — derive a site-specific correction factor +2. **Team expertise**: If the team has prior SAS experience, apply 0.8x multiplier +3. **Code quality**: If SAS code is well-documented with clear data flow, apply 0.9x +4. **Testing requirements**: If full regression testing required (not just compilation), add 30% to effort diff --git a/plugin/skills/migration/sas/assess-sas-migration/references/threshold-calibration.md b/plugin/skills/migration/sas/assess-sas-migration/references/threshold-calibration.md new file mode 100644 index 0000000..44e30da --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/references/threshold-calibration.md @@ -0,0 +1,102 @@ +# Threshold Calibration + +> **Scope:** These thresholds tune only the **complexity** (LOW/MEDIUM/HIGH) and **volume** axes — +> a sizing/effort signal. They do NOT affect the **translation tier** (Tier 1/2/3). Tier is set by +> the strict "any-block" rule in `../../references/block-tiering-spec.md` (any Tier-3 block → Tier 3, +> else any Tier-2 → Tier 2, else Tier 1) and is not calibratable. + +## Current Defaults + +### Complexity Thresholds + +| Level | Score Range | Meaning | +|-------|-------------|---------| +| LOW | 0 - 50 | Straightforward SQL migration, high automation potential | +| MEDIUM | 51 - 150 | Moderate complexity, may need stored procedures or manual review | +| HIGH | > 150 | Likely needs PySpark/SCOS, multiple stored procedures, or significant manual effort | + +### Volume Thresholds (Line Count) + +| Level | Lines | Meaning | +|-------|-------|---------| +| LOW | 0 - 250 | Small file, typically 1-3 blocks | +| MEDIUM | 251 - 1000 | Medium file, multiple logical steps | +| HIGH | > 1000 | Large file, often multi-phase pipeline or extensive macro library | + +--- + +## How to Calibrate + +### Step 1: Run Against Your Corpus + +```bash +python assess_sas.py /path/to/customer/sas/files --output ./calibration_run +``` + +### Step 2: Check Distribution + +Open `assessment.json` and examine `portfolio_summary.complexity_distribution` and `portfolio_summary.volume_distribution`. + +**Target distribution** (empirically validated): +- Complexity: ~50% LOW, ~35% MEDIUM, ~15% HIGH +- Volume: ~50% LOW, ~35% MEDIUM, ~15% HIGH + +### Step 3: Adjust Thresholds + +If the distribution is off, create a custom `config.json`: + +**Too many HIGH files (>25%):** +```json +{ + "complexity_thresholds": { "low_max": 60, "medium_max": 180 }, + "volume_thresholds": { "low_max": 300, "medium_max": 1200 } +} +``` + +**Too many LOW files (>70%):** +```json +{ + "complexity_thresholds": { "low_max": 35, "medium_max": 100 }, + "volume_thresholds": { "low_max": 150, "medium_max": 600 } +} +``` + +### Step 4: Re-run + +```bash +python assess_sas.py /path/to/customer/sas/files --config config.json --output ./calibration_v2 +``` + +--- + +## Known Corpus Benchmarks + +| Corpus | Files | LOW% | MED% | HIGH% | Notes | +|--------|-------|------|------|-------|-------| +| Generic (1483 files) | 1483 | 62% | 37% | 0.1% | Mix of customers, default thresholds | +| DI Studio corpus (262 files) | 262 | 7% | 92% | 0% | All SAS DI Studio — boilerplate inflates MEDIUM | +| Training scripts (112 files) | 112 | ~60% | ~30% | ~10% | Representative sample | + +--- + +## Scoring Components Explained + +The total score is: `base_score + feature_score + structure_score` + +### Base Score +- Sum of block-type weights (PROC SQL=2, DATA STEP=2, MACRO=4, etc.) +- Log-damped above 50 to prevent runaway scores from many simple blocks + +### Feature Score +- Pattern-matched against SAS constructs that indicate translation difficulty +- Tier 3 patterns (HASH, CALL EXECUTE, statistical PROCs): +6-8 points each +- Tier 2 patterns (CALL SYMPUT, PROC TRANSPOSE): +3 points each +- Tier 1 advanced (RETAIN, ARRAY, MERGE, FIRST./LAST.): +1-3 points each +- Each pattern capped at 3 occurrences (prevents repetitive code from dominating) +- Boilerplate files get 50% discount on Tier 2/3 feature scores + +### Structure Score +- Macro count: +2 per macro (max 10) +- Nested macros: +3 each (max 5) +- Macro variables: +1 each (max 10) +- Boilerplate files: subtract 5 macros and 10 vars before scoring diff --git a/plugin/skills/migration/sas/assess-sas-migration/references/wave-planning.md b/plugin/skills/migration/sas/assess-sas-migration/references/wave-planning.md new file mode 100644 index 0000000..fc5904a --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/references/wave-planning.md @@ -0,0 +1,100 @@ +# Migration Wave Planning + +How to break a SAS portfolio into **blocks of scripts** and sequence them into **migration +waves**. This reference is purely about *what migrates together and in what order* — it contains +**no effort, no staffing, and no durations**. (Effort and staffing live in `sizing-model.md`.) + +The wave plan is the single source of truth for sequencing. It is generated **once** and embedded +verbatim in both the main report's "Part 5 — Migration Wave Plan" and the separate +`effort_staffing_plan.md` deliverable. + +--- + +## Wave Grouping Principles + +Assign every script to a wave using these rules, in priority order: + +1. **Keep dependency clusters intact.** Scripts that share intermediate tables or form a connected + sub-DAG migrate together, so each wave is independently testable and deployable. A cluster + moves to the wave of its latest-ready / highest-tier member. +2. **Isolate externally-blocked scripts.** Any script that reads an external DB source + (Oracle/DB2/Teradata), a dynamic `%INCLUDE`, or an unverified control table that is not yet + available in Snowflake goes into a later wave, gated on that source becoming available. +3. **Sequence by tier, then confidence.** Within what remains, migrate Tier 1 before Tier 2 before + Tier 3, and HIGH-confidence before LOW-confidence — front-loading fast, automatable wins. +4. **Pilot first.** Carve a small representative slice out of the above into a Wave 0 pilot to + validate the approach before bulk conversion. + +--- + +## Standard Wave Sequence + +| Wave | Scope | Purpose | +|------|-------|---------| +| Wave 0 — Pilot | 5-10 representative scripts (mix of tiers + 1 complex stress test + 1 cross-file dependency) | Validate the conversion approach and patterns end-to-end before scaling | +| Wave 1 — Independent Tier 1 | Tier 1 scripts with no external blockers, in self-contained clusters | High automation, fast throughput, low risk | +| Wave 2 — Procedural | Tier 2 stored-procedure scripts and their dependency clusters | Manual review of state/BY-group logic | +| Wave 3 — Complex / Externally-dependent | Tier 3, highest-complexity, and external-DB-dependent scripts | Hardest work last; gated on source availability | + +Waves are **logical groupings**, not a fixed count. A small portfolio may collapse to 2 waves; a +large one with several independent clusters may split Wave 1 into 1a/1b/1c so each cluster ships on +its own. Do not invent waves that have no scripts. + +--- + +## Per-Script Wave Assignment Procedure + +For each script, derive four attributes and bucket accordingly: + +1. `cluster_id` — the connected sub-DAG it belongs to (from the dependency graph). +2. `tier` — TIER_1 / TIER_2 / TIER_3. +3. `confidence` — HIGH / MEDIUM / LOW. +4. `external_blocked` — true if it reads an external/unavailable source. + +Assignment: +- If the script is in the curated pilot sample → **Wave 0**. +- Else if `external_blocked` → **final wave** (gated on source availability). +- Else assign by the cluster's highest tier: TIER_1 → Wave 1, TIER_2 → Wave 2, TIER_3 → Wave 3. +- Keep every member of a `cluster_id` in the **same** wave (use the latest wave any member requires). + +--- + +## Pilot Selection Criteria + +Select 5-10 scripts for Wave 0 that cover: +- At least 1 script from each tier present in the portfolio +- At least 1 HIGH complexity script +- At least 1 script with cross-file dependencies +- At least 1 script with external database references (if any) +- The most business-critical script (ask the customer) + +--- + +## Wave Gate Criteria (qualitative — no calendar time) + +| Transition | Gate | +|------------|------| +| Wave 0 → Wave 1 | Pilot scripts compile and pass validation; conversion patterns confirmed; no systemic issues | +| Wave 1 → Wave 2 | All Wave 1 terminal tables validate; Tier 1 throughput pattern is stable | +| Wave 2 → Wave 3 | Procedural scripts validated; external-source access for the final wave is confirmed available | + +Each gate is a readiness condition, not a date. A wave begins when the prior wave's terminal +outputs validate and its blocking dependencies are resolved. + +--- + +## Output Table Format (for the report) + +Render the wave plan as: + +```markdown +| Wave | Scripts | Tier mix | Rationale | Entry gate | +|------|---------|----------|-----------|------------| +| 0 — Pilot | {n}: {names or cluster} | {tier mix} | {why these first} | Start of engagement | +| 1 — Independent Tier 1 | {n}: {cluster} | Tier 1 | {independent, fast wins} | Pilot validated | +| 2 — Procedural | {n}: {cluster} | Tier 2 | {state/BY-group review} | Wave 1 terminals validate | +| 3 — Complex / External | {n}: {names} | Tier 2/3 | {gated on source X} | Wave 2 validated + source ready | +``` + +Tailor wave count and membership to the actual DAG and tier mix. Note any cluster that ships as its +own sub-wave, and call out the external source each gated wave depends on. diff --git a/plugin/skills/migration/sas/assess-sas-migration/templates/assessment-report.md b/plugin/skills/migration/sas/assess-sas-migration/templates/assessment-report.md new file mode 100644 index 0000000..3a92a70 --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/templates/assessment-report.md @@ -0,0 +1,286 @@ +# Assessment Report Template + +The skill renders this report when presenting results. It produces a single **merged report** +that combines the CLI's quantitative metrics (Parts 1-3) with the CoCo-native qualitative +analysis, migration wave plan, open questions, and recommendations (Parts 4-7). + +**Output file:** `/assessment_complete.md` (the merged report). The CLI's raw +artifacts (`assessment.json`, `assessment_report.md`, `dependency_dag.mmd`) live alongside it. +Effort and staffing are NOT in this report — when requested they go to a separate +`effort_staffing_plan.md` (see `templates/effort-staffing-report.md`), which embeds the same +Migration Wave Plan shown in Part 5 here. + +**Standalone mode (CLI not run):** Parts 1-3 are generated from direct source reading using +`references/complexity-analysis.md`. Title the report **"CoCo-native estimate (CLI not run)"** +and note that quantitative counts are LLM-derived. + +--- + +## Report Header + +```markdown +# SAS Migration Assessment — {portfolio_name} + +**Source:** `{source_path}` +**CLI outputs:** `assessment.json`, `assessment_report.md`, `dependency_dag.mmd` (same folder) +**Date:** {date} +``` + +--- + +## Part 1 — Executive Summary + +```markdown +# Part 1 — Executive Summary + +| Metric | Value | +|--------|-------| +| Total SAS Files | {total_files} | +| Total Lines of Code | {total_lines:,} | +| Business Blocks | {total_blocks} ({n} macro-def · {n} DATA step · {n} PROC SQL) | +| Recommended Migration Waves | {wave_count} | + +### Translation Tier Split + +| Tier | Files | % | Approach | +|------|-------|---|----------| +| Tier 1 (SQL) | {t1_count} | {t1_pct}% | Pure SQL — CTAS / CTEs / window functions | +| Tier 2 (Stored Proc) | {t2_count} | {t2_pct}% | Snowflake stored procedures | +| Tier 3 (PySpark) | {t3_count} | {t3_pct}% | PySpark via Snowpark Connect | + +### Complexity / Volume / Confidence + +| Complexity | Count | % | | Volume | Count | % | +|---|---|---|---|---|---|---| +| LOW | {n} | {pct}% | | LOW | {n} | {pct}% | +| MEDIUM | {n} | {pct}% | | MEDIUM | {n} | {pct}% | +| HIGH | {n} | {pct}% | | HIGH | {n} | {pct}% | + +- **Confidence:** {dist summary — e.g. MEDIUM 41 (100%), no LOW-confidence files} +- **Boilerplate:** {count flagged DI Studio / DataFlow-generated; note LOC inflation} + +### Complexity × Volume Matrix + +| | Low Volume | Medium Volume | High Volume | +|--|-----------|---------------|-------------| +| LOW Complexity | {n} | {n} | {n} | +| MEDIUM Complexity | {n} | {n} | {n} | +| HIGH Complexity | {n} | {n} | {n} | +``` + +--- + +## Part 2 — CLI Quantitative Detail + +> Populate from `assessment.json`. In standalone mode, derive from direct source scanning and +> label counts as estimates. + +```markdown +# Part 2 — CLI Quantitative Detail + +## Block Type Distribution + +| Block Type | Count | +|-----------|-------| +| DATA_STEP | {n} | +| PROC_SQL | {n} | +| MACRO_DEF | {n} | + +## Top SAS Functions Used + +| Function | Occurrences | | Function | Occurrences | +|----------|-------------|--|----------|-------------| +| {fn} | {n} | | {fn} | {n} | + +## Per-File Details + +| File | Lines | Blocks | Score | Complexity | Volume | Tier | Confidence | +|------|-------|--------|-------|------------|--------|------|------------| +| {filename} | {n} | {n} | {score} | {LOW/MED/HIGH} | {LOW/MED/HIGH} | {TIER_n} | {conf} | +``` + +--- + +## Part 3 — Dependency DAG + +```markdown +# Part 3 — Dependency DAG ({node_count} nodes / {edge_count} edges) + +{One-line topology description — e.g. "Fan-in funnel converging on the merge layer."} +Flow: `{stage1 → stage2 → ... → sink}`. +See `dependency_dag.mmd` for the full Mermaid diagram. External inputs: {count}. + +| Most depended-on (fan-in) | Depends-on-most (fan-out) | +|---|---| +| {file} ({n}) | {file} ({n}) | + +{Note the terminal sink / widest join if notable.} + +### External Dependencies (referenced, not created in scope) + +{Describe external inputs: macro variables, DICTIONARY tables, and — critically — real +external DB sources (Oracle/DB2/Teradata). Distinguish genuine external DB deps from +cross-file/control tables.} +``` + +--- + +## Part 4 — Complexity Analysis (CoCo-Native) + +> Qualitative layer from reading `.sas` source directly (see `references/complexity-analysis.md`). +> Portfolio-level themes and hotspots — NOT a per-block dump. + +```markdown +# Part 4 — Complexity Analysis + +### Construct Hotspots + +| Construct | Files | Recommended target | +|---|---|---| +| HASH objects (`declare hash`) | {n} | {SQL JOIN / PySpark} | +| CALL EXECUTE (dynamic loops) | {n} | {Stored proc / removable scaffolding} | +| PROC TRANSPOSE | {n} | CASE-WHEN pivot | +| PROC MEANS / SUMMARY (aggregation) | {n} | GROUP BY | +| INTNX / INTCK date logic | {n} occ | SQL date funcs (verify alignment) | +| External {engine} LIBNAME | {n} | Passthrough / pre-ingest to Snowflake | + +### Top Complex Files + +| File | Score | Tier | Dominant driver | +|---|---|---|---| +| {filename} | {score} | {T1/T2/T3} | {one-line driver} | + +### Boilerplate Note + +{e.g. "All N files are DataFlow-generated scaffolding (high line count, low business +complexity). After stripping M macro-def boilerplate blocks, ~K genuine business blocks remain."} + +### Cross-Cutting Risks + +1. {risk — e.g. single external dependency blocking DAG root} +2. {risk — e.g. CALL EXECUTE scaffolding, mostly deletable} +3. {risk — e.g. no statistical modeling → zero PySpark, de-risked} +4. {risk — e.g. uniform MEDIUM confidence drivers} +``` + +--- + +## Part 5 — Migration Wave Plan + +> Apply `references/wave-planning.md`. Break the portfolio into dependency-aware blocks of scripts +> and sequence them into migration waves. NO effort, staffing, or durations here — those (when +> requested) go to the separate `effort_staffing_plan.md`, which embeds this same wave plan. + +```markdown +# Part 5 — Migration Wave Plan + +{One-line sequencing rationale — e.g. "Phased by dependency cluster: independent Tier 1 first, +external-DB-dependent scripts last (gated on source availability)."} + +| Wave | Scripts | Tier mix | Rationale | Entry gate | +|------|---------|----------|-----------|------------| +| 0 — Pilot | {n}: {names or cluster} | {tier mix} | {representative slice to validate approach} | Start of engagement | +| 1 — Independent Tier 1 | {n}: {cluster} | Tier 1 | {self-contained, high automation} | Pilot validated | +| 2 — Procedural | {n}: {cluster} | Tier 2 | {state / BY-group review} | Wave 1 terminals validate | +| 3 — Complex / External | {n}: {names} | Tier 2/3 | {gated on external source X} | Wave 2 validated + source ready | + +### Pilot Wave (Wave 0) Scripts + +{List the 5-10 pilot scripts and why each was chosen — one from each tier, one HIGH complexity, +one cross-file dependency, one external-DB reference, the most business-critical.} + +### Wave Gates + +| Transition | Gate (qualitative — no calendar time) | +|------------|---------------------------------------| +| 0 → 1 | Pilot scripts compile and validate; patterns confirmed; no systemic issues | +| 1 → 2 | Wave 1 terminal tables validate; Tier 1 throughput stable | +| 2 → 3 | Procedural scripts validated; external-source access confirmed for the final wave | +``` + +> Tailor the wave count and membership to the actual DAG and tier mix. Collapse to fewer waves for +> small portfolios; split Wave 1 into per-cluster sub-waves (1a/1b/...) when independent clusters +> can ship separately. Do not invent empty waves. + +--- + +## Part 6 — Open Questions & Dependencies to Clarify + +> Items the customer must answer or provide before/early in the migration. Focus on +> **source availability and access gaps** — anything that would block conversion, compilation, +> or validation if unresolved. Derive these from the DAG's external inputs, missing %INCLUDE +> references, external DB LIBNAMEs, and unknown source schemas/volumes. + +```markdown +# Part 6 — Open Questions & Dependencies to Clarify + +### Source Availability & Access + +| # | Question / Dependency | Why it matters | Blocks | Owner | +|---|----------------------|----------------|--------|-------| +| 1 | {e.g. Can we get read access + connection details for the Oracle source database?} | {DAG root depends on it} | Pilot | Customer | +| 2 | {e.g. Are the N external source tables (`X`, `Y`...) available in Snowflake, or must they be ingested?} | {Compilation will fail without them} | Pilot | Customer | +| 3 | {e.g. %INCLUDE 'lib.sas' references — can the source be provided?} | {Logic unavailable; stub only} | Conversion | Customer | +| 4 | {e.g. What are the row counts / refresh cadence of the top source tables?} | {Warehouse sizing + orchestration cadence} | Orchestration | Customer | +| 5 | {e.g. Are control tables (`&ETLS_CONTROLTABLE`, GROUP_1..6) static reference data or runtime-populated?} | {Affects whether they're seeded or pipeline-built} | Conversion | Customer | + +### Missing / Unverified Inputs + +| Input | Type | Status | Action Needed | +|-------|------|--------|---------------| +| {table/file/macro var} | {External DB / %INCLUDE / control table} | {Not in scope / unverified} | {Provide / grant access / confirm} | + +### Resolution Priority + +| Priority | Item | Needed By | +|----------|------|-----------| +| P1 (blocks pilot) | {external DB access, root source tables} | Before Phase 1 | +| P2 (blocks batch) | {schemas, volumes, control-table semantics} | Before Phase 2 | +| P3 (clarification) | {refresh cadence, naming conventions} | During pilot | +``` + +--- + +## Part 7 — Recommendations & Next Steps + +```markdown +# Part 7 — Recommendations & Next Steps + +1. **{Migration approach}** — {e.g. phased by dependency cluster given fan-in funnel}. +2. **{Risk posture}** — {e.g. de-risked: 0 Tier-3, 0 statistical modeling, 0 LOW-confidence}. +3. **{Unblock first}** — {e.g. ingest the N external source tables before pilot}. + +### Pilot File Suggestions (representative mix) + +- {file} — Tier 1, baseline +- {file} — Tier 2, stored proc +- {file} — HIGH complexity, stress test +- {file} — cross-file dependency, integration test + +### Handoff to Conversion + +To begin conversion, ask to **convert the SAS programs to Snowflake** — the migration router +loads the `convert-sas-to-snowflake` skill. + +The conversion skill consumes `assessment.json` at Step 3 to skip re-classification, saving +time on large portfolios. +``` + +--- + +## Generation Rules + +1. **Always produce the merged report** as a single `assessment_complete.md`, using the part + ordering above. +2. **Parts 1-3 (quantitative):** populate from `assessment.json` when the CLI ran; otherwise + derive from direct source reading and label as a CoCo-native estimate. +3. **Part 4 (complexity):** portfolio-level themes and hotspots only — no per-block table. +4. **Part 5 (wave plan):** apply `references/wave-planning.md`; break scripts into dependency-aware + blocks and sequence into waves. NO effort/staffing in this report. When the user opted into + effort & staffing, also write `effort_staffing_plan.md` (per `templates/effort-staffing-report.md`) + embedding this same wave plan. +5. **Part 6 (open questions):** ALWAYS include. Focus on source & access gaps that would block + pilot/conversion/validation. If nothing is outstanding, state "No open dependencies — all + sources available in scope." +6. **Part 7 (recommendations):** tailor to the data (tier mix, DAG shape, external deps). +7. **Mermaid:** reference `dependency_dag.mmd`; do not inline large diagrams. diff --git a/plugin/skills/migration/sas/assess-sas-migration/templates/effort-staffing-report.md b/plugin/skills/migration/sas/assess-sas-migration/templates/effort-staffing-report.md new file mode 100644 index 0000000..e5766af --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/templates/effort-staffing-report.md @@ -0,0 +1,94 @@ +# Effort & Staffing Report Template + +Defines the **separate** deliverable `effort_staffing_plan.md`. This file holds the effort and +staffing estimates that used to live in the main report's Part 5, plus an embedded copy of the +**same Migration Wave Plan** so it is self-contained. + +**Output file:** `/effort_staffing_plan.md` (alongside `assessment_complete.md`). + +**When produced:** only when the user opts in at the Step 4 stopping point. The Migration Wave Plan +in the main report (Part 5) is always produced; this file additionally layers effort + staffing +onto that same plan. + +**Sourcing:** +- Effort + staffing → apply formulas in `references/sizing-model.md`. +- Migration Wave Plan section → embed the **identical** wave plan generated for the main report's + Part 5 (from `references/wave-planning.md`). Do not regenerate a different plan — the two must match. + +--- + +## Report Body + +```markdown +# Effort & Staffing Plan — {portfolio_name} + +**Companion to:** `assessment_complete.md` +**Source:** `{source_path}` +**Date:** {date} + +> Effort and staffing are estimates derived from tier/complexity heuristics, not commitments. +> Calibrate after the pilot wave. Sequencing follows the Migration Wave Plan below (identical to +> Part 5 of the assessment report). + +## Summary + +| Metric | Value | +|--------|-------| +| Total SAS Files | {total_files} | +| Calibrated Effort | {low}–{high} hr | +| Recommended Migration Waves | {wave_count} | + +## Effort by Tier + +| Tier | Files | Avg Effort/File | Total Hours | +|------|-------|-----------------|-------------| +| Tier 1 | {n} | {avg} hrs | {total} hrs | +| Tier 2 | {n} | {avg} hrs | {total} hrs | +| Tier 3 | {n} | {avg} hrs | {total} hrs | +| **Total (×1.15 overhead)** | **{n}** | | **{grand_total} hrs** | + +## Staffing + +| Role | Allocation | Duration | +|------|------------|----------| +| Snowflake SE (advisory) | {se_fte} FTE | {duration} | +| Developer(s) | {dev_count} FTE | {duration} | +| QA / Validation | {qa_fte} FTE | {duration} | + +> Timeline derived from `calendar_weeks = total_effort_hours / (developer_count × 30 × 0.8)` +> (see `references/sizing-model.md`). + +## Migration Wave Plan + +> EMBED the identical wave plan from the assessment report's Part 5 here, verbatim. +> See `references/wave-planning.md` for the format. + +| Wave | Scripts | Tier mix | Rationale | Entry gate | +|------|---------|----------|-----------|------------| +| 0 — Pilot | {n}: {names or cluster} | {tier mix} | {why these first} | Start of engagement | +| 1 — Independent Tier 1 | {n}: {cluster} | Tier 1 | {independent, fast wins} | Pilot validated | +| 2 — Procedural | {n}: {cluster} | Tier 2 | {state/BY-group review} | Wave 1 terminals validate | +| 3 — Complex / External | {n}: {names} | Tier 2/3 | {gated on source X} | Wave 2 validated + source ready | + +### Effort by Wave (optional roll-up) + +| Wave | Files | Effort (hrs) | +|------|-------|--------------| +| 0 — Pilot | {n} | {hrs} | +| 1 — Independent Tier 1 | {n} | {hrs} | +| 2 — Procedural | {n} | {hrs} | +| 3 — Complex / External | {n} | {hrs} | +``` + +--- + +## Generation Rules + +1. Produce this file **only** when the user opted into effort & staffing at the Step 4 stopping + point. Otherwise skip it silently — the wave plan still appears in the main report's Part 5. +2. Apply `references/sizing-model.md` for all effort and staffing figures; always show the ×1.15 + overhead total. +3. The "Migration Wave Plan" section MUST be identical to the main report's Part 5 — generate the + wave plan once and embed the same content in both files. +4. The optional "Effort by Wave" roll-up maps the per-tier effort onto the waves; include it only + if it adds clarity for the portfolio. diff --git a/plugin/skills/migration/sas/assess-sas-migration/tool/README.md b/plugin/skills/migration/sas/assess-sas-migration/tool/README.md new file mode 100644 index 0000000..b364b2b --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/tool/README.md @@ -0,0 +1,74 @@ +# SAS Migration Assessment Tool + +Standalone CLI for assessing SAS-to-Snowflake migration complexity and volume. + +## Usage + +```bash +cd assess-sas-migration/tool + +# Analyze a directory of .sas files +python assess_sas.py /path/to/sas/files --output ./results + +# Analyze a single file +python assess_sas.py /path/to/script.sas --output ./results + +# Use custom thresholds +python assess_sas.py /path/to/sas/ --config custom_config.json + +# JSON only +python assess_sas.py /path/to/sas/ --format json --output ./results + +# HTML report only +python assess_sas.py /path/to/sas/ --format html --output ./results +``` + +## Outputs + +| File | Description | +|------|-------------| +| `assessment.json` | Machine-readable per-file metrics, portfolio summary, dependency graph | +| `assessment_report.md` | Human-readable report with tables and distribution analysis | +| `assessment_report.html` | Self-contained, SCAI-themed HTML report (KPIs, tier mix, complexity/volume charts, dependency DAG, per-file detail). Open in a browser. | +| `dependency_dag.mmd` | Mermaid diagram showing cross-file data flow dependencies | + +`--format` accepts `json`, `md`, `html`, or `all` (default). The HTML report renders fully offline except the dependency diagram, which uses the Mermaid CDN when opened in a browser (an edges table is the offline fallback). + +## Threshold Tuning + +Edit `config.json` to adjust classification boundaries: + +```json +{ + "complexity_thresholds": { + "low_max": 50, + "medium_max": 150 + }, + "volume_thresholds": { + "low_max": 250, + "medium_max": 1000 + } +} +``` + +**Target distribution** (empirically validated against 1400+ real SAS files): +- ~50% LOW, ~35% MEDIUM, ~15% HIGH + +## Dependencies + +Python 3.8+ with standard library only. No pip install required. + +## How It Works + +1. **Parser** — Extracts typed blocks (DATA steps, PROC SQL, macros, etc.) from SAS source +2. **Scorer** — 3-component complexity score: base (block weights) + feature (tier-specific patterns) + structure (macros/nesting). Produces the **complexity** axis (LOW/MEDIUM/HIGH), used for sizing — independent of the translation tier. +3. **Classifier** — SQL-first **translation tier** assignment aligned with the conversion skill (Tier 1 SQL → Tier 2 SP → Tier 3 PySpark). A file's tier follows the strict "any-block" rule: any Tier-3 block → Tier 3, else any Tier-2 block → Tier 2, else Tier 1. No proportion thresholds. +4. **Dependency Tracker** — Builds cross-file DAG from dataset CREATES/READS +5. **Reporter** — Generates JSON, Markdown, and Mermaid outputs + +**Block counting & tiering are governed by the shared canonical spec** +`../references/block-tiering-spec.md`: macros are flattened (each inner DATA/PROC step counts as +a block), DI Studio / DataFlow boilerplate is excluded from counts and tiering, and all reported +counts (portfolio total, per-file, per-tier) are computed over the same block set so they +reconcile. This is the same logic the `convert-sas-to-snowflake` skill applies, so assessment and +conversion agree on block counts and tiers. diff --git a/plugin/skills/migration/sas/assess-sas-migration/tool/assess_sas.py b/plugin/skills/migration/sas/assess-sas-migration/tool/assess_sas.py new file mode 100644 index 0000000..c74a6ec --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/tool/assess_sas.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +""" +SAS Migration Assessment Tool + +Standalone CLI that analyzes SAS files and produces: + - assessment.json (structured metrics per file + portfolio) + - assessment_report.md (human-readable report) + - assessment_report.html (self-contained, SCAI-themed HTML report) + - dependency_dag.mmd (Mermaid diagram of cross-file dependencies) + +Usage: + python assess_sas.py /path/to/sas/files --output ./assessment_output + python assess_sas.py /path/to/single_file.sas --output ./results + python assess_sas.py /path/to/sas/ --config custom_config.json +""" + +import argparse +import json +import os +import sys +from pathlib import Path + +from sas_analyzer import SASParser, ComplexityScorer, TierClassifier, DependencyTracker, AssessmentReporter + + +def find_sas_files(source: str) -> list: + source_path = Path(source) + if source_path.is_file() and source_path.suffix.lower() == '.sas': + return [source_path] + elif source_path.is_dir(): + files = sorted(source_path.rglob('*.sas')) + return files + else: + print(f"Error: '{source}' is not a .sas file or directory.", file=sys.stderr) + sys.exit(1) + + +def load_config(config_path: str = None) -> dict: + default_config = { + 'complexity_thresholds': { + 'low_max': 50, + 'medium_max': 150, + }, + 'volume_thresholds': { + 'low_max': 250, + 'medium_max': 1000, + }, + } + if config_path and os.path.exists(config_path): + with open(config_path) as f: + user_config = json.load(f) + for key in default_config: + if key in user_config: + default_config[key].update(user_config[key]) + return default_config + + +def main(): + ap = argparse.ArgumentParser(description='SAS Migration Assessment Tool') + ap.add_argument('source', help='Path to .sas file or directory containing .sas files') + ap.add_argument('--output', '-o', default='./assessment_output', help='Output directory for results') + ap.add_argument('--config', '-c', help='Path to config.json with custom thresholds') + ap.add_argument('--format', choices=['json', 'md', 'html', 'all'], default='all', help='Output format') + args = ap.parse_args() + + config = load_config(args.config) + + sas_files = find_sas_files(args.source) + if not sas_files: + print("No .sas files found.", file=sys.stderr) + sys.exit(1) + + print(f"Found {len(sas_files)} SAS file(s) to analyze...") + + parser = SASParser() + scorer = ComplexityScorer(config) + classifier = TierClassifier() + dep_tracker = DependencyTracker() + reporter = AssessmentReporter(config) + + scripts = [] + scores = [] + classifications = [] + file_analyses = {} + + for sas_file in sas_files: + try: + content = sas_file.read_text(encoding='utf-8', errors='replace') + except Exception as e: + print(f" Warning: Could not read {sas_file}: {e}", file=sys.stderr) + continue + + script = parser.parse(content, filename=sas_file.name) + score = scorer.score_script(script) + classification = classifier.classify_file(script) + deps = dep_tracker.analyze_file(script) + + scripts.append(script) + scores.append(score) + classifications.append(classification) + file_analyses[script.filename] = deps + + parser = SASParser() + + if not scripts: + print("Error: No files could be parsed.", file=sys.stderr) + sys.exit(1) + + graph = dep_tracker.build_cross_file_graph(file_analyses) + graph['data_source_inventory'] = dep_tracker.build_source_inventory(file_analyses) + mermaid_str = dep_tracker.generate_mermaid_with_externals(graph, file_analyses) + + assessment = reporter.generate_assessment(scripts, scores, classifications, graph, file_analyses) + + output_dir = Path(args.output) + output_dir.mkdir(parents=True, exist_ok=True) + + if args.format in ('json', 'all'): + json_path = output_dir / 'assessment.json' + reporter.write_json(assessment, str(json_path)) + print(f" Written: {json_path}") + + if args.format in ('md', 'all'): + md_path = output_dir / 'assessment_report.md' + reporter.write_markdown(assessment, str(md_path)) + print(f" Written: {md_path}") + + if args.format in ('html', 'all'): + html_path = output_dir / 'assessment_report.html' + reporter.write_html(assessment, mermaid_str, str(html_path)) + print(f" Written: {html_path}") + + dag_path = output_dir / 'dependency_dag.mmd' + reporter.write_mermaid_dag(mermaid_str, str(dag_path)) + print(f" Written: {dag_path}") + + total = len(scripts) + tier_dist = assessment['portfolio_summary']['tier_distribution'] + complexity_dist = assessment['portfolio_summary']['complexity_distribution'] + + print(f"\n{'='*60}") + print(f" ASSESSMENT COMPLETE: {total} files analyzed") + print(f"{'='*60}") + print(f" Tier 1 (SQL): {tier_dist.get('TIER_1_SQL', 0):>4} ({tier_dist.get('TIER_1_SQL', 0)/total*100:.0f}%)") + print(f" Tier 2 (Stored Proc): {tier_dist.get('TIER_2_SP', 0):>4} ({tier_dist.get('TIER_2_SP', 0)/total*100:.0f}%)") + print(f" Tier 3 (PySpark): {tier_dist.get('TIER_3_PYSPARK', 0):>4} ({tier_dist.get('TIER_3_PYSPARK', 0)/total*100:.0f}%)") + print(f" ---") + print(f" Complexity LOW: {complexity_dist.get('LOW', 0):>4} ({complexity_dist.get('LOW', 0)/total*100:.0f}%)") + print(f" Complexity MEDIUM: {complexity_dist.get('MEDIUM', 0):>4} ({complexity_dist.get('MEDIUM', 0)/total*100:.0f}%)") + print(f" Complexity HIGH: {complexity_dist.get('HIGH', 0):>4} ({complexity_dist.get('HIGH', 0)/total*100:.0f}%)") + print(f"{'='*60}") + + +if __name__ == '__main__': + main() diff --git a/plugin/skills/migration/sas/assess-sas-migration/tool/config.json b/plugin/skills/migration/sas/assess-sas-migration/tool/config.json new file mode 100644 index 0000000..6278323 --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/tool/config.json @@ -0,0 +1,10 @@ +{ + "complexity_thresholds": { + "low_max": 50, + "medium_max": 150 + }, + "volume_thresholds": { + "low_max": 250, + "medium_max": 1000 + } +} diff --git a/plugin/skills/migration/sas/assess-sas-migration/tool/requirements.txt b/plugin/skills/migration/sas/assess-sas-migration/tool/requirements.txt new file mode 100644 index 0000000..3f6e162 --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/tool/requirements.txt @@ -0,0 +1,2 @@ +# No external dependencies required - uses Python standard library only +# Python >= 3.8 diff --git a/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/__init__.py b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/__init__.py new file mode 100644 index 0000000..49f5291 --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/__init__.py @@ -0,0 +1,14 @@ +from .parser import SASParser, SASScript, SASBlock, BlockType +from .constants import ( + BOILERPLATE_MACRO_NAMES, + BOILERPLATE_INDICATORS, + SKIP_TYPES, + is_boilerplate_macro, + iter_countable_blocks, +) +from .scorer import ComplexityScorer +from .classifier import TierClassifier +from .dependency import DependencyTracker +from .reporter import AssessmentReporter + +__version__ = "1.0.0" diff --git a/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/assets/snowconvert_ai_logo.svg b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/assets/snowconvert_ai_logo.svg new file mode 100644 index 0000000..5fa777c --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/assets/snowconvert_ai_logo.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/assets/snowflake_logo.svg b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/assets/snowflake_logo.svg new file mode 100644 index 0000000..99f73fc --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/assets/snowflake_logo.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/classifier.py b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/classifier.py new file mode 100644 index 0000000..b794268 --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/classifier.py @@ -0,0 +1,122 @@ +from typing import Dict +from .parser import SASScript, SASBlock, BlockType +from .constants import iter_countable_blocks + + +class TierClassifier: + + TIER3_TRIGGERS = [ + ('declare hash', 'HASH objects - no SQL equivalent'), + ('call execute', 'CALL EXECUTE - dynamic code generation'), + ] + + # Statistical-modeling PROCs -> Tier 3 (PySpark/SCOS). Canonical list, kept in + # sync with references/block-tiering-spec.md and the conversion skill. + TIER3_STAT_PROCS = [ + 'proc reg', 'proc glm', 'proc logistic', 'proc cluster', + 'proc factor', 'proc phreg', 'proc lifetest', 'proc surveyselect', + 'proc mixed', 'proc genmod', 'proc nlmixed', + ] + + # External DB engines -> LOW confidence. Canonical union list. + EXTERNAL_ENGINES = [ + 'sqlsvr', 'mssql', 'sql server', 'oracle', 'teradata', + 'odbc', 'oledb', 'db2', 'postgres', 'mysql', 'dsn=', + ] + + def classify_block(self, block: SASBlock) -> Dict: + content_lower = block.content.lower() + + for trigger, reason in self.TIER3_TRIGGERS: + if trigger in content_lower: + return {'tier': 3, 'label': 'TIER_3_PYSPARK', 'reason': reason, 'confidence': 'HIGH'} + + if ('do until' in content_lower or 'do while' in content_lower): + if 'symput' in content_lower or content_lower.count('call ') > 2: + return {'tier': 3, 'label': 'TIER_3_PYSPARK', 'reason': 'DO loop with external state', 'confidence': 'HIGH'} + + for proc in self.TIER3_STAT_PROCS: + if proc in content_lower: + return {'tier': 3, 'label': 'TIER_3_PYSPARK', 'reason': f'Statistical modeling: {proc}', 'confidence': 'HIGH'} + + if 'retain ' in content_lower and 'first.' in content_lower: + if '= 0' in content_lower or '= .' in content_lower: + return {'tier': 2, 'label': 'TIER_2_SP', 'reason': 'RETAIN with conditional reset', 'confidence': 'HIGH'} + + if ('first.' in content_lower or 'last.' in content_lower): + if content_lower.count('output ') > 1: + return {'tier': 2, 'label': 'TIER_2_SP', 'reason': 'FIRST./LAST. with multiple OUTPUT', 'confidence': 'HIGH'} + + if content_lower.count('output ') > 1 and 'output;' not in content_lower: + return {'tier': 2, 'label': 'TIER_2_SP', 'reason': 'Multiple OUTPUT datasets', 'confidence': 'MEDIUM'} + + # Procedural IF/THEN/ELSE and SELECT/WHEN branching only exists in DATA + # steps. A PROC SQL CASE WHEN is pure SQL (Tier 1) no matter how many + # WHEN clauses, so gate this rule to DATA steps. See block-tiering-spec.md. + if block.block_type == BlockType.DATA_STEP and ( + content_lower.count('if ') > 5 or content_lower.count('when ') > 5): + return {'tier': 2, 'label': 'TIER_2_SP', 'reason': 'Complex branching (>5 IF/WHEN) in DATA step', 'confidence': 'MEDIUM'} + + dml_count = sum(1 for kw in ['delete ', 'insert ', 'update '] if kw in content_lower) + if dml_count >= 3: + return {'tier': 2, 'label': 'TIER_2_SP', 'reason': '3+ sequential DML operations', 'confidence': 'MEDIUM'} + + confidence = self._assess_confidence(content_lower) + return {'tier': 1, 'label': 'TIER_1_SQL', 'reason': 'SQL-translatable', 'confidence': confidence} + + def _assess_confidence(self, content_lower: str) -> str: + if content_lower.count('%macro') > 2: + return 'LOW' + if '%include' in content_lower and '&' in content_lower: + return 'LOW' + if any(eng in content_lower for eng in self.EXTERNAL_ENGINES): + return 'LOW' + if 'intck' in content_lower or 'intnx' in content_lower or 'datepart' in content_lower: + return 'MEDIUM' + if 'notsorted' in content_lower: + return 'MEDIUM' + return 'HIGH' + + def classify_file(self, script: SASScript) -> Dict: + block_classifications = [] + tier_counts = {1: 0, 2: 0, 3: 0} + confidence_counts = {'HIGH': 0, 'MEDIUM': 0, 'LOW': 0} + + for block in iter_countable_blocks(script): + classification = self.classify_block(block) + block_classifications.append({ + 'block_type': block.block_type.value, + 'start_line': block.start_line, + 'end_line': block.end_line, + **classification + }) + tier_counts[classification['tier']] += 1 + confidence_counts[classification['confidence']] += 1 + + # Strict "any-block" rule: the file's tier is driven by its most demanding + # block, matching the conversion skill (one Tier-3 block -> notebook). + # See references/block-tiering-spec.md Section 3. + if tier_counts[3] > 0: + primary_tier = 'TIER_3_PYSPARK' + elif tier_counts[2] > 0: + primary_tier = 'TIER_2_SP' + else: + primary_tier = 'TIER_1_SQL' + + if confidence_counts['LOW'] > 0: + overall_confidence = 'LOW' + elif confidence_counts['MEDIUM'] > 0: + overall_confidence = 'MEDIUM' + else: + overall_confidence = 'HIGH' + + return { + 'primary_tier': primary_tier, + 'confidence': overall_confidence, + 'tier_distribution': { + 'TIER_1_SQL': tier_counts[1], + 'TIER_2_SP': tier_counts[2], + 'TIER_3_PYSPARK': tier_counts[3], + }, + 'block_classifications': block_classifications, + } diff --git a/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/constants.py b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/constants.py new file mode 100644 index 0000000..4427d75 --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/constants.py @@ -0,0 +1,98 @@ +"""Shared constants and the canonical countable-block iterator. + +Single source of truth for block counting so the portfolio total, per-file count, +and per-tier distribution are always computed over the SAME set of blocks and +reconcile. Mirrors `references/block-tiering-spec.md` (Section 1). +""" + +from typing import Iterator + +from .parser import SASScript, SASBlock, BlockType + + +# DI Studio / DataFlow scaffolding macro names — excluded from counts and tiering. +BOILERPLATE_MACRO_NAMES = frozenset({ + 'rcset', 'rcsetds', 'etls_startperformancestats', 'etls_setdebug', + 'etls_recordcount', 'etls_endperformancestats', 'etls_recordtable', + 'etls_getrecordcount', 'etls_jobstatus', 'etls_logerror', +}) + +# File-level boilerplate indicators (used by the complexity scorer's discount). +BOILERPLATE_INDICATORS = ( + 'etls_', + 'sas data integration studio', + '%macro etls_', + '%macro rcset', + 'perfinit', + 'log4sas', + 'armsubsys', + '%macro etls_startperformancestats', +) + +# Block types that are never counted as code blocks. +SKIP_TYPES = frozenset({ + BlockType.LET_STATEMENT, + BlockType.COMMENT, + BlockType.LIBNAME, + BlockType.MACRO_CALL, +}) + + +def is_boilerplate_macro(block: SASBlock) -> bool: + """True if a MACRO_DEF block is DI Studio / DataFlow scaffolding.""" + content_lower = block.content.lower() + return any(f'%macro {name}' in content_lower for name in BOILERPLATE_MACRO_NAMES) + + +def iter_countable_blocks(script: SASScript) -> Iterator[SASBlock]: + """Canonical countable-block set for a parsed SAS file. + + Every block count reported by the assessment (portfolio total, per-file + count, per-tier distribution) MUST iterate this generator so the numbers + reconcile and match how the conversion skill enumerates blocks. + + The parser surfaces each DATA/PROC step at the TOP level even when it lives + inside a macro (its extraction regexes scan the whole file). So macros are + already "flattened": we count those top-level steps directly and never + descend into ``sub_blocks`` (that would double-count). We then: + + - skip non-code types (LET/COMMENT/LIBNAME/MACRO_CALL); + - skip a MACRO_DEF wrapper when it has inner blocks (they are counted at the + top level); count the wrapper once only for a pure macro-language macro + with no DATA/PROC step inside; + - exclude boilerplate macros AND every block whose line falls within a + boilerplate macro's range (its inner steps also appear at the top level). + + See ``references/block-tiering-spec.md`` Section 1. + """ + boilerplate_ranges = [ + (b.start_line, b.end_line) + for b in script.blocks + if b.block_type == BlockType.MACRO_DEF and is_boilerplate_macro(b) + ] + + def within_boilerplate(block: SASBlock) -> bool: + return any(lo <= block.start_line <= hi for lo, hi in boilerplate_ranges) + + for block in script.blocks: + if block.block_type in SKIP_TYPES: + continue + + if block.block_type == BlockType.MACRO_DEF: + if is_boilerplate_macro(block): + continue + if block.sub_blocks: + # Inner DATA/PROC steps are already surfaced at the top level; + # skip the wrapper to avoid double counting. + continue + # Pure macro-language logic (no DATA/PROC inside) counts once. + yield block + continue + + if within_boilerplate(block): + # A DATA/PROC step belonging to a boilerplate macro, surfaced at the + # top level by the parser's global extraction. + continue + + yield block + diff --git a/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/dependency.py b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/dependency.py new file mode 100644 index 0000000..d01c1a3 --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/dependency.py @@ -0,0 +1,200 @@ +import re +from typing import List, Dict, Set, Tuple +from .parser import SASScript, SASBlock, BlockType + + +class DependencyTracker: + + EXTERNAL_LIBRARIES = { + 'oracle', 'teradata', 'db2', 'sqlsvr', 'odbc', 'oledb', + 'hadoop', 'spark', 'redshift', 'snowflake', 'postgres', 'mysql', 'mssql', + } + + # Librefs that are local scratch or SAS-supplied metadata, never a data source. + LOCAL_LIBREFS = {'WORK', 'SWORK'} + SYSTEM_LIBREFS = {'DICTIONARY', 'SASHELP', 'SASUSER', 'MAPS', 'MAPSGFK', 'MAPSSAS'} + _LIBREF_RE = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$') + + def analyze_file(self, script: SASScript) -> Dict: + creates: Set[str] = set() + reads: Set[str] = set() + external_sources: List[Dict] = [] + + for block in script.blocks: + md = block.metadata + for ds in md.get('output_datasets', []): + creates.add(self._normalize_dataset(ds)) + for ds in md.get('input_datasets', []): + reads.add(self._normalize_dataset(ds)) + + for lib_name, lib_path in script.libraries.items(): + path_lower = lib_path.lower() + for engine in self.EXTERNAL_LIBRARIES: + if engine in path_lower or engine in lib_name.lower(): + external_sources.append({ + 'libname': lib_name, + 'engine': engine, + 'path': lib_path, + }) + break + + full_content = '\n'.join(b.content for b in script.blocks).lower() + connect_pattern = r'connect\s+to\s+(\w+)' + for match in re.finditer(connect_pattern, full_content): + engine = match.group(1) + if engine in self.EXTERNAL_LIBRARIES: + external_sources.append({ + 'libname': 'PASSTHROUGH', + 'engine': engine, + 'path': f'CONNECT TO {engine}', + }) + + return { + 'creates': sorted(creates), + 'reads': sorted(reads), + 'external_sources': external_sources, + } + + def build_cross_file_graph(self, file_analyses: Dict[str, Dict]) -> Dict: + nodes = list(file_analyses.keys()) + edges: List[Dict] = [] + all_creates: Dict[str, str] = {} + + for filename, analysis in file_analyses.items(): + for dataset in analysis['creates']: + all_creates[dataset] = filename + + for filename, analysis in file_analyses.items(): + for dataset in analysis['reads']: + if dataset in all_creates and all_creates[dataset] != filename: + edges.append({ + 'from': all_creates[dataset], + 'to': filename, + 'via': dataset, + }) + + external_inputs = set() + for filename, analysis in file_analyses.items(): + for dataset in analysis['reads']: + if dataset not in all_creates: + external_inputs.add(dataset) + + return { + 'nodes': nodes, + 'edges': edges, + 'external_inputs': sorted(external_inputs), + } + + def build_source_inventory(self, file_analyses: Dict[str, Dict]) -> List[Dict]: + """Aggregate qualified external libraries into a source inventory. + + One row per non-local libref (e.g. OWDATA, SFNGGRE) with its table count + and read/write direction. Local WORK, unqualified, and SAS metadata + librefs are dropped — they are scratch, not data sources. + """ + engine_by_lib: Dict[str, str] = {} + for analysis in file_analyses.values(): + for src in analysis.get('external_sources', []): + engine_by_lib[src['libname'].upper()] = src['engine'] + + reads_by_lib: Dict[str, Set[str]] = {} + writes_by_lib: Dict[str, Set[str]] = {} + for analysis in file_analyses.values(): + for ds in analysis.get('reads', []): + lib, table = self._split_libref(ds) + if lib: + reads_by_lib.setdefault(lib, set()).add(table) + for ds in analysis.get('creates', []): + lib, table = self._split_libref(ds) + if lib: + writes_by_lib.setdefault(lib, set()).add(table) + + inventory: List[Dict] = [] + for lib in set(reads_by_lib) | set(writes_by_lib): + if lib in self.LOCAL_LIBREFS or lib in self.SYSTEM_LIBREFS: + continue + if not self._LIBREF_RE.match(lib): + continue + rd = reads_by_lib.get(lib, set()) + wr = writes_by_lib.get(lib, set()) + direction = 'Read + Write' if rd and wr else ('Read' if rd else 'Write') + inventory.append({ + 'source': lib, + 'engine': engine_by_lib.get(lib, 'External SAS library'), + 'tables': len(rd | wr), + 'direction': direction, + 'table_names': sorted(rd | wr), + }) + + inventory.sort(key=lambda r: (-r['tables'], r['source'])) + return inventory + + def _split_libref(self, name: str): + if '.' not in name: + return None, name + lib, _, table = name.partition('.') + return lib.upper().strip(), table.strip() + + def generate_mermaid(self, graph: Dict) -> str: + lines = ['graph LR'] + + node_ids = {} + for i, node in enumerate(graph['nodes']): + node_id = f'F{i}' + node_ids[node] = node_id + safe_label = node.replace('.sas', '').replace(' ', '_') + lines.append(f' {node_id}["{safe_label}"]') + + for ext in graph.get('external_inputs', [])[:20]: + ext_id = f'EXT_{ext.replace(".", "_").replace(" ", "")}' + lines.append(f' {ext_id}[("{ext}")]') + + for edge in graph['edges']: + from_id = node_ids.get(edge['from'], '') + to_id = node_ids.get(edge['to'], '') + if from_id and to_id: + lines.append(f' {from_id} -->|"{edge["via"]}"| {to_id}') + + for ext in graph.get('external_inputs', [])[:20]: + ext_id = f'EXT_{ext.replace(".", "_").replace(" ", "")}' + for filename, analysis in []: + pass + + return '\n'.join(lines) + + def generate_mermaid_with_externals(self, graph: Dict, file_analyses: Dict[str, Dict]) -> str: + lines = ['graph LR'] + + node_ids = {} + for i, node in enumerate(graph['nodes']): + node_id = f'F{i}' + node_ids[node] = node_id + safe_label = node.replace('.sas', '').replace(' ', '_') + lines.append(f' {node_id}["{safe_label}"]') + + ext_node_ids = {} + for i, ext in enumerate(graph.get('external_inputs', [])[:20]): + ext_id = f'EXT{i}' + ext_node_ids[ext] = ext_id + lines.append(f' {ext_id}[("{ext}")]') + + for edge in graph['edges']: + from_id = node_ids.get(edge['from'], '') + to_id = node_ids.get(edge['to'], '') + if from_id and to_id: + lines.append(f' {from_id} -->|"{edge["via"]}"| {to_id}') + + for filename, analysis in file_analyses.items(): + file_id = node_ids.get(filename, '') + if not file_id: + continue + for dataset in analysis['reads']: + if dataset in ext_node_ids: + lines.append(f' {ext_node_ids[dataset]} --> {file_id}') + + return '\n'.join(lines) + + def _normalize_dataset(self, name: str) -> str: + clean = re.sub(r'\([^)]*\)', '', name).strip() + clean = clean.strip("'\"") + return clean.upper() diff --git a/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/html_report.py b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/html_report.py new file mode 100644 index 0000000..cbdc327 --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/html_report.py @@ -0,0 +1,419 @@ +"""Self-contained HTML assessment report for the SAS migration skill. + +Renders the assessment dict (from ``AssessmentReporter.generate_assessment``) as +a single-file report styled after the SnowConvert AI assessment multi-report: a +dark navy sidebar with the Snowflake / SnowConvert AI logos and vertical nav, +and a light content area with KPI cards, distribution charts, a complexity x +volume matrix, the dependency DAG, and per-file detail. No third-party Python +deps. Everything renders offline except the dependency diagram, which uses the +Mermaid CDN when opened in a browser (an edges table is the offline fallback). +""" + +from __future__ import annotations + +import html +import re +from pathlib import Path +from typing import Dict, List + +PRIMARY = "#29B5E8" +PRIMARY_DARK = "#005C8F" +SIDEBAR = "#102E46" +MUTED = "#64748B" +BORDER = "#E2E8F0" +SEV = {"LOW": "#16a34a", "MEDIUM": "#f59e0b", "HIGH": "#ef4444"} +TIER_COLOR = {"TIER_1_SQL": "#29B5E8", "TIER_2_SP": "#005C8F", "TIER_3_PYSPARK": "#102E46"} +TIER_LABEL = { + "TIER_1_SQL": "Tier 1 · SQL", + "TIER_2_SP": "Tier 2 · Stored Proc", + "TIER_3_PYSPARK": "Tier 3 · PySpark", +} +TIER_APPROACH = { + "TIER_1_SQL": "Pure SQL — CTAS + CTEs + window functions", + "TIER_2_SP": "Snowflake stored procedures (Snowflake Scripting)", + "TIER_3_PYSPARK": "PySpark / Snowpark notebook", +} +# Each translation tier maps to a plain-language conversion-effort level so a +# reader who doesn't know the SQL/SP/PySpark taxonomy can still read the mix. +EFFORT_LABEL = {"TIER_1_SQL": "Low effort", "TIER_2_SP": "Medium effort", "TIER_3_PYSPARK": "High effort"} +EFFORT_TARGET = {"TIER_1_SQL": "SQL", "TIER_2_SP": "Stored proc", "TIER_3_PYSPARK": "PySpark"} +EFFORT_SEV = {"TIER_1_SQL": "LOW", "TIER_2_SP": "MEDIUM", "TIER_3_PYSPARK": "HIGH"} + +CODE_BLOCK_DEF = ( + "A code block is one parsed SAS unit — a PROC step, DATA step, or macro definition. " + "%LET, LIBNAME, comments, and macro calls are not counted." +) + +_ASSETS = Path(__file__).parent / "assets" + + +def _load_logo(name: str) -> str: + """Return the inline SVG for a bundled logo, sized by CSS (inline width stripped).""" + path = _ASSETS / name + try: + svg = path.read_text(encoding="utf-8") + except OSError: + return "" + return re.sub(r'(]*?)\s+style="[^"]*"', r"\1", svg, count=1) + + +def _e(value) -> str: + return html.escape(str(value), quote=True) + + +def _pct(n: int, total: int) -> float: + return (n / total * 100) if total else 0.0 + + +def _kpi(value, label, accent=PRIMARY, hint="") -> str: + title = f' title="{_e(hint)}"' if hint else "" + return ( + f'
{_e(value)}
' + f'
{_e(label)}
' + ) + + +def _bars(dist: Dict[str, int], order: List[str], colors: Dict[str, str], total: int) -> str: + rows = [] + for key in order: + count = dist.get(key, 0) + pct = _pct(count, total) + rows.append( + '
' + f'
{_e(key.title())}
' + '
' + f'
' + f'
{count} · {pct:.0f}%
' + '
' + ) + return '\n'.join(rows) + + +def _tier_donut(tier_dist: Dict[str, int], total: int) -> str: + stops, acc = [], 0.0 + for key in ("TIER_1_SQL", "TIER_2_SP", "TIER_3_PYSPARK"): + pct = _pct(tier_dist.get(key, 0), total) + if pct <= 0: + continue + stops.append(f"{TIER_COLOR[key]} {acc:.2f}% {acc + pct:.2f}%") + acc += pct + gradient = ", ".join(stops) if stops else f"{BORDER} 0% 100%" + legend = "".join( + f'
' + f'{_e(EFFORT_LABEL[k])} · {_e(EFFORT_TARGET[k])} {tier_dist.get(k, 0)}
' + for k in ("TIER_1_SQL", "TIER_2_SP", "TIER_3_PYSPARK") + ) + return ( + '
' + f'
' + f'
{total}files
' + f'
{legend}
' + '
' + ) + + +def _findings(assessment: Dict) -> List[str]: + p = assessment["portfolio_summary"] + total = assessment["metadata"]["total_files"] + tier = p["tier_distribution"] + higher = tier.get("TIER_2_SP", 0) + tier.get("TIER_3_PYSPARK", 0) + high_cx = p["complexity_distribution"].get("HIGH", 0) + ext = len(assessment["dependency_graph"].get("external_inputs", [])) + edges = len(assessment["dependency_graph"].get("edges", [])) + macros = p["block_type_distribution"].get("MACRO_DEF", 0) + out = [] + if total: + out.append( + f"{tier.get('TIER_1_SQL', 0)} of {total} files " + f"({_pct(tier.get('TIER_1_SQL', 0), total):.0f}%) are Tier-1 — convertible to pure Snowflake SQL." + ) + if higher: + out.append(f"{higher} file(s) need a procedural rewrite (stored procedure or PySpark).") + else: + out.append("No files require stored-procedure or PySpark rewrites in this portfolio.") + if high_cx: + out.append(f"{high_cx} file(s) are HIGH complexity — plan for extra review and testing.") + if ext: + out.append(f"{ext} external source table(s) are referenced but not created in scope — provision or ingest them first.") + if edges: + out.append(f"{edges} cross-file dependency edge(s) detected — migrate producers before consumers (see Dependencies).") + if macros: + out.append(f"{macros} macro definition(s) found — factor shared macros into reusable Snowflake objects.") + files = assessment.get("files", []) + if files: + top = max(files, key=lambda f: f.get("complexity_score", 0)) + out.append(f"Highest-scoring file: {_e(top['filename'])} (score {top['complexity_score']}).") + return out + + +def render_html(assessment: Dict, mermaid_str: str = "") -> str: + meta = assessment["metadata"] + p = assessment["portfolio_summary"] + files = assessment.get("files", []) + graph = assessment.get("dependency_graph", {}) + total = meta["total_files"] + tier = p["tier_distribution"] + higher_tier = tier.get("TIER_2_SP", 0) + tier.get("TIER_3_PYSPARK", 0) + high_cx = p["complexity_distribution"].get("HIGH", 0) + + snowflake_logo = _load_logo("snowflake_logo.svg") + snowconvert_logo = _load_logo("snowconvert_ai_logo.svg") + + kpis = "".join([ + _kpi(total, "SAS files"), + _kpi(f"{p['total_lines']:,}", "Lines of code"), + _kpi(f"{p['total_blocks']:,}", "Code blocks", hint=CODE_BLOCK_DEF), + _kpi(f"{_pct(tier.get('TIER_1_SQL', 0), total):.0f}%", "SQL-ready (Tier 1)", "#16a34a"), + _kpi(higher_tier, "Need procedural rewrite", PRIMARY_DARK), + _kpi(high_cx, "High complexity", SEV["HIGH"] if high_cx else MUTED), + ]) + + complexity_bars = _bars(p["complexity_distribution"], ["LOW", "MEDIUM", "HIGH"], SEV, total) + volume_bars = _bars(p["volume_distribution"], ["LOW", "MEDIUM", "HIGH"], SEV, total) + donut = _tier_donut(tier, total) + findings = "".join(f"
  • {f}
    • " for f in _findings(assessment)) + + block_rows = "".join( + f"    " + for bt, c in sorted(p["block_type_distribution"].items(), key=lambda x: -x[1]) + ) + func_rows = "".join( + f"" + for fn, c in list(p.get("function_usage", {}).items())[:20] + ) or "" + + tier_rows = "".join( + f"" + f"" + f"" + f"" + for k in ("TIER_1_SQL", "TIER_2_SP", "TIER_3_PYSPARK") + ) + + matrix = {} + for f in files: + matrix[(f["complexity_level"], f["volume_level"])] = matrix.get((f["complexity_level"], f["volume_level"]), 0) + 1 + matrix_rows = "" + for cl in ("LOW", "MEDIUM", "HIGH"): + cells = "" + for vl in ("LOW", "MEDIUM", "HIGH"): + n = matrix.get((cl, vl), 0) + cells += f"" + matrix_rows += f"{cells}" + + file_rows = "" + for f in sorted(files, key=lambda x: x["complexity_score"], reverse=True): + file_rows += ( + "" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + "" + ) + + edges = graph.get("edges", []) + edge_rows = "".join( + f"" + for ed in edges + ) or "" + ext_inputs = graph.get("external_inputs", []) + ext_list = "".join(f"
  • {_e(x)}
    • " for x in ext_inputs[:200]) or "
  • None — every referenced table is produced within scope.
    • " + ext_more = f"

    … and {len(ext_inputs) - 200} more.

    " if len(ext_inputs) > 200 else "" + + inventory = graph.get("data_source_inventory", []) + inv_rows = "".join( + f"    " + f"" + for r in inventory + ) or "" + + n_nodes = len(graph.get("nodes", [])) + if not mermaid_str.strip(): + mermaid_block = '

    No dependency graph generated.

    ' + elif n_nodes > 40 or len(edges) > 60: + mermaid_block = ( + f'

    The dependency graph is large ({n_nodes} nodes, {len(edges)} edges) — ' + 'an inline diagram would be unreadable here. Use the edges table below, or open ' + 'dependency_dag.mmd in a Mermaid viewer for the full graph.

    ' + ) + else: + mermaid_block = f'
    {_e(mermaid_str)}
    ' + + nav_items = [ + ("overview", "Overview"), + ("tiers", "Complexity & Tiers"), + ("deps", "Dependencies"), + ("files", "Per-File Detail"), + ("funcs", "Functions"), + ] + nav = "".join( + f'' + for i, (tid, label) in enumerate(nav_items) + ) + + return f""" + + +SAS → Snowflake Migration Assessment · SnowConvert AI + + +
    + +
    +
    +

    SAS → Snowflake Migration Assessment

    +
    {total} SAS files · {p['total_lines']:,} lines
    Not SnowConvert / AIM registry
    +
    +
    +
    {kpis}
    + +
    +

    Key findings

      {findings}
    +
    +

    Conversion effort mix

    {donut} +

    Effort tier by conversion target — Low = SQL, Medium = stored procedure, High = PySpark / Snowpark.

    +

    Block types

    {_e(bt)}{c}
    {_e(fn)}{c}
    No common SAS functions detected.
    {_e(TIER_LABEL[k])}{tier.get(k, 0)}{_pct(tier.get(k, 0), total):.0f}%{_e(EFFORT_LABEL[k])}{_e(TIER_APPROACH[k])}
    {n}
    {cl} complexity
    {_e(f['filename'])}{f['lines']}{f['blocks']}{f['complexity_score']}{_e(f['complexity_level'])}{_e(f['volume_level'])}{_e(TIER_LABEL.get(f['primary_tier'], f['primary_tier']))}{_e(f['confidence'])}
    {_e(ed['from'])}{_e(ed['to'])}{_e(ed.get('via', ''))}
    No cross-file dependencies detected.
    {_e(r['source'])}{_e(r['engine'])}{r['tables']}{_e(r['direction'])}
    No external source libraries detected — every referenced table is local WORK or produced within scope.
    {block_rows}
    Block typeCount
    +

    {_e(CODE_BLOCK_DEF)}

    +
    +
    +

    Complexity distribution

    {complexity_bars}
    +

    Volume distribution

    {volume_bars}
    +
    + + +
    +

    Translation tiers

    + {tier_rows}
    TierFilesShareEffortRecommended approach
    +
    +

    Complexity × Volume matrix

    + {matrix_rows}
    Low volumeMedium volumeHigh volume
    +
    +
    + +
    +

    Dependency graph

    {mermaid_block}
    +

    Data source inventory ({len(inventory)})

    + {inv_rows}
    SourceEngine / TypeTablesDirection
    +

    External libraries the module reads from or writes to, aggregated by library. Local WORK, unqualified, and SAS dictionary datasets are excluded. Engine / type is shown when a LIBNAME or CONNECT declares it; libraries resolved at runtime via %assign_libname appear as “External SAS library”.

    +
    +
    +

    Dependency edges ({len(edges)})

    {edge_rows}
    ProducerConsumerVia table
    +

    External source tables ({len(ext_inputs)})

      {ext_list}
    {ext_more}
    +
    +
    + +
    +

    Per-file detail ({total} files, sorted by complexity score)

    +
    + {file_rows}
    FileLinesBlocksScoreComplexityVolumeTierConfidence
    +
    +
    + +
    +

    SAS functions used

    + {func_rows}
    FunctionOccurrences
    +
    +
    + +
    Preview — generated by the SAS-to-Snowflake assessment skill (SnowConvert AI, tool v{_e(meta['tool_version'])}). + Estimates are heuristic and intended for planning; validate against a sample conversion before committing to a plan.
    + + + + + +""" diff --git a/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/parser.py b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/parser.py new file mode 100644 index 0000000..4356144 --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/parser.py @@ -0,0 +1,387 @@ +import re +from dataclasses import dataclass, field +from typing import List, Dict, Optional +from enum import Enum + + +class BlockType(Enum): + PROC_SQL = "PROC_SQL" + DATA_STEP = "DATA_STEP" + PROC_SORT = "PROC_SORT" + PROC_DATASETS = "PROC_DATASETS" + PROC_APPEND = "PROC_APPEND" + PROC_MEANS = "PROC_MEANS" + PROC_SUMMARY = "PROC_SUMMARY" + PROC_FREQ = "PROC_FREQ" + PROC_TRANSPOSE = "PROC_TRANSPOSE" + PROC_FORMAT = "PROC_FORMAT" + PROC_PRINT = "PROC_PRINT" + PROC_IMPORT = "PROC_IMPORT" + PROC_EXPORT = "PROC_EXPORT" + PROC_CONTENTS = "PROC_CONTENTS" + PROC_OTHER = "PROC_OTHER" + MACRO_DEF = "MACRO_DEF" + MACRO_CALL = "MACRO_CALL" + LET_STATEMENT = "LET_STATEMENT" + LIBNAME = "LIBNAME" + COMMENT = "COMMENT" + UNKNOWN = "UNKNOWN" + + +@dataclass +class SASBlock: + block_type: BlockType + content: str + start_line: int + end_line: int + raw_text: str + complexity_score: int = 0 + sub_blocks: List['SASBlock'] = field(default_factory=list) + metadata: Dict = field(default_factory=dict) + + +@dataclass +class SASScript: + filename: str + total_lines: int + blocks: List[SASBlock] + macros: Dict[str, SASBlock] + macro_variables: Dict[str, str] + libraries: Dict[str, str] + complexity_score: int = 0 + + +class SASParser: + + PROC_PATTERNS = { + BlockType.PROC_SQL: r'(?i)^\s*PROC\s+SQL\b', + BlockType.PROC_SORT: r'(?i)^\s*PROC\s+SORT\b', + BlockType.PROC_DATASETS: r'(?i)^\s*PROC\s+DATASETS\b', + BlockType.PROC_APPEND: r'(?i)^\s*PROC\s+APPEND\b', + BlockType.PROC_MEANS: r'(?i)^\s*PROC\s+MEANS\b', + BlockType.PROC_SUMMARY: r'(?i)^\s*PROC\s+SUMMARY\b', + BlockType.PROC_FREQ: r'(?i)^\s*PROC\s+FREQ\b', + BlockType.PROC_TRANSPOSE: r'(?i)^\s*PROC\s+TRANSPOSE\b', + BlockType.PROC_FORMAT: r'(?i)^\s*PROC\s+FORMAT\b', + BlockType.PROC_PRINT: r'(?i)^\s*PROC\s+PRINT\b', + BlockType.PROC_IMPORT: r'(?i)^\s*PROC\s+IMPORT\b', + BlockType.PROC_EXPORT: r'(?i)^\s*PROC\s+EXPORT\b', + BlockType.PROC_CONTENTS: r'(?i)^\s*PROC\s+CONTENTS\b', + } + + def __init__(self): + self.macro_vars: Dict[str, str] = {} + self.libraries: Dict[str, str] = {} + + def parse(self, content: str, filename: str = "unknown.sas") -> SASScript: + lines = content.split('\n') + total_lines = len(lines) + + content = self._remove_comments_preserve_structure(content) + blocks = self._extract_blocks(content) + + macros = {b.metadata.get('name', ''): b for b in blocks if b.block_type == BlockType.MACRO_DEF} + + for block in blocks: + block.complexity_score = self._calculate_block_complexity(block) + + total_complexity = sum(b.complexity_score for b in blocks) + + return SASScript( + filename=filename, + total_lines=total_lines, + blocks=blocks, + macros=macros, + macro_variables=self.macro_vars.copy(), + libraries=self.libraries.copy(), + complexity_score=total_complexity + ) + + def _remove_comments_preserve_structure(self, content: str) -> str: + content = re.sub(r'/\*.*?\*/', '', content, flags=re.DOTALL) + content = re.sub(r'^\s*\*[^;]*;', '', content, flags=re.MULTILINE) + return content + + def _extract_blocks(self, content: str) -> List[SASBlock]: + blocks = [] + + libname_pattern = r"(?i)LIBNAME\s+(\w+)\s+['\"]?([^;'\"]+)['\"]?\s*;" + for match in re.finditer(libname_pattern, content): + lib_name = match.group(1) + lib_path = match.group(2).strip() + self.libraries[lib_name.upper()] = lib_path + blocks.append(SASBlock( + block_type=BlockType.LIBNAME, + content=match.group(0), + start_line=content[:match.start()].count('\n') + 1, + end_line=content[:match.end()].count('\n') + 1, + raw_text=match.group(0), + metadata={'name': lib_name, 'path': lib_path} + )) + + let_pattern = r"(?i)%LET\s+(\w+)\s*=\s*([^;]+);" + for match in re.finditer(let_pattern, content): + var_name = match.group(1) + var_value = match.group(2).strip() + self.macro_vars[var_name.upper()] = var_value + blocks.append(SASBlock( + block_type=BlockType.LET_STATEMENT, + content=match.group(0), + start_line=content[:match.start()].count('\n') + 1, + end_line=content[:match.end()].count('\n') + 1, + raw_text=match.group(0), + metadata={'name': var_name, 'value': var_value} + )) + + macro_blocks = self._extract_macro_definitions(content) + blocks.extend(macro_blocks) + + proc_sql_blocks = self._extract_proc_sql_blocks(content) + blocks.extend(proc_sql_blocks) + + data_step_blocks = self._extract_data_step_blocks(content) + blocks.extend(data_step_blocks) + + other_proc_blocks = self._extract_other_proc_blocks(content) + blocks.extend(other_proc_blocks) + + blocks.sort(key=lambda b: b.start_line) + return blocks + + def _extract_macro_definitions(self, content: str) -> List[SASBlock]: + blocks = [] + macro_pattern = r"(?i)%MACRO\s+(\w+)(?:\s*\([^)]*\))?\s*;(.*?)%MEND\s*(?:\1)?\s*;" + + for match in re.finditer(macro_pattern, content, re.DOTALL): + macro_name = match.group(1) + macro_body = match.group(2) + inner_blocks = self._extract_blocks(macro_body) + + blocks.append(SASBlock( + block_type=BlockType.MACRO_DEF, + content=match.group(0), + start_line=content[:match.start()].count('\n') + 1, + end_line=content[:match.end()].count('\n') + 1, + raw_text=match.group(0), + sub_blocks=inner_blocks, + metadata={'name': macro_name} + )) + return blocks + + def _extract_proc_sql_blocks(self, content: str) -> List[SASBlock]: + blocks = [] + proc_sql_pattern = r"(?i)(PROC\s+SQL\b[^;]*;)(.*?)(QUIT\s*;|(?=PROC\s+|DATA\s+|%MACRO\s+|$))" + + for match in re.finditer(proc_sql_pattern, content, re.DOTALL): + proc_header = match.group(1) + sql_body = match.group(2) + terminator = match.group(3) or '' + full_content = proc_header + sql_body + terminator + + metadata = self._parse_sql_metadata(sql_body, proc_header) + + blocks.append(SASBlock( + block_type=BlockType.PROC_SQL, + content=full_content, + start_line=content[:match.start()].count('\n') + 1, + end_line=content[:match.end()].count('\n') + 1, + raw_text=full_content, + metadata=metadata + )) + return blocks + + def _parse_sql_metadata(self, sql_body: str, proc_header: str) -> Dict: + metadata = { + 'output_datasets': [], + 'input_datasets': [], + 'noprint': 'noprint' in proc_header.lower() + } + + create_pattern = r"(?i)CREATE\s+TABLE\s+(\S+)" + for match in re.finditer(create_pattern, sql_body): + metadata['output_datasets'].append(match.group(1).rstrip('(')) + + from_pattern = r"(?i)\bFROM\s+(\w+\.?\w*)" + join_pattern = r"(?i)\bJOIN\s+(\w+\.?\w*)" + for match in re.finditer(from_pattern, sql_body): + tbl = match.group(1) + if tbl.upper() not in ('DUAL', 'DICTIONARY'): + metadata['input_datasets'].append(tbl) + for match in re.finditer(join_pattern, sql_body): + metadata['input_datasets'].append(match.group(1)) + + return metadata + + def _extract_data_step_blocks(self, content: str) -> List[SASBlock]: + blocks = [] + data_step_pattern = r"(?i)(DATA\s+([^;]+)\s*;)(.*?)(RUN\s*;)" + + for match in re.finditer(data_step_pattern, content, re.DOTALL): + data_header = match.group(1) + output_datasets = match.group(2) + data_body = match.group(3) + run_stmt = match.group(4) + full_content = data_header + data_body + run_stmt + + data_metadata = self._parse_data_step_body(data_body, output_datasets) + + blocks.append(SASBlock( + block_type=BlockType.DATA_STEP, + content=full_content, + start_line=content[:match.start()].count('\n') + 1, + end_line=content[:match.end()].count('\n') + 1, + raw_text=full_content, + metadata=data_metadata + )) + return blocks + + def _parse_data_step_body(self, body: str, output_datasets: str) -> Dict: + out_datasets = [] + for ds in output_datasets.split(): + clean = re.sub(r'\([^)]*\)', '', ds).strip() + if clean and clean.upper() != '_NULL_': + out_datasets.append(clean) + + metadata = { + 'output_datasets': out_datasets, + 'input_datasets': [], + 'has_retain': bool(re.search(r'(?i)\bRETAIN\b', body)), + 'has_array': bool(re.search(r'(?i)\bARRAY\b', body)), + 'has_merge': False, + 'has_first_last': bool(re.search(r'(?i)\b(FIRST\.|LAST\.)', body)), + 'by_vars': [], + } + + set_match = re.search(r'(?i)SET\s+([^;]+)\s*;', body) + if set_match: + datasets = set_match.group(1) + for ds in re.split(r'\s+', datasets): + clean = re.sub(r'\([^)]*\)', '', ds).strip() + if clean and not clean.upper().startswith('END=') and not clean.upper().startswith('NOBS='): + metadata['input_datasets'].append(clean) + + merge_match = re.search(r'(?i)MERGE\s+([^;]+)\s*;', body) + if merge_match: + metadata['has_merge'] = True + datasets = merge_match.group(1) + for ds in re.split(r'\s+', datasets): + clean = re.sub(r'\([^)]*\)', '', ds).strip() + if clean: + metadata['input_datasets'].append(clean) + + by_match = re.search(r'(?i)BY\s+([^;]+)\s*;', body) + if by_match: + metadata['by_vars'] = [v.strip() for v in by_match.group(1).split() + if v.strip() and v.upper() not in ('DESCENDING', 'ASCENDING')] + + return metadata + + def _extract_other_proc_blocks(self, content: str) -> List[SASBlock]: + blocks = [] + + for block_type, pattern in self.PROC_PATTERNS.items(): + if block_type == BlockType.PROC_SQL: + continue + + # Strip the leading ^\s* anchor: these patterns are matched WITHOUT + # re.MULTILINE, so ^ would only match the very start of the file and + # silently drop every PROC after the first statement. Use a word + # boundary so we still match PROC anywhere in the content. + clean_pattern = pattern.replace('(?i)', '').replace(r'^\s*', r'\b') + proc_pattern = rf"({clean_pattern}[^;]*;)(.*?)((?:QUIT|RUN)\s*;)" + + for match in re.finditer(proc_pattern, content, re.DOTALL | re.IGNORECASE): + proc_header = match.group(1) + proc_body = match.group(2) + terminator = match.group(3) + full_content = proc_header + proc_body + terminator + + metadata = self._parse_proc_metadata(proc_header, proc_body, block_type) + + blocks.append(SASBlock( + block_type=block_type, + content=full_content, + start_line=content[:match.start()].count('\n') + 1, + end_line=content[:match.end()].count('\n') + 1, + raw_text=full_content, + metadata=metadata + )) + + # Catch-all for any PROC not covered by a specific pattern above (e.g. + # PROC REG / GLM / LOGISTIC / TABULATE / REPORT / SGPLOT ...). Without + # this, statistical and other PROCs are invisible to counting and + # tiering, diverging from how the conversion skill enumerates blocks. + known = 'SQL|SORT|DATASETS|APPEND|MEANS|SUMMARY|FREQ|TRANSPOSE|FORMAT|PRINT|IMPORT|EXPORT|CONTENTS' + generic_pattern = rf"(\bPROC\s+(?!(?:{known})\b)\w+\b[^;]*;)(.*?)((?:QUIT|RUN)\s*;)" + for match in re.finditer(generic_pattern, content, re.DOTALL | re.IGNORECASE): + proc_header = match.group(1) + proc_body = match.group(2) + terminator = match.group(3) + full_content = proc_header + proc_body + terminator + + metadata = self._parse_proc_metadata(proc_header, proc_body, BlockType.PROC_OTHER) + + blocks.append(SASBlock( + block_type=BlockType.PROC_OTHER, + content=full_content, + start_line=content[:match.start()].count('\n') + 1, + end_line=content[:match.end()].count('\n') + 1, + raw_text=full_content, + metadata=metadata + )) + return blocks + + def _parse_proc_metadata(self, header: str, body: str, block_type: BlockType) -> Dict: + metadata: Dict = {'input_datasets': [], 'output_datasets': []} + + data_match = re.search(r'(?i)DATA\s*=\s*(\S+)', header) + out_match = re.search(r'(?i)OUT\s*=\s*(\S+)', header) + + if data_match: + metadata['input_datasets'].append(re.sub(r'\([^)]*\)', '', data_match.group(1))) + if out_match: + metadata['output_datasets'].append(re.sub(r'\([^)]*\)', '', out_match.group(1))) + + if block_type == BlockType.PROC_SORT: + metadata['nodupkey'] = 'nodupkey' in header.lower() + elif block_type == BlockType.PROC_APPEND: + base_match = re.search(r'(?i)BASE\s*=\s*(\S+)', header + body) + if base_match: + metadata['output_datasets'].append(re.sub(r'\([^)]*\)', '', base_match.group(1))) + + output_match = re.search(r'(?i)OUTPUT\s+OUT\s*=\s*(\S+)', body) + if output_match: + metadata['output_datasets'].append(re.sub(r'\([^)]*\)', '', output_match.group(1))) + + return metadata + + def _calculate_block_complexity(self, block: SASBlock) -> int: + type_scores = { + BlockType.PROC_SQL: 3, + BlockType.DATA_STEP: 3, + BlockType.PROC_SORT: 1, + BlockType.MACRO_DEF: 4, + BlockType.PROC_TRANSPOSE: 2, + BlockType.PROC_MEANS: 2, + BlockType.PROC_SUMMARY: 2, + BlockType.PROC_FREQ: 2, + } + score = type_scores.get(block.block_type, 1) + + content_lower = block.content.lower() + score += content_lower.count('join') * 2 + score += content_lower.count('case when') * 1 + score += content_lower.count('group by') * 1 + score += content_lower.count('having') * 1 + score += len(re.findall(r'(?i)\bif\b', content_lower)) + score += content_lower.count('%do') * 2 + score += content_lower.count('array') * 2 + score += content_lower.count('retain') * 1 + + if block.metadata.get('has_merge'): + score += 3 + if block.metadata.get('has_first_last'): + score += 2 + + score += len(block.sub_blocks) * 2 + return score diff --git a/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/reporter.py b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/reporter.py new file mode 100644 index 0000000..e700b7a --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/reporter.py @@ -0,0 +1,261 @@ +import json +from datetime import datetime +from typing import List, Dict +from collections import Counter +from .parser import SASScript +from .constants import iter_countable_blocks + + +class AssessmentReporter: + + def __init__(self, config: Dict = None): + self.config = config or {} + + def generate_assessment(self, scripts: List[SASScript], scores: List[Dict], + classifications: List[Dict], graph: Dict, + file_analyses: Dict[str, Dict]) -> Dict: + portfolio = self._build_portfolio_summary(scripts, scores, classifications) + files = self._build_file_details(scripts, scores, classifications, file_analyses) + + return { + 'metadata': { + 'tool_version': '1.0.0', + 'generated_at': datetime.now().isoformat(), + 'total_files': len(scripts), + 'config': self.config, + }, + 'portfolio_summary': portfolio, + 'files': files, + 'dependency_graph': graph, + } + + def _build_portfolio_summary(self, scripts: List[SASScript], scores: List[Dict], + classifications: List[Dict]) -> Dict: + total_lines = sum(s.total_lines for s in scripts) + + # All block counts use the ONE canonical countable-block set so the + # portfolio total, per-file counts, and tier distribution reconcile. + total_blocks = sum(1 for s in scripts for _ in iter_countable_blocks(s)) + + complexity_dist = Counter(s['complexity_level'] for s in scores) + volume_dist = Counter(s['volume_level'] for s in scores) + tier_dist = Counter(c['primary_tier'] for c in classifications) + + block_type_dist = Counter() + for s in scripts: + for b in iter_countable_blocks(s): + block_type_dist[b.block_type.value] += 1 + + func_usage = self._get_function_usage(scripts) + + return { + 'total_lines': total_lines, + 'total_blocks': total_blocks, + 'complexity_distribution': dict(complexity_dist), + 'volume_distribution': dict(volume_dist), + 'tier_distribution': dict(tier_dist), + 'block_type_distribution': dict(block_type_dist), + 'function_usage': dict(sorted(func_usage.items(), key=lambda x: -x[1])[:20]), + } + + def _build_file_details(self, scripts: List[SASScript], scores: List[Dict], + classifications: List[Dict], file_analyses: Dict[str, Dict]) -> List[Dict]: + files = [] + for script, score, classification in zip(scripts, scores, classifications): + deps = file_analyses.get(script.filename, {'creates': [], 'reads': [], 'external_sources': []}) + files.append({ + 'filename': script.filename, + 'lines': script.total_lines, + 'blocks': sum(1 for _ in iter_countable_blocks(script)), + 'complexity_score': score['overall_score'], + 'complexity_level': score['complexity_level'], + 'volume_level': score['volume_level'], + 'primary_tier': classification['primary_tier'], + 'confidence': classification['confidence'], + 'tier_distribution': classification['tier_distribution'], + 'is_boilerplate': score['is_boilerplate'], + 'dependencies': { + 'creates': deps['creates'], + 'reads': deps['reads'], + }, + 'external_sources': deps['external_sources'], + }) + return files + + def _get_function_usage(self, scripts: List[SASScript]) -> Dict[str, int]: + import re + function_counts: Counter = Counter() + common_functions = [ + 'INPUT', 'PUT', 'SUBSTR', 'TRIM', 'COMPRESS', 'UPCASE', 'LOWCASE', + 'SUM', 'MEAN', 'COUNT', 'MAX', 'MIN', 'ROUND', 'ABS', + 'INTCK', 'INTNX', 'TODAY', 'DATETIME', 'YEAR', 'MONTH', 'DAY', + 'CAT', 'CATS', 'CATX', 'SCAN', 'INDEX', 'TRANWRD', + 'COALESCE', 'IFN', 'IFC', 'MISSING', 'MDY', 'DATEPART', + ] + for script in scripts: + full_content = '\n'.join(b.content for b in script.blocks).upper() + for func in common_functions: + count = len(re.findall(rf'\b{func}\s*\(', full_content)) + if count > 0: + function_counts[func] += count + return dict(function_counts) + + def write_json(self, assessment: Dict, output_path: str): + with open(output_path, 'w') as f: + json.dump(assessment, f, indent=2, default=str) + + def write_markdown(self, assessment: Dict, output_path: str): + md = self._render_markdown(assessment) + with open(output_path, 'w') as f: + f.write(md) + + def write_mermaid_dag(self, mermaid_str: str, output_path: str): + with open(output_path, 'w') as f: + f.write(mermaid_str) + + def write_html(self, assessment: Dict, mermaid_str: str, output_path: str): + from .html_report import render_html + with open(output_path, 'w') as f: + f.write(render_html(assessment, mermaid_str)) + + def _render_markdown(self, assessment: Dict) -> str: + meta = assessment['metadata'] + portfolio = assessment['portfolio_summary'] + files = assessment['files'] + + lines = [] + lines.append('# SAS Migration Assessment Report') + lines.append('') + lines.append(f'**Generated:** {meta["generated_at"]}') + lines.append(f'**Total Files:** {meta["total_files"]}') + lines.append('') + + lines.append('## Portfolio Summary') + lines.append('') + lines.append(f'| Metric | Value |') + lines.append(f'|--------|-------|') + lines.append(f'| Total SAS Files | {meta["total_files"]} |') + lines.append(f'| Total Lines | {portfolio["total_lines"]:,} |') + lines.append(f'| Total Blocks | {portfolio["total_blocks"]:,} |') + lines.append('') + + lines.append('## Complexity Distribution') + lines.append('') + lines.append('| Level | Count | Percentage |') + lines.append('|-------|-------|------------|') + total = meta['total_files'] + for level in ['LOW', 'MEDIUM', 'HIGH']: + count = portfolio['complexity_distribution'].get(level, 0) + pct = (count / total * 100) if total > 0 else 0 + lines.append(f'| {level} | {count} | {pct:.1f}% |') + lines.append('') + + lines.append('## Volume Distribution') + lines.append('') + lines.append('| Level | Count | Percentage |') + lines.append('|-------|-------|------------|') + for level in ['LOW', 'MEDIUM', 'HIGH']: + count = portfolio['volume_distribution'].get(level, 0) + pct = (count / total * 100) if total > 0 else 0 + lines.append(f'| {level} | {count} | {pct:.1f}% |') + lines.append('') + + lines.append('## Translation Tier Distribution') + lines.append('') + lines.append('_Translation tier is the migration approach per file (independent of the ' + 'complexity score above). A file is Tier 3 if it has any Tier-3 block, else ' + 'Tier 2 if any Tier-2 block, else Tier 1 — matching the conversion skill._') + lines.append('') + lines.append('| Tier | Count | Percentage | Approach |') + lines.append('|------|-------|------------|----------|') + tier_map = { + 'TIER_1_SQL': ('Tier 1', 'Pure SQL (CTAS + CTEs + Window Functions)'), + 'TIER_2_SP': ('Tier 2', 'Snowflake Stored Procedures'), + 'TIER_3_PYSPARK': ('Tier 3', 'PySpark/SCOS Notebook'), + } + for tier_key, (label, approach) in tier_map.items(): + count = portfolio['tier_distribution'].get(tier_key, 0) + pct = (count / total * 100) if total > 0 else 0 + lines.append(f'| {label} | {count} | {pct:.1f}% | {approach} |') + lines.append('') + + lines.append('## Block Type Distribution') + lines.append('') + lines.append('| Block Type | Count |') + lines.append('|-----------|-------|') + for btype, count in sorted(portfolio['block_type_distribution'].items(), key=lambda x: -x[1]): + lines.append(f'| {btype} | {count} |') + lines.append('') + + if portfolio.get('function_usage'): + lines.append('## Top SAS Functions Used') + lines.append('') + lines.append('| Function | Occurrences |') + lines.append('|----------|-------------|') + for func, count in list(portfolio['function_usage'].items())[:15]: + lines.append(f'| {func} | {count} |') + lines.append('') + + lines.append('## Complexity x Volume Matrix') + lines.append('') + matrix = {} + for f in files: + key = (f['complexity_level'], f['volume_level']) + matrix[key] = matrix.get(key, 0) + 1 + lines.append('| | Low Volume | Medium Volume | High Volume |') + lines.append('|--|-----------|---------------|-------------|') + for cl in ['LOW', 'MEDIUM', 'HIGH']: + row = f'| {cl} Complexity |' + for vl in ['LOW', 'MEDIUM', 'HIGH']: + row += f' {matrix.get((cl, vl), 0)} |' + lines.append(row) + lines.append('') + + lines.append('## Per-File Details') + lines.append('') + lines.append('| File | Lines | Blocks | Score | Complexity | Volume | Tier | Confidence |') + lines.append('|------|-------|--------|-------|------------|--------|------|------------|') + sorted_files = sorted(files, key=lambda x: x['complexity_score'], reverse=True) + for f in sorted_files: + lines.append( + f'| {f["filename"]} | {f["lines"]} | {f["blocks"]} | ' + f'{f["complexity_score"]} | {f["complexity_level"]} | {f["volume_level"]} | ' + f'{f["primary_tier"]} | {f["confidence"]} |' + ) + lines.append('') + + inventory = assessment['dependency_graph'].get('data_source_inventory', []) + if inventory: + lines.append('## Data Source Inventory') + lines.append('') + lines.append('External libraries the module reads from or writes to, aggregated by library ' + '(local WORK, unqualified, and SAS dictionary datasets excluded):') + lines.append('') + lines.append('| Source | Engine / Type | Tables | Direction |') + lines.append('|--------|---------------|-------:|-----------|') + for row in inventory: + lines.append(f'| {row["source"]} | {row["engine"]} | {row["tables"]} | {row["direction"]} |') + lines.append('') + + ext_inputs = assessment['dependency_graph'].get('external_inputs', []) + if ext_inputs: + lines.append('## External Dependencies') + lines.append('') + lines.append('Tables referenced but not created by any file in scope:') + lines.append('') + for ext in ext_inputs[:30]: + lines.append(f'- `{ext}`') + lines.append('') + + graph = assessment['dependency_graph'] + if graph.get('edges'): + lines.append('## Dependency DAG') + lines.append('') + lines.append('See `dependency_dag.mmd` for the full Mermaid diagram.') + lines.append('') + lines.append(f'- **Files:** {len(graph["nodes"])}') + lines.append(f'- **Dependencies:** {len(graph["edges"])}') + lines.append(f'- **External Inputs:** {len(graph.get("external_inputs", []))}') + lines.append('') + + return '\n'.join(lines) diff --git a/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/scorer.py b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/scorer.py new file mode 100644 index 0000000..987fd99 --- /dev/null +++ b/plugin/skills/migration/sas/assess-sas-migration/tool/sas_analyzer/scorer.py @@ -0,0 +1,177 @@ +import math +from typing import List, Dict +from .parser import SASScript, SASBlock, BlockType +from .constants import ( + BOILERPLATE_INDICATORS, + iter_countable_blocks, +) + + +class ComplexityScorer: + + COMPLEXITY_WEIGHTS = { + BlockType.PROC_SQL: 2, + BlockType.DATA_STEP: 2, + BlockType.MACRO_DEF: 4, + BlockType.PROC_SORT: 1, + BlockType.PROC_APPEND: 1, + BlockType.PROC_DATASETS: 1, + BlockType.PROC_MEANS: 2, + BlockType.PROC_SUMMARY: 2, + BlockType.PROC_FREQ: 2, + BlockType.PROC_TRANSPOSE: 3, + BlockType.PROC_FORMAT: 1, + BlockType.PROC_IMPORT: 1, + BlockType.PROC_EXPORT: 1, + BlockType.PROC_PRINT: 0, + BlockType.PROC_CONTENTS: 0, + BlockType.PROC_OTHER: 2, + BlockType.LET_STATEMENT: 0, + BlockType.LIBNAME: 0, + BlockType.COMMENT: 0, + BlockType.MACRO_CALL: 0, + BlockType.UNKNOWN: 0, + } + + TIER3_PATTERNS = { + 'declare hash': 8, + 'call execute': 6, + 'proc reg ': 6, + 'proc glm ': 6, + 'proc logistic ': 6, + 'proc cluster ': 6, + 'proc factor ': 6, + 'proc phreg ': 6, + 'proc lifetest ': 6, + } + + TIER2_PATTERNS = { + 'call symput': 3, + 'symget': 3, + 'proc transpose': 3, + } + + TIER1_ADVANCED_PATTERNS = { + 'retain ': 2, + 'array ': 2, + 'merge ': 2, + 'first.': 1, + 'last.': 1, + '%do ': 1, + '%if ': 1, + 'infile ': 3, + 'ods ': 1, + } + + FEATURE_CAP = 3 + + def __init__(self, config: Dict = None): + config = config or {} + thresholds = config.get('complexity_thresholds', {}) + self.low_max = thresholds.get('low_max', 50) + self.medium_max = thresholds.get('medium_max', 150) + + volume = config.get('volume_thresholds', {}) + self.volume_low_max = volume.get('low_max', 250) + self.volume_medium_max = volume.get('medium_max', 1000) + + def score_script(self, script: SASScript) -> Dict: + is_boilerplate = self._detect_boilerplate(script) + # Score over the canonical countable-block set (flattened macros, + # boilerplate + non-code types excluded) so complexity and tier are + # computed on the same block universe. See constants.iter_countable_blocks. + business_blocks = list(iter_countable_blocks(script)) + + raw_base = sum(self.COMPLEXITY_WEIGHTS.get(b.block_type, 0) for b in business_blocks) + base_score = (min(raw_base, 50) + max(0, int(math.log2(max(raw_base - 50, 1))))) if raw_base > 50 else raw_base + + feature_score = self._calculate_feature_score(business_blocks, is_boilerplate) + structure_score = self._calculate_structure_score(script, is_boilerplate) + + overall_score = base_score + feature_score + structure_score + complexity_level = self._get_complexity_level(overall_score) + volume_level = self._get_volume_level(script.total_lines) + + return { + 'overall_score': overall_score, + 'complexity_level': complexity_level, + 'volume_level': volume_level, + 'base_score': base_score, + 'feature_score': feature_score, + 'structure_score': structure_score, + 'is_boilerplate': is_boilerplate, + 'business_block_count': len(business_blocks), + } + + def _detect_boilerplate(self, script: SASScript) -> bool: + full_content = '\n'.join(b.content for b in script.blocks).lower() + matches = sum(1 for ind in BOILERPLATE_INDICATORS if ind in full_content) + return matches >= 3 + + def _calculate_feature_score(self, business_blocks: List[SASBlock], is_boilerplate: bool) -> int: + score = 0 + full_content = '\n'.join(b.content for b in business_blocks).lower() + boilerplate_discount = 0.5 if is_boilerplate else 1.0 + + for pattern, weight in self.TIER3_PATTERNS.items(): + count = min(full_content.count(pattern), self.FEATURE_CAP) + score += int(count * weight * boilerplate_discount) + + for pattern, weight in self.TIER2_PATTERNS.items(): + count = min(full_content.count(pattern), self.FEATURE_CAP) + score += int(count * weight * boilerplate_discount) + + for pattern, weight in self.TIER1_ADVANCED_PATTERNS.items(): + count = min(full_content.count(pattern), self.FEATURE_CAP) + score += count * weight + + return score + + def _calculate_structure_score(self, script: SASScript, is_boilerplate: bool) -> int: + score = 0 + + if is_boilerplate: + effective_macros = max(len(script.macros) - 5, 0) + effective_vars = max(len(script.macro_variables) - 10, 0) + else: + effective_macros = len(script.macros) + effective_vars = len(script.macro_variables) + + score += min(effective_macros, 10) * 2 + nested_macros = sum( + 1 for b in script.blocks + if b.block_type == BlockType.MACRO_DEF and b.sub_blocks + ) + score += min(nested_macros, 5) * 3 + score += min(effective_vars, 10) + return score + + def _get_complexity_level(self, score: int) -> str: + if score <= self.low_max: + return "LOW" + elif score <= self.medium_max: + return "MEDIUM" + else: + return "HIGH" + + def _get_volume_level(self, lines: int) -> str: + if lines <= self.volume_low_max: + return "LOW" + elif lines <= self.volume_medium_max: + return "MEDIUM" + else: + return "HIGH" + + def identify_hotspots(self, script: SASScript) -> List[Dict]: + hotspots = [] + for block in script.blocks: + if block.complexity_score >= 5: + hotspots.append({ + 'block_type': block.block_type.value, + 'start_line': block.start_line, + 'end_line': block.end_line, + 'complexity_score': block.complexity_score, + 'content_preview': block.content[:200] + '...' if len(block.content) > 200 else block.content, + }) + hotspots.sort(key=lambda x: x['complexity_score'], reverse=True) + return hotspots[:10] diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/SKILL.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/SKILL.md new file mode 100644 index 0000000..e768651 --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/SKILL.md @@ -0,0 +1,760 @@ +--- +name: convert-sas-to-snowflake +parent_skill: sas +description: "Preview. Convert SAS code to Snowflake SQL and stored procedures. Use for: DATA steps, PROC SQL, macros, PROC steps, EGP projects (extracted .sas files only). Triggers: convert SAS, migrate SAS, SAS to Snowflake, translate SAS code, SAS migration, SAS analysis, SAS dependency diagram, validate SAS conversion. Note: For .egp files, extract the embedded .sas code first using SAS Enterprise Guide export." +license: Proprietary. See License-Skills for complete terms +--- + +# Convert SAS to Snowflake + +> © Snowflake Inc. This skill and its contents are the proprietary intellectual property of Snowflake Inc. + +Expert SAS to Snowflake migration - **ALWAYS outputs `.sql` by default**, with PySpark/SCOS as last resort. + +**Output:** `.sql` by default (Tier 1 SQL + Tier 2 Stored Procedures). PySpark/SCOS notebook = LAST RESORT (HASH objects, CALL EXECUTE, statistical procs only). See SQL-First Classification below for full tier tables. + +**Snowflake Interaction Policy:** ALL Snowflake operations (object creation, compilation, testing) require explicit user confirmation. See `workflows/steps-8-10-post-conversion.md` for confirmation patterns and testing mode selection. + +**User Consent Propagation:** When a user confirms a high-level decision (validation scope, compilation approval), downstream sub-decisions that are direct consequences auto-resolve without re-prompting. See "Consent Propagation Rules" section below. + +--- + +## Validation Prompt Policy (EXPLICIT — No Auto-Propagation) + +Each validation phase requires its own explicit user confirmation. No phase auto-proceeds based on a prior selection. Snowflake operations ALWAYS show credit impact before executing. + +| Phase | Prompt | Credits | Can Skip? | +|-------|--------|---------|-----------| +| Phase 1: Synthetic Data | Always runs (no prompt needed) | Zero | No | +| Phase 2: LLM Trace | "Run LLM trace?" | Zero | Yes | +| Phase 3: Snowflake Compilation | "Compile on Snowflake?" (with credit estimate) | Minimal | Yes | +| Phase 4: Snowflake Execution (Tier 2/3) | "Execute on Snowflake?" (with credit estimate) | Moderate | Yes | + +**Hard Artifact Gates:** Each phase produces artifacts on disk. The NEXT phase CANNOT begin unless the prior phase's gate passes (artifacts verified to exist). State file updates without corresponding disk artifacts are INVALID. + +| Phase | Required Artifacts | +|-------|-------------------| +| Phase 1 | `source_table_ddl.sql` + `synthetic_data.sql` | +| Phase 2 | `tests//expected/expected_.csv` for each traced file | +| Phase 3 | `compilation_results.json` with per-file pass/fail | +| Phase 4 | `snowflake_execution_results.json` | + +**Consent propagation is ONLY retained for:** +- Fix-and-retry loop within Phase 3 (auto-retry compilation fixes without re-prompting) +- Auto-drop of temporary compilation environment after Phase 3 completes +- These are sub-operations WITHIN a phase the user already approved + +**No batch-mode amplifier.** Batch mode (10+ files) follows the same prompt rules as interactive mode. + +--- + +## Intent Detection + +``` +Start + ↓ +Analyze User Request + ↓ + ├─→ Convert intent → Follow Conversion Workflow (below) + │ (convert, migrate, translate, transform) + │ + ├─→ Assess intent → Load ../assess-sas-migration/SKILL.md + │ (assess, analyze, size, estimate, complexity, volume, LOE, readiness, heatmap) + │ + └─→ Validate intent → Load validate-sas-conversion/SKILL.md + (validate, verify, test conversion, check migration) +``` + +--- + +## When to Use + +- Convert SAS DATA steps to Snowflake SQL or stored procedures +- Translate PROC SQL to Snowflake SQL +- Migrate SAS macros to Snowflake scripting +- Convert PROC steps (SORT, MEANS, FREQ, TRANSPOSE, etc.) +- Validate converted code against test data + +## Prerequisites + +- Active Snowflake connection (for validation) +- Target schema exists (or user confirms schema) + +--- + +## SQL-First Classification + +### TIER 1: Pure SQL (Default - ALWAYS Try First) + +**Use SQL for:** + +| SAS Pattern | Snowflake SQL | +|-------------|---------------| +| PROC SQL | Direct SQL (strip PROC SQL/QUIT) | +| PROC SORT | `ORDER BY` or `QUALIFY ROW_NUMBER()` for NODUPKEY | +| Simple DATA step (SET, WHERE, IF≤3) | `CREATE TABLE AS SELECT` | +| PROC FREQ | `GROUP BY` with `COUNT(*)` | +| PROC MEANS (no OUTPUT) | Aggregate functions (AVG, SUM, etc.) | +| MERGE with simple BY | `JOIN` | +| FIRST./LAST. flags | Window functions: `ROW_NUMBER()`, `LEAD/LAG` | +| RETAIN (running totals) | `SUM() OVER (ROWS UNBOUNDED PRECEDING)` | +| Simple ARRAY (same operation) | `GREATEST()`, `LEAST()`, `COALESCE()` | + +### TIER 2: Stored Procedures (When Pure SQL Insufficient) + +**Use Snowflake Scripting when:** + +| Pattern | Detection | Why Stored Procedure | +|---------|-----------|---------------------| +| RETAIN with conditional reset | `retain` + `if first.` + reset logic | State management across rows | +| FIRST./LAST. with complex calc | `first.` + accumulator + `last.` output | Multi-step BY-group logic | +| Multiple OUTPUT datasets | `output table1; output table2;` | Conditional routing | +| >5 IF/WHEN branches in a DATA step | DATA-step SELECT/WHEN or IF/THEN (not PROC SQL CASE WHEN) | Complex business logic | +| Row-by-row state changes | Previous row affects current | Cursor-like processing | +| Iterative calculations | Values build on prior iterations | Sequential dependency | + +**Stored Procedure Pattern:** +```sql +CREATE OR REPLACE PROCEDURE process_sas_logic(p_input STRING, p_output STRING) +RETURNS STRING +LANGUAGE SQL +AS +$$ +DECLARE + v_row_count INTEGER; +BEGIN + -- Use CTEs and window functions where possible + CREATE OR REPLACE TABLE IDENTIFIER(:p_output) AS + WITH ranked AS ( + SELECT *, + ROW_NUMBER() OVER (PARTITION BY group_col ORDER BY sort_col) AS rn, + SUM(amount) OVER (PARTITION BY group_col ORDER BY sort_col + ROWS UNBOUNDED PRECEDING) AS running_total + FROM IDENTIFIER(:p_input) + ) + SELECT * FROM ranked WHERE some_condition; + + SELECT COUNT(*) INTO :v_row_count FROM IDENTIFIER(:p_output); + RETURN 'Processed ' || v_row_count || ' rows'; +END; +$$; +``` + +### TIER 3: PySpark via SCOS (LAST RESORT ONLY) + +**Use PySpark with Snowpark Connect (SCOS) ONLY when SQL/Stored Procedures CANNOT work:** + +| Pattern | Detection | Why PySpark Required | +|---------|-----------|---------------------| +| HASH objects | `declare hash` | In-memory key-value lookups | +| CALL EXECUTE | `call execute` | Dynamic code generation at runtime | +| DO UNTIL/WHILE + external | `do until` + `symput`/`call` | True iteration with external state | +| External file I/O | `infile`/`file` non-Snowflake | Reading/writing local files | +| Complex SYMPUT chains | Multiple `call symput` with dependencies | Cross-step variable passing | +| PROC REG/GLM/LOGISTIC/CLUSTER/FACTOR/PHREG/LIFETEST/SURVEYSELECT/MIXED/GENMOD/NLMIXED | Statistical modeling | Regression/ML models | + +**⚠️ These are SQL, NOT PySpark:** +- ARRAY iteration → CASE expressions +- RETAIN → window functions +- FIRST./LAST. → ROW_NUMBER() +- KEY= lookup → LEFT JOIN +- %DO loops → GENERATOR + +See `references/pyspark-fallback.md` for full SCOS notebook setup template and patterns. + +**Before using PySpark, ask:** "Can this be done with window functions, CTEs, or a stored procedure?" + +--- + +## Critical Conversion Rules + +**Load `references/conversion-rules.md` at Steps 5-6** (code generation and self-check). Contains the rules governing semantic correctness. + +**Key rules summary (always in context):** +- NEVER generate converter scripts — LLM reads and converts directly +- NEVER truncate or use "similar pattern" shortcuts +- SAS missing (.) = NULL with different comparison semantics +- MERGE != JOIN (many-to-many risk, IN= flags, source precedence) +- WORK tables = TEMPORARY, bare name, no schema prefix +- Snowflake Scripting: `:var` only in SQL statements, SELECT INTO, named exceptions only +- Migrated-source type traps: cast join keys to a common type, `TRY_TO_DOUBLE` VARCHAR measures, `TO_DATE()` VARCHAR date columns +- Validation anti-joins must use master reference tables (never derived/subset) +- Output column count must match SAS; never silently drop columns +- Consolidation must preserve all output tables + +**Known conflict — NOT adopted (pending verification):** An external ruleset claims `$VAR` session-variable syntax *fails* inside stored procedures and that `GETVARIABLE()` is mandatory. This is **not** adopted here — the verified guidance in `references/common-patterns.md` (Platform Constraints) is that `$VAR` **works** inside an `EXECUTE AS CALLER` procedure body, with `GETVARIABLE('VAR')` as an equivalent alternative. `GETVARIABLE()` is genuinely *required* only when the variable name is **computed at runtime** (you can write `GETVARIABLE('VAR' || i)` but never `$VAR || i`). Do not switch to mandatory `GETVARIABLE()` for the static case unless re-verified empirically on Snowflake. + +--- + +## Batch Mode (10+ Files) + +When 10+ files detected, load `references/batch-mode.md` for batch-specific rules, +context window recovery, and Step 8a compilation with stub tables. + +--- + +## Context Budget Management + +The LLM should minimize context usage by: +1. **Only load references relevant to the current step** — never pre-load all references at Step 2 +2. **Do NOT re-read completed files** — use `conversion_state.json` to skip already-converted files +3. **Write state to disk before each step transition** — enables resume without context replay +4. **For batch mode, process in groups of 10-20 files** — write .sql to disk between groups +5. **When context exceeds 60% consumed:** finish current file, write state, inform user +6. **On resume:** read ONLY `conversion_state.json` + the reference files needed for the current step + +**Step-to-reference mapping (load ONLY these at each step):** + +| Step | Required References | +|------|-------------------| +| 1-2 | `common-patterns.md` only | +| 1 (state init) | `state-tracker-schema.md` + `checkpoint-logging.md` (load once; lightweight, governs all checkpoint writes) | +| 2.5 | `vendor-passthrough.md` (only if external LIBNAME/CONNECT TO detected) | +| 3-4 | `classification-logic.md` (if needed) | +| 5 | `conversion-rules.md` + construct-specific refs (data-steps, proc-sql, etc.); `multi-block-orchestration.md` if program has 3+ dependent blocks / macro `%DO` loops / `&&var&i` / `&SYSERR` / `FILEEXIST` / trigger-file gates | +| 6 | `conversion-rules.md` (for self-check items) | +| 7 | `steps-7-validation.md` + `schema-inference.md` + `synthetic-data-rules.md` | +| 8 | `steps-8-10-post-conversion.md` + `batch-mode.md` (if batch) | +| 9 | `steps-8-10-post-conversion.md` + `comparison-rules.md` | +| 10 | `steps-8-10-post-conversion.md` + `templates/conversion-report.md` | + +**Unload after use:** After completing a step, the reference files loaded for that step can be considered "consumed" — their content is materialized in the output artifacts. Do not re-read them unless the step needs to be re-executed. + +--- + +## Conversion State Tracker (`conversion_state.json`) + +A persistent JSON file written incrementally throughout the workflow to track per-file progress. Enables resume from ANY step across context windows. See `references/state-tracker-schema.md` for the full JSON schema, enforcement rules, and valid `current_step` values. + +**Location:** `/conversion_state.json` + +**Key fields:** `metadata.current_step`, `files..status`, `gates.*`, `trace_progress`, `execution_progress`, `compilation_env` + +**Resume behavior (Step 1 pre-check):** If `conversion_state.json` exists, read it and offer Resume/Restart. + +**File status progression:** `classified → converted → self_checked → compiled → validated → complete` + +**Enforcement rule:** State writes are BLOCKING PREREQUISITES for step transitions. The NEXT step CANNOT begin until the state file reflects the current step's completion. Context pressure does NOT exempt state writes. + +--- + +## Conversion Workflow + +### Step 1: Gather Input + +**Pre-check:** If `/conversion_state.json` exists, a prior session has in-progress conversion. Read the state file and offer: +- **Resume** from `current_step` (skip completed files/steps) +- **Restart fresh** (delete state file, begin from scratch) + +**Ask user:** +1. SAS code source (file path, directory, or paste) +2. Target schema (e.g., `DATABASE.SCHEMA`) +3. Migration mode: + - **1:1 Direct** (default) — Each SAS file converts to one .sql file. PySpark notebook generated ONLY for true Tier 3 edge cases (HASH objects, CALL EXECUTE, statistical procs). + - **Optimize & Consolidate** — Merges related SAS files into fewer .sql files where dependencies allow. Same SQL-first tiering applies. + +**⚠️ STOP**: Confirm understanding of SAS code. + +**MANDATORY STATE WRITE (Step 1 → Step 2 transition):** Immediately after gathering input, write `conversion_state.json` with initial metadata: +```json +{ + "metadata": { + "output_dir": "", + "target_schema": "", + "migration_mode": "<1:1 or consolidate>", + "created_at": "", + "last_updated": "", + "current_step": "1", + "total_files": 0, + "validation_scope": "pending_selection", + "pre_approved_compilation": false, + "pre_approved_validation": null + } +} +``` +This ensures metadata survives context loss between Steps 1 and 3. Step 2 CANNOT begin until this write is confirmed on disk. + +**MANDATORY CHECKPOINT (every transition):** Alongside each `conversion_state.json` write throughout the workflow, append one line to `/checkpoint_log.jsonl` (create it here at Step 1 with the first `STARTED` line). This append-only audit trail records what happened and when — see `references/checkpoint-logging.md` for the schema and the per-checkpoint list. Appending the checkpoint line is a BLOCKING prerequisite to advancing, exactly like the state write. + +### Step 2: Load References + +**Always load:** `references/common-patterns.md` (Variable Binding Rules) + +**Deferred load** (do NOT load at Step 2 — loaded when needed): +- `workflows/validation-pipeline.md` → loaded after Step 6a (single validation orchestrator) +- `workflows/steps-8-10-post-conversion.md` → loaded at Step 10 only (report generation) +- `templates/conversion-report.md` → loaded at Step 10 +- `references/schema-inference.md` → loaded at Phase 1 (synthetic data gen) +- `references/synthetic-data-rules.md` → loaded at Phase 1 +- `references/validation-execution.md` → loaded at Phase 2 (LLM trace) +- `references/snowflake-compile.md` → loaded at Phase 3 (Snowflake compilation) +- `references/snowflake-execution.md` → loaded at Phase 4 (Snowflake Tier 2/3 execution) +- `references/state-tracker-schema.md` → loaded on resume or when first writing state +- `references/checkpoint-logging.md` → loaded once at Step 1 (governs append-only `checkpoint_log.jsonl` written at every checkpoint) +- `references/batch-mode.md` → loaded only when 10+ files detected +- `references/classification-logic.md` → loaded at Step 3 if detailed pseudocode needed +- `references/comparison-rules.md` → loaded at Phase 4 (result comparison) +- `references/auto-validation.md` → loaded at Phase 2 (pipeline orchestration) +- `references/e2e-orchestration-test.md` → loaded only when Phase 5 (Integration & E2E) runs +- `references/consolidation-patterns.md` → loaded only in Optimize & Consolidate mode +- `references/multi-block-orchestration.md` → loaded at Step 5 only for multi-block programs (3+ dependent blocks, macro `%DO` loops, `&&var&i` indexed vars, `&SYSERR` gating, `FILEEXIST`/trigger-file gates) +- `references/pyspark-fallback.md` → loaded only when Tier 3 blocks detected +- `references/mermaid-diagrams.md` → loaded only at Step 10 (DAG generation) + +**Then load based on constructs (Step 5 only):** + +| Construct | Reference | +|-----------|-----------| +| DATA steps | `references/data-steps.md` | +| PROC SQL | `references/proc-sql.md` | +| Macros | `references/macros.md` | +| Multi-block programs / macro `%DO` loops / cross-block state (`&&var&i`, `&SYSERR`, `FILEEXIST`, trigger-file gates) | `references/multi-block-orchestration.md` | +| PROC steps | `references/proc-steps.md` | +| Function mappings | `references/function-mappings.md` | +| Oracle/DB2/SQL Server passthrough | `references/vendor-passthrough.md` + `references/vendor-function-mappings.md` | +| Large files (>500 lines) | `references/large-file-rules.md` | + +### Step 2.5: External Source Resolution + +**Scan SAS code for external database references:** +1. `LIBNAME` statements with engine keywords: `sqlsvr`, `oracle`, `teradata`, `odbc`, `oledb`, `postgres`, `mysql`, `mssql` +2. `PROC SQL CONNECT TO` statements: `connect to sqlsvr`, `connect to oracle`, etc. +3. Any `DSN=`, `authdomain=`, `qualifier=`, `schema=` parameters + +**For each external reference found, extract:** +- Engine type (sqlsvr, oracle, etc.) +- DSN / qualifier (maps to source database name) +- Schema (maps to source schema name) +- Tables referenced via this LIBNAME (scan for `LIBNAME.table` patterns) + +**Prompt user ONCE with all external sources:** +``` +External data sources detected in SAS code: +| LIBNAME | Engine | DSN/Qualifier | Schema | Tables Referenced | +|---------|--------|---------------|--------|-------------------| +| EUDB | sqlsvr | EUDB | dbo | Table1, Table2 | + +These tables are sourced from an EXTERNAL database (not Snowflake). +They will eventually be migrated to Snowflake. + +Please provide the target Snowflake location for these tables: +a) Use target schema already specified: {TARGET_SCHEMA} +b) Provide a different DATABASE.SCHEMA for these external tables: ___ +c) Use placeholder names: {PLACEHOLDER_DB}.{PLACEHOLDER_SCHEMA}.{table} +``` + +**Store mapping in `conversion_state.json` under `source_mappings`** (see state-tracker-schema.md). + +**During Step 5 (Generation):** Use the mapping to resolve external table references. If LIBNAME `EUDB` with `qualifier=ChargemasterEnforcement_Prod schema=dbo` references `dbo.MyTable`, convert to the user-provided `TARGET_DB.TARGET_SCHEMA.MYTABLE`. + +**If NO external sources detected:** Skip this step silently. No prompt needed. + +### Step 3: Classify Each Block (with Dependency Tracking) + +**Assessment Auto-Discovery (no prompt):** A prior `/assess-sas-migration` run may have produced an +`assessment.json`. Search these candidate locations **in order** and use the FIRST readable match +(if several match, pick the one with the most recent modification time): + +1. `/assessment.json` +2. `/assessment.json` +3. `/assessment_output/assessment.json` (the assess CLI's default `--output ./assessment_output`) +4. `/../assessment_output/assessment.json` +5. a shallow glob (depth ≤ 2) for `assessment.json` under `` and `` + +A match is valid only if it parses as JSON and contains `metadata.generated_at` and a `files[]` +array; otherwise treat it as not found (record `reason: "unreadable"`). + +- **If found:** seed per-file `primary_tier` / `tier_distribution` and the dependency graph from it + to skip re-scanning. Note which path was used. If `metadata.generated_at` is older than the newest + source-file modification time, set `stale: true` (still consume — the reconciliation below will + surface any drift). +- **If NONE found:** proceed silently with fresh classification. **Do NOT prompt** for an assessment + file. Record `assessment.consumed: false, reason: "not_found"` in state. + +Record the outcome in `conversion_state.json` under `assessment` (see `references/state-tracker-schema.md`). + +**Still verify the dependency map is current** — source files may have changed since assessment. + +**CRITICAL: Before classifying, build a dependency map:** +1. Track which tables/datasets each block CREATES (output tables) +2. Track which tables/datasets each block READS (input tables) +3. Track macro variables set (CALL SYMPUT, %LET, INTO :var) and where they are consumed +4. Identify inter-block dependencies: if block N creates table X and block M reads table X, note the dependency +5. For repeated macro invocations: track parameter combinations separately — do NOT assume same behavior + +**Input Table Existence Check (optional, requires active Snowflake connection):** + +After building the dependency map, identify all TRUE external inputs — tables that appear in READ lists but are NOT created by any block in this conversion: +1. Collect unique source tables from all blocks' READ lists +2. Subtract tables that appear in any block's CREATES list (these are intermediate/temp tables) +3. The remainder = true external inputs that must pre-exist in Snowflake +4. If a Snowflake connection is active, verify these tables exist: `SHOW TABLES LIKE '' IN SCHEMA ` +5. If tables are NOT found, prompt user ONCE: + ``` + These source tables are referenced but not found in {TARGET_SCHEMA}: + - TABLE_A (used in blocks 1, 3) + - TABLE_B (used in block 2) + + Options: + a) Provide the correct DATABASE.SCHEMA for these tables + b) They will be created before this script runs (proceed with current names) + c) They were already mapped in Step 2.5 (use those mappings) + d) Skip check (proceed as-is, may fail at compilation) + ``` +6. Store any user-provided mappings in `conversion_state.json` under `source_mappings` + +**If no Snowflake connection available:** Skip silently. Compilation (Phase 4) will surface missing tables later. + +**For EACH block, apply SQL-First classification using the quick-reference table below.** For detailed classification pseudocode and confidence assessment logic, load `references/classification-logic.md`. + +**Block counting (must match the assessment CLI):** Follow the canonical +`../references/block-tiering-spec.md`. Enumerate **each DATA/PROC step inside a macro** as its +own block, and **exclude DI Studio / DataFlow boilerplate** (macros named `etls_*`, `rcset`, +`rcsetds`; content markers `etls_`, `perfinit`, `log4sas`, `armsubsys`, `sas data integration +studio`) from both the block count and tiering. This keeps the conversion's block count and +per-file tier identical to what `/assess-sas-migration` reports. + +**Quick Classification:** + +| SAS Pattern | Tier | Snowflake Approach | +|-------------|------|--------------------| +| `declare hash` | 3-SCOS | No SQL equivalent | +| `call execute` | 3-SCOS | Dynamic code generation | +| `do until/while` + `symput` + external state | 3-SCOS | Iteration with external state | +| `proc reg/glm/logistic/cluster/factor/phreg/lifetest/surveyselect/mixed/genmod/nlmixed` | 3-SCOS | Statistical modeling | +| `retain` + `first.` + reset | 2-SP | State management across rows | +| `first./last.` + multiple `output` | 2-SP | Multi-step BY-group logic | +| Multiple `output` destinations | 2-SP | Conditional routing | +| >5 `if`/`when` branches in a DATA step (not PROC SQL CASE WHEN) | 2-SP | Complex business logic | +| 3+ sequential dependent DML (DELETE→INSERT→UPDATE) | 2-SP | Orchestration + error handling | +| Everything else (PROC SQL, SORT, simple DATA step, MERGE, ARRAY, RETAIN running total, %DO loops) | 1-SQL | Window functions, CTEs, GENERATOR | + +**Confidence:** LOW = nested macros, %INCLUDE with dynamic path, external LIBNAME. MEDIUM = INTCK/INTNX, NOTSORTED. HIGH = everything else. + +**Assessment Reconciliation (only if `assessment.json` was consumed):** After computing this +conversion's own per-file block count and tier, compare them against the assessment as a baseline. +Both skills follow the same canonical `../references/block-tiering-spec.md`, so a current assessment +should match; a mismatch signals a stale assessment (source changed since it ran) or a bug — it is +**informational and does NOT block** conversion. + +For each file present in BOTH this conversion's scope and the assessment, compare: +- `blocks` (countable block count) +- `primary_tier` +- `tier_distribution` (T1 / T2 / T3 counts) + +Also note coverage deltas: files in the assessment but not in this conversion's scope, and vice versa. +Record the result in `conversion_state.json` under `assessment.reconciliation` (files_compared, +files_matched, files_mismatched, and a `mismatches` list of `{file, field, assessment, convert}`). +Present the summary in Step 4. + +**PREREQUISITE FOR STEP 4:** Write `conversion_state.json` with initial entries for all files (`status: "classified"`, tier, block count). Set `current_step: "3"`. Also persist the dependency map per file: `"dependencies": {"creates": [...], "reads": [...]}`, and the `assessment` object (discovery outcome + reconciliation, or `consumed: false`). Verify the file exists on disk before presenting block analysis. Step 4 CANNOT begin until this write is confirmed. + +### Step 4: Present Block Analysis + +```markdown +## Block Analysis: script_name.sas + +| Block | Type | Tier | Lines | Creates | Reads | Reason | +|-------|------|------|-------|---------|-------|--------| +| 1 | PROC SQL | 🔷 SQL | 10-25 | staging_data | source_table | Direct SQL mapping | +| 2 | DATA step | 🔷 SQL | 27-45 | summary | staging_data | Window functions | +| 3 | DATA step | 🔶 Stored Proc | 47-80 | flagged | summary | RETAIN + FIRST./LAST. | +| 4 | DATA step | 🔴 PySpark | 82-95 | lookup | ext_file | HASH object | + +**Dependencies:** Block 2 depends on Block 1 (staging_data). Block 3 depends on Block 2 (summary). +**Output:** script_name.sql (all blocks as SQL/SP, block 4 commented if truly needs Python) +``` + +**Assessment reconciliation (only when `assessment.json` was consumed in Step 3):** Present a +compact comparison so drift from the assessment baseline is visible. Omit this section entirely +when no assessment was consumed. + +```markdown +## Assessment Reconciliation (baseline: , generated ) + +| File | Assessment (blocks / tier) | Conversion (blocks / tier) | Match? | +|------|----------------------------|----------------------------|--------| +| a.sas | 22 / TIER_1_SQL | 22 / TIER_1_SQL | ✅ | +| b.sas | 22 / TIER_2_SP | 20 / TIER_1_SQL | ⚠️ blocks, tier | + +**Summary:** 8 matched, 1 mismatched. (⚠️ assessment is STALE — generated before the latest +source edit; re-run `/assess-sas-migration` to refresh.) [show the stale note only if `stale: true`] +Coverage: 0 files in assessment but not in this conversion; 0 the other way. +``` + +Mismatches are informational (source may legitimately have changed) and do not block conversion. + +**⚠️ STOP**: Review classification. If user disagrees with PySpark assignment, attempt SQL/Stored Proc alternative. + +**Permission Check (when Tier 2 blocks exist):** + +If any blocks are classified as Tier 2 (Stored Procedure), prompt: +``` +Some blocks require stored procedures (CREATE PROCEDURE). +Does your Snowflake role have CREATE PROCEDURE privilege on the target schema? +- Yes → proceed with stored procedure generation +- No → fall back to PySpark/SCOS notebook for these blocks (preserves logic fidelity) +- Unsure → proceed with stored procedures; compilation (Phase 4) will catch permission errors +``` + +**If user selects "No":** +- Reclassify affected blocks from Tier 2 to Tier 3 (PySpark/SCOS notebook) +- Generate a `.ipynb` notebook for those blocks using the SCOS template from `references/pyspark-fallback.md` +- Tier 1 (SQL) blocks remain as `.sql` output +- Update `conversion_state.json`: set `metadata.sp_permission: false` + +**If user selects "Unsure":** +- Proceed with Tier 2 stored procedures as normal +- Phase 4 (Snowflake Compilation) will surface permission errors +- At that point, offer the same PySpark/SCOS fallback + +**STATE WRITE on reclassification:** If the user reclassifies a file's tier (e.g., Tier 3 → Tier 2), update `conversion_state.json` immediately with the revised tier before proceeding to Step 5. + +### Step 5: Generate Conversion + +**For TIER 1 (SQL) blocks:** +- Use `references/function-mappings.md` +- Generate clean SQL with CREATE OR REPLACE TABLE +- Use explicit column lists (not SELECT *) when ROW_NUMBER() helpers are used +- WORK datasets → CREATE OR REPLACE TEMPORARY TABLE (no schema prefix) + +**For TIER 2 (Stored Procedure) blocks:** +- Follow Variable Binding Rules from `references/common-patterns.md` +- Use CTEs and window functions inside the procedure +- Include proper EXCEPTION handling (named exceptions ONLY) +- Use SELECT INTO :var (NEVER LET := SELECT) +- Persist cross-block variables via EXECUTE IMMEDIATE SET + +**For TIER 3 (PySpark/SCOS) blocks:** +- ONLY for HASH objects, CALL EXECUTE, DO UNTIL with external state, or PROC REG/GLM +- Use Snowpark Connect (SCOS) template from TIER 3 section +- Generate .ipynb notebook with SCOS setup cell +- Keep PySpark scope minimal - only the blocks that truly need it + +**Generation Rules:** +- Convert EVERY column derivation — no "similar pattern" shortcuts +- Convert EVERY branch in conditional logic — no skipped branches +- Preserve the exact output table count from the SAS block +- Preserve all CALL SYMPUT/SYMPUTX assignments +- For PROC FORMAT: use CASE expressions for simple value mappings, lookup tables for complex ranges +- For Oracle/DB2 passthrough (CONNECT USING): strip vendor wrappers, convert vendor-specific functions (DECODE→COALESCE, NVL→COALESCE, TO_DATE format strings) to Snowflake native SQL +- For %INCLUDE without source: generate CALL to stored procedure stub with MANUAL_REVIEW_REQUIRED +- For external macros without source: generate parameterized stub comment with inferred behavior +- **NEVER** delegate conversion to a subagent that writes a converter script — if using subagents for context management, each subagent must read SAS files and write SQL directly (not write a program that writes SQL) + +**PREREQUISITE FOR STEP 6:** As each file's .sql is written to disk, update its entry in `conversion_state.json` to `status: "converted"`. Set `current_step: "5"`. Step 6 CANNOT begin until all converted files are reflected in the state file. + +### Step 6: Post-Generation Self-Check (MANDATORY before compilation) + +Before submitting to Snowflake for compilation, verify the generated SQL for: +1. **No qualified WORK/temp table names** — temp tables must be bare names, never DATABASE.SCHEMA.TABLE +2. **TEMPORARY keyword preserved** — every CREATE OR REPLACE for a WORK dataset must include TEMPORARY +3. **No RAISE USING MESSAGE syntax** — only named exceptions with RAISE +4. **No unresolved SAS macro variables** — no `&var` or `&&var` in output +5. **No `:var` in scripting control flow** — `:var` only in SQL statements, not in IF/WHILE/assignment +6. **Cross-block variable persistence** — if a variable set in block N is used in block N+1, it must be persisted via SET session variable +7. **No SELECT * leaking helper columns** — ROW_NUMBER() dedup patterns must use explicit column lists +8. **All output tables accounted for** — count matches SAS block's output tables +9. **No vendor-specific SQL syntax remaining** — no DECODE, NVL, CONNECT TO, DISCONNECT FROM +10. **No "similar pattern" / "same as above" shortcuts** — every derivation must be explicit +11. **All validation source tables preserved** — every source table in the SAS validation block appears in the converted SQL validation query; no sources silently dropped +12. **Output table count matches SAS** — every CREATE TABLE/INSERT INTO target in the converted SQL maps to a SAS output dataset; count(DISTINCT output tables in SQL) >= count(DISTINCT output datasets in SAS) +13. **Output column count matches SAS** — column count of each converted output table >= column count of the corresponding SAS output dataset; no columns silently dropped +14. **No files dropped from prior version** (batch/re-run mode) — if re-running a conversion, verify the new output directory has >= the file count of the previous version; any reduction must be explicitly justified by consolidation +15. **Validation reference tables match SAS** — for each anti-join validation check, the reference table in the converted SQL must match the reference table in the SAS source; a master table (e.g., `CUSTOMER_MASTER`) must not be replaced with a derived/subset table (e.g., `CUSTOMER_SUBSET`) +16. **Referential checks not downgraded to value checks** — if SAS checks whether a key EXISTS in a reference table (anti-join), the converted SQL must also use an anti-join, not a simple `WHERE value IS NULL OR value = 0` check +17. **No unresolved stored procedure references** — if the converted SQL calls a stored procedure (e.g., `CALL SP_NAME()`), that SP must either be defined in the output files or explicitly flagged as MANUAL_REVIEW_REQUIRED +18. **No invented elements** — every variable, parameter, column name, and filter condition in the output must trace to the source SAS code; nothing fabricated or "assumed helpful" (see conversion-rules.md Anti-Hallucination rule) +19. **No session variables in VALUES()** — `$VAR` is invalid in `INSERT ... VALUES(...)`; use `INSERT ... SELECT` instead (see common-patterns.md Platform Constraints) +20. **EXECUTE AS CALLER on procs using session vars/temp tables** — any procedure that reads/sets a session variable or shares temp tables across the orchestration chain must be `EXECUTE AS CALLER`, never owner's rights +21. **Reserved-word columns quoted** — SAS column names that are Snowflake reserved words (DESC, TYPE, VALUE, LOCATION, NAME, FILE, STATUS, etc.) must be double-quoted +22. **XLSX I/O uses stored proc + CALL** — PROC IMPORT/EXPORT for XLSX must generate a Python stored procedure + CALL (never `COPY INTO`/`INFER_SCHEMA` on a native `.xlsx`); CSV/pipe stays native SQL +23. **Type-safe joins on migrated keys** — when joining key columns from tables migrated from different sources, both sides are cast to a common type (`::VARCHAR`) to avoid silent NUMBER-vs-VARCHAR zero-row failures (see conversion-rules.md → Migrated-Source Type Traps) +24. **Numeric aggregation guards VARCHAR measures** — `SUM`/`AVG`/etc. on measure columns from migrated tables are wrapped in `TRY_TO_DOUBLE`/`TRY_TO_NUMBER` when the column may be VARCHAR +25. **VARCHAR date columns wrapped in TO_DATE** — date-range comparisons on migrated `*_DT` columns wrap the **column** in `TO_DATE(col, fmt)`, not just the compared value + +If any check fails, fix the SQL before proceeding to compilation. + +**PREREQUISITE FOR STEP 6a:** After self-check passes for each file, update `conversion_state.json` with `self_check_passed: true`. Set `current_step: "6"`. Step 6a CANNOT begin until all files show `self_check_passed: true`. + +### Step 6a: Artifact Presence Verification (HARD GATE) + +Before proceeding to Step 7, verify these artifacts exist on disk: +- All converted `.sql` files written to `/` +- Count of output `.sql` files matches expected count from Step 4 classification + +**Enforcement:** +1. Check each artifact via filesystem read +2. If ANY missing: + - Print: `"BLOCKED: Cannot proceed to Step 7. Missing: [list of missing files]"` + - Update `conversion_state.json`: set `gates.step_6a_artifacts: "BLOCKED"` + - Attempt to generate missing files (re-run Step 5 for the specific missing file) + - If generation fails: surface to user with specific remediation action + - DO NOT proceed until gate passes +3. If all present: + - Update `conversion_state.json`: set `gates.step_6a_artifacts: "PASSED"` + - Proceed to Step 7 + +### Step 6b: State Persistence (automatic — no separate step needed) + +The `conversion_state.json` file has been incrementally updated throughout Steps 3-6a. At this point it contains the full classification, conversion status, and gate results. If context is exhausted here, the next session can resume from the earliest incomplete step by reading this file (see Step 1 pre-check). + +### Validation Decision Gate (MANDATORY — cannot be silently skipped) + +**Load `workflows/validation-pipeline.md`** — the single authoritative validation orchestrator. + +After Step 6a passes, present the **two-tier consent gate** (validation runs end-to-end by default once approved — the user does NOT hand-pick phases): +- **Tier-1 (zero cost):** "Run local validation (synthetic data + LLM trace)?" → Yes runs Phase 1 + Phase 2 automatically. +- **Tier-2 (Snowflake credits):** after Phase 2, ONE combined prompt: "Run full Snowflake validation end-to-end (compile + execute + integration/E2E orchestration)?" → Yes runs Phase 3 + Phase 4 + Phase 5 automatically, no further per-phase prompts. + +Phases: +- Phase 1: Synthetic Data (always runs under Tier-1) — with range-join/unit alignment (see `synthetic-data-rules.md`) +- Phase 2: LLM Trace (0 credits) +- Phase 3: Snowflake Compilation (minimal credits) +- Phase 4: Snowflake Execution — Tier 2/3 procedures (moderate credits) +- Phase 5: Integration & End-to-End Orchestration Test (moderate credits) — generates a reusable orchestration harness, runs the full pipeline in DAG order, asserts terminal tables non-empty + +Each phase has a HARD ARTIFACT GATE that verifies output files exist on disk before allowing the next phase to begin. At each gate, append a line to `checkpoint_log.jsonl`. See `workflows/validation-pipeline.md` for full protocol. + +**Phase-specific reference loading:** + +| Phase | Reference to Load | +|-------|-------------------| +| Phase 1 | `references/schema-inference.md` + `references/synthetic-data-rules.md` | +| Phase 2 | `references/validation-execution.md` Phase 3A | +| Phase 3 | `references/snowflake-compile.md` | +| Phase 4 | `references/snowflake-execution.md` | +| Phase 5 | `references/e2e-orchestration-test.md` | + +**PREREQUISITE:** The user's tier consent decisions must be recorded in `conversion_state.json` under `validation_selections` (`tier1_approved`, `tier2_approved`). + +### TRANSITION GATE: Steps 7-10 Are NOT Optional + +⛔ The conversion workflow is **NOT complete** after Step 6. +Steps 7, 8, 9, and 10 are **MANDATORY continuations** — not optional quality gates: + +- **Validation Pipeline**: Load `workflows/validation-pipeline.md`. Present the two-tier consent gate. Once a tier is approved, its phases execute sequentially and automatically (no per-phase prompts). At every checkpoint, append a line to `checkpoint_log.jsonl`. + - Phase 1 (Synthetic Data): Runs under Tier-1. Zero credits. HARD GATE on `source_table_ddl.sql` + `synthetic_data.sql`. + - Phase 2 (LLM Trace): Runs under Tier-1. Zero credits. HARD GATE on `tests//expected/*.csv`. + - Phase 3 (Snowflake Compilation): Runs under Tier-2. Minimal credits. HARD GATE on `compilation_results.json`. + - Phase 4 (Snowflake Execution Tier 2/3): Runs under Tier-2. Moderate credits. HARD GATE on `snowflake_execution_results.json`. + - Phase 5 (Integration & E2E Orchestration): Runs under Tier-2. Moderate credits. Generates reusable harness (`orchestration/sp_e2e_pipeline.sql`, `task_dag.sql`, `adf_pipeline.sql`, `testing/setup_test_data.sql`, `testing/expected_results.sql`). HARD GATE on `e2e_test_results.json`. +- **Step 10**: Generate `conversion_report.md` with ALL sections. Generate `conversion_report.docx` (best effort). + +**DO NOT** present final output to the user until Step 10 is complete. + +If context window is nearly exhausted after Step 6, **save all converted files to disk**. The `conversion_state.json` file already captures progress — the next session will resume from the earliest incomplete step. + +**Load `workflows/steps-8-10-post-conversion.md`** for Step 10 (Report Generation) only. Steps 8 and 9 are now handled by the modular validation pipeline. + +### POST-STEP-10 ARTIFACT GATE (HARD GATE) + +After Step 10, verify these artifacts exist on disk: + +**Required artifacts:** +- `conversion_state.json` with `current_step: "complete"` (the state tracker is the primary proof of workflow completion) +- `checkpoint_log.jsonl` (append-only audit trail; must contain at least the Step 1 STARTED line and a final `complete` line) +- `compilation_results.json` (even if compilation skipped, file must exist with status) +- `source_table_ddl.sql` (required in batch mode) +- `synthetic_data.sql` (required in batch mode) +- `e2e_test_results.json` + `orchestration/sp_e2e_pipeline.sql` (required when Phase 5 ran) +- `conversion_report.md` with all 6 mandatory sections + +**Best-effort artifacts:** +- `conversion_report.docx` (if missing, print: `"DOCX not generated. Install with: pip install python-docx OR brew install pandoc"`) + +**Enforcement:** +1. Check each required artifact via filesystem read +2. If ANY required artifact missing: + - Print: `"BLOCKED: Conversion incomplete. Missing: [list]"` + - Update `conversion_state.json`: set `gates.step_10_final_artifacts: "BLOCKED"` + - Attempt to generate missing artifacts (re-run the specific generation step) + - If generation fails after 1 attempt: surface to user with specific action + - DO NOT declare conversion complete until gate passes +3. If all required artifacts present: + - Update `conversion_state.json`: set `current_step: "complete"`, `gates.step_10_final_artifacts: "PASSED"` + - Append final checkpoint line to `checkpoint_log.jsonl`: `step:"complete" status:"PASSED" notes:"conversion complete"` + - Print: `"All required artifacts verified. Conversion complete."` + +--- + +## Output Formats + +See `templates/output-sql.md` for pure SQL and stored procedure output format, and `templates/output-mixed.md` for mixed SQL + PySpark. +All output files MUST include the standard file header (Converted from, Target Schema, Translation Tier). + +--- + +## Stopping Points + +**Interactive mode (< 10 files):** +- ✋ Step 1: Confirm SAS code understanding +- ✋ Step 4: Review block classification and dependency map (especially any PySpark assignments) +- ✋ Step 7: Review LLM tracing results, confirm proceed to compilation +- ✋ Step 8: Confirm compilation results +- ✋ Step 9: User choice on validation testing +- ✋ Step 10: Present conversion report + +**Batch mode (10+ files):** +- ✋ Step 1: Confirm target schema and migration mode (ONCE for all files) +- ✋ Step 10: Present consolidated conversion report (ONCE at end) + +## Success Criteria + +- ✅ ALL blocks attempted as SQL first +- ✅ Stored Procedures used before PySpark +- ✅ PySpark only for true edge cases (HASH, CALL EXECUTE, statistical procs) +- ✅ All code compiles successfully +- ✅ Confidence levels documented +- ✅ Low-confidence blocks clearly marked with MANUAL_REVIEW_REQUIRED +- ✅ Inter-block dependencies tracked and correct +- ✅ Every column derivation explicit — no "similar pattern" shortcuts +- ✅ Every macro branch converted — no skipped branches +- ✅ WORK tables use TEMPORARY keyword with no schema prefix +- ✅ Snowflake Scripting syntax correct (named exceptions, :var only in SQL, SELECT INTO) +- ✅ Post-generation self-check passed (Step 6) +- ✅ Conversion report generated with all applicable sections (Step 10) +- ✅ Cross-file DAG and orchestration recommendation provided (when 2+ files) +- ✅ All manual remediation items catalogued with priority +- ✅ Validation checks use data-driven lookups — no hardcoded value lists replacing dimension table references +- ✅ All validation source tables preserved — no sources silently dropped from SAS checks +- ✅ Output column count matches SAS — no columns silently dropped from output tables +- ✅ Consolidation preserves all output tables — no SAS output datasets lost during file merging +- ✅ All mandatory artifacts present (Step 6a checklist passed) + +## Mandatory Output Artifacts + +At the end of every conversion, verify these files exist before declaring completion: + +| Artifact | File | Required | +|----------|------|----------| +| Converted SQL | `/*.sql` | Always | +| Conversion state | `/conversion_state.json` | Always | +| Checkpoint log | `/checkpoint_log.jsonl` | Always (append-only audit trail) | +| Source table DDL | `/source_table_ddl.sql` | Batch mode | +| Synthetic data | `/synthetic_data.sql` | Batch mode | +| Compilation results | `/compilation_results.json` | Always (even if skipped: status field required) | +| Snowflake execution results | `/snowflake_execution_results.json` | When Phase 4 runs | +| E2E test results | `/e2e_test_results.json` | When Phase 5 runs | +| E2E orchestration harness | `/orchestration/sp_e2e_pipeline.sql`, `task_dag.sql`, `adf_pipeline.sql` | When Phase 5 runs | +| E2E test harness | `/testing/setup_test_data.sql`, `expected_results.sql` | When Phase 5 runs | +| Conversion report | `/conversion_report.md` | Always | +| Conversion report DOCX | `/conversion_report.docx` | Best effort | + +**Enforced by POST-STEP-10 HARD GATE** — if any required artifact is missing, the gate blocks completion and attempts generation. + +--- + +## Output + +Snowflake SQL or Stored Procedures with: +- Tier classification documented +- Conversion notes per block +- Compilation status per file (from `compilation_results.json`) +- Test artifacts (`source_table_ddl.sql`, `synthetic_data.sql`) +- Conversion report (always generated) with: + - Difficult/unremediated block inventory + - Test results summary + - Cross-file DAG diagram (when applicable) + - Snowflake-native orchestration recommendation with SQL + - Manual remediation action items with priorities diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/auto-validation.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/auto-validation.md new file mode 100644 index 0000000..616a11d --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/auto-validation.md @@ -0,0 +1,321 @@ +# Automated Validation Pipeline (No Real Data Required) + +Orchestrates schema inference, synthetic data generation, SAS logic tracing, Snowflake compilation, and Snowflake execution into a single automated pipeline. + +## When to Load + +Load this file at Step 7 (Phase 2: SAS Logic Trace) and Step 9 (Phase 4: Snowflake Execution). Also loaded when the user requests validation testing without real data. + +--- + +## Overview + +``` +Phase 1: DDL Inference → source_table_ddl.sql +Phase 2: Synthetic Data Gen → synthetic_data.sql +Phase 3: SAS Logic Trace → tests//expected/*.csv +Phase 4: Snowflake Compilation → compilation_results.json +Phase 5: Snowflake Execution → snowflake_execution_results.json +``` + +This pipeline runs WITHOUT real data. The expected baseline comes from manually tracing SAS logic against synthetic data. The actual result comes from executing converted Snowflake SQL on Snowflake against the same synthetic data. + +--- + +## Phase 1: DDL Inference + +**Input:** All converted `.sql` files + original `.sas` source files +**Output:** `/source_table_ddl.sql` + +### 1.1 Extract Source Table References + +Parse every converted SQL file. Collect every table name that appears in FROM or JOIN clauses but is NOT created in the same file (i.e., it is a source/input table, not an intermediate). + +```python +import re, os, glob + +def extract_source_tables(sql_dir): + created_tables = set() + referenced_tables = set() + + for sql_file in sorted(glob.glob(os.path.join(sql_dir, '*.sql'))): + with open(sql_file) as f: + content = f.read() + + # Tables created in this file + for m in re.finditer(r'CREATE\s+(?:OR\s+REPLACE\s+)?(?:TEMPORARY\s+)?TABLE\s+(\w+)', content, re.IGNORECASE): + created_tables.add(m.group(1).upper()) + for m in re.finditer(r'INSERT\s+INTO\s+(\w+)', content, re.IGNORECASE): + created_tables.add(m.group(1).upper()) + + # Tables referenced in FROM/JOIN + for m in re.finditer(r'(?:FROM|JOIN)\s+(\w+)', content, re.IGNORECASE): + tbl = m.group(1).upper() + if tbl not in ('SELECT', 'WHERE', 'ON', 'AND', 'OR', 'LEFT', 'RIGHT', 'FULL', 'INNER', 'OUTER', 'CROSS', 'LATERAL'): + referenced_tables.add(tbl) + + # Source tables = referenced but never created + source_tables = referenced_tables - created_tables + return source_tables +``` + +### 1.2 Infer Column Types + +For each source table, scan the SAS source files AND the converted SQL to infer columns and types: + +1. **From converted SQL:** Extract column names from SELECT lists, WHERE clauses, JOIN conditions, and GROUP BY +2. **From SAS source:** Extract FORMAT/INFORMAT statements, PROC IMPORT attributes, INPUT statement column definitions +3. **Apply type mapping** from `references/schema-inference.md` +4. **Apply fallback heuristics** based on column naming conventions (_ID→VARCHAR(50), _AMT→NUMBER(18,4), _DT→DATE, etc.) + +### 1.3 Generate DDL File + +Write one CREATE TABLE IF NOT EXISTS per source table: + +```sql +-- Auto-generated source table DDL +-- Inferred from SAS code analysis +-- Generated by: /convert-sas-to-snowflake Step 7a (auto-validation) + +CREATE OR REPLACE TEMPORARY TABLE ( + , -- inferred: + , -- inferred: + ... +); +``` + +--- + +## Phase 2: Synthetic Data Generation + +**Input:** `source_table_ddl.sql` + converted SQL dependency graph +**Output:** `/synthetic_data.sql` + +### 2.1 Build Table Dependency Graph + +Parse all converted SQL files to determine which tables depend on which: +- File A creates TABLE_X from SOURCE_1 and SOURCE_2 +- File B creates TABLE_Y from TABLE_X and SOURCE_3 +- Therefore: File B depends on File A + +### 2.2 Generate Per-Table Data + +For each source table, apply `references/synthetic-data-rules.md`: + +| Rule | Implementation | +|------|----------------| +| Row count | 5-8 rows per table | +| Join keys | For every JOIN pair, include: matching keys, left-only key, right-only key, duplicate key | +| NULL rows | At least 1 row per table with NULL in numeric and character columns | +| Boundary values | 0, -1, empty string, year-start/end dates | +| BY-group testing | At least 1 key with 3+ rows (for FIRST./LAST. and GROUP BY) | +| RETAIN/LAG testing | At least 2 partitions with 4+ rows each | + +### 2.3 Cross-Table Key Alignment + +When two tables are joined on a key: +```sql +-- TABLE_A has keys: 1, 2, 3, 4 (key 4 = left-only) +-- TABLE_B has keys: 1, 2, 3, 5 (key 5 = right-only) +-- Key 2 has duplicates in TABLE_A (tests one-to-many) +``` + +### 2.4 Output Format + +Write INSERT statements grouped by table: +```sql +-- Synthetic data for validation testing +-- Generated by: /convert-sas-to-snowflake Step 7a (auto-validation) + +-- Table: CUSTOMER_DIM (source for: 02, 03, 04) +INSERT INTO CUSTOMER_DIM VALUES (...); +INSERT INTO CUSTOMER_DIM VALUES (...); +... +``` + +--- + +## Phase 3: SAS Logic Trace (Expected Baseline) — Runs at Step 7 + +**Input:** Original SAS source code + synthetic data (from Step 7a) +**Output:** `tests//expected/expected_
    .csv` + +This is the critical differentiator. The LLM reads the original SAS code and mentally executes it against the small synthetic dataset to produce expected output. **This phase runs as part of Step 7 (Local LLM Tracing Validation), BEFORE Snowflake compilation.** + +### 3.1 Trace Rules + +Follow `references/validation-execution.md` Phase 3A rules: + +- **DATA Step:** Initialize PDV, process row-by-row, apply SAS missing semantics (missing < any value), track RETAIN, apply FIRST./LAST. flags +- **PROC SQL:** Standard SQL semantics (not DATA step), NULL propagation +- **MERGE:** NOT equivalent to JOIN. Interleave within BY-groups. Track IN= flags. + +### 3.2 Batch Mode Priority Selection + +For batch mode (10+ files), trace a **representative subset** instead of all files: + +| Priority | Selection Criteria | Target Count | +|----------|-------------------|--------------| +| P1 | Anti-join validation patterns (most common) | 2-3 files | +| P2 | GROUP BY / aggregation patterns | 2-3 files | +| P3 | Multi-table JOIN + UNION ALL | 2-3 files | +| P4 | CASE expression / product mapping | 1-2 files | +| P5 | Window function / ROW_NUMBER dedup | 1-2 files | +| P6 | Complex multi-step (>3 blocks) | 1-2 files | + +Target: 10-15 files covering all major patterns. + +### 3.3 Iterative Batch Execution (Context-Aware) + +For large batches (50+ files), process the priority subset in **batches of 5 files**: + +1. Initialize `trace_progress` in `conversion_state.json` with selected files in `files_remaining` +2. Process 5 files at a time: + - Read SAS source + synthetic data + - Trace logic row-by-row + - Write expected CSV/SQL to `tests//expected/` + - Move file from `files_remaining` to `files_traced` in state +3. After each batch: check context budget + - If sufficient: continue to next batch + - If nearing limit: save state, exit gracefully (next session resumes) +4. Resume: read `conversion_state.json`, skip `files_traced`, process `files_remaining` + +**State schema:** +```json +{ + "trace_progress": { + "total_files_in_scope": 15, + "files_traced": ["file_a", "file_b"], + "files_remaining": ["file_c", "file_d"], + "current_batch": 1, + "batch_size": 5, + "concerns_found": ["description of issue"] + } +} +``` + +### 3.4 Output Format + +For each traced output table, write: +- CSV file: `tests//expected/expected_.csv` +- Reference SQL: `tests//expected/expected_.sql` + +--- + +## Phase 4: Snowflake Compilation — Runs at Step 8 + +**Input:** Converted Snowflake SQL files +**Output:** `compilation_results.json` + +Follow `references/snowflake-compile.md`: +1. Probe: compile one simple file with `only_compile=true` +2. If target schema missing: auto-create temp environment +3. Compile all files +4. Fix-and-retry loop (max 3 per file) +5. Write `compilation_results.json` +6. Drop temp environment (if created) + +--- + +## Phase 5: Snowflake Execution — Runs at Step 9 + +**Input:** Converted Snowflake SQL + synthetic data + expected baselines +**Output:** `snowflake_execution_results.json` + +### 5.1 Setup + +Execute `source_table_ddl.sql` and `synthetic_data.sql` on Snowflake to create temporary tables with test data. + +### 5.2 Execute Each Converted File (Dependency Order) + +Build dependency order from cross-file DAG. Execute each file sequentially against the synthetic data on Snowflake. + +### 5.3 Capture Output and Compare + +For each output table created by the converted SQL: +1. SELECT * from the output table +2. Compare against the expected baseline from Phase 3 (`tests//expected/expected_
    .csv`) +3. Apply `references/comparison-rules.md` for equivalence rules + +### 5.4 Mismatch Classification + +Use the classification table from `references/comparison-rules.md`: + +| Pattern | Likely Cause | +|---------|-------------| +| NULL vs non-NULL | Missing value handling error | +| Row count actual > expected | Many-to-many join | +| Row count actual < expected | Wrong JOIN type or extra WHERE | +| Numeric diff < 0.01 | Float precision (acceptable) | +| Date off by 1 | INTNX alignment | +| Duplicate rows | Missing DISTINCT/QUALIFY | + +### 5.5 Result Format + +```json +{ + "validation_date": "", + "total_files_tested": 12, + "passed": 10, + "failed": 2, + "results": [ + { + "file": "02_to_04_validate_customers.sql", + "tables_tested": 1, + "schema_match": true, + "row_count_match": true, + "value_match": true, + "status": "PASS" + }, + { + "file": "05_build_final_tables.sql", + "tables_tested": 13, + "schema_match": true, + "row_count_match": false, + "value_match": false, + "status": "FAIL", + "mismatches": [ + {"table": "FINAL_SUMMARY", "type": "row_count", "expected": 6, "actual": 8, "cause": "Many-to-many join on CUSTOMER_KEY"} + ] + } + ] +} +``` + +### 5.6 Cleanup + +Drop all temporary tables and objects created during testing. + +--- + +## Batch Mode Execution Strategy + +For 10+ files, the pipeline uses subagents: + +### Subagent 1: DDL + Synthetic Data (Phases 1-2) +- Parse all SQL files for source tables +- Infer types from SAS source +- Generate source_table_ddl.sql +- Generate synthetic_data.sql + +### Subagent 2-N: Trace per Priority File (Phase 3) +- Each subagent handles 2-3 files +- Traces SAS logic against synthetic data +- Returns expected baselines + +### Aggregator: Final Report +- Merge results from all subagents +- Write snowflake_execution_results.json +- Update conversion_report.md Section 3 + +--- + +## Integration with Steps 7 and 9 + +This pipeline spans two skill steps: + +| Step | Phases | What Happens | +|------|--------|--------------| +| Step 7 (Local LLM Tracing) | Phases 1-3 | DDL inference, synthetic data gen, SAS logic trace → expected baselines | +| Step 8 (Compilation) | Phase 4 | Snowflake compilation with fix-and-retry | +| Step 9 (Snowflake Execution) | Phase 5 | Snowflake execution → actual results, comparison against baselines | diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/batch-mode.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/batch-mode.md new file mode 100644 index 0000000..1384b36 --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/batch-mode.md @@ -0,0 +1,168 @@ +# Batch Mode (10+ Files) + +When converting 10 or more SAS files in a single session, **batch mode** is automatically enabled. Batch mode reduces interactive stopping points to keep large conversions efficient. + +**NOTE**: All Snowflake operations (compilation, stub tables) require user confirmation per the Snowflake Interaction Policy in `workflows/steps-8-10-post-conversion.md`. Testing mode (Local or Snowflake) is selected by user prompt in Step 9b. + +## Batch Mode Rules + +| Step | Interactive (Default) | Batch Mode | +|------|----------------------|------------| +| Step 1: Gather Input | Ask per file | Gather ONCE for entire directory; apply same schema/mode to all files | +| Step 4: Block Analysis | STOP for review | Log classification per file; do NOT stop for confirmation. Present aggregate summary at end. | +| Step 7: LLM Tracing | N/A | Step 7a (test artifacts) always runs. For 7b (logic trace): prompt user via Validation Decision Gate ONCE (see SKILL.md) — user selects Full / Trace only / Skip. Once selected, auto-execute without further prompts (consent propagation). | +| Step 7c: Trace Results | STOP for review | Present summary. If compilation pre-approved (consent propagation): auto-proceed to Step 8 without "Proceed to compilation?" prompt. | +| Step 8: Compilation | STOP for results | Prompt ONCE. If user says Yes and DB doesn't exist, auto-create temporary compilation environment via Step 8-pre (consent propagation). Do NOT re-prompt for Step 8-pre options. Do NOT stop on first failure. | +| Step 8a: Stub Tables | N/A | If consent propagation active: auto-create stubs, auto-drop after compilation. Otherwise: confirm before creating stubs; confirm before dropping stubs. | +| Step 9: Validation | Ask per file | Follows validation scope from the Validation Decision Gate. Executes on Snowflake with credit confirmation. No separate "Select testing mode" prompt needed — scope is already determined. | +| Step 10: Report | STOP for review | Generate single consolidated report for all files with ALL 6 sections. Present once at end. | + +## Batch Mode Detection + +Batch mode activates when: +- User provides a **directory path** containing 10+ `.sas` files +- User provides a **file list** with 10+ entries +- User says "convert all", "batch convert", or "convert the directory" + +## Context Budget (Large Batch) + +When converting 50+ files, the context window may be exhausted during Steps 1-6. Split the workflow into phases: + +**CONVERSION METHOD (MANDATORY):** Each SAS file MUST be individually read by the LLM, semantically analyzed for SAS-specific behavior (missing values, BY-group processing, MERGE semantics, RETAIN state), and converted to Snowflake SQL by the LLM directly writing the .sql output. Do NOT write a script/program to batch-convert files. When context is limited, process 10-20 files per context window and resume via `conversion_state.json`. + +- **Phase A** (Steps 1-6): Convert all files, write `.sql` to disk +- **Phase B** (Step 7): LLM Tracing — generate DDL + synthetic data, trace SAS logic, produce expected baselines +- **Phase C** (Step 8): Read back files from disk, compile all, write `compilation_results.json` +- **Phase D** (Steps 9-10): Snowflake execution, comparison, conversion report + +If context is exhausted during Phase A, save progress to disk and prompt: +> "Conversion files written to ``. Continue with compilation and testing? (Steps 7-9)" + +This ensures Steps 7-9 always execute, even across multiple context windows. + +**CRITICAL**: If context is exhausted before Steps 7-10, the `conversion_state.json` state tracker ensures the next session resumes from the earliest incomplete step. The prompt-based gates in Steps 7c and 8 still apply when resuming. + +## Context Window Recovery + +When a new context window starts after Step 6 (or mid-conversion): +1. Read `/` to list existing `.sql` files — confirm conversion files exist +2. Check for `compilation_results.json` — if missing, start at Step 7 (LLM tracing first, then compilation) +3. Check for `source_table_ddl.sql` + `synthetic_data.sql` — if missing, start at Step 7a +4. Check for `conversion_report.md` — if missing, start at Step 10 +5. **Always pick up at the earliest missing step** — never re-run completed steps +6. If re-running a conversion (same source, new output), verify the new output has >= the file count of the previous version +7. **Check intra-phase progress** — read `conversion_state.json` for `trace_progress` and `execution_progress` fields. If either has non-empty `files_remaining`, resume within that phase. + +## Intra-Phase Checkpointing (Large Batch) + +When a phase cannot complete in a single context window, use these checkpoints to enable resume: + +### Phase B (Step 7b — LLM Trace): +- **Batch size:** 5 files per checkpoint +- **Checkpoint trigger:** After each batch of 5 files traced and written to disk +- **State field:** `conversion_state.json` → `trace_progress` +- **Resume condition:** `trace_progress.files_remaining` is non-empty +- **Recovery:** Read `files_traced` list, skip those, continue with `files_remaining` +- **Output per batch:** `tests//expected/expected_
    .csv` files written to disk + +### Phase C (Step 8 — Compilation): +- **Batch size:** 20 files per checkpoint +- **Checkpoint trigger:** After each batch of 20 files compiled +- **State field:** `conversion_state.json` → per-file `compilation_status` +- **Resume condition:** Any file has `compilation_status: "PENDING"` +- **Recovery:** Compile only PENDING files (skip those already SUCCESS/FAILED) + +### Phase D (Step 9 — Snowflake Execution): +- **Batch size:** 10-15 files per checkpoint (respecting dependency order) +- **Checkpoint trigger:** After each batch executed and compared +- **State field:** `conversion_state.json` → per-file `validation_status` +- **Resume condition:** Any Tier 2/3 file has `validation_status: "PENDING"` +- **Recovery:** Execute only PENDING files (skip those already PASS/FAIL) + +### Context Budget Heuristic + +The LLM should monitor its own context consumption and apply this heuristic: + +``` +IF context_consumed > 60% AND current_phase is NOT complete: + 1. Finish current batch (do NOT leave mid-file — partial traces are invalid) + 2. Write all state to disk (conversion_state.json + any output CSVs) + 3. Inform user: "Context budget nearing limit. [N]/[total] files processed. + Progress saved to conversion_state.json. Resume in next session." + 4. If user says "continue": attempt ONE more batch, then re-evaluate + 5. If user says "stop" or context exhausted: exit gracefully +``` + +**Rules:** +- NEVER leave a file half-traced or half-executed — complete the current file before checkpointing +- ALWAYS write state to disk before yielding — unsaved progress is lost +- The batch sizes (5 for trace, 20 for compile) are defaults — adjust downward if files are very large (>500 lines) or context is tight + +## Batch Mode Output + +In batch mode, the conversion report includes: +- **Aggregate compilation status** (X/Y files compiled, Z failures) +- **Per-phase summary table** (not per-file — group by pipeline phase) +- **Top error categories** (e.g., "15 files: PROC IMPORT stub", "3 files: PROC TRANSPOSE manual") +- **Single consolidated DAG** (not per-file DAGs) +- **Single orchestration recommendation** with full Tasks DAG SQL + +--- + +## Step 8a: Batch-Mode Compilation (when target tables do not exist) + +When compiling against an empty schema (greenfield migration), target tables referenced in FROM/JOIN clauses will not exist. Use stub tables to unblock compilation: + +1. **Scan all output `.sql` files** for table references in FROM, JOIN, and IDENTIFIER() clauses +2. **Build a dependency-ordered list** of tables — tables created by earlier scripts are inputs to later scripts +3. **For each referenced table not yet created by a prior script**: + - Infer column names and types from the SELECT that populates it upstream (CREATE TABLE AS SELECT) + - If no upstream definition exists (external source table), generate a stub using `references/schema-inference.md` + - Prepare the stub DDL: `CREATE TABLE IF NOT EXISTS .
    ( VARIANT, ...)` +4. **Confirm stub creation (per Snowflake Interaction Policy):** + ``` + SNOWFLAKE OBJECT CREATION + + Action: Create [N] temporary stub tables for compilation + Schema: TARGET_SCHEMA + Tables: + - STUB_TABLE_1 + - STUB_TABLE_2 + - ... + Purpose: Scaffolding only -- will be dropped after compilation + Credit impact: Minimal + + Proceed? [Yes] / [No - Skip stub compilation] / [Show DDL] + ``` +5. If approved: **execute stubs in dependency order** (using the DAG from Step 10) +6. If declined: skip stub creation and compile without stubs (expect missing-table errors) +7. **Compile each `.sql` file** with `only_compile=true` +8. **Record pass/fail per file** — include in the conversion report (Section 1 Compilation Status column) +9. **Confirm stub cleanup:** + ``` + SNOWFLAKE CLEANUP + + Action: Drop [N] stub tables created for compilation + Tables: + - STUB_TABLE_1 + - STUB_TABLE_2 + - ... + + Proceed? [Yes - Drop All] / [No - Keep for debugging] + ``` +10. If approved: **drop all stub tables** (they were scaffolding only) +11. If declined: keep stubs for user inspection + +```sql +-- Stub table pattern (when column types cannot be inferred) +CREATE TABLE IF NOT EXISTS TARGET_SCHEMA.SOURCE_TABLE ( + _STUB_COL VARIANT +); +-- NOTE: This stub exists only for compilation. Drop after validation. +``` + +**Batch compilation rules:** +- Do NOT stop on first failure — compile ALL files and report aggregate results +- Group compilation errors by type (missing table, syntax error, type mismatch) +- If >80% of files compile successfully, report as PASS with exceptions listed +- If <80% compile, flag as WARNING and list the top error categories diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/checkpoint-logging.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/checkpoint-logging.md new file mode 100644 index 0000000..494b45e --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/checkpoint-logging.md @@ -0,0 +1,119 @@ +# Checkpoint Logging (`checkpoint_log.jsonl`) + +## When to Load + +Load once, early (Step 1), and keep the append rule in context for the whole conversion. This file is lightweight — it defines an append-only audit trail that complements `conversion_state.json`. + +--- + +## Purpose + +`conversion_state.json` is a **current-state snapshot** — it is overwritten at every transition, so it cannot answer "what happened, and when?" across a multi-session conversion. + +`checkpoint_log.jsonl` is the **immutable audit trail** — one JSON object per line, appended (never overwritten) at every checkpoint. It enables: +- Reconstructing exactly what passed/failed/skipped across context windows and sessions +- Debugging a stalled or failed conversion without replaying it +- Auditing credit-consuming operations (when each Snowflake phase ran) + +The two files are complementary and BOTH required: + +| File | Role | Write mode | +|------|------|-----------| +| `conversion_state.json` | Current state (resume point, per-file status, gates) | Overwrite | +| `checkpoint_log.jsonl` | Historical event log (audit trail) | Append-only | + +**Location:** `/checkpoint_log.jsonl` + +--- + +## Line Schema + +Each line is a single complete JSON object (JSON Lines format — no array wrapper, no trailing commas): + +```json +{"ts": "2026-06-23T10:15:30Z", "step": "6a", "gate": "step_6a_artifacts", "status": "PASSED", "artifacts_written": ["customers_clean.sql"], "rows_affected": null, "notes": "12 .sql files verified on disk"} +``` + +| Field | Type | Required | Meaning | +|-------|------|----------|---------| +| `ts` | string (ISO 8601 UTC) | Yes | When the checkpoint occurred | +| `step` | string | Yes | Workflow step or phase (e.g. `"1"`, `"3"`, `"6a"`, `"phase_2"`, `"phase_5"`) | +| `gate` | string or null | Yes | Gate name if this checkpoint corresponds to a gate (e.g. `"phase_3_snowflake_compile"`), else null | +| `status` | string | Yes | `STARTED` \| `PASSED` \| `FAILED` \| `BLOCKED` \| `SKIPPED` \| `INFO` | +| `artifacts_written` | string[] | No | Files written/updated at this checkpoint (basenames) | +| `rows_affected` | integer or null | No | Row count when relevant (e.g. synthetic inserts, execution output) | +| `notes` | string | No | Short human-readable context (keep < 200 chars) | + +--- + +## Append Rule (BLOCKING — append BEFORE advancing) + +At every point where the workflow writes `conversion_state.json` (see the transition table in `references/state-tracker-schema.md`), also append ONE line to `checkpoint_log.jsonl` **before** beginning the next step. + +Order of operations at any checkpoint: +1. Complete the step's work (artifacts on disk) +2. Update `conversion_state.json` (current-state snapshot) +3. Append a line to `checkpoint_log.jsonl` (audit trail) +4. Only then advance to the next step + +If `checkpoint_log.jsonl` does not exist yet, create it with the first line. Never rewrite existing lines — append only. + +--- + +## Append Helper + +Use the Bash tool to append (do NOT read-modify-write the whole file — that defeats append-only and risks corruption): + +```python +import json, os +from datetime import datetime, timezone + +def append_checkpoint(output_dir, step, status, gate=None, artifacts=None, rows=None, notes=None): + entry = { + "ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "step": step, + "gate": gate, + "status": status, + "artifacts_written": artifacts or [], + "rows_affected": rows, + "notes": notes or "" + } + path = os.path.join(output_dir, "checkpoint_log.jsonl") + with open(path, "a") as f: + f.write(json.dumps(entry) + "\n") +``` + +--- + +## Checkpoints to Log (minimum) + +| Step / Phase | status values to emit | +|--------------|----------------------| +| Step 1 (input gathered) | `STARTED` (conversion begins) | +| Step 2.5 (source resolution) | `PASSED` or `SKIPPED` | +| Step 3 (classification) | `PASSED` | +| Step 5 (conversion) | `INFO` per batch, `PASSED` when all converted | +| Step 6 (self-check) | `PASSED` | +| Step 6a (artifact gate) | `PASSED` or `BLOCKED` | +| Phase 1 (synthetic data) | `STARTED`, then `PASSED`/`BLOCKED` | +| Phase 2 (LLM trace) | `STARTED`, then `PASSED`/`SKIPPED`/`BLOCKED` | +| Phase 3 (compilation) | `STARTED`, then `PASSED`/`SKIPPED`/`BLOCKED` | +| Phase 4 (Tier 2/3 execution) | `STARTED`, then `PASSED`/`SKIPPED`/`BLOCKED` | +| Phase 5 (E2E orchestration) | `STARTED`, then `PASSED`/`SKIPPED`/`BLOCKED` | +| Step 10 (report) | `PASSED` | +| Complete | `PASSED` with notes "conversion complete" | + +For credit-consuming phases (3, 4, 5), also log an `INFO` line capturing the user's consent decision (e.g. `notes: "Tier-2 gate: user approved"`). + +At Step 3, the `PASSED` line should carry an assessment note capturing baseline consumption and +reconciliation, e.g. `notes: "assessment consumed=true matched=8 mismatched=1 stale=false"` (or +`notes: "assessment consumed=false reason=not_found"` when none was discovered). + +--- + +## Reading the Log (resume / audit) + +On resume, the log is informational only — `conversion_state.json` remains the authoritative resume point. Use the log to: +- Show the user a timeline: `tail` the file and render each line as `[ts] step status — notes` +- Confirm a credit phase actually ran (search for the phase's `PASSED` line) +- Diagnose where a prior session stopped (last line = last completed checkpoint) diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/classification-logic.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/classification-logic.md new file mode 100644 index 0000000..ddbe5c9 --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/classification-logic.md @@ -0,0 +1,133 @@ +# Classification Logic (Deferred Reference) + +## When to Load + +Load this file at Step 3 (Classify Each Block) if you need the detailed classification pseudocode, confidence assessment, or output-type determination logic. + +> **Canonical source:** The block-counting and tiering rules below mirror +> `../../references/block-tiering-spec.md`. That spec is the single source of truth shared with +> the `assess-sas-migration` CLI — keep this file in sync with it. In particular: +> **enumerate each DATA/PROC step inside a macro as its own block**, and **exclude DI Studio / +> DataFlow boilerplate** (macros named `etls_*`, `rcset`, `rcsetds`; content markers `etls_`, +> `perfinit`, `log4sas`, `armsubsys`, `sas data integration studio`) from both the block count +> and tiering. + +--- + +## SQL-First Block Classification + +```python +def classify_block_sql_first(block_content, block_type='DATA_STEP'): + """ + SQL-FIRST classification. Returns: ('sql' | 'stored_proc' | 'pyspark', reason) + PySpark via SCOS is LAST RESORT only. + `block_type` is the parsed block kind (e.g. 'DATA_STEP', 'PROC_SQL') — used to + scope the branch-count rule so PROC SQL CASE WHEN is not mistaken for procedural logic. + """ + content_lower = block_content.lower() + + # === TIER 3: PySpark/SCOS (ONLY when SQL/SP truly cannot work) === + if 'declare hash' in content_lower: + return ('pyspark', 'HASH objects - no SQL equivalent, use SCOS') + + if 'call execute' in content_lower: + return ('pyspark', 'CALL EXECUTE - dynamic code generation, use SCOS') + + if ('do until' in content_lower or 'do while' in content_lower): + if 'symput' in content_lower or content_lower.count('call ') > 2: + return ('pyspark', 'DO loop with external state - use SCOS') + + # Statistical modeling (canonical list — keep in sync with block-tiering-spec.md) + stats_procs = ['proc reg', 'proc glm', 'proc logistic', 'proc cluster', + 'proc factor', 'proc phreg', 'proc lifetest', 'proc surveyselect', + 'proc mixed', 'proc genmod', 'proc nlmixed'] + for proc in stats_procs: + if proc in content_lower: + return ('pyspark', f'Statistical modeling: {proc} - use SCOS') + + # === TIER 2: Stored Procedure === + if 'retain ' in content_lower and 'first.' in content_lower: + if '= 0' in content_lower or '= .' in content_lower: + return ('stored_proc', 'RETAIN with conditional reset - use SP') + + if ('first.' in content_lower or 'last.' in content_lower): + if content_lower.count('output ') > 1: + return ('stored_proc', 'FIRST./LAST. with multiple OUTPUT - use SP') + + if content_lower.count('output ') > 1 and 'output;' not in content_lower: + return ('stored_proc', 'Multiple OUTPUT datasets - use SP') + + # Branch-count rule applies to DATA steps ONLY. A PROC SQL CASE WHEN is pure + # SQL (Tier 1) however many WHEN clauses it has. See block-tiering-spec.md. + if block_type == 'DATA_STEP' and ( + content_lower.count('if ') > 5 or content_lower.count('when ') > 5): + return ('stored_proc', 'Complex branching (>5 IF/WHEN in DATA step) - use SP') + + # 3+ sequential DML operations -> orchestration + error handling + if sum(1 for kw in ['delete ', 'insert ', 'update '] if kw in content_lower) >= 3: + return ('stored_proc', '3+ sequential DML operations - use SP') + + # === TIER 1: SQL (Default - EVERYTHING ELSE) === + return ('sql', 'SQL-translatable with window functions/CTEs') + + +def determine_output_type(blocks, user_requested_notebook=False): + """ + Determine output type. DEFAULT is ALWAYS .sql + Returns: 'sql' | 'notebook' + """ + if user_requested_notebook: + return 'notebook' + + for block in blocks: + # `block` carries its parsed type; pass it so the branch-count rule is + # scoped to DATA steps. + tier, _ = classify_block_sql_first(block.content, block.block_type) + if tier == 'pyspark': + return 'notebook' + + return 'sql' + + +def assess_confidence(block_content): + """Assess translation confidence.""" + content_lower = block_content.lower() + + # LOW CONFIDENCE + if content_lower.count('%macro') > 2: + return ('low', 'Nested macros - complex resolution') + if '%include' in content_lower and '&' in content_lower: + return ('low', '%INCLUDE with dynamic path') + # External DB engines (canonical union list — keep in sync with block-tiering-spec.md) + if any(eng in content_lower for eng in + ['sqlsvr', 'mssql', 'sql server', 'oracle', 'teradata', + 'odbc', 'oledb', 'db2', 'postgres', 'mysql', 'dsn=']): + return ('low', 'External database engine reference') + + # MEDIUM CONFIDENCE + if any(func in content_lower for func in ['intck', 'intnx', 'datepart']): + return ('medium', 'Complex date manipulation - verify') + if 'notsorted' in content_lower: + return ('medium', 'BY-group with NOTSORTED') + + # HIGH CONFIDENCE + return ('high', 'Standard pattern with direct mapping') +``` + +## File-Level Tier (strict "any-block" rule) + +A file's overall tier is driven by its most demanding block — identical to +`determine_output_type` above and to the assessment CLI: + +- **any** block is `pyspark` → file is **Tier 3** (emit a notebook) +- else **any** block is `stored_proc` → file is **Tier 2** +- else → file is **Tier 1** (pure SQL) + +There are no proportion thresholds. Boilerplate blocks are excluded *before* this rule is +applied (see the canonical-source note above), so a stray scaffolding HASH does not promote a +whole generated file to Tier 3. + +> **PROC SQL CASE WHEN is Tier 1.** A large `CASE WHEN ... END` in PROC SQL is pure Snowflake +> SQL, regardless of how many `WHEN` clauses it has. The ">5 IF/WHEN" Tier-2 rule fires only for +> **DATA-step** procedural branching (`IF/THEN/ELSE`, `SELECT/WHEN`) — never for PROC SQL. + diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/common-patterns.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/common-patterns.md new file mode 100644 index 0000000..ec5f79d --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/common-patterns.md @@ -0,0 +1,497 @@ +# Common SAS to Snowflake Patterns + +## When to Load + +Load this reference for **all conversions** - contains critical patterns for stored procedures, function mappings, and troubleshooting. + +--- + +## Key Conversion Principles + +### SAS → Snowflake Mental Model + +| SAS Concept | Snowflake Equivalent | +|-------------|---------------------| +| DATA step | SQL INSERT/SELECT or Stored Procedure | +| SAS dataset | Table or View | +| Observation | Row | +| Variable | Column | +| RETAIN | Variable declaration in procedure OR window functions | +| BY group processing | PARTITION BY / window functions | +| Missing (.) | NULL | +| SAS date (days since 1960) | `DATEADD('day', sas_date, '1960-01-01')` | +| SAS datetime (seconds since 1960) | `DATEADD('second', sas_datetime, '1960-01-01')` | +| PROC SQL | SQL (mostly 1:1) | +| SAS Macro | Stored Procedure, UDF, or Jinja templating | + +--- + +## Translation Priority + +1. **SQL First** - Rule-based conversion to Snowflake SQL +2. **Window Functions** - For RETAIN, FIRST./LAST. patterns +3. **Stored Procedures** - For complex iteration, state management +4. **Flag for Review** - Unknown patterns with clear comments + +### When to Use Pure SQL + +- Simple column transformations +- Filtering (WHERE) +- Aggregations (PROC MEANS/SUMMARY → GROUP BY) +- Joins (MERGE with BY → JOIN) +- Sorting (PROC SORT → ORDER BY) +- Running totals with RETAIN → `SUM() OVER (ROWS UNBOUNDED PRECEDING)` + +### When to Use Stored Procedures + +- Row-by-row conditional logic with state +- RETAIN variables that accumulate with complex conditions +- Complex FIRST./LAST. BY-group processing beyond simple window functions +- Multiple output datasets from one pass +- Iterative/looping constructs (`DO WHILE`, `DO UNTIL`) +- Dynamic SQL generation + +--- + +## Variable Binding Rules (CRITICAL) + +**⚠️ ALWAYS review before generating stored procedures.** + +In Snowflake Scripting stored procedures: + +| Context | Use `:` prefix? | Example | +|---------|-----------------|---------| +| SQL statement (SELECT, INSERT, etc.) | YES | `SELECT * FROM t WHERE id = :my_var` | +| INTO clause | YES | `SELECT col INTO :my_var FROM t` | +| IDENTIFIER() for dynamic names | YES | `FROM IDENTIFIER(:table_name)` | +| Assignment (`:=`) | NO | `my_var := 'value'` | +| RETURN statement | NO | `RETURN my_var` | +| String concatenation | NO | `sql_stmt := 'SELECT ' \|\| col_name` | +| FOR loop variable | NO | `FOR i IN 1 TO 10` | +| IF condition | NO | `IF (my_var > 0) THEN` | + +**Common Mistakes:** +```sql +-- WRONG: Missing colon in SQL +SELECT * FROM table WHERE id = my_var; + +-- CORRECT: +SELECT * FROM table WHERE id = :my_var; + +-- WRONG: Colon in assignment +:my_var := 'value'; + +-- CORRECT: +my_var := 'value'; +``` + +--- + +## Error Handling Pattern (CRITICAL) + +Always wrap stored procedures with EXCEPTION handling: + +```sql +CREATE OR REPLACE PROCEDURE process_data(input_table STRING) +RETURNS STRING +LANGUAGE SQL +AS +$$ +DECLARE + invalid_input EXCEPTION (-20001, 'Invalid input parameter'); + rows_processed INTEGER DEFAULT 0; +BEGIN + -- Validate inputs + IF (input_table IS NULL OR input_table = '') THEN + RAISE invalid_input; + END IF; + + -- Main logic here + CREATE OR REPLACE TABLE output AS SELECT * FROM IDENTIFIER(:input_table); + SELECT COUNT(*) INTO :rows_processed FROM output; + + RETURN 'Success: Processed ' || rows_processed || ' rows'; +EXCEPTION + WHEN STATEMENT_ERROR THEN + RETURN OBJECT_CONSTRUCT('status', 'ERROR', 'code', sqlcode, 'message', sqlerrm)::STRING; + WHEN invalid_input THEN + RETURN OBJECT_CONSTRUCT('status', 'ERROR', 'code', sqlcode, 'message', sqlerrm)::STRING; + WHEN OTHER THEN + RETURN OBJECT_CONSTRUCT('status', 'ERROR', 'code', sqlcode, 'message', sqlerrm)::STRING; +END; +$$; +``` + +**Exception types:** +- `STATEMENT_ERROR` — SQL execution failed +- `EXPRESSION_ERROR` — Expression evaluation failed +- Custom exceptions — Declare with `EXCEPTION (code, 'message')` +- `OTHER` — Catch-all + +**Built-in error variables:** +- `SQLCODE` — Error code (integer) +- `SQLERRM` — Error message (string) +- `SQLSTATE` — ANSI SQL state code + +--- + +## Stored Procedure Best Practices + +```sql +CREATE OR REPLACE PROCEDURE schema_name.proc_name( + p_input_table STRING, + p_output_table STRING, + p_run_date DATE DEFAULT CURRENT_DATE() +) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +DECLARE + -- 1. Exceptions + err_invalid_input EXCEPTION (-20001, 'Invalid input'); + err_no_data EXCEPTION (-20002, 'No data found'); + + -- 2. Variables + v_row_count INTEGER DEFAULT 0; + v_start_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP(); +BEGIN + -- 3. Validation + IF (p_input_table IS NULL) THEN + RAISE err_invalid_input; + END IF; + + -- 4. Processing + CREATE OR REPLACE TABLE IDENTIFIER(:p_output_table) AS + SELECT * FROM IDENTIFIER(:p_input_table) + WHERE process_date = :p_run_date; + + -- 5. Metrics + SELECT COUNT(*) INTO :v_row_count FROM IDENTIFIER(:p_output_table); + + IF (v_row_count = 0) THEN + RAISE err_no_data; + END IF; + + -- 6. Return success + RETURN OBJECT_CONSTRUCT( + 'status', 'SUCCESS', + 'rows_processed', v_row_count, + 'duration_sec', DATEDIFF('second', v_start_time, CURRENT_TIMESTAMP()) + )::STRING; + +EXCEPTION + WHEN err_invalid_input THEN + RETURN OBJECT_CONSTRUCT('status', 'ERROR', 'code', sqlcode, 'message', sqlerrm)::STRING; + WHEN err_no_data THEN + RETURN OBJECT_CONSTRUCT('status', 'WARNING', 'code', sqlcode, 'message', sqlerrm)::STRING; + WHEN OTHER THEN + RETURN OBJECT_CONSTRUCT('status', 'ERROR', 'code', sqlcode, 'message', sqlerrm)::STRING; +END; +$$; +``` + +**Naming conventions:** +- Procedures: `verb_noun` (e.g., `process_sales`) +- Parameters: `p_` prefix +- Variables: `v_` prefix +- Exceptions: `err_` prefix +- Custom error codes: -20001 to -20999 + +--- + +## Common Pattern Translations + +### Running Total (RETAIN) + +**SAS:** +```sas +DATA running; + SET transactions; + BY customer_id; + RETAIN running_sum 0; + IF FIRST.customer_id THEN running_sum = 0; + running_sum = running_sum + amount; +RUN; +``` + +**Snowflake:** +```sql +SELECT *, + SUM(amount) OVER ( + PARTITION BY customer_id + ORDER BY transaction_date + ROWS UNBOUNDED PRECEDING + ) AS running_sum +FROM transactions; +``` + +### FIRST./LAST. Detection + +**SAS:** +```sas +DATA flagged; + SET sorted; + BY group_var; + first_flag = FIRST.group_var; + last_flag = LAST.group_var; +RUN; +``` + +**Snowflake:** +```sql +SELECT *, + CASE WHEN ROW_NUMBER() OVER (PARTITION BY group_var ORDER BY sort_col) = 1 + THEN 1 ELSE 0 END AS first_flag, + CASE WHEN ROW_NUMBER() OVER (PARTITION BY group_var ORDER BY sort_col DESC) = 1 + THEN 1 ELSE 0 END AS last_flag +FROM sorted; +``` + +### Keep Last Row per Group + +**SAS:** +```sas +DATA last_only; + SET sorted; + BY customer_id; + IF LAST.customer_id; +RUN; +``` + +**Snowflake:** +```sql +SELECT * FROM sorted +QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY sort_col DESC) = 1; +``` + +### Conditional Output to Multiple Tables + +**SAS:** +```sas +DATA high_value low_value; + SET customers; + IF total_spend > 1000 THEN OUTPUT high_value; + ELSE OUTPUT low_value; +RUN; +``` + +**Snowflake (two statements):** +```sql +CREATE TABLE high_value AS +SELECT * FROM customers WHERE total_spend > 1000; + +CREATE TABLE low_value AS +SELECT * FROM customers WHERE total_spend <= 1000; +``` + +--- + +## Untranslated Section Comment Template + +```sql +-- ============================================================ +-- WARNING: UNTRANSLATED/LOW CONFIDENCE SECTION +-- Reason: [specific reason - e.g., "Complex ARRAY with DO loop"] +-- Original SAS Code: +-- [original SAS code here] +-- ============================================================ +-- TODO: Manual review required +``` + +--- + +## Cross-Block Variable Persistence (CRITICAL) + +Snowflake Scripting variables declared in DECLARE...BEGIN...END **DO NOT persist** after the block ends. + +### When to Persist + +If block N computes a value (via SELECT INTO :var or := assignment) that block N+1 or later needs, you MUST persist it as a session variable. + +### How to Persist + +```sql +-- At end of block N: +EXECUTE IMMEDIATE 'SET MY_VAR = ''' || REPLACE(v_local_var, '''', '''''') || ''''; + +-- For numeric values: +EXECUTE IMMEDIATE 'SET MY_COUNT = ' || CAST(v_count AS VARCHAR); + +-- In block N+1, reference as: +SELECT * FROM my_table WHERE col = $MY_VAR; +``` + +### Rules +- Use `REPLACE(val, '''', '''''')` for proper single-quote escaping — **NEVER** backslash escaping +- Scripting variables (:var) only work within the same DECLARE...BEGIN...END block +- Session variables ($var) persist for the entire Snowflake session +- When converting SAS `PROC SQL INTO :macrovar`, always persist as session variable if any downstream block references `¯ovar` + +--- + +## LISTAGG and String Aggregation Rules + +When converting SAS RETAIN+concatenation loops to Snowflake: +- Use `LISTAGG(column, delimiter) WITHIN GROUP (ORDER BY ...)` +- The ORDER BY clause MUST reference only columns that exist in the source table +- If no natural ordering column exists, ORDER BY the value column itself or omit ORDER BY +- LISTAGG has a 1MB output limit — flag MANUAL_REVIEW_REQUIRED for large concatenations +- Pattern: strip leading delimiter then wrap → `'(' || LISTAGG(TRIM(col), ' or ') WITHIN GROUP (ORDER BY col) || ')'` + +### LISTAGG(DISTINCT ...) with a computed ORDER BY + +Snowflake rejects a **computed expression** in `ORDER BY` when the aggregate is `LISTAGG(DISTINCT ...)` — it fails at runtime with *"not a valid order by expression"*. A plain column reference is fine inline; any function/CONCAT/TRIM/SPLIT_PART/operator in the ordered expression must be pushed into a `SELECT DISTINCT` subquery first. + +```sql +-- WRONG (fails): computed expr in both DISTINCT and ORDER BY +SELECT LISTAGG(DISTINCT TRIM(col), '~') WITHIN GROUP (ORDER BY TRIM(col)) FROM t; + +-- CORRECT: distinct the computed value in a subquery, then aggregate +SELECT LISTAGG(val, '~') WITHIN GROUP (ORDER BY val) +FROM (SELECT DISTINCT TRIM(col) AS val FROM t); +``` +This is common when converting SAS `PROC SQL SELECT ... INTO :var SEPARATED BY` over a derived expression. + +--- + +## Snowflake Platform Constraints (common runtime traps) + +These are generic Snowflake behaviors that silently break otherwise-plausible converted code. + +### Session variables inside stored procedures (EXECUTE AS CALLER) + +A stored procedure created with the **default owner's rights cannot access session variables at all** — it can neither read a `$VAR` set before the call nor `SET` one for later use. To use session variables inside a procedure, it MUST be created with `EXECUTE AS CALLER`. + +```sql +CREATE OR REPLACE PROCEDURE my_proc() +RETURNS VARCHAR +LANGUAGE SQL +EXECUTE AS CALLER -- REQUIRED to read/set session variables +AS +$$ +BEGIN + -- $RDT resolves correctly here only under EXECUTE AS CALLER: + CREATE OR REPLACE TEMPORARY TABLE out AS SELECT * FROM src WHERE run_dt = $RDT; + RETURN 'ok'; +END; +$$; +``` + +- `$VAR` syntax **does** work inside an `EXECUTE AS CALLER` procedure body — you do NOT need `GETVARIABLE()` to read it. (`GETVARIABLE('VAR')` is an equivalent alternative, not a requirement.) +- The orchestration pattern (one driver proc calling sub-procs, sharing temp tables + session vars) only works if **every** proc in the chain is `EXECUTE AS CALLER` — owner's-rights procs run in an isolated scope and cannot see the caller's temp tables or session variables. + +### Referencing a session variable SET in the SAME scripting block + +If a SQL statement references a session variable that was `SET` earlier in the **same** `DECLARE...BEGIN...END` (or `EXECUTE IMMEDIATE`) block, that statement must itself run via `EXECUTE IMMEDIATE` — a static statement in the same block fails with *"Session variable '$VAR' does not exist"*. + +```sql +-- WRONG (fails in the same block): +BEGIN + SET val = (SELECT 100); + INSERT INTO my_table VALUES ($val); +END; + +-- CORRECT: +BEGIN + SET val = (SELECT 100); + EXECUTE IMMEDIATE 'INSERT INTO my_table VALUES ($val)'; +END; +``` + +### Session variables are NOT allowed in a VALUES() clause + +`$VAR` works in `SELECT`, `WHERE`, and `SET` contexts but NOT inside `INSERT ... VALUES(...)`. Use `INSERT ... SELECT` instead. + +```sql +-- WRONG: +INSERT INTO t (c1, c2) VALUES ('lit', $RSTRDT); +-- CORRECT: +INSERT INTO t (c1, c2) SELECT 'lit', TO_DATE($RSTRDT, 'YYYY-MM-DD'); +``` + +### Reserved-word column names must be double-quoted + +Many ordinary SAS column names are Snowflake reserved keywords and must be double-quoted wherever used as identifiers (CREATE, INSERT column list, SELECT, alias). When unsure, quote — over-quoting is safe. + +Common offenders from SAS sources: `DESC`, `TYPE`, `VALUE`, `LOCATION`, `NAME`, `FILE`, `STATUS`, `ORDER`, `GROUP`, `DATE`, `TIME`, `KEY`, `COMMENT`, `START`, `END`, `ROW`, `ROWS`, `SHARE`, `ROLE`, `USER`, `SCHEMA`, `COLUMN`, `INDEX`, `REPLACE`. + +```sql +CREATE OR REPLACE TABLE t ("DESC" VARCHAR, "TYPE" NUMBER, "VALUE" NUMBER); +SELECT "DESC", "TYPE" FROM t; +``` + +### Temp/session-table metadata: use `DESCRIBE TABLE`, not `INFORMATION_SCHEMA` + +`INFORMATION_SCHEMA.COLUMNS` does **not** reliably show temporary/session-scoped tables — it may return no rows, or stale metadata from a since-dropped permanent table of the same name. To discover columns/types of a table that may be temporary (created earlier in the same session), use `DESCRIBE TABLE`. + +```sql +-- CORRECT for temp/session tables: +DESCRIBE TABLE MY_TABLE; -- row["name"], row["type"], row["null?"], ... +-- WRONG (unreliable for temp tables): +SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 'MY_TABLE'; +``` +`INFORMATION_SCHEMA` remains fine for permanent tables in other schemas/databases and for counting/pattern queries. + +### `COPY FILES` does not support user stages (`@~`) + +`COPY FILES INTO ... FROM @~/...` fails — user stages are not valid sources/targets. Use a **named internal stage** as the intermediate, single-quote any path containing spaces, and `REMOVE` temp files afterward so they don't accumulate. + +```sql +-- WRONG: COPY FILES INTO @target FROM @~/temp ... +-- CORRECT: +CREATE STAGE IF NOT EXISTS ..NAMED_STG; +COPY FILES INTO @..NAMED_STG FROM @source ...; +-- paths with spaces must be single-quoted: COPY FILES INTO '@STG/My Folder/Sub Dir' FROM ...; +REMOVE @..NAMED_STG/; -- cleanup +``` + +--- + +## WORK / Temporary Table Rules + +| Rule | Details | +|------|---------| +| SAS WORK dataset | → `CREATE OR REPLACE TEMPORARY TABLE name AS ...` | +| Schema prefix | **NEVER** — use bare table name only | +| Wrong | `CREATE OR REPLACE TEMPORARY TABLE DB.SCHEMA.MY_TABLE` | +| Correct | `CREATE OR REPLACE TEMPORARY TABLE MY_TABLE` | +| TEMPORARY keyword | **ALWAYS** preserve on rewrites/deduplication | +| Permanence | Temp tables persist for the session — available to all subsequent blocks | +| FROM reference | `FROM MY_TABLE` — no schema prefix | + +--- + +## Oracle / DB2 Passthrough SQL Conversion + +See `references/vendor-passthrough.md` for full vendor function mapping tables and passthrough conversion steps. + +--- + +## Troubleshooting + +### Unsupported SAS Constructs + +| Construct | Issue | Workaround | +|-----------|-------|------------| +| HASH objects | No direct equivalent | Use JOINs or temp tables | +| INFILE/FILE | File I/O | Use Snowflake stages | +| CALL SYMPUT | Dynamic macro vars | Use stored procedure variables | +| CALL EXECUTE | Dynamic code | Use EXECUTE IMMEDIATE | +| DO loops with state | Complex iteration | Stored procedure with cursors | + +### Common Compile Errors + +| Error | Cause | Fix | +|-------|-------|-----| +| "Invalid identifier" | Missing colon in SQL | Add `:` before variable | +| "Object does not exist" | Dynamic table name | Use `IDENTIFIER(:var)` | +| "Type mismatch" | SAS is type-flexible | Add explicit `CAST()` | +| "Reserved word" | Column name is keyword | Quote with `"column"` | + +### Performance Concerns + +- SAS row-by-row → Use window functions, not cursors +- Large RETAIN patterns → Consider incremental/staging approach +- Many small queries → Batch into fewer larger queries diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/comparison-rules.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/comparison-rules.md new file mode 100644 index 0000000..987b63a --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/comparison-rules.md @@ -0,0 +1,180 @@ +# SAS-to-Snowflake Output Comparison Rules + +Rules for determining equivalence between SAS expected output and Snowflake actual output during validation testing. + +## Value Equivalence Rules + +| SAS Value | Snowflake Value | Equivalent? | Rule | +|-----------|----------------|-------------|------| +| `.` (numeric missing) | `NULL` | YES | SAS numeric missing maps to SQL NULL | +| `' '` (character missing) | `NULL` | YES | SAS character missing maps to NULL | +| `' '` (character missing) | `''` (empty string) | YES | Both represent absent character values | +| `''` (empty string) | `NULL` | YES | Treat empty string and NULL as equivalent for character columns | +| `0` | `NULL` | NO | Zero is a valid numeric value, not missing | +| `0` | `0.0` | YES | Integer and float representations of zero are equivalent | + +## Numeric Precision + +| Scenario | Tolerance | Rationale | +|----------|-----------|-----------| +| Integer arithmetic | Exact match | No precision loss expected | +| Float arithmetic (SUM, MEAN) | abs(diff) < 1e-10 | SAS 8-byte float vs Snowflake NUMBER(38,x) | +| Division results | abs(diff) < 1e-8 | Division may differ at low-order digits | +| Currency/money columns | abs(diff) < 0.01 | Round to 2 decimal places before comparing | +| Percentage calculations | abs(diff) < 1e-6 | Intermediate rounding differences | + +### Comparison SQL Pattern for Numeric Tolerance + +```sql +SELECT + e.row_key, + e.col_name AS column_name, + e.val AS expected, + a.val AS actual, + ABS(e.val - a.val) AS diff +FROM expected_unpivoted e +JOIN actual_unpivoted a ON e.row_key = a.row_key AND e.col_name = a.col_name +WHERE ABS(e.val - a.val) > 1e-10; +``` + +## Character Value Rules + +| Rule | Behavior | +|------|----------| +| Trailing spaces | TRIM both sides before comparing (SAS fixed-length pads with spaces) | +| Leading spaces | Preserve -- leading spaces are significant in SAS | +| Case sensitivity | Case-sensitive comparison (SAS preserves case in character values) | +| Character encoding | UTF-8 assumed on both sides | + +## Date Value Rules + +| SAS Representation | Snowflake Representation | Equivalence | +|--------------------|--------------------------|-------------| +| SAS date value (days since 1960-01-01) | DATE type | Convert SAS numeric to calendar date for comparison | +| SAS datetime value (seconds since 1960-01-01) | TIMESTAMP | Convert SAS numeric to timestamp for comparison | +| SAS time value (seconds since midnight) | TIME | Convert SAS numeric to time for comparison | +| Formatted date string (`'01JAN2024'`) | `'2024-01-01'` | Compare as DATE after parsing both formats | + +### Date Conversion Reference + +``` +SAS date value 0 = 1960-01-01 +SAS date value 23376 = 2024-01-01 +Formula: Snowflake DATE = DATEADD(day, sas_date_value, '1960-01-01') +``` + +## Sort Order and Row Ordering + +| Rule | Details | +|------|---------| +| Row ordering | Do NOT compare row order -- sort both result sets by key columns before comparison | +| If no key columns exist | Sort by ALL columns (left to right) to produce deterministic order | +| NULL sort position | SAS sorts missing LOW (before all values); Snowflake sorts NULL HIGH (after all values) by default. Sorting differences are NOT comparison failures. | + +### Key Column Selection for Sorting + +Priority order for choosing sort keys: +1. Primary key columns (if identifiable from schema) +2. Columns used in BY statements in the original SAS code +3. All non-computed columns (exclude aggregated/derived columns) +4. All columns as last resort + +## Schema Comparison Rules + +| Check | Pass Condition | +|-------|---------------| +| Column count | Must match exactly | +| Column names | Case-insensitive match required | +| Column order | Must match (SAS preserves column order from PDV) | +| Column types | Compatible types pass (see compatibility table below) | + +### Type Compatibility Matrix + +| Expected (SAS-derived) | Actual (Snowflake) | Compatible? | +|------------------------|--------------------|-------------| +| NUMBER | NUMBER | YES | +| NUMBER | FLOAT | YES | +| NUMBER | DECIMAL | YES | +| NUMBER | INTEGER | YES | +| VARCHAR | VARCHAR | YES | +| VARCHAR | STRING | YES (same in Snowflake) | +| VARCHAR | CHAR | YES (compare after TRIM) | +| DATE | DATE | YES | +| DATE | TIMESTAMP_NTZ | YES (compare date part only) | +| TIMESTAMP | TIMESTAMP_NTZ | YES | +| BOOLEAN | NUMBER | YES (0/1 mapping) | + +## Row Count Rules + +| Scenario | Expected Behavior | +|----------|-------------------| +| Simple filter (WHERE) | Output rows <= input rows | +| GROUP BY / aggregation | Output rows <= distinct groups | +| JOIN (inner) | Output rows <= product of input rows | +| LEFT JOIN | Output rows >= left table rows | +| UNION / SET operations | Depends on UNION vs UNION ALL | +| MERGE (SAS) | Output rows = max(left rows, right rows) for one-to-one; flag if > max | +| Many-to-many join | Flag if output_rows > 2x max(input_rows) | + +## Comparison Execution SQL + +### Full Table Comparison (MINUS/EXCEPT approach) + +```sql +-- Rows in expected but NOT in actual (missing from Snowflake output) +SELECT 'MISSING_FROM_ACTUAL' AS diff_type, * +FROM expected_
    +MINUS +SELECT 'MISSING_FROM_ACTUAL' AS diff_type, * +FROM actual_
    ; + +-- Rows in actual but NOT in expected (extra in Snowflake output) +SELECT 'EXTRA_IN_ACTUAL' AS diff_type, * +FROM actual_
    +MINUS +SELECT 'EXTRA_IN_ACTUAL' AS diff_type, * +FROM expected_
    ; +``` + +### Column-Level Comparison (for numeric tolerance) + +```sql +WITH expected_numbered AS ( + SELECT *, ROW_NUMBER() OVER (ORDER BY ) AS _rn + FROM expected_
    +), +actual_numbered AS ( + SELECT *, ROW_NUMBER() OVER (ORDER BY ) AS _rn + FROM actual_
    +) +SELECT + e._rn AS row_num, + '' AS column_name, + e.AS expected_val, + a.AS actual_val +FROM expected_numbered e +JOIN actual_numbered a ON e._rn = a._rn +WHERE NOT ( + (e.IS NULL AND a.IS NULL) + OR (TRIM(e.::VARCHAR) = TRIM(a.::VARCHAR)) + OR (TRY_TO_DOUBLE(e.) IS NOT NULL + AND ABS(e.- a.) < 1e-10) +); +``` + +## Mismatch Classification + +When reporting mismatches, classify the likely root cause: + +| Mismatch Pattern | Likely Root Cause | +|------------------|-------------------| +| NULL vs non-NULL value | Missing value handling (SAS `.` not mapped to NULL, or COALESCE missing) | +| NULL vs empty string | Character missing value conversion | +| Row count: actual > expected | Many-to-many join (SAS MERGE vs SQL JOIN) | +| Row count: actual < expected | Incorrect INNER JOIN (should be LEFT/FULL) or WHERE filter too restrictive | +| Numeric difference < 0.01 | Float precision (acceptable for most cases) | +| Numeric difference > 0.01 | Wrong aggregation, missing COALESCE(x,0), or SAS sum statement vs assignment | +| Date off by 1 | INTNX alignment parameter (B vs E vs S) | +| All values shifted by N rows | Window function ROWS vs RANGE, or missing ORDER BY | +| Duplicate rows in actual | Missing DISTINCT or QUALIFY (SAS NODUPKEY equivalent) | +| Column missing | PDV variable not carried forward (check KEEP/DROP/RENAME) | diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/consolidation-patterns.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/consolidation-patterns.md new file mode 100644 index 0000000..61f0e8b --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/consolidation-patterns.md @@ -0,0 +1,433 @@ +# Consolidation Patterns for SAS to Snowflake Migration + +## When to Load + +Load when user selects **Optimize & Consolidate** migration mode. + +--- + +## Dependency Analysis + +### Building the Dependency Graph + +For each script, extract: + +```python +def extract_dependencies(sas_content): + """Extract input/output tables from SAS code""" + inputs = set() + outputs = set() + + # DATA step outputs + for match in re.findall(r'(?i)data\s+(\w+(?:\.\w+)?)\s*;', sas_content): + outputs.add(match.lower()) + + # DATA step inputs (SET, MERGE) + for match in re.findall(r'(?i)(?:set|merge)\s+(\w+(?:\.\w+)?)', sas_content): + inputs.add(match.lower()) + + # PROC SQL CREATE TABLE + for match in re.findall(r'(?i)create\s+table\s+(\w+(?:\.\w+)?)', sas_content): + outputs.add(match.lower()) + + # PROC SQL FROM + for match in re.findall(r'(?i)from\s+(\w+(?:\.\w+)?)', sas_content): + inputs.add(match.lower()) + + return {'inputs': inputs, 'outputs': outputs} +``` + +### Dependency Matrix + +| Script | Inputs | Outputs | Depends On | +|--------|--------|---------|------------| +| script_a.sas | source_data | staging_1 | - | +| script_b.sas | staging_1 | staging_2 | script_a | +| script_c.sas | staging_2 | final_output | script_b | + +--- + +## Consolidation Patterns + +### Pattern 1: Sequential Pipeline Merge + +**Detection:** +- Script A output is Script B's only input +- Script B output is Script C's only input +- Linear chain of dependencies + +**Before (3 scripts):** +```sas +/* script_a.sas */ +DATA staging_1; + SET source_data; + new_col = col1 * 2; +RUN; + +/* script_b.sas */ +DATA staging_2; + SET staging_1; + WHERE new_col > 100; +RUN; + +/* script_c.sas */ +PROC SQL; + CREATE TABLE final_output AS + SELECT * FROM staging_2 WHERE status = 'ACTIVE'; +QUIT; +``` + +**After (1 script with CTEs):** +```sql +-- consolidated_pipeline.sql +CREATE OR REPLACE TABLE final_output AS +WITH staging_1 AS ( + -- From script_a.sas + SELECT *, col1 * 2 AS new_col + FROM source_data +), +staging_2 AS ( + -- From script_b.sas + SELECT * FROM staging_1 + WHERE new_col > 100 +) +-- From script_c.sas +SELECT * FROM staging_2 +WHERE status = 'ACTIVE'; +``` + +**Savings:** 2 intermediate tables eliminated, 1 file instead of 3 + +--- + +### Pattern 2: Shared Source Consolidation + +**Detection:** +- Multiple scripts read from same source table +- Apply different filters/transformations +- Create separate output tables + +**Before (2 scripts):** +```sas +/* active_customers.sas */ +DATA active_customers; + SET customers; + WHERE status = 'ACTIVE'; + segment = 'Active'; +RUN; + +/* vip_customers.sas */ +DATA vip_customers; + SET customers; + WHERE total_spend > 10000; + segment = 'VIP'; +RUN; +``` + +**After (1 script):** +```sql +-- customer_segments.sql +CREATE OR REPLACE TABLE active_customers AS +SELECT *, 'Active' AS segment +FROM customers +WHERE status = 'ACTIVE'; + +CREATE OR REPLACE TABLE vip_customers AS +SELECT *, 'VIP' AS segment +FROM customers +WHERE total_spend > 10000; + +-- OR if both needed in single table: +CREATE OR REPLACE TABLE customer_segments AS +SELECT *, + CASE + WHEN status = 'ACTIVE' THEN 'Active' + WHEN total_spend > 10000 THEN 'VIP' + END AS segment +FROM customers +WHERE status = 'ACTIVE' OR total_spend > 10000; +``` + +--- + +### Pattern 3: Duplicate PROC SORT Elimination + +**Detection:** +- Same `PROC SORT DATA=X BY Y` appears in multiple scripts +- Sorted result used for different purposes + +**Before (3 scripts with same sort):** +```sas +/* report_1.sas */ +PROC SORT DATA=transactions; + BY customer_id date; +RUN; +/* ... use sorted data ... */ + +/* report_2.sas */ +PROC SORT DATA=transactions; + BY customer_id date; +RUN; +/* ... use sorted data ... */ + +/* report_3.sas */ +PROC SORT DATA=transactions; + BY customer_id date; +RUN; +/* ... use sorted data ... */ +``` + +**After (1 sorted view, 3 scripts reference it):** +```sql +-- 00_common_views.sql (run first) +CREATE OR REPLACE VIEW transactions_sorted AS +SELECT * FROM transactions +ORDER BY customer_id, date; + +-- report_1.sql +SELECT ... FROM transactions_sorted ...; + +-- report_2.sql +SELECT ... FROM transactions_sorted ...; + +-- report_3.sql +SELECT ... FROM transactions_sorted ...; +``` + +**Savings:** Sort executed once instead of 3x + +--- + +### Pattern 4: Intermediate Table to CTE + +**Detection:** +- Table created in one block +- Used in immediately following block +- Never referenced elsewhere + +**Before:** +```sas +DATA temp_calc; + SET source; + calculated_field = complex_formula; +RUN; + +DATA final; + SET temp_calc; + WHERE calculated_field > threshold; +RUN; +``` + +**After:** +```sql +CREATE OR REPLACE TABLE final AS +WITH temp_calc AS ( + SELECT *, complex_formula AS calculated_field + FROM source +) +SELECT * FROM temp_calc +WHERE calculated_field > threshold; +``` + +**Savings:** 1 intermediate table eliminated + +--- + +### Pattern 5: Inline Single-Use Macros + +**Detection:** +- `%MACRO name` defined +- `%name` called only once in codebase +- Macro is simple enough to inline + +**Before:** +```sas +%MACRO calc_tax(rate); + tax_amount = gross_amount * &rate; +%MEND; + +DATA with_tax; + SET orders; + %calc_tax(0.08); +RUN; +``` + +**After:** +```sql +CREATE OR REPLACE TABLE with_tax AS +SELECT *, + gross_amount * 0.08 AS tax_amount +FROM orders; +``` + +--- + +### Pattern 6: Redundant Table Creation + +**Detection:** +- Same table name created with identical logic in multiple scripts +- Usually indicates copy-paste pattern + +**Before (table created in 3 places):** +```sas +/* script_1.sas */ +DATA customer_base; + SET customers; + WHERE active_flag = 'Y'; +RUN; + +/* script_2.sas */ +DATA customer_base; + SET customers; + WHERE active_flag = 'Y'; +RUN; + +/* script_3.sas */ +DATA customer_base; + SET customers; + WHERE active_flag = 'Y'; +RUN; +``` + +**After (single source of truth):** +```sql +-- 00_base_tables.sql (run once) +CREATE OR REPLACE TABLE customer_base AS +SELECT * FROM customers +WHERE active_flag = 'Y'; + +-- Other scripts reference customer_base directly +``` + +--- + +### Pattern 7: PROC MEANS/FREQ Consolidation + +**Detection:** +- Multiple scripts run similar PROC MEANS/FREQ on same data +- Different variables but same source + +**Before:** +```sas +/* stats_1.sas */ +PROC MEANS DATA=sales; + VAR revenue; +RUN; + +/* stats_2.sas */ +PROC MEANS DATA=sales; + VAR quantity; +RUN; + +/* stats_3.sas */ +PROC FREQ DATA=sales; + TABLES region; +RUN; +``` + +**After:** +```sql +-- sales_statistics.sql +-- Revenue stats +SELECT + COUNT(revenue) AS revenue_n, + AVG(revenue) AS revenue_mean, + STDDEV(revenue) AS revenue_std, + MIN(revenue) AS revenue_min, + MAX(revenue) AS revenue_max +FROM sales; + +-- Quantity stats +SELECT + COUNT(quantity) AS quantity_n, + AVG(quantity) AS quantity_mean, + STDDEV(quantity) AS quantity_std, + MIN(quantity) AS quantity_min, + MAX(quantity) AS quantity_max +FROM sales; + +-- Region frequency +SELECT region, COUNT(*) AS count +FROM sales +GROUP BY region +ORDER BY region; +``` + +--- + +## Consolidation Report Template + +```markdown +# SAS Migration Consolidation Report + +## Summary +| Metric | Before | After | Savings | +|--------|--------|-------|---------| +| Scripts | X | Y | Z fewer | +| Tables Created | A | B | C fewer | +| PROC SORTs | D | E | F fewer | +| Lines of Code | G | H | I fewer | + +## Consolidation Groups + +### Group 1: [Name] +**Pattern:** Sequential Pipeline Merge +**Scripts Merged:** script_a.sas, script_b.sas, script_c.sas +**Output:** consolidated_pipeline.sql +**Tables Eliminated:** staging_1, staging_2 + +### Group 2: [Name] +**Pattern:** Duplicate PROC SORT Elimination +**Scripts Affected:** report_1.sas, report_2.sas, report_3.sas +**Output:** 00_common_views.sql + 3 report scripts +**Sorts Reduced:** 3 → 1 + +## Traceability Matrix +| Original File | Original Block | Target File | Target Location | +|---------------|----------------|-------------|-----------------| +| script_a.sas | DATA staging_1 | pipeline.sql | CTE staging_1 | +| script_b.sas | DATA staging_2 | pipeline.sql | CTE staging_2 | +| script_c.sas | PROC SQL | pipeline.sql | Final SELECT | + +## Behavioral Notes +- [Any differences in execution behavior] +- [Order dependencies that must be preserved] +- [Data that may differ due to consolidation] +``` + +--- + +## Consolidation Decision Matrix + +| Scenario | Consolidate? | Reason | +|----------|--------------|--------| +| A → B → C linear chain | ✅ Yes | Perfect CTE candidate | +| A → B, A → C (fan-out) | ⚠️ Partial | Keep A, may merge B+C if similar | +| A → C, B → C (fan-in) | ⚠️ Partial | May combine A+B if similar sources | +| Independent scripts | ❌ No | No benefit | +| Same sort in N scripts | ✅ Yes | Create shared view | +| Macro called once | ✅ Yes | Inline | +| Macro called N times | ❌ No | Keep as UDF | +| Temp table used once | ✅ Yes | Convert to CTE | +| Temp table used N times | ❌ No | Keep as table | + +--- + +## Risk Considerations + +### When NOT to Consolidate + +1. **Audit Requirements** - Need 1:1 traceability for compliance +2. **Different Schedules** - Scripts run at different times +3. **Different Owners** - Managed by different teams +4. **Error Isolation** - Want failures to be isolated +5. **Performance Testing** - Need to measure individual components + +### Consolidation Risks + +| Risk | Mitigation | +|------|------------| +| Behavioral change | Compare output row counts and checksums | +| Missing data | Validate all source tables exist | +| Order dependency | Document execution order | +| Increased complexity | Balance consolidation vs readability | diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/conversion-rules.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/conversion-rules.md new file mode 100644 index 0000000..766a4dc --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/conversion-rules.md @@ -0,0 +1,225 @@ +# Critical Conversion Rules + +## When to Load + +Load at **Steps 5-6** (code generation and self-check). These rules govern semantic correctness of the converted SQL. + +--- + +### Rule: No Programmatic Delegation (NEVER Generate Converter Scripts) +- **NEVER** generate a Python, bash, Node.js, or any other script to perform SAS-to-SQL conversion programmatically +- **NEVER** delegate file conversion to a subagent with instructions to write a converter/parser program +- **NEVER** use regex-based find-and-replace as a substitute for semantic understanding of SAS logic +- The LLM MUST read each SAS file, understand its procedural semantics, and write the Snowflake SQL directly +- Regex parsers CANNOT correctly handle: missing value comparison semantics, MERGE vs JOIN distinctions, RETAIN with conditional resets, BY-group processing order dependencies, or conditional OUTPUT behavior +- For large batches (50+ files): process files in groups of 10-20 per context window, writing completed .sql files to disk between groups. Use `conversion_state.json` to track progress across windows. This is slower but produces correct output. +- The only acceptable use of Python in this workflow is for post-conversion utilities (e.g., generating `conversion_report.docx` via python-docx) + +### Rule: Output Completeness (NEVER Truncate) +- **NEVER** summarize repeated logic with "similar pattern", "same as above", "repeat for other tables", or "adjusted joins/filters" +- **NEVER** replace logic with pseudocode or placeholders +- If a block has many column derivations, output EVERY derivation +- If PROC SQL creates multiple tables, generate EACH CREATE TABLE fully +- If a macro has multiple branches, convert EVERY branch +- Prefer **completeness and explicitness** over brevity +- **Filter Dataset Preservation:** If SAS defines a dataset and uses it as a filter (via IN() subquery, INNER JOIN, WHERE ... IN (SELECT ... FROM dataset), or macro variable populated from that dataset), the converted SQL MUST: + 1. Create/populate the equivalent of that filter dataset (as a CTE, temp table, or subquery) + 2. Apply the same filtering relationship in the main query + 3. NEVER silently drop a filtering dataset or its associated WHERE/JOIN condition +- Common pattern: SAS creates a small lookup/filter table (e.g., `REGION_LIST` with specific REGION_IDs), then uses those IDs in a WHERE clause like `WHERE T1.REGION_ID IN (®ION_ID.)`. The converted SQL must preserve BOTH the filter dataset creation AND its application as a WHERE/JOIN filter. +- If a macro variable is populated via `SELECT ... INTO :var` from a filter dataset, the converted SQL must preserve that filter logic (as a subquery, CTE, session variable, or equivalent) +- During Step 6 self-check: for each dataset created in the SAS source, verify it is either: (a) present as output, or (b) consumed as a filter/join in the converted SQL. If it exists in SAS but is absent from the conversion, it is a logic omission. + +### Rule: Missing Value Semantics +- SAS numeric missing (`.`) → `NULL` +- SAS `missing(.A)` through `missing(.Z)` → flag `MANUAL_REVIEW_REQUIRED` +- SAS treats missing as **less than** any value in comparisons; SQL NULLs propagate differently +- Review WHERE/IF/CASE/JOIN conditions involving missing values — behavior may differ +- Convert `MISSING()`, `NMISS()`, `CMISS()` explicitly +- Preserve defaulting logic built from missing-value checks +- Use `NULLS FIRST` in ORDER BY when SAS sort order depends on missing values sorting low +- When SAS checks `IF var NOT IN ('x','X')`, the SQL equivalent **MUST** also handle NULLs explicitly with `OR var IS NULL`, because SQL `NOT IN` evaluates to NULL (not TRUE) when `var` is NULL — SAS treats missing as not-in-list (TRUE). Pattern: `WHERE var NOT IN ('x','X') OR var IS NULL` + +### Rule: DATA Step Row-by-Row Semantics +- Treat DATA step as **row-by-row procedural execution**, not simple set-based SQL +- Preserve implicit OUTPUT at end of iteration (when no explicit OUTPUT statement) +- Preserve explicit OUTPUT, DELETE, RETURN, and STOP behavior exactly +- Preserve `_N_` semantics when used → `ROW_NUMBER() OVER (ORDER BY 1)` +- If one input row can create zero, one, or multiple output rows, preserve that behavior +- Preserve ordering dependencies that influence retained values or lag behavior + +### Rule: BY-Group Processing +- BY-group processing REQUIRES explicit ordering equivalent to SAS sort order +- If SAS assumes a prior PROC SORT, preserve that ordering dependency +- Do NOT assume input order is stable unless SAS explicitly sorted +- Preserve group resets, running totals by group, and first/last-row logic +- Preserve duplicate handling within BY groups + +### Rule: RETAIN / LAG / Sum Statement +- SAS sum statement `x + y;` (with trailing semicolon, no equals sign) **differs from assignment** — it adds y to x's retained value and treats missing as 0 +- Convert `x + y;` sum statement to: `COALESCE(x, 0) + COALESCE(y, 0)` with RETAIN semantics +- Preserve LAG behavior exactly; LAG in SAS always reads from the **input queue** regardless of WHERE +- Preserve multiple retained variables with interdependent updates +- If RETAIN or LAG depends on sorted groups, preserve same partitioning and order + +### Rule: MERGE vs JOIN Semantics +- SAS MERGE is **NOT** automatically equivalent to a SQL JOIN +- Detect one-to-many and many-to-many row multiplication risk +- Preserve IN= dataset flag behavior explicitly +- Preserve source precedence when multiple datasets contribute overlapping columns +- Preserve row survival rules when unmatched keys exist +- Do NOT silently deduplicate unless SAS explicitly does so +- For UPDATE/MODIFY semantics: preserve in-place update intent + +### Rule: WORK / Temp Tables +- SAS WORK datasets → `CREATE OR REPLACE TEMPORARY TABLE` (no schema prefix) +- **NEVER** qualify temp tables with DATABASE.SCHEMA — use bare name only +- **ALWAYS** preserve the `TEMPORARY` keyword on rewrites +- Temporary tables persist for the Snowflake session — available to ALL subsequent blocks +- Do NOT recreate temp tables that were already created upstream + +### Rule: Block Separation of Concerns +- Each converted block MUST correspond exactly to its SAS block +- A DATA step block MUST NOT also perform the logic of a subsequent PROC SORT +- A PROC SORT block MUST only dedup/sort, not recreate the table from source +- Temp tables created upstream will still exist when downstream blocks execute +- A temp table created in block N may be consumed by block N+2 or later — not just N+1 + +### Rule: Snowflake Scripting Syntax +- Variable assignment: use `SELECT col INTO :var FROM table` — **NEVER** `LET var := (SELECT ...)` +- Use `:var` prefix only inside SQL statements (SELECT, INSERT, UPDATE, WHERE, SET) +- Do NOT use `:var` in IF conditions, assignments, concatenations, or RETURN statements +- For RAISE: **ONLY** named exceptions declared in DECLARE section — **NEVER** `RAISE USING MESSAGE` +- For cursors in FOR loop: do NOT use explicit OPEN/CLOSE (FOR opens/closes automatically) + +### Rule: PROC SORT with NODUPKEY +- When using ROW_NUMBER() for deduplication, the outer SELECT **MUST use an explicit column list** +- **NEVER** use `SELECT *` if it would leak the helper column (rn) into the output table +- If source is TEMPORARY TABLE, the deduplicated output MUST also be TEMPORARY + +### Rule: Cross-Block Variable Persistence +- Snowflake Scripting variables in DECLARE...BEGIN...END **DO NOT persist** after block ends +- If a variable from block N is needed by block N+1, persist as session variable: + `EXECUTE IMMEDIATE 'SET VAR_NAME = ''' || REPLACE(v_local_var, '''', '''''') || '''';` +- Reference downstream as `$VAR_NAME` + +### Rule: Validation Checks Must Be Data-Driven +- When SAS validates values against a dimension/lookup table (e.g., `IF Status_Code NOT IN (SELECT ... FROM dim_table)`), the converted SQL **MUST** preserve the table-based lookup +- **NEVER** replace a SAS dimension-table lookup with a hardcoded value list (e.g., `NOT IN ('A','B','C')`) +- Hardcoded lists become stale when dimension members change — they are a silent correctness regression +- Pattern: SAS `IF var NOT IN (dimension values)` → SQL `LEFT JOIN dimension_table WHERE match IS NULL` +- If the dimension table reference is unclear, flag as MANUAL_REVIEW_REQUIRED rather than hardcoding values + +### Rule: Validation Reference Table Preservation +- When SAS validation checks anti-join against a **master/reference table** (e.g., `Ref.CUSTOMER_MASTER`, `Ref.PRODUCT_MASTER`, `Ref.ACCOUNT_MASTER`), the converted SQL **MUST** anti-join against the **same semantic equivalent** — not a derived, subset, or working table +- A master hierarchy table (e.g., `CUSTOMER_MASTER`) and a working/derived table (e.g., `CUSTOMER_SUBSET`) are **NOT interchangeable** — the master is the authoritative source; the derived table is a user-entered subset +- **Referential checks** ("does this key exist in the master?") must use anti-join against the master table +- **Value checks** ("is this value null/zero/invalid?") may use simple WHERE on the target table +- **NEVER** downgrade a referential check to a value check — checking `WHERE Rate IS NULL` is NOT equivalent to checking `WHERE key NOT IN (SELECT key FROM rate_table)` because the latter catches entirely missing rows +- During Step 6 self-check, verify: for each anti-join validation, the reference table in the converted SQL matches the reference table in the SAS source — not a derived/subset alternative + +### Rule: Validation Check Source Completeness +- When converting SAS validation scripts (anti-join checks, existence checks, referential integrity), **preserve ALL source tables** referenced in the original SAS check +- If SAS validates Key_ID against tables A, B, and C, the converted SQL must also validate against A, B, and C — not just A +- Dropping a validation source is equivalent to removing a quality gate — it is a silent correctness regression +- During Step 6 self-check, verify: every source table in the SAS validation block appears in the converted SQL validation query + +### Rule: Output Column Completeness +- The converted SQL must produce the **same columns** as the SAS output dataset +- If the SAS DATA step or PROC SQL outputs columns A through Z, the converted CREATE TABLE must include A through Z +- **NEVER silently drop output columns** — even if they appear redundant or derivable +- If a column cannot be translated (e.g., depends on unavailable external data), include it with a NULL placeholder and MANUAL_REVIEW_REQUIRED comment +- During Step 6 self-check, verify: column count of converted output >= column count of SAS output + +### Rule: Consolidation Must Preserve All Output Tables +- When consolidating multiple SAS scripts into fewer SQL files (Optimize & Consolidate mode), the **total set of output tables** across all consolidated files **MUST equal** the total set from the unconsolidated conversion +- Before declaring consolidation complete, cross-check: + 1. List every CREATE TABLE / INSERT INTO target from the unconsolidated version + 2. List every CREATE TABLE / INSERT INTO target from the consolidated version + 3. Any target in (1) but not in (2) is a **dropped output** — this is a P1 bug +- If a SAS script's logic is merged into another file, the merged file MUST contain ALL output tables from the original script +- **NEVER silently drop a SAS script's output tables during consolidation** +- Consolidation **MUST NOT change which reference table** a validation check validates against — if SAS check N anti-joins against `Ref.CUSTOMER_MASTER`, the consolidated SQL must also anti-join against `CUSTOMER_MASTER` (or its schema-qualified equivalent), not a derived table like `CUSTOMER_SUBSET` + +### Rule: Tier Consistency in Consolidation +- When consolidating files in Optimize & Consolidate mode, **do not downgrade a file's tier** unless consolidation genuinely eliminates the need for procedural control flow +- If a SAS file has 3+ sequential dependent DML statements (e.g., DELETE → INSERT → UPDATE on related tables), it MUST remain Tier 2 with SP wrapping — even if each individual statement is simple SQL +- SP wrapping provides: transaction scope, error handling via EXCEPTION, orchestration via CALL, and row-count reporting via RETURN + +### Rule: No Invented Elements (Anti-Hallucination) +- **NEVER** introduce variables, parameters, columns, macro variables, or filter values that do not exist in the source SAS code +- The converted SQL must be a FAITHFUL translation — not an "improved" or "augmented" version +- If a variable name appears in the output SQL, it MUST trace back to the source SAS (as a column, macro parameter, macro variable, or SAS variable) +- If a WHERE clause or filter condition appears in the output, it MUST correspond to a condition in the source SAS +- Specifically prohibited: + - Adding parameters to macros/procedures that aren't in the SAS `%MACRO` definition + - Adding WHERE clauses or JOIN conditions not present in the SAS source + - Inventing column names based on assumed data patterns + - Adding "helper" filters that seem logical but aren't in the source + - Adding variables to make the code "more flexible" when the SAS source is hardcoded +- During Step 6 self-check: for each variable/parameter in the output, verify it has a source in the input SAS — if it doesn't, it is a hallucination and must be removed +- Specifically: if a file's logic involves a DELETE-then-INSERT-then-UPDATE chain where later statements depend on earlier ones succeeding, flat Tier 1 SQL is insufficient — wrap in a stored procedure with CALL at the bottom + +### Rule: BOOLEAN Flag/Indicator Column Detection +- SAS commonly stores flags as character `'Y'`/`'N'`, but the migrated Snowflake column may be a native `BOOLEAN` +- When converting `WHERE flag = 'Y'` / `flag = 'N'` (or `IF`/`CASE` on such a column): + - If the target column is known `BOOLEAN`: emit `WHERE flag = TRUE` / `flag = FALSE` + - If the target column is known `VARCHAR`: keep `WHERE flag = 'Y'` / `'N'` + - If the type is **unknown** and the column name ends in `_IND` or `_FLAG`: prefer `BOOLEAN` (TRUE/FALSE) and add a comment: `-- NOTE: verify column type (BOOLEAN vs VARCHAR) for ` +- **NEVER** blindly copy SAS `'Y'`/`'N'` comparisons without considering a BOOLEAN target — a string compare against a BOOLEAN column errors or silently matches nothing + +### Rule: Loop / Delimited-String NULL Guards +- When converting a SAS loop that iterates over a delimited string (e.g., `%DO` over a comma list, or cursor over `SPLIT_PART`), guard against NULL/empty before iterating: + ```sql + IF (v_list IS NOT NULL AND TRIM(v_list) <> '') THEN + -- loop body; also skip blank items: IF (TRIM(v_item) <> '') THEN ... + END IF; + ``` +- When building an iteration list from a control/config table, filter out null/empty keys: `WHERE key_col IS NOT NULL AND TRIM(key_col) <> ''` +- This prevents malformed dynamic SQL (empty `IN ()` clauses, broken concatenation) and wasted iterations on empty data + +--- + +## Migrated-Source Type Traps + +These three rules address a common failure class: SAS treats data types permissively, and source engines (Teradata, Oracle, DB2, SQL Server) often coerced types implicitly. When those tables land in Snowflake — which is **strict** about types — otherwise-faithful conversions silently return wrong results (0 rows) or raise errors. Apply these whenever a converted query touches a table that was migrated from a non-Snowflake source. + +### Rule: Type-Safe JOIN/Comparison on Migrated Keys +- The **same logical key** can have **different physical types** across tables from different source systems (e.g., an ID is `NUMBER` in one table and `VARCHAR` in another). +- When joining or comparing key columns, cast **both sides to the same type** (`::VARCHAR` is the safe default for identifiers) whenever: + 1. The column is an identifier/key (an `*_ID`, `*_GUID`, `*_SK`, customer/account/order key, etc.), **and** + 2. The two tables originate from **different source systems or schemas**, **and** + 3. There is any possibility of a `NUMBER` vs `VARCHAR` mismatch. + ```sql + -- CORRECT: cast both sides + ... ON A.CUSTOMER_ID::VARCHAR = B.CUSTOMER_ID::VARCHAR + WHERE HDR.ACCOUNT_KEY::VARCHAR = DTL.ACCOUNT_KEY::VARCHAR + -- WRONG: raw compare fails silently (0 rows) if one side is NUMBER, other VARCHAR + ... ON A.CUSTOMER_ID = B.CUSTOMER_ID + ``` +- **Why:** Teradata/Oracle perform permissive implicit conversion; Snowflake does not. A mismatched-type join yields zero rows or a "Numeric value is not recognized" error. +- Alternative for repeated joins: pre-cast into a clean temp table (`SELECT DISTINCT key::VARCHAR AS key ... WHERE key IS NOT NULL`) and join to that. + +### Rule: Numeric Aggregation on Migrated VARCHAR Columns +- Migration tools sometimes store numeric measures as `VARCHAR` (to preserve precision). A direct `SUM()`/`AVG()`/`MIN()`/`MAX()` on such a column raises "Numeric value is not recognized". +- Wrap aggregations with `TRY_TO_DOUBLE` (or `TRY_TO_NUMBER`) — plus `COALESCE(..., 0)` to match SAS's "missing + missing = 0 in SUM" behavior — when the source table was migrated **and** the column type cannot be confirmed numeric **and** the name suggests a measure (`*_AMT`, `*_CNT`, `*_QTY`, `*_DAYS`, `*_COST`, `*_CHRG`, `PAID_*`, `SPLY_*`). + ```sql + -- CORRECT + COALESCE(SUM(TRY_TO_DOUBLE(CPVAL.PAID_AMT::VARCHAR)), 0) AS TOT_PAID + -- WRONG (errors if PAID_AMT is VARCHAR) + COALESCE(SUM(CPVAL.PAID_AMT), 0) AS TOT_PAID + ``` +- `TRY_TO_*` returns NULL (not an error) for non-numeric text, so bad rows are skipped rather than aborting the query. + +### Rule: VARCHAR Date-Column Range Comparison +- When SAS compares a date column against a date range (e.g., `PAID_DT >= &DTS`), determine whether the **target Snowflake column** is a native `DATE`/`TIMESTAMP` or a `VARCHAR` holding a date string. +- If the column is (or may be) `VARCHAR`, wrap the **column itself** in `TO_DATE(col, fmt)` — not just the comparison value: + ```sql + -- CORRECT (column stored as 'DDMONYYYY') + WHERE TO_DATE(DTL.CLM_LN_PAID_DT, 'DDMONYYYY') >= TO_DATE($DTS, 'YYYY-MM-DD') + AND TO_DATE(DTL.CLM_LN_PAID_DT, 'DDMONYYYY') <= TO_DATE($DTE, 'YYYY-MM-DD') + -- WRONG: string comparison, meaningless as a date ('15JAN2024' >= '2024-01-01' is alphabetic) + WHERE DTL.CLM_LN_PAID_DT >= $DTS + ``` +- Detection heuristics: SAS uses `INPUT(col, date_fmt.)` before comparing → column is VARCHAR; source DDL shows `CHAR`/`VARCHAR`; a `*_DT` column from a Teradata/DB2-migrated fact table → assume VARCHAR `DDMONYYYY` unless context says otherwise. +- When unsure, wrapping the column in `TO_DATE()` is safe for both native `DATE` and `VARCHAR` columns. diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/data-steps.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/data-steps.md new file mode 100644 index 0000000..4798c77 --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/data-steps.md @@ -0,0 +1,505 @@ +# SAS DATA Step to Snowflake Conversion + +## When to Load + +Load when SAS code contains: `DATA` step, `SET`, `MERGE`, `RETAIN`, `BY`-group processing, `OUTPUT`, arrays, or `FIRST.`/`LAST.` logic. + +--- + +## SQL-First Decision Tree + +``` +DATA step detected + │ + ▼ +┌──────────────────────────────────────────────────────────────┐ +│ Does it use RETAIN, FIRST./LAST., ARRAY, or complex IF? │ +└──────────────────────────────────────────────────────────────┘ + │ + ├── NO ──► Pure SQL (SELECT/CREATE TABLE AS) + │ + └── YES ──► Check pattern below + │ + ┌────────────┴────────────┐ + │ │ + ▼ ▼ + Can use Window Needs row-by-row + Functions? state management? + │ │ + ├── YES ──► SQL └── Stored Procedure + │ (LAG, SUM OVER, (DECLARE, BEGIN/END) + │ ROW_NUMBER) + │ + └── NO ──► Stored Procedure +``` + +--- + +## TIER 1: Pure SQL Patterns + +### Simple Column Transformations + +**SAS:** +```sas +DATA customers_clean; + SET customers; + full_name = CATX(' ', first_name, last_name); + age = INTCK('YEAR', birth_date, TODAY()); +RUN; +``` + +**Snowflake SQL:** +```sql +CREATE OR REPLACE TABLE customers_clean AS +SELECT *, + CONCAT_WS(' ', first_name, last_name) AS full_name, + DATEDIFF('YEAR', birth_date, CURRENT_DATE()) AS age +FROM customers; +``` + +### Filtering (WHERE/IF) + +**SAS:** +```sas +DATA active; + SET customers; + WHERE status = 'ACTIVE'; + IF age >= 18; +RUN; +``` + +**Snowflake SQL:** +```sql +CREATE OR REPLACE TABLE active AS +SELECT * FROM customers +WHERE status = 'ACTIVE' AND age >= 18; +``` + +### RETAIN - Running Totals (SQL with Window Functions) + +**SAS:** +```sas +DATA running_totals; + SET transactions; + BY customer_id; + RETAIN running_sum 0; + IF FIRST.customer_id THEN running_sum = 0; + running_sum = running_sum + amount; +RUN; +``` + +**Snowflake SQL (NOT stored procedure):** +```sql +CREATE OR REPLACE TABLE running_totals AS +SELECT *, + SUM(amount) OVER ( + PARTITION BY customer_id + ORDER BY transaction_date + ROWS UNBOUNDED PRECEDING + ) AS running_sum +FROM transactions; +``` + +### FIRST./LAST. Flags (SQL with Window Functions) + +**SAS:** +```sas +DATA first_last; + SET sorted_data; + BY group_var; + IF FIRST.group_var THEN first_flag = 1; ELSE first_flag = 0; + IF LAST.group_var THEN last_flag = 1; ELSE last_flag = 0; +RUN; +``` + +**Snowflake SQL:** +```sql +CREATE OR REPLACE TABLE first_last AS +SELECT *, + CASE WHEN ROW_NUMBER() OVER (PARTITION BY group_var ORDER BY sort_col) = 1 + THEN 1 ELSE 0 END AS first_flag, + CASE WHEN ROW_NUMBER() OVER (PARTITION BY group_var ORDER BY sort_col DESC) = 1 + THEN 1 ELSE 0 END AS last_flag +FROM sorted_data; +``` + +### Keep Only Last Row per Group (SQL with QUALIFY) + +**SAS:** +```sas +DATA last_per_group; + SET data; + BY customer_id; + IF LAST.customer_id THEN OUTPUT; +RUN; +``` + +**Snowflake SQL:** +```sql +CREATE OR REPLACE TABLE last_per_group AS +SELECT * FROM data +QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY sort_col DESC) = 1; +``` + +### MERGE (Joins) - SQL + +**SAS:** +```sas +DATA merged; + MERGE table_a (IN=a) table_b (IN=b); + BY key_var; + IF a AND b; /* Inner join */ +RUN; +``` + +**Snowflake SQL:** +```sql +-- IF a AND b = INNER JOIN +CREATE OR REPLACE TABLE merged AS +SELECT a.*, b.col1, b.col2 +FROM table_a a +INNER JOIN table_b b ON a.key_var = b.key_var; + +-- IF a = LEFT JOIN +SELECT a.*, b.col1, b.col2 +FROM table_a a +LEFT JOIN table_b b ON a.key_var = b.key_var; + +-- IF a OR b = FULL OUTER JOIN +SELECT COALESCE(a.key_var, b.key_var) AS key_var, a.*, b.* +FROM table_a a +FULL OUTER JOIN table_b b ON a.key_var = b.key_var; +``` + +### MERGE with IN= Conditional (SQL with CASE) + +**SAS:** +```sas +DATA result; + MERGE master(IN=a) updates(IN=b); + BY id; + IF a AND NOT b THEN source = 'MASTER_ONLY'; + ELSE IF NOT a AND b THEN source = 'UPDATE_ONLY'; + ELSE source = 'BOTH'; +RUN; +``` + +**Snowflake SQL:** +```sql +CREATE OR REPLACE TABLE result AS +SELECT + COALESCE(a.id, b.id) AS id, + a.*, + b.*, + CASE + WHEN a.id IS NOT NULL AND b.id IS NULL THEN 'MASTER_ONLY' + WHEN a.id IS NULL AND b.id IS NOT NULL THEN 'UPDATE_ONLY' + ELSE 'BOTH' + END AS source +FROM master a +FULL OUTER JOIN updates b ON a.id = b.id; +``` + +### Simple Arrays (Same Operation) - SQL + +**SAS:** +```sas +DATA transformed; + SET input; + ARRAY nums[3] var1 var2 var3; + DO i = 1 TO 3; + IF nums[i] < 0 THEN nums[i] = 0; + END; +RUN; +``` + +**Snowflake SQL:** +```sql +CREATE OR REPLACE TABLE transformed AS +SELECT + * EXCLUDE (var1, var2, var3), + GREATEST(var1, 0) AS var1, + GREATEST(var2, 0) AS var2, + GREATEST(var3, 0) AS var3 +FROM input; +``` + +### LAG/LEAD Comparisons - SQL + +**SAS:** +```sas +DATA with_prev; + SET data; + prev_value = LAG(value); + change = value - prev_value; +RUN; +``` + +**Snowflake SQL:** +```sql +CREATE OR REPLACE TABLE with_prev AS +SELECT *, + LAG(value) OVER (ORDER BY sort_col) AS prev_value, + value - LAG(value) OVER (ORDER BY sort_col) AS change +FROM data; +``` + +### OUTPUT to Multiple Datasets - SQL + +**SAS:** +```sas +DATA high_value low_value; + SET customers; + IF total_spend > 1000 THEN OUTPUT high_value; + ELSE OUTPUT low_value; +RUN; +``` + +**Snowflake SQL (separate statements):** +```sql +CREATE OR REPLACE TABLE high_value AS +SELECT * FROM customers WHERE total_spend > 1000; + +CREATE OR REPLACE TABLE low_value AS +SELECT * FROM customers WHERE total_spend <= 1000; +``` + +--- + +## TIER 2: Stored Procedures (When SQL Window Functions Insufficient) + +Use stored procedures ONLY when: +- Multiple complex calculations depend on each other row-by-row +- RETAIN resets conditionally based on complex logic (not just BY group) +- Multiple OUTPUT with overlapping conditions +- >5 branching conditions with state + +### Complex RETAIN with Conditional Reset + +**SAS:** +```sas +DATA flagged; + SET transactions; + BY customer_id; + RETAIN prev_amount consecutive_count; + IF FIRST.customer_id THEN DO; + prev_amount = .; + consecutive_count = 0; + END; + IF amount > prev_amount > . THEN consecutive_count + 1; + ELSE consecutive_count = 0; + flag = (consecutive_count >= 3); + prev_amount = amount; +RUN; +``` + +**Snowflake Stored Procedure:** +```sql +CREATE OR REPLACE PROCEDURE sp_flag_consecutive() +RETURNS STRING +LANGUAGE SQL +AS +$$ +BEGIN + CREATE OR REPLACE TABLE flagged AS + WITH lagged AS ( + SELECT *, + LAG(amount) OVER (PARTITION BY customer_id ORDER BY trans_date) AS prev_amount, + ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY trans_date) AS rn + FROM transactions + ), + increasing_flag AS ( + SELECT *, + CASE WHEN amount > prev_amount AND prev_amount IS NOT NULL THEN 1 ELSE 0 END AS is_increasing + FROM lagged + ), + streak_groups AS ( + SELECT *, + SUM(CASE WHEN is_increasing = 0 THEN 1 ELSE 0 END) + OVER (PARTITION BY customer_id ORDER BY trans_date) AS streak_id + FROM increasing_flag + ), + streak_counts AS ( + SELECT *, + SUM(is_increasing) OVER ( + PARTITION BY customer_id, streak_id + ORDER BY trans_date + ) AS consecutive_count + FROM streak_groups + ) + SELECT + customer_id, trans_date, amount, prev_amount, + consecutive_count, + CASE WHEN consecutive_count >= 3 THEN TRUE ELSE FALSE END AS flag + FROM streak_counts; + + RETURN 'Success'; +END; +$$; +``` + +### Multiple OUTPUT with Complex Conditions + +**SAS:** +```sas +DATA good bad review; + SET applications; + IF score >= 700 AND income > 50000 THEN OUTPUT good; + ELSE IF score < 500 OR debt_ratio > 0.5 THEN OUTPUT bad; + ELSE OUTPUT review; +RUN; +``` + +**Snowflake Stored Procedure:** +```sql +CREATE OR REPLACE PROCEDURE sp_categorize_applications() +RETURNS STRING +LANGUAGE SQL +AS +$$ +DECLARE + v_good INT; v_bad INT; v_review INT; +BEGIN + CREATE OR REPLACE TABLE good AS + SELECT * FROM applications WHERE score >= 700 AND income > 50000; + + CREATE OR REPLACE TABLE bad AS + SELECT * FROM applications + WHERE NOT (score >= 700 AND income > 50000) + AND (score < 500 OR debt_ratio > 0.5); + + CREATE OR REPLACE TABLE review AS + SELECT * FROM applications + WHERE NOT (score >= 700 AND income > 50000) + AND NOT (score < 500 OR debt_ratio > 0.5); + + SELECT COUNT(*) INTO :v_good FROM good; + SELECT COUNT(*) INTO :v_bad FROM bad; + SELECT COUNT(*) INTO :v_review FROM review; + + RETURN 'Good: ' || v_good || ', Bad: ' || v_bad || ', Review: ' || v_review; +END; +$$; +``` + +--- + +## Date Handling + +| SAS Function | Snowflake Equivalent | +|--------------|---------------------| +| `TODAY()` | `CURRENT_DATE()` | +| `DATETIME()` | `CURRENT_TIMESTAMP()` | +| `INTCK('DAY', a, b)` | `DATEDIFF('DAY', a, b)` | +| `INTNX('MONTH', date, 1)` | `DATEADD('MONTH', 1, date)` | +| `INTNX('MONTH', date, 1, 'B')` | `DATE_TRUNC('month', DATEADD('month', 1, date))` | +| `INTNX('MONTH', date, 1, 'E')` | `LAST_DAY(DATEADD('month', 1, date))` | +| `INTNX('MONTH', date, 0, 'B')` | `DATE_TRUNC('month', date)` | +| `INTNX('MONTH', date, 0, 'E')` | `LAST_DAY(date)` | +| `YEAR(date)` | `YEAR(date)` | +| `MONTH(date)` | `MONTH(date)` | +| `DAY(date)` | `DAY(date)` | +| `WEEKDAY(date)` | `DAYOFWEEK(date)` | +| `MDY(m, d, y)` | `DATE_FROM_PARTS(y, m, d)` | + +**INTNX alignment parameter:** +- `'B'` (Beginning) → `DATE_TRUNC` on the interval +- `'M'` (Middle) → `DATEADD` then adjust to midpoint (flag MANUAL_REVIEW if complex) +- `'E'` (End) → `LAST_DAY` for month, or `DATEADD - 1 day` from next period start +- `'S'` (Same day, default) → plain `DATEADD` + +**SAS date literal conversion:** +- SAS dates = days since Jan 1, 1960 +- Convert: `DATEADD('DAY', sas_date_num, '1960-01-01')` + +--- + +## SAS Sum Statement (CRITICAL) + +The SAS sum statement `x + y;` (with `+` and `;`, no `=` sign) is NOT a simple assignment: +- It **adds y to x's retained value** +- It **treats missing values as 0** (unlike regular SAS arithmetic where missing propagates) + +```sas +/* SAS sum statement - x accumulates, missing treated as 0 */ +x + amount; /* equivalent to: RETAIN x 0; x = SUM(x, amount); */ +``` + +**Snowflake equivalent:** +```sql +-- In window function context: +SUM(COALESCE(amount, 0)) OVER (PARTITION BY group_col ORDER BY sort_col ROWS UNBOUNDED PRECEDING) AS x + +-- In stored procedure context: +v_x := COALESCE(v_x, 0) + COALESCE(v_amount, 0); +``` + +--- + +## DELETE / RETURN / STOP Behavior + +| SAS Statement | Meaning | Snowflake Equivalent | +|---------------|---------|---------------------| +| `DELETE;` | Remove current row from output | Add `WHERE NOT (condition)` to exclude | +| `RETURN;` | Skip remaining statements, go to next iteration | Restructure with CASE or separate CTEs | +| `STOP;` | Stop processing immediately | Not needed in set-based SQL; flag if logic depends on partial processing | +| `OUTPUT;` (explicit) | Write current row to output | Row passes the WHERE/CASE filter | +| No explicit OUTPUT | Implicit output at end of DATA step | Default SELECT behavior | + +When multiple of these interact, use a stored procedure with explicit cursor logic. + +--- + +## _N_ and _ERROR_ Automatic Variables + +| SAS Variable | Meaning | Snowflake Equivalent | +|-------------|---------|---------------------| +| `_N_` | Current iteration number (1-based) | `ROW_NUMBER() OVER (ORDER BY 1)` or explicit ordering | +| `_ERROR_` | Error flag (0/1) | No direct equivalent; use TRY_* functions | +| `END=last_obs` | Flag set on last observation | `ROW_NUMBER() OVER (ORDER BY sort_col DESC) = 1` | + +--- + +## MERGE with UPDATE Semantics + +SAS UPDATE statement differs from MERGE: +```sas +DATA master; + UPDATE master transactions; + BY key_col; +RUN; +``` + +This **overwrites** master columns with non-missing transaction values. Snowflake equivalent: +```sql +MERGE INTO master m +USING transactions t ON m.key_col = t.key_col +WHEN MATCHED THEN UPDATE SET + m.col1 = COALESCE(t.col1, m.col1), + m.col2 = COALESCE(t.col2, m.col2); +``` + +### Many-to-Many MERGE Warning + +SAS MERGE with many-to-many BY keys produces unexpected results (last value wins per group). Add MANUAL_REVIEW_REQUIRED when: +- Both datasets have duplicates on the BY key +- The SAS log would show "NOTE: MERGE statement has more than one data set with repeats of BY values" + +--- + +## Missing Value Handling + +| SAS | Snowflake | +|-----|-----------| +| `.` (numeric missing) | `NULL` | +| `' '` (character missing) | `NULL` or `''` | +| `IF var = .` | `WHERE var IS NULL` | +| `IF var NE .` | `WHERE var IS NOT NULL` | +| `COALESCE(a, b)` | `COALESCE(a, b)` | +| `MISSING(var)` | `var IS NULL` | + +**Important:** SAS treats missing as less than any value. Snowflake NULLs sort last by default. Use `NULLS FIRST` if needed: + +```sql +SELECT * FROM data ORDER BY col NULLS FIRST; +``` diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/e2e-orchestration-test.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/e2e-orchestration-test.md new file mode 100644 index 0000000..8b02470 --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/e2e-orchestration-test.md @@ -0,0 +1,188 @@ +# Phase 5: Integration & End-to-End Orchestration Test + +## When to Load + +Load ONLY when Phase 5 runs (Tier-2 consent approved). This is the heaviest validation phase — it proves the full pipeline works end-to-end, not just that individual files compile/execute. + +## Why Phase 5 exists (distinct from Phase 4) + +| | Phase 4 (Execution) | Phase 5 (E2E Orchestration) | +|---|---|---| +| Scope | Each Tier 2/3 file in isolation | **Entire pipeline as one integrated flow** | +| Dependency handling | Per-file mini chain | **Full topological DAG order** | +| Output check | Compare each file's output to baseline | **Assert terminal tables non-empty + compare** | +| Artifact | `snowflake_execution_results.json` | **Reusable harness on disk + `e2e_test_results.json`** | + +A pipeline can pass Phase 4 (every SP runs) yet still be broken end-to-end (terminal tables empty because an intermediate join produced zero rows). Phase 5 catches that. + +--- + +## Step 1: Build the Topological Layer Order + +Read the cross-file dependency DAG (built at Step 3, stored per file under `files..dependencies.{creates,reads}` in `conversion_state.json`). + +```python +# Kahn topological sort over files using creates/reads +def topo_layers(files): + # node = file; edge A->B if B.reads intersects A.creates + produced = {} # table -> file that creates it + for f, info in files.items(): + for t in info["dependencies"]["creates"]: + produced[t.upper()] = f + deps = {f: set() for f in files} + for f, info in files.items(): + for t in info["dependencies"]["reads"]: + src = produced.get(t.upper()) + if src and src != f: + deps[f].add(src) + layers, placed = [], set() + while len(placed) < len(files): + layer = [f for f in files if f not in placed and deps[f] <= placed] + if not layer: # cycle or external-only inputs left + layer = [f for f in files if f not in placed] + layers.append(sorted(layer)) + placed.update(layer) + return layers +``` + +Files whose `reads` are ALL external source tables (no upstream converted file) land in Layer 0. Terminal files = those whose `creates` tables appear in NO other file's `reads`. + +--- + +## Step 2: Generate the Reusable Harness (on disk) + +Generate five files. These are deliverables — they let the user re-run the integration test and deploy orchestration without regenerating. + +### 2.1 `orchestration/sp_e2e_pipeline.sql` + +A master stored procedure that runs every converted file in DAG layer order. For Tier-2 files, `CALL` the procedure; for Tier-1 files, inline the file's primary `CREATE TABLE AS` statement (or `CALL` a thin wrapper). Track step count and catch errors. + +```sql +CREATE OR REPLACE PROCEDURE .SP_E2E_PIPELINE() +RETURNS STRING LANGUAGE SQL EXECUTE AS CALLER AS +$$ +DECLARE + v_step STRING DEFAULT ''; + v_passed INTEGER DEFAULT 0; + v_count INTEGER; +BEGIN + -- Layer 1 + v_step := 'L1: '; CALL .SP_...(); v_passed := v_passed + 1; + -- ... one statement per file, in layer order ... + -- Layer N (terminal) + v_step := 'VALIDATION'; + SELECT COUNT(*) INTO :v_count FROM .; + RETURN OBJECT_CONSTRUCT('status','SUCCESS','steps_passed',v_passed, + 'steps_failed',0,'last_step',v_step, + 'terminal_rows',v_count)::STRING; +EXCEPTION WHEN OTHER THEN + RETURN OBJECT_CONSTRUCT('status','FAILED','failed_at',v_step, + 'steps_passed',v_passed,'code',sqlcode,'message',sqlerrm)::STRING; +END; +$$; +``` + +### 2.2 `orchestration/task_dag.sql` + +Snowflake Task DAG mirroring the layers: a root task + `AFTER` children, one task (or one bundled task per layer). Tasks created `SUSPENDED`; include `ALTER TASK ... RESUME` lines bottom-up. Include a comment noting `EXECUTE TASK` requires the `EXECUTE TASK` account privilege (ACCOUNTADMIN grant). + +### 2.3 `orchestration/adf_pipeline.sql` + +ADF master-orchestrator definition (JSON in a comment block): a Script activity that CALLs the staging SP, then a Script activity that CALLs `SP_E2E_PIPELINE()`, then a Lookup activity that validates a terminal table row count. ADF is the master scheduler; Snowflake Tasks handle intra-DAG parallelism. + +### 2.4 `testing/setup_test_data.sql` + +The depth/unit-aligned synthetic loader (from Phase 1 output). MUST follow the range-join + unit-consistency rules in `references/synthetic-data-rules.md` so multi-hop joins survive to terminal tables. Include a `TRUNCATE` block so it is idempotent. + +### 2.5 `testing/expected_results.sql` + +Expected terminal-table row counts and key values (from Phase 2 baselines), as comments + sample assertion `SELECT`s. + +--- + +## Step 3: Execute End-to-End + +1. Ensure source tables + aligned synthetic data are loaded (run `testing/setup_test_data.sql`). +2. Deploy all Tier-2 procedures (if not already created in Phase 4). +3. `CREATE` then `CALL .SP_E2E_PIPELINE()`. +4. Parse the returned JSON. Require `steps_failed = 0`. If a step failed, the `failed_at` field localizes the broken file — fix the converted SQL (not the SAS interpretation) and re-run. + +--- + +## Step 4: Terminal-Output Assertion (the key check) + +For every **terminal** table (no downstream consumer), assert row count > 0: + +```sql +SELECT '' AS TBL, COUNT(*) AS ROW_CNT FROM . +UNION ALL ...; +``` + +Classify each terminal table: +- **POPULATED** (> 0 rows): pass. +- **EMPTY — JUSTIFIED**: the table is empty only because a genuine source/reference table is empty by design (e.g. an unused regional reference). Record the empty upstream in `notes`. +- **EMPTY — DEFECT**: empty despite populated sources. This is a real failure (almost always a join that did not overlap — see the range-join/unit-consistency rules). Flag for fix. + +A run where any terminal table is `EMPTY — DEFECT` does NOT pass Phase 5. + +--- + +## Step 5: Compare Terminal Outputs to Baselines + +For terminal tables that have a Phase 2 expected baseline, compare row counts and key values per `references/comparison-rules.md`. Apply the trivial-pass rule from `references/snowflake-execution.md`: `expected = 0 AND actual = 0` is a WARNING (`TRIVIAL_PASS`), not a silent pass. + +--- + +## Step 6: Write Results + +```python +e2e = { + "executed_at": "", + "warehouse": "", + "layers": , + "steps_total": , "steps_passed": , "steps_failed": , + "terminal_tables": [ + {"table": "", "rows": , "status": "POPULATED|EMPTY_JUSTIFIED|EMPTY_DEFECT", + "empty_source": "
    ", "baseline_match": "PASS|FAIL|NO_BASELINE|TRIVIAL_PASS"} + ], + "harness_files": ["orchestration/sp_e2e_pipeline.sql","orchestration/task_dag.sql", + "orchestration/adf_pipeline.sql","testing/setup_test_data.sql", + "testing/expected_results.sql"], + "status": "PASS|FAIL" +} +# write /e2e_test_results.json +``` + +`status = PASS` requires: `steps_failed == 0` AND no terminal table is `EMPTY_DEFECT`. + +--- + +## Step 7: Cleanup + +If a temporary validation database was created for Phase 5, prompt to drop it (or auto-drop under consent propagation), same pattern as Phase 4. + +--- + +## Modular Execution (Context-Aware) + +Generating the master SP for a large pipeline (40+ files) can be large. Build it incrementally: +1. Generate the SP layer-by-layer, appending each layer's CALLs. +2. After writing the harness files, update `execution_progress` in `conversion_state.json` and append a checkpoint line. +3. If context nears the limit mid-generation: save the partial harness + state, inform the user, exit. Resume reads `execution_progress` and continues from the next layer. + +--- + +## HARD GATE (Phase 5) + +```python +import os, sys, json +out = "" +need = [os.path.join(out, "e2e_test_results.json"), + os.path.join(out, "orchestration", "sp_e2e_pipeline.sql")] +missing = [p for p in need if not os.path.exists(p)] +if missing: + print(f"BLOCKED: Phase 5 artifacts missing: {missing}"); sys.exit(1) +print("Phase 5 GATE PASSED") +``` + +Update `conversion_state.json`: `gates.phase_5_e2e_orchestration: "PASSED"`. Append checkpoint `step:"phase_5" gate:"phase_5_e2e_orchestration" status:"PASSED"`. diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/function-mappings.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/function-mappings.md new file mode 100644 index 0000000..07c40f7 --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/function-mappings.md @@ -0,0 +1,442 @@ +# SAS to Snowflake Function Mappings + +## When to Load + +Load for **all conversions** - comprehensive function translation reference. + +--- + +## String Functions + +| SAS Function | Snowflake Equivalent | Notes | +|--------------|---------------------|-------| +| `SUBSTR(str, pos, len)` | `SUBSTR(str, pos, len)` | Direct mapping | +| `SUBSTRN(str, pos, len)` | `SUBSTR(str, pos, len)` | Same as SUBSTR | +| `SCAN(str, n, 'delim')` | `SPLIT_PART(str, 'delim', n)` | Word extraction | +| `INDEX(str, substr)` | `CHARINDEX(substr, str)` | Find position | +| `FIND(str, substr, pos, 'i')` | `CHARINDEX(LOWER(substr), LOWER(str), pos)` | Case-insensitive with 'i' modifier | +| `UPCASE(str)` | `UPPER(str)` | | +| `LOWCASE(str)` | `LOWER(str)` | | +| `PROPCASE(str)` | `INITCAP(str)` | Title case | +| `TRIM(str)` | `TRIM(str)` | | +| `LEFT(str)` | `LTRIM(str)` | Left trim | +| `RIGHT(str)` | `RTRIM(str)` | Right trim | +| `STRIP(str)` | `TRIM(str)` | Both sides | +| `LENGTH(str)` | `LENGTH(str)` | | +| `LENGTHN(str)` | `LENGTH(str)` | | +| `LENGTHC(str)` | `LENGTH(str)` | | +| `REVERSE(str)` | `REVERSE(str)` | | +| `REPEAT(str, n)` | `REPEAT(str, n)` | | +| `TRANSLATE(str, to, from)` | `TRANSLATE(str, from, to)` | ⚠️ Argument order differs | +| `SOUNDEX(str)` | `SOUNDEX(str)` | | +| `BYTE(n)` | `CHR(n)` | Character from ASCII | +| `RANK(char)` | `ASCII(char)` | ASCII from character | + +## Concatenation Functions + +| SAS Function | Snowflake Equivalent | Notes | +|--------------|---------------------|-------| +| `CAT(a, b, c)` | `CONCAT(a, b, c)` | Simple concat | +| `CATS(a, b, c)` | `CONCAT(TRIM(COALESCE(a,'')), TRIM(COALESCE(b,'')), TRIM(COALESCE(c,'')))` | Strip all then concat | +| `CATT(a, b, c)` | `CONCAT(RTRIM(COALESCE(a,'')), RTRIM(COALESCE(b,'')), RTRIM(COALESCE(c,'')))` | Trailing trim then concat | +| `CATX('sep', a, b, c)` | `CONCAT_WS('sep', TRIM(COALESCE(a,'')), TRIM(COALESCE(b,'')), TRIM(COALESCE(c,'')))` | With separator | +| `a \|\| b` | `a \|\| b` | Direct mapping | + +## COMPRESS Function (Special Handling) + +**1-argument form (remove ALL whitespace):** + +SAS `COMPRESS(str)` with NO second argument removes **all** whitespace characters — spaces, tabs, newlines, carriage returns, etc. — NOT just spaces. + +```sql +-- SAS: COMPRESS(str) +-- Snowflake (CORRECT — removes all whitespace): +REGEXP_REPLACE(str, '\\s', '') +``` + +**⚠️ FORBIDDEN:** Do NOT map `COMPRESS(str)` to `REPLACE(str, ' ', '')` — that only strips spaces and silently leaves tabs/newlines/CR. Also do NOT expand it into a chain of nested `REPLACE(...)` calls per character — that is verbose, error-prone, and misses characters. A single `REGEXP_REPLACE(expr, '\\s', '')` matches SAS default `COMPRESS()` behavior exactly. + +**2-argument form (remove specific chars):** +```sql +-- SAS: COMPRESS(str, 'abc') +-- Snowflake: +REGEXP_REPLACE(str, '[abc]', '') +``` + +**3-argument form with modifiers:** + +| Modifier | Meaning | Snowflake Pattern | +|----------|---------|-------------------| +| `a`, `i` | Alphabetic | `[A-Za-z]` | +| `d` | Digits | `[0-9]` | +| `n` | Alphanumeric + underscore | `[A-Za-z0-9_]` | +| `l` | Lowercase | `[a-z]` | +| `u` | Uppercase | `[A-Z]` | +| `f` | Underscore + letters | `[A-Za-z_]` | +| `k` | **KEEP** these chars (invert) | Use `[^...]` | + +```sql +-- SAS: COMPRESS(str, '', 'kd') -- Keep only digits +-- Snowflake: +REGEXP_REPLACE(str, '[^0-9]', '') + +-- SAS: COMPRESS(str, '', 'd') -- Remove digits +-- Snowflake: +REGEXP_REPLACE(str, '[0-9]', '') +``` + +## COMPBL Function + +```sql +-- SAS: COMPBL(str) -- Compress multiple blanks to single +-- Snowflake: +REGEXP_REPLACE(str, ' +', ' ') +``` + +## Conditional Functions + +| SAS Function | Snowflake Equivalent | Notes | +|--------------|---------------------|-------| +| `IFC(cond, true_str, false_str)` | `CASE WHEN cond THEN true_str ELSE false_str END` | Character IF | +| `IFN(cond, true_num, false_num)` | `CASE WHEN cond THEN true_num ELSE false_num END` | Numeric IF | +| `COALESCE(a, b, c)` | `COALESCE(a, b, c)` | First non-null | +| `COALESCEC(a, b, c)` | `COALESCE(a, b, c)` | Character version | + +**Nested IFC/IFN handling:** +Parse with balanced parentheses to handle: +```sas +IFC(cond1, IFC(cond2, 'A', 'B'), 'C') +``` +→ +```sql +CASE WHEN cond1 THEN + CASE WHEN cond2 THEN 'A' ELSE 'B' END +ELSE 'C' END +``` + +## Date/Time Functions + +| SAS Function | Snowflake Equivalent | Notes | +|--------------|---------------------|-------| +| `TODAY()` | `CURRENT_DATE()` | | +| `DATE()` | `CURRENT_DATE()` | | +| `DATETIME()` | `CURRENT_TIMESTAMP()` | | +| `TIME()` | `CURRENT_TIME()` | | +| `YEAR(date)` | `YEAR(date)` | | +| `MONTH(date)` | `MONTH(date)` | | +| `DAY(date)` | `DAY(date)` | | +| `HOUR(dt)` | `HOUR(dt)` | | +| `MINUTE(dt)` | `MINUTE(dt)` | | +| `SECOND(dt)` | `SECOND(dt)` | | +| `WEEK(date)` | `WEEK(date)` | | +| `QTR(date)` | `QUARTER(date)` | | +| `WEEKDAY(date)` | `DAYOFWEEK(date)` | | +| `DATEPART(datetime)` | `TO_DATE(datetime)` | Extract date from datetime | +| `TIMEPART(datetime)` | `TO_TIME(datetime)` | Extract time from datetime | +| `MDY(m, d, y)` | `DATE_FROM_PARTS(y, m, d)` | ⚠️ Argument order differs | +| `YMD(y, m, d)` | `DATE_FROM_PARTS(y, m, d)` | | +| `HMS(h, m, s)` | `TIME_FROM_PARTS(h, m, s)` | | +| `DHMS(date, h, m, s)` | `TIMESTAMP_FROM_PARTS(...)` | Complex conversion | + +## INTCK (Date Intervals) + +```sql +-- SAS: INTCK('MONTH', start_date, end_date) +-- Snowflake: +DATEDIFF('month', start_date, end_date) +``` + +| SAS Interval | Snowflake Interval | +|--------------|-------------------| +| `'YEAR'` | `'year'` | +| `'MONTH'` | `'month'` | +| `'DAY'` | `'day'` | +| `'WEEK'` | `'week'` | +| `'HOUR'` | `'hour'` | +| `'MINUTE'` | `'minute'` | +| `'SECOND'` | `'second'` | +| `'QTR'` | `'quarter'` | +| `'QUARTER'` | `'quarter'` | + +## INTNX (Date Arithmetic) + +```sql +-- SAS: INTNX('MONTH', date, 3) -- Default alignment = 'S' (same day) +-- Snowflake: +DATEADD('month', 3, date) + +-- SAS: INTNX('MONTH', date, 3, 'B') -- Beginning of interval +-- Snowflake: +DATE_TRUNC('month', DATEADD('month', 3, date)) + +-- SAS: INTNX('MONTH', date, 3, 'E') -- End of interval +-- Snowflake: +LAST_DAY(DATEADD('month', 3, date)) + +-- SAS: INTNX('MONTH', date, 0, 'B') -- Beginning of CURRENT month +-- Snowflake: +DATE_TRUNC('month', date) + +-- SAS: INTNX('MONTH', date, 0, 'E') -- End of CURRENT month +-- Snowflake: +LAST_DAY(date) + +-- SAS: INTNX('YEAR', date, 1, 'B') -- Beginning of next year +-- Snowflake: +DATE_TRUNC('year', DATEADD('year', 1, date)) + +-- SAS: INTNX('YEAR', date, 0, 'E') -- End of current year +-- Snowflake: +DATEADD('day', -1, DATE_TRUNC('year', DATEADD('year', 1, date))) + +-- SAS: INTNX('WEEK', date, 0, 'B') -- Beginning of current week +-- Snowflake: +DATE_TRUNC('week', date) + +-- SAS: INTNX('QTR', date, 1, 'B') -- Beginning of next quarter +-- Snowflake: +DATE_TRUNC('quarter', DATEADD('quarter', 1, date)) +``` + +**INTNX alignment parameter reference:** + +| Alignment | Meaning | Snowflake Pattern | +|-----------|---------|-------------------| +| `'S'` (default) | Same day within new interval | `DATEADD(interval, n, date)` | +| `'B'` | Beginning of target interval | `DATE_TRUNC(interval, DATEADD(interval, n, date))` | +| `'E'` | End of target interval | For month: `LAST_DAY(DATEADD(...))`. For others: `DATEADD('day', -1, DATE_TRUNC(interval, DATEADD(interval, n+1, date)))` | +| `'M'` | Middle of target interval | Flag `MANUAL_REVIEW_REQUIRED` — complex midpoint logic | + +**⚠️ CRITICAL:** Do NOT assume `DATEADD` alone is equivalent to `INTNX` without checking the alignment parameter. The default ('S') maps to DATEADD, but 'B' and 'E' require DATE_TRUNC/LAST_DAY wrapping. + +## INPUT/PUT (Type Conversion) + +**INPUT (string to typed value):** +```sql +-- SAS: INPUT(str, 8.) -- To number +-- Snowflake: +TRY_TO_NUMBER(str) + +-- SAS: INPUT(str, DATE9.) -- To date +-- Snowflake: +TRY_TO_DATE(str) + +-- SAS: INPUT(str, DATETIME20.) +-- Snowflake: +TRY_TO_TIMESTAMP(str) + +-- SAS: INPUT(str, $20.) -- Character (just trim) +-- Snowflake: +TRIM(str) +``` + +**PUT (typed value to string):** +```sql +-- SAS: PUT(num, 8.) +-- Snowflake: +TO_VARCHAR(num) + +-- SAS: PUT(date, DATE9.) +-- Snowflake: +TO_VARCHAR(date, 'DDMONYYYY') + +-- SAS: PUT(num, Z5.) -- Zero-padded +-- Snowflake: +LPAD(TO_VARCHAR(num), 5, '0') + +-- SAS: PUT(num, COMMA12.2) +-- Snowflake: +TO_VARCHAR(num, '999,999,999.99') +``` + +## Regular Expression Functions + +| SAS Function | Snowflake Equivalent | +|--------------|---------------------| +| `PRXMATCH('/pattern/', str)` | `CASE WHEN REGEXP_LIKE(str, 'pattern') THEN 1 ELSE 0 END` | +| `PRXCHANGE('s/pat/repl/', -1, str)` | `REGEXP_REPLACE(str, 'pat', 'repl')` | +| `PRXCHANGE('s/pat/repl/', 1, str)` | `REGEXP_REPLACE(str, 'pat', 'repl', 1, 1)` | + +## Word Functions + +| SAS Function | Snowflake Equivalent | +|--------------|---------------------| +| `COUNTW(str)` | `ARRAY_SIZE(SPLIT(TRIM(str), ' '))` | +| `COUNTW(str, 'delim')` | `ARRAY_SIZE(SPLIT(str, 'delim'))` | +| `TRANWRD(str, find, replace)` | `REPLACE(str, find, replace)` | + +## VERIFY Function + +```sql +-- SAS: VERIFY(str, 'valid_chars') +-- Returns 0 if all chars valid, else position of first invalid +-- Snowflake: +CASE + WHEN REGEXP_LIKE(str, '^[valid_chars]*$') THEN 0 + ELSE REGEXP_INSTR(str, '[^valid_chars]') +END +``` + +## Numeric Functions + +| SAS Function | Snowflake Equivalent | +|--------------|---------------------| +| `ABS(x)` | `ABS(x)` | +| `ROUND(x, n)` | `ROUND(x, n)` | +| `CEIL(x)` | `CEIL(x)` | +| `CEILING(x)` | `CEIL(x)` | +| `FLOOR(x)` | `FLOOR(x)` | +| `INT(x)` | `TRUNC(x)` | +| `MOD(x, y)` | `MOD(x, y)` | +| `SQRT(x)` | `SQRT(x)` | +| `LOG(x)` | `LN(x)` | +| `LOG10(x)` | `LOG(10, x)` | +| `LOG2(x)` | `LOG(2, x)` | +| `EXP(x)` | `EXP(x)` | +| `POWER(x, y)` | `POWER(x, y)` | +| `SIGN(x)` | `SIGN(x)` | +| `RANUNI(seed)` | `RANDOM()` | +| `RAND('UNIFORM')` | `RANDOM()` | + +## Aggregate Functions + +| SAS Function | Snowflake Equivalent | +|--------------|---------------------| +| `SUM(x)` | `SUM(x)` | +| `MEAN(x)` | `AVG(x)` | +| `MIN(x)` | `MIN(x)` | +| `MAX(x)` | `MAX(x)` | +| `COUNT(x)` | `COUNT(x)` | +| `N(x)` | `COUNT(x)` | +| `NMISS(x)` | `COUNT_IF(x IS NULL)` | +| `STD(x)` | `STDDEV(x)` | +| `VAR(x)` | `VARIANCE(x)` | +| `MEDIAN(x)` | `MEDIAN(x)` | +| `SKEWNESS(x)` | `SKEW(x)` | +| `KURTOSIS(x)` | `KURTOSIS(x)` | + +## Window/Lag Functions + +| SAS Function | Snowflake Equivalent | +|--------------|---------------------| +| `LAG(var)` | `LAG(var) OVER (ORDER BY ...)` | +| `LAG1(var)` | `LAG(var, 1) OVER (ORDER BY ...)` | +| `LAG2(var)` | `LAG(var, 2) OVER (ORDER BY ...)` | +| `DIF(var)` | `var - LAG(var) OVER (ORDER BY ...)` | + +## Missing Value Handling + +| SAS Construct | Snowflake Equivalent | +|---------------|---------------------| +| `.` (numeric missing) | `NULL` | +| `var = .` | `var IS NULL` | +| `var NE .` | `var IS NOT NULL` | +| `MISSING(var)` | `var IS NULL` | +| `NOTMISSING(var)` | `var IS NOT NULL` | + +## Operator Conversions + +| SAS Operator | Snowflake Operator | +|--------------|-------------------| +| `EQ` | `=` | +| `NE` | `<>` | +| `LT` | `<` | +| `LE` | `<=` | +| `GT` | `>` | +| `GE` | `>=` | +| `^=` | `<>` | +| `~=` | `<>` | +| `AND` | `AND` | +| `OR` | `OR` | +| `NOT` | `NOT` | + +## Date Literal Conversions + +```sql +-- SAS: '01JAN2024'd +-- Snowflake: +TO_DATE('01JAN2024', 'DDMONYYYY') + +-- SAS: '01JAN2024:12:30:00'dt +-- Snowflake: +TO_TIMESTAMP('01JAN2024:12:30:00', 'DDMONYYYY:HH24:MI:SS') +``` + +## SAS Date Format to Snowflake Format + +| SAS Format | Snowflake Format | +|------------|-----------------| +| `YYMMDD10.` | `YYYY-MM-DD` | +| `DDMMYY10.` | `DD/MM/YYYY` | +| `MMDDYY10.` | `MM/DD/YYYY` | +| `DATE9.` | `DDMONYYYY` | +| `DATETIME20.` | `YYYY-MM-DD HH24:MI:SS` | +| `TIME8.` | `HH24:MI:SS` | +| `MONYY7.` | `MONYYYY` | +| `YEAR4.` | `YYYY` | +| `COMMA12.2` | `999,999,999.99` | +| `DOLLAR12.2` | `$999,999,999.99` | +| `PERCENT8.2` | `999.99%` | + +## Special Variables + +| SAS Variable | Snowflake Equivalent | +|--------------|---------------------| +| `_N_` | `ROW_NUMBER() OVER (ORDER BY 1)` | +| `MONOTONIC()` | `ROW_NUMBER() OVER ()` | +| `_ERROR_` | No direct equivalent; use TRY_* functions | +| `END=last` (SET option) | `ROW_NUMBER() OVER (ORDER BY sort_col DESC) = 1` | + +--- + +## SAS Automatic (Predefined) Macro Variables + +SAS supplies automatic macro variables resolved at compile time. Convert each to its Snowflake equivalent — never leave a raw `&SYS...` reference in the output. + +| SAS Automatic Variable | Snowflake Equivalent | +|------------------------|----------------------| +| `&SYSDATE` / `&SYSDATE9` | `TO_CHAR(CURRENT_DATE(), 'DDMONYY')` / `'DDMONYYYY'` | +| `&SYSTIME` | `TO_CHAR(CURRENT_TIME(), 'HH24:MI')` | +| `&SYSDAY` | `DAYNAME(CURRENT_DATE())` | +| `&SYSUSERID` | `CURRENT_USER()` | +| `&SYSPROCESSID` / `&SYSJOBID` | `CURRENT_SESSION()` | +| `&SQLOBS` | `SQLROWCOUNT` (rows from last DML) or `COUNT(*)` of the result | +| `&SQLRC` / `&SYSERR` | Scripting exception state (`SQLCODE` / `SQLERRM`); see multi-block-orchestration.md for `&SYSERR` flow gating | + +Macro-language string/index functions (complements `%SCAN`, `%EVAL`, `%SYSFUNC` in macros.md): + +| SAS Macro Function | Snowflake Equivalent | +|--------------------|----------------------| +| `%SUBSTR(str, pos, len)` | `SUBSTR(str, pos, len)` | +| `%SCAN(str, n, delim)` | `SPLIT_PART(str, delim, n)` | +| `%INDEX(source, sub)` | `POSITION(sub IN source)` | +| `%LENGTH(str)` | `LENGTH(str)` | +| `%UPCASE` / `%LOWCASE` | `UPPER` / `LOWER` | + +--- + +## SAS Sum Statement (x + y;) + +**CRITICAL:** The SAS sum statement `x + y;` (variable + expression followed by semicolon, no assignment operator) is NOT regular arithmetic: +- It **RETAINS x** across rows (initializes to 0, not missing) +- It **adds y to x**, treating missing values as 0 +- Equivalent to: `RETAIN x 0; x = SUM(x, y);` + +```sql +-- SAS: total + amount; +-- Snowflake (window function): +SUM(COALESCE(amount, 0)) OVER ( + PARTITION BY group_col + ORDER BY sort_col + ROWS UNBOUNDED PRECEDING +) AS total + +-- SAS: total + amount; (with BY group reset) +-- If FIRST.group resets total, the PARTITION BY handles it automatically +``` + +--- + +## Oracle / DB2 Passthrough Function Mappings + +See `references/vendor-function-mappings.md` for full Oracle→Snowflake and DB2→Snowflake function mapping tables and SQL passthrough conversion steps. diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/large-file-rules.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/large-file-rules.md new file mode 100644 index 0000000..25db5e5 --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/large-file-rules.md @@ -0,0 +1,121 @@ +# Large File Conversion Rules + +## When to Load + +Load when SAS file exceeds **500 lines**, **50K characters**, or **20 blocks**. + +--- + +## Chunked Conversion Strategy + +Large SAS files must be converted completely. Never truncate output. + +### Approach: Dependency-Aware Block Groups + +1. **Build full dependency graph** before converting any block +2. **Group blocks** into independent clusters that can be converted together +3. **Convert each group** in dependency order +4. **Validate inter-group references** — ensure table/variable names match + +### Conversion Order + +``` +1. Parse all blocks → build dependency map (creates/reads/variables) +2. Identify independent clusters (blocks with no cross-dependencies) +3. Convert Tier 1 (pure SQL) blocks first — they are fastest +4. Convert Tier 2 (stored procedure) blocks next +5. Convert Tier 3 (PySpark) blocks last +6. Validate cross-references between all converted blocks +``` + +--- + +## Output Completeness Rules + +### NEVER Truncate + +- If a block has 100+ lines: convert ALL lines +- If a block creates 10 tables: generate ALL 10 CREATE TABLE statements +- If a macro has 20 internal steps: convert ALL 20 steps +- If an ARRAY iterates over 50 columns: generate ALL 50 column expressions + +### NEVER Summarize + +These shortcuts are FORBIDDEN in output: +- "similar pattern for remaining columns" +- "same as above" +- "repeat for other tables" +- "adjusted joins/filters" +- "... and so on for columns X through Z" +- Any ellipsis (`...`) replacing actual logic + +### Verify Completeness + +After generating conversion for each block: +1. Count output tables in SAS → verify same count in Snowflake +2. Count CALL SYMPUT/SYMPUTX assignments → verify all persisted +3. Count %IF branches → verify all branches converted +4. Count column derivations → verify all present + +--- + +## Repeated Macro Invocations + +When a macro is called multiple times with different parameters: + +1. **Track each invocation separately** — different parameters may trigger different branches +2. **Do NOT assume** all invocations produce the same output structure +3. For %IF/%THEN branches controlled by parameters, verify which branch each invocation takes +4. Generate separate SQL for each invocation if outputs differ +5. If macro is called >5 times, consider converting to a stored procedure called multiple times + +### Example: Macro called with different date parameters + +```sas +%process_month(month=JAN, year=2024); +%process_month(month=FEB, year=2024); +%process_month(month=MAR, year=2024); +``` + +Convert to: +```sql +CALL sp_process_month('JAN', 2024); +CALL sp_process_month('FEB', 2024); +CALL sp_process_month('MAR', 2024); +``` + +NOT to a single call or summary comment. + +--- + +## Inter-Block Dependency Rules + +### Table Dependencies + +- A temp table created in block N may be consumed by block N+2, N+3, or later +- Do NOT assume consumption is always N+1 +- Session variables persist across ALL blocks in the session +- Temporary tables persist across ALL blocks in the session +- Do NOT recreate a temp table that was already created upstream + +### Variable Dependencies + +- Macro variables set via CALL SYMPUT in one step may be referenced in distant downstream steps +- In Snowflake: persist via `EXECUTE IMMEDIATE 'SET ...'` and reference as `$VAR_NAME` +- Track ALL variable assignments and their downstream consumers + +### Column Dependencies + +- Verify column names match between producer and consumer blocks +- If block N renames a column (e.g., RENAME in SAS), downstream blocks must use the new name +- If block N adds a computed column, downstream blocks may reference it + +--- + +## Error Recovery for Large Files + +If conversion fails partway through: +1. Save completed blocks to the output file +2. Mark the failing block with MANUAL_REVIEW_REQUIRED +3. Continue converting remaining blocks +4. Report: "X of Y blocks converted successfully. Block Z requires manual review: [reason]" diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/macros.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/macros.md new file mode 100644 index 0000000..9ed42ed --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/macros.md @@ -0,0 +1,572 @@ +# SAS Macros to Snowflake Conversion + +## When to Load + +Load when SAS code contains: `%MACRO`, `%MEND`, `%LET`, `&variable` references, `%DO` loops, `%IF`/`%THEN`, `%SYSFUNC`, or autocall macro calls. + +--- + +## Overview + +SAS macros generate code at compile time. Snowflake alternatives: +1. **Stored Procedures** - For procedural logic +2. **UDFs** - For reusable calculations +3. **Jinja templates** - For SQL generation (in dbt or external tools) +4. **JavaScript UDFs** - For complex string manipulation + +## Macro Variables + +### Simple Variable Substitution + +**SAS:** +```sas +%LET start_date = 2024-01-01; +%LET table_name = sales; + +PROC SQL; + SELECT * FROM &table_name + WHERE date >= "&start_date"d; +QUIT; +``` + +**Snowflake (Session Variables):** +```sql +SET start_date = '2024-01-01'; +SET table_name = 'sales'; + +SELECT * FROM IDENTIFIER($table_name) +WHERE date >= $start_date::DATE; +``` + +**Snowflake (Stored Procedure Parameters):** +```sql +CREATE OR REPLACE PROCEDURE query_table(table_name STRING, start_date DATE) +RETURNS TABLE() +LANGUAGE SQL +AS +$$ +DECLARE + result RESULTSET; +BEGIN + result := (SELECT * FROM IDENTIFIER(:table_name) WHERE date >= :start_date); + RETURN TABLE(result); +END; +$$; + +CALL query_table('sales', '2024-01-01'::DATE); +``` + +### Macro Variable from Query + +**SAS:** +```sas +PROC SQL NOPRINT; + SELECT MAX(date) INTO :max_date FROM transactions; +QUIT; +%PUT Max date is &max_date; +``` + +**Snowflake:** +```sql +CREATE OR REPLACE PROCEDURE get_max_date() +RETURNS STRING +LANGUAGE SQL +AS +$$ +DECLARE + max_date DATE; +BEGIN + SELECT MAX(date) INTO :max_date FROM transactions; + RETURN 'Max date is ' || max_date::STRING; +END; +$$; + +CALL get_max_date(); +``` + +## Simple Macro Programs + +### Macro Without Parameters + +**SAS:** +```sas +%MACRO clean_data; + DATA cleaned; + SET raw; + name = PROPCASE(STRIP(name)); + IF age < 0 THEN age = .; + RUN; +%MEND; + +%clean_data; +``` + +**Snowflake Stored Procedure:** +```sql +CREATE OR REPLACE PROCEDURE clean_data() +RETURNS STRING +LANGUAGE SQL +AS +$$ +DECLARE + err_no_source EXCEPTION (-20001, 'Source table raw does not exist'); + v_row_count INTEGER; +BEGIN + -- Validate source exists + SELECT COUNT(*) INTO :v_row_count FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_NAME = 'RAW' AND TABLE_SCHEMA = CURRENT_SCHEMA(); + IF (v_row_count = 0) THEN + RAISE err_no_source; + END IF; + + CREATE OR REPLACE TABLE cleaned AS + SELECT *, + INITCAP(TRIM(name)) AS name_clean, + CASE WHEN age < 0 THEN NULL ELSE age END AS age_clean + FROM raw; + + SELECT COUNT(*) INTO :v_row_count FROM cleaned; + RETURN 'Data cleaned: ' || v_row_count || ' rows'; +EXCEPTION + WHEN err_no_source THEN + RETURN OBJECT_CONSTRUCT('status', 'ERROR', 'code', sqlcode, 'message', sqlerrm)::STRING; + WHEN OTHER THEN + RETURN OBJECT_CONSTRUCT('status', 'ERROR', 'code', sqlcode, 'message', sqlerrm)::STRING; +END; +$$; + +CALL clean_data(); +``` + +### Macro With Parameters + +**SAS:** +```sas +%MACRO summarize(input=, output=, by_var=); + PROC MEANS DATA=&input NOPRINT; + CLASS &by_var; + VAR amount; + OUTPUT OUT=&output SUM=total MEAN=average; + RUN; +%MEND; + +%summarize(input=sales, output=sales_summary, by_var=region); +``` + +**Snowflake Stored Procedure:** +```sql +CREATE OR REPLACE PROCEDURE summarize( + p_input_table STRING, + p_output_table STRING, + p_by_var STRING +) +RETURNS STRING +LANGUAGE SQL +AS +$$ +DECLARE + err_invalid_input EXCEPTION (-20001, 'Input parameters cannot be NULL'); + v_sql_stmt STRING; + v_row_count INTEGER; +BEGIN + -- Validate inputs + IF (p_input_table IS NULL OR p_output_table IS NULL OR p_by_var IS NULL) THEN + RAISE err_invalid_input; + END IF; + + v_sql_stmt := 'CREATE OR REPLACE TABLE ' || p_output_table || ' AS + SELECT ' || p_by_var || ', + SUM(amount) AS total, + AVG(amount) AS average, + COUNT(*) AS n + FROM ' || p_input_table || ' + GROUP BY ' || p_by_var; + EXECUTE IMMEDIATE v_sql_stmt; + + v_sql_stmt := 'SELECT COUNT(*) FROM ' || p_output_table; + EXECUTE IMMEDIATE v_sql_stmt INTO :v_row_count; + + RETURN OBJECT_CONSTRUCT('status', 'SUCCESS', 'table', p_output_table, 'rows', v_row_count)::STRING; +EXCEPTION + WHEN err_invalid_input THEN + RETURN OBJECT_CONSTRUCT('status', 'ERROR', 'code', sqlcode, 'message', sqlerrm)::STRING; + WHEN STATEMENT_ERROR THEN + RETURN OBJECT_CONSTRUCT('status', 'ERROR', 'code', sqlcode, 'message', sqlerrm)::STRING; + WHEN OTHER THEN + RETURN OBJECT_CONSTRUCT('status', 'ERROR', 'code', sqlcode, 'message', sqlerrm)::STRING; +END; +$$; + +CALL summarize('sales', 'sales_summary', 'region'); +``` + +## Conditional Logic (%IF) + +**SAS:** +```sas +%MACRO process(include_nulls=N); + PROC SQL; + SELECT * FROM data + %IF &include_nulls = Y %THEN %DO; + /* include all */ + %END; + %ELSE %DO; + WHERE value IS NOT NULL + %END; + ; + QUIT; +%MEND; +``` + +**Snowflake Stored Procedure:** +```sql +CREATE OR REPLACE PROCEDURE process(include_nulls BOOLEAN) +RETURNS TABLE() +LANGUAGE SQL +AS +$$ +DECLARE + result RESULTSET; +BEGIN + IF (:include_nulls) THEN + result := (SELECT * FROM data); + ELSE + result := (SELECT * FROM data WHERE value IS NOT NULL); + END IF; + RETURN TABLE(result); +END; +$$; + +CALL process(FALSE); +``` + +## Looping (%DO) + +**SAS:** +```sas +%MACRO create_monthly_tables; + %DO month = 1 %TO 12; + DATA sales_&month; + SET sales; + WHERE MONTH(date) = &month; + RUN; + %END; +%MEND; +``` + +**Snowflake Stored Procedure:** +```sql +CREATE OR REPLACE PROCEDURE create_monthly_tables() +RETURNS STRING +LANGUAGE SQL +AS +$$ +DECLARE + month INTEGER; + sql_stmt STRING; +BEGIN + FOR month IN 1 TO 12 DO + sql_stmt := 'CREATE OR REPLACE TABLE sales_' || month::STRING || + ' AS SELECT * FROM sales WHERE MONTH(date) = ' || month::STRING; + EXECUTE IMMEDIATE sql_stmt; + END FOR; + RETURN 'Created 12 monthly tables'; +END; +$$; + +CALL create_monthly_tables(); +``` + +## Macro Functions + +### %SYSFUNC + +**SAS:** +```sas +%LET today = %SYSFUNC(TODAY(), DATE9.); +%LET file_count = %SYSFUNC(COUNTW(&file_list)); +``` + +**Snowflake:** +```sql +-- Session variables (outside procedures): +SET today = CURRENT_DATE()::STRING; +SET file_count = ARRAY_SIZE(SPLIT('file1 file2 file3', ' ')); + +-- Or in stored procedures: +CREATE OR REPLACE PROCEDURE sysfunc_example(file_list STRING) +RETURNS OBJECT +LANGUAGE SQL +AS +$$ +DECLARE + today STRING := CURRENT_DATE()::STRING; + file_count INTEGER := ARRAY_SIZE(SPLIT(:file_list, ' ')); +BEGIN + RETURN OBJECT_CONSTRUCT('today', today, 'file_count', file_count); +END; +$$; +``` + +### %EVAL and %SYSEVALF + +**SAS:** +```sas +%LET result = %EVAL(10 + 5); +%LET pct = %SYSEVALF(100 * &num / &denom); +``` + +**Snowflake:** +```sql +-- Session variables: +SET result = 10 + 5; +SET pct = 100.0 * $num / $denom; + +-- Or in stored procedures: +CREATE OR REPLACE PROCEDURE eval_example(num FLOAT, denom FLOAT) +RETURNS OBJECT +LANGUAGE SQL +AS +$$ +DECLARE + result INTEGER := 10 + 5; + pct FLOAT := 100.0 * :num / :denom; +BEGIN + RETURN OBJECT_CONSTRUCT('result', result, 'pct', pct); +END; +$$; +``` + +## Complex Macro: JavaScript UDF Alternative + +For complex text manipulation that macros do: + +**SAS:** +```sas +%MACRO parse_name(full_name); + %LET first = %SCAN(&full_name, 1, %STR( )); + %LET last = %SCAN(&full_name, -1, %STR( )); +%MEND; +``` + +**Snowflake JavaScript UDF:** +```sql +CREATE OR REPLACE FUNCTION parse_name(full_name STRING) +RETURNS OBJECT +LANGUAGE JAVASCRIPT +AS +$$ + var parts = FULL_NAME ? FULL_NAME.trim().split(/\s+/) : []; + return { + first: parts[0] || null, + last: parts.length > 1 ? parts[parts.length - 1] : null + }; +$$; + +SELECT parse_name('John Michael Smith')['first'] AS first_name, + parse_name('John Michael Smith')['last'] AS last_name; +``` + +## Autocall Macros / Reusable Libraries + +**SAS autocall library:** +```sas +/* In autocall library */ +%MACRO stdize(var); + (&var - MEAN(&var)) / STD(&var) +%MEND; +``` + +**Snowflake UDF:** +```sql +CREATE OR REPLACE FUNCTION stdize(val FLOAT, mean_val FLOAT, std_val FLOAT) +RETURNS FLOAT +AS +$$ + (val - mean_val) / NULLIF(std_val, 0) +$$; + +-- Usage with window function: +SELECT *, + stdize(value, AVG(value) OVER(), STDDEV(value) OVER()) AS standardized +FROM data; +``` + +## Dynamic SQL Generation + +**SAS (macro building SQL):** +```sas +%MACRO build_select(varlist); + %LET n = %SYSFUNC(COUNTW(&varlist)); + SELECT + %DO i = 1 %TO &n; + %SCAN(&varlist, &i) + %IF &i < &n %THEN ,; + %END +%MEND; + +PROC SQL; + %build_select(name age salary) + FROM employees; +QUIT; +``` + +**Snowflake Stored Procedure:** +```sql +CREATE OR REPLACE PROCEDURE build_select(varlist STRING, table_name STRING) +RETURNS TABLE() +LANGUAGE SQL +AS +$$ +DECLARE + sql_stmt STRING; + result RESULTSET; +BEGIN + sql_stmt := 'SELECT ' || varlist || ' FROM ' || table_name; + result := (EXECUTE IMMEDIATE sql_stmt); + RETURN TABLE(result); +END; +$$; + +CALL build_select('name, age, salary', 'employees'); +``` + +## Conversion Decision Guide + +| SAS Macro Pattern | Snowflake Approach | +|-------------------|-------------------| +| Simple variable substitution | Session variables (`SET var = value`) | +| Parameterized code blocks | Stored Procedure with parameters | +| Reusable calculations | SQL UDF or JavaScript UDF | +| Dynamic table/column names | Stored Procedure with EXECUTE IMMEDIATE | +| Complex text processing | JavaScript UDF | +| Code generation | External tool (dbt Jinja, Python) | +| Complex string manipulation (mixed case, word parsing) | JavaScript UDF | +| PROC FORMAT lookup macros | Lookup tables with LEFT JOIN | +| Filter/validation macros | Views or SQL UDFs | + +--- + +## Macro Internal Completeness Rules (CRITICAL) + +Every SAS statement inside a macro MUST have a corresponding Snowflake equivalent. Do NOT skip any. + +### Branch Completeness +- For `%IF/%THEN/%ELSE`: convert BOTH the IF branch AND the ELSE branch completely +- Never skip a branch even if it appears to be an edge case + +### Statement Coverage +| SAS Statement in Macro | Snowflake Conversion | +|-----------------------|---------------------| +| `%SYSEXEC` (shell commands) | `MANUAL_REVIEW_REQUIRED` with Snowflake stage commands | +| `%INCLUDE` (external SAS file) | `CALL stored_procedure_name()` + `MANUAL_REVIEW_REQUIRED` if not yet converted | +| `%sysfunc(fileexist(path))` | LIST @stage with RESULT_SCAN pattern in BEGIN...EXCEPTION block | +| `ENDSAS` / `%ABORT` | Named EXCEPTION with RAISE | +| LIBNAME inside macros | Resolve to fully qualified DATABASE.SCHEMA per mapping | +| Nested macro calls `%macroname(args)` | `CALL procedure_name(args)` | + +### Repeated Macro Invocations +- When a macro is called multiple times with different parameters, track each invocation separately +- Different parameter combinations may trigger different `%IF` branches +- Generate separate output for each invocation if results differ +- If macro is called >5 times with varying parameters, prefer a stored procedure called in a loop + +### CALL SYMPUT / SYMGET Cross-Block Persistence + +```sql +-- SAS: CALL SYMPUT('my_var', value); +-- ... later in another step: WHERE col = "&my_var" + +-- Snowflake (within same stored procedure): +v_my_var := value; + +-- Snowflake (across anonymous blocks): +-- At end of block N: +EXECUTE IMMEDIATE 'SET MY_VAR = ''' || REPLACE(v_my_var, '''', '''''') || ''''; + +-- At start of block N+1: +-- Reference as $MY_VAR in SQL +SELECT * FROM table WHERE col = $MY_VAR; +``` + +### Macro-Variable IN-Lists → `SPLIT_TO_TABLE` + +SAS frequently builds a comma-separated list in a macro variable (via `CATS`/`CATX` in a loop, or `SELECT ... INTO :var SEPARATED BY ','`) and injects it into an `IN (&var)` clause. SAS macro resolution expands `&var` into quoted literals; Snowflake has no macro resolution, so the idiomatic equivalent is to store the value as **unquoted, plain comma-separated** text and expand it at query time with `SPLIT_TO_TABLE`. + +```sql +-- Build the list UNQUOTED (comma only, no embedded quotes): +-- v_list := v_list || ',' || v_item; -- e.g. '202301,01/2023,12/2022' + +-- Consume with SPLIT_TO_TABLE + TRIM (works in static SQL and dynamic SQL): +WHERE col IN ( + SELECT TRIM(VALUE) FROM TABLE(SPLIT_TO_TABLE($MY_LIST, ',')) +) +``` + +- **Store unquoted.** Adding `'` around each value (`'202301',''01/2023''`) breaks `SPLIT_TO_TABLE` — the quotes become part of each value and every comparison fails. +- **Always `TRIM(VALUE)`.** Leading/trailing whitespace from the CSV causes silent zero-match failures. +- Only embed quotes when the value is used as a raw injected IN-list (`IN (' || v_list || ')`), where `v_list` must itself already contain `'a','b'`. Prefer the `SPLIT_TO_TABLE` form — it is the safe default. + +### %INCLUDE Without Source Available + +When `%INCLUDE` references a file whose source is not available: + +```sql +-- MANUAL_REVIEW_REQUIRED: %INCLUDE '/path/to/macros.sas' +-- This file was not available for conversion. +-- Expected to define: [inferred macro names from usage context] +-- Action required: Convert the included file separately and create the +-- corresponding stored procedures/UDFs, then update CALL statements below. +CALL sp_included_macro_name(); -- Stub: replace with actual procedure +``` + +--- + +## PROC FORMAT as Macro → Lookup Table or UDF + +### Simple Value Mapping → CASE or Lookup Table +```sql +-- SAS: PROC FORMAT; VALUE status_fmt 1='Active' 2='Inactive'; RUN; +-- Then: PUT(status, status_fmt.) + +-- Option 1: Inline CASE +CASE status WHEN 1 THEN 'Active' WHEN 2 THEN 'Inactive' END + +-- Option 2: Lookup table (for complex or reusable formats) +CREATE OR REPLACE TABLE LKP_STATUS (code INT, description STRING); +INSERT INTO LKP_STATUS VALUES (1, 'Active'), (2, 'Inactive'); +-- Usage: LEFT JOIN LKP_STATUS l ON t.status = l.code +``` + +### Range-Based Mapping → CASE with ranges +```sql +-- SAS: VALUE age_grp LOW-17='Youth' 18-64='Adult' 65-HIGH='Senior'; +CASE + WHEN age <= 17 THEN 'Youth' + WHEN age BETWEEN 18 AND 64 THEN 'Adult' + WHEN age >= 65 THEN 'Senior' +END +``` + +### PROC FORMAT with CNTLIN= (data-driven format) +```sql +-- SAS: PROC FORMAT CNTLIN=format_data; RUN; +-- Snowflake: Create a lookup table from the format dataset +CREATE OR REPLACE TABLE LKP_FORMAT AS +SELECT start AS code, label AS description FROM format_data; +``` + +### Complex String Manipulation Macro → JavaScript UDF +For macros that perform complex string operations (mixed case conversion, word parsing, pattern matching), use JavaScript UDFs: + +```sql +CREATE OR REPLACE FUNCTION fn_complex_string_op(input_str STRING) +RETURNS STRING +LANGUAGE JAVASCRIPT +AS +$$ + if (!INPUT_STR) return null; + // Complex string logic here + return result; +$$; +``` diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/mermaid-diagrams.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/mermaid-diagrams.md new file mode 100644 index 0000000..13b85d1 --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/mermaid-diagrams.md @@ -0,0 +1,279 @@ +# SAS Dependency Mermaid Diagrams + +## When to Load + +Load when user requests: +- Dependency diagram / visualization +- Data flow analysis +- Script interaction mapping +- Codebase structure overview + +--- + +## Diagram Types + +### 1. Script-Level Dependency Diagram + +Shows how SAS scripts interact through shared datasets: + +```mermaid +flowchart TD + subgraph "01_extract.sas" + E1["DATA raw_data
    SET source_table"] + E2["DATA cleaned_data
    SET raw_data"] + end + + subgraph "02_transform.sas" + T1["DATA enriched_data
    MERGE cleaned_data ref_table"] + T2["PROC SQL: summary_data
    FROM enriched_data"] + end + + subgraph "03_load.sas" + L1["DATA final_output
    SET summary_data"] + end + + source_table[(source_table)] --> E1 + ref_table[(ref_table)] --> T1 + E1 --> E2 + E2 --> T1 + T1 --> T2 + T2 --> L1 + L1 --> final_output[(final_output)] +``` + +### 2. Block-Level Flow (Single Script) + +Shows DATA steps and PROCs within one script: + +```mermaid +flowchart TB + subgraph "process_data.sas" + direction TB + B1["1. DATA step: temp1
    • Input: raw
    • Transforms: filter, calc"] + B2["2. PROC SORT
    • BY: customer_id"] + B3["3. DATA step: temp2
    • MERGE: temp1 + lookup
    • BY: customer_id"] + B4["4. PROC SQL: output
    • Aggregation
    • GROUP BY region"] + end + + raw[(raw)] --> B1 + lookup[(lookup)] --> B3 + B1 --> B2 --> B3 --> B4 + B4 --> output[(output)] +``` + +### 3. Macro Dependency Diagram + +Shows macro calls and dependencies: + +```mermaid +flowchart TD + subgraph "Macros" + M1["%MACRO process_month"] + M2["%MACRO calc_metrics"] + M3["%MACRO export_results"] + end + + subgraph "Main Script" + S1["Set parameters
    %LET year=2024"] + S2["%process_month(jan)"] + S3["%process_month(feb)"] + S4["%export_results"] + end + + M1 --> M2 + S1 --> S2 + S2 --> S3 + S3 --> S4 + S4 --> M3 +``` + +--- + +## Generation Rules + +### Node Naming Convention + +| Block Type | Node Format | +|------------|-------------| +| DATA step | `"DATA: output_table
    SET: input_table"` | +| PROC SQL | `"PROC SQL: table_name
    FROM: source"` | +| PROC SORT | `"PROC SORT
    DATA=input OUT=output
    BY: vars"` | +| PROC MEANS | `"PROC MEANS
    DATA=input
    OUTPUT: stats"` | +| Macro | `"%MACRO name(params)"` | +| Macro call | `"%macro_name(args)"` | + +### Edge Rules + +1. **Data dependency**: Input table → Processing block +2. **Output flow**: Processing block → Output table +3. **Sequential**: Block N → Block N+1 (if no explicit dependency) +4. **Macro calls**: Calling block → Macro definition + +### Subgraph Organization + +```mermaid +flowchart TD + subgraph "script_name.sas" + direction TB + %% Blocks go here + end + + %% External tables as cylinders + external_input[(external_input)] + external_output[(external_output)] +``` + +--- + +## Extracting Dependencies from SAS Code + +### DATA Step Dependencies + +```sas +DATA output_table; + SET input_table1 input_table2; /* Inputs */ + MERGE table_a table_b; /* Also inputs */ + BY key_var; +RUN; +``` + +**Extract:** +- Outputs: `output_table` +- Inputs: `input_table1`, `input_table2`, `table_a`, `table_b` +- BY vars: `key_var` + +### PROC SQL Dependencies + +```sas +PROC SQL; + CREATE TABLE output_sql AS + SELECT a.*, b.field + FROM table_a a + LEFT JOIN table_b b ON a.key = b.key; +QUIT; +``` + +**Extract:** +- Output: `output_sql` +- Inputs: `table_a`, `table_b` + +### Macro Dependencies + +```sas +%MACRO process(input=, output=); + DATA &output; + SET &input; + RUN; +%MEND; + +%process(input=raw_data, output=processed_data); +``` + +**Extract:** +- Macro: `process` with params `input`, `output` +- Call resolves to: input=`raw_data`, output=`processed_data` + +--- + +## Complex Example + +**Input SAS files:** + +``` +project/ +├── 01_extract.sas # Extracts from source +├── 02_transform.sas # Cleans and enriches +├── 03_aggregate.sas # Creates summaries +├── macros/ +│ ├── common_macros.sas +│ └── report_macros.sas +└── 04_report.sas # Generates final output +``` + +**Generated Mermaid:** + +```mermaid +flowchart TD + subgraph "macros/common_macros.sas" + M1["%MACRO clean_data"] + M2["%MACRO validate"] + end + + subgraph "macros/report_macros.sas" + M3["%MACRO format_report"] + end + + subgraph "01_extract.sas" + E1["DATA raw_extract
    SET source_db.transactions"] + end + + subgraph "02_transform.sas" + T1["%clean_data(raw_extract)"] + T2["DATA enriched
    MERGE cleaned lookup"] + end + + subgraph "03_aggregate.sas" + A1["PROC SQL: daily_summary
    FROM enriched
    GROUP BY date"] + A2["PROC SQL: monthly_summary
    FROM daily_summary"] + end + + subgraph "04_report.sas" + R1["%format_report(monthly_summary)"] + R2["DATA final_report
    SET formatted_data"] + end + + source_db.transactions[(source_db.transactions)] --> E1 + lookup[(lookup)] --> T2 + + E1 --> T1 + M1 -.-> T1 + T1 --> T2 + T2 --> A1 + A1 --> A2 + A2 --> R1 + M3 -.-> R1 + R1 --> R2 + R2 --> final_report[(final_report)] + + style M1 fill:#f9f,stroke:#333 + style M2 fill:#f9f,stroke:#333 + style M3 fill:#f9f,stroke:#333 +``` + +--- + +## Presentation Tips + +1. **Start simple**: Show high-level script dependencies first +2. **Drill down**: Offer to expand individual scripts +3. **Highlight complexity**: Use colors for complex blocks (RETAIN, ARRAY) +4. **Mark external**: Use cylinder shapes `[( )]` for external tables +5. **Show macros**: Use dotted lines `-..->` for macro dependencies + +--- + +## Color Coding (Optional) + +```mermaid +%%{init: {'theme': 'base', 'themeVariables': { 'primaryColor': '#fff'}}}%% +flowchart TD + classDef dataStep fill:#b3d9ff,stroke:#0066cc + classDef procSql fill:#c2f0c2,stroke:#009900 + classDef procSort fill:#ffffb3,stroke:#999900 + classDef macro fill:#ffb3ff,stroke:#990099 + classDef external fill:#f0f0f0,stroke:#666666 + + D1[DATA step]:::dataStep + P1[PROC SQL]:::procSql + S1[PROC SORT]:::procSort + M1[%MACRO]:::macro + E1[(External)]:::external +``` + +| Color | Meaning | +|-------|---------| +| Blue | DATA step | +| Green | PROC SQL | +| Yellow | PROC SORT/MEANS/FREQ | +| Pink | Macro | +| Gray | External table | diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/multi-block-orchestration.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/multi-block-orchestration.md new file mode 100644 index 0000000..e477055 --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/multi-block-orchestration.md @@ -0,0 +1,197 @@ +# Multi-Block Program Orchestration + +## When to Load + +Load at **Step 5** when a SAS program is **multi-block** and the blocks share state — specifically when any of these are present: +- A program that produces **3+ blocks** with temp-table / macro-variable dependencies between them +- Macro `%DO` loops that wrap multiple operations +- Indexed macro-array variables (`&&var&i`) +- `&SYSERR`-driven error branching +- `%SYSFUNC(FILEEXIST(...))` / `FEXIST()` file-existence gates +- Trigger-file gates (`%if not %sysfunc(fileexist(trigger)) %then endsas`) + +These patterns are about **stitching converted blocks into one runnable Snowflake program**. All examples use placeholders (``, ``, `@`, `
    `) — resolve them from the conversion context / library mapping. Never hard-code database, schema, stage, or table names. + +--- + +## 1. Orchestration Wrapper — `SP__MAIN()` + +A converted SAS program that produces more than ~3 dependent blocks SHOULD ship with a top-level orchestration procedure that runs every block **in one session, in order**. Without it, the user receives disconnected blocks and must manually determine execution order, session scoping, and error handling — which is the bulk of post-conversion effort. + +```sql +CREATE OR REPLACE PROCEDURE ..SP__MAIN() +RETURNS VARCHAR +LANGUAGE SQL +EXECUTE AS CALLER -- REQUIRED: shares temp tables + session vars across the chain +AS +BEGIN + USE DATABASE ; + USE SCHEMA ; + + -- Block B00001: environment / macro-parameter setup (inline small blocks) + -- Block B00002: CREATE OR REPLACE TEMPORARY TABLE ... ; + -- Block B00003: CALL ..SP_(); -- large blocks → sub-procs + -- ... blocks in dependency order ... + + -- CLEANUP (only after ALL blocks complete) + DROP TABLE IF EXISTS ; + DROP TABLE IF EXISTS ; + + RETURN 'Program completed successfully'; +EXCEPTION + WHEN OTHER THEN + RETURN 'FAILED: ' || SQLERRM; +END; +``` + +Rules: +- Name it `SP__MAIN()`; make it the **last** item in the output. +- **Inline** small blocks (<~20 lines); **extract** large/looping blocks as sub-procs and `CALL` them. +- **Every** proc in the chain (this one and all it calls) MUST be `EXECUTE AS CALLER` — owner's-rights procs run in an isolated scope and cannot see the caller's temp tables or session variables, breaking the flow. (See `common-patterns.md` → Platform Constraints.) +- **Temp-table lifetime:** never `DROP` a temp table until no later block reads it. Before emitting any `DROP`, scan all subsequent blocks for that table name; if found, defer the drop to the cleanup section at the end. +- **Macro-parameter persistence:** if the SAS program is a macro with parameters, `SET` each parameter as a session variable at the top so every sub-block can read it; never let a sub-block silently overwrite one. + +This is a report/skeleton recommendation — actually creating objects still follows the skill's Snowflake Interaction Policy (explicit user confirmation). + +--- + +## 2. Indexed Macro Variables (`&&var&i`) → `LOOP_VARS` Temp Table + +SAS macro arrays use doubly-resolved names (`&&file_extn&i`, `&&tin&i`). Snowflake session variables **cannot be dynamically named** at runtime (`SELECT $VAR || i` does not work). Replace the whole indexed-variable scheme with a temp table keyed by an index column, then iterate it with a cursor. + +```sql +-- Replaces all CALL SYMPUTX(cats('var', i), value) assignments: +CREATE OR REPLACE TEMPORARY TABLE LOOP_VARS AS +SELECT + ROW_NUMBER() OVER (ORDER BY ) AS IDX, + , , -- one column per &&var&i prefix +FROM +WHERE ; + +-- Replaces CALL SYMPUTX('cnt', _n_): +-- SELECT COUNT(*) FROM LOOP_VARS; + +-- Replaces %DO i=1 %TO &cnt (access columns directly, no dynamic var reads): +DECLARE cur CURSOR FOR SELECT * FROM LOOP_VARS ORDER BY IDX; +BEGIN + FOR rec IN cur DO + -- use rec., rec. directly + END FOR; +END; +``` + +- **NEVER** emit `EXECUTE IMMEDIATE 'SELECT $STG' || CAST(v_i AS VARCHAR)` — dynamic session-variable names do not resolve in Snowflake. +- Simplest variant: if the loop just copies columns straight from a source table and consumes them together, skip `LOOP_VARS` and put a cursor directly over the source table. +- Exception: a single indexed value used once may stay a plain session variable. + +--- + +## 3. `%DO` Loop Unification → One `WHILE` Block + +When a SAS `%DO i=1 %TO &cnt` loop wraps several operations (data steps, queries, exports, `%IF`/`%THEN` branches), produce **one** Snowflake Scripting `WHILE` block (or one stored proc) containing the **entire** loop body — never separate disconnected blocks per operation. + +```sql +DECLARE + v_cnt INTEGER; + v_i INTEGER DEFAULT 1; +BEGIN + SELECT COUNT(*) INTO :v_cnt FROM LOOP_VARS; + WHILE (v_i <= v_cnt) DO + BEGIN + -- ALL operations from the SAS %DO body, in sequence: + -- 1. variable/row resolution 2. query/extract 3. export/file op + -- 4. status update 5. %IF/%THEN branches → IF/ELSEIF/ELSE + EXCEPTION + WHEN OTHER THEN NULL; -- one iteration's failure must not abort the loop + END; + v_i := v_i + 1; + END WHILE; +END; +``` + +- Nested `%DO` → nested `WHILE`. All `%IF/%THEN/%ELSE` branches inside the loop become `IF/ELSEIF/ELSE` inside the same iteration. +- **Anti-pattern (forbidden):** emitting "Block A = txt export", "Block B = csv export" when they are branches of one `%IF/%ELSE` inside a single `%DO` loop; or a bare comment like `-- call this inside a loop` without providing the loop. + +--- + +## 4. `&SYSERR` Error Propagation → `ERROR_STATE` Temp Table + +SAS uses `&SYSERR` (`%LET error = &syserr; %IF &error %THEN ...`) to gate flow on the last step's status. For programs with 3+ error checkpoints, track state in a temp table (session variables are fragile for complex flows). + +```sql +CREATE OR REPLACE TEMPORARY TABLE ERROR_STATE ( + BLOCK_ID VARCHAR, STEP_NAME VARCHAR, + ERROR_CODE INTEGER DEFAULT 0, ERROR_MSG VARCHAR DEFAULT '', + OCCURRED_AT TIMESTAMP_NTZ DEFAULT CURRENT_TIMESTAMP() +); + +BEGIN + -- ... run step ... + INSERT INTO ERROR_STATE (BLOCK_ID, STEP_NAME, ERROR_CODE) VALUES ('B00010', '', 0); +EXCEPTION + WHEN OTHER THEN + INSERT INTO ERROR_STATE (BLOCK_ID, STEP_NAME, ERROR_CODE, ERROR_MSG) + SELECT 'B00010', '', SQLCODE, SQLERRM; -- INSERT ... SELECT: functions not allowed in VALUES +END; + +-- Check before the next step (equivalent of %IF &error %THEN): +DECLARE v_last_error INTEGER; +BEGIN + SELECT ERROR_CODE INTO :v_last_error FROM ERROR_STATE + WHERE BLOCK_ID = 'B00010' ORDER BY OCCURRED_AT DESC LIMIT 1; + IF (v_last_error != 0) THEN + NULL; -- SAS error branch + ELSE + NULL; -- SAS normal branch + END IF; +END; +``` + +- `%IF &error %THEN` → `IF (v_last_error != 0) THEN`; `%IF NOT &error` → `IF (v_last_error = 0) THEN`. +- For 1–2 checks a session variable `SET ERROR = 0` is acceptable. Never silently drop a `&SYSERR` check — it controls SAS program flow. + +--- + +## 5. `FILEEXIST` / `FEXIST` → Stage `LIST` + `RESULT_SCAN` + +SAS `%SYSFUNC(FILEEXIST(path))` / `FEXIST(fileref)` check the filesystem; in Snowflake, files live on stages. Convert to a `LIST ... PATTERN` count, wrapped in an exception handler so a missing stage / permission error returns "not found" rather than aborting. + +```sql +DECLARE + v_file_exists BOOLEAN DEFAULT FALSE; + v_file_count INTEGER DEFAULT 0; +BEGIN + BEGIN + LET rs RESULTSET := (EXECUTE IMMEDIATE + 'LIST @' || :v_stage || ' PATTERN=''.*' || :v_filename || '.*'''); + LET cur CURSOR FOR rs; + FOR rec IN cur DO + v_file_count := v_file_count + 1; + END FOR; + EXCEPTION + WHEN OTHER THEN v_file_count := 0; + END; + v_file_exists := (v_file_count > 0); +END; +``` + +- `%IF %SYSFUNC(FILEEXIST(path)) %THEN` → `IF (v_file_exists) THEN`; the `%ELSE`/`ENDSAS` branch → a named exception `RAISE` (never a bare comment or silent continue). +- Resolve the SAS filesystem path to a stage from the conversion context's stage/library mapping. If no mapping is provided, flag `MANUAL_REVIEW_REQUIRED` for that path — do **not** invent a stage name. +- For file **matching** driven by a lookup (does *my expected* file exist?), match lookup→stage: `LIST @ PATTERN='.*.*'`. Never list all stage files and compare each against one expected name (produces false errors for unrelated files). + +--- + +## 6. Trigger-File Gates → Exclude / Recommend `TASK ... WHEN` + +SAS batch programs often gate execution on a scheduler trigger file: +```sas +%if not %sysfunc(fileexist(/path/TRIGGER_FILE.txt)) %then %do; endsas; %end; +``` +This is a **batch-scheduling mechanism**, not business logic. In Snowflake, notebooks/procedures execute their steps sequentially in one session — there is no scheduler dropping trigger files. + +Rules: +- **Exclude** the trigger-check block from executable output; replace with a comment: + `-- Trigger check excluded — sequential execution in Snowflake` +- If job gating is genuinely required, recommend a Snowflake **`TASK` with a `WHEN` clause** or **stream-based triggering** (e.g., `WHEN SYSTEM$STREAM_HAS_DATA('')`) rather than a file gate. +- Detection: block contains `fileexist` + `TRIGGER` + `endsas` (typically a `%if ... fileexist ... endsas` macro). +- Distinguish from rule 5: a **data-dependent** `FILEEXIST` gate (does an input data file exist before processing it?) must be converted per rule 5; only **scheduler trigger** files are excluded. diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/proc-sql.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/proc-sql.md new file mode 100644 index 0000000..7a9ed6b --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/proc-sql.md @@ -0,0 +1,305 @@ +# SAS PROC SQL to Snowflake Conversion + +## When to Load + +Load when SAS code contains: `PROC SQL`, `QUIT`, `CALCULATED` keyword, or SAS-specific SQL functions. + +--- + +## Overview + +PROC SQL is mostly ANSI SQL compliant, so conversion is often straightforward. Key differences are SAS-specific extensions and functions. + +## Basic Structure + +**SAS:** +```sas +PROC SQL; + CREATE TABLE output AS + SELECT column1, column2 + FROM input + WHERE condition; +QUIT; +``` + +**Snowflake:** +```sql +CREATE TABLE output AS +SELECT column1, column2 +FROM input +WHERE condition; +``` + +## Key Differences + +### Table Creation + +**SAS:** +```sas +PROC SQL; + CREATE TABLE new_table AS + SELECT * FROM source; +QUIT; +``` + +**Snowflake:** +```sql +CREATE OR REPLACE TABLE new_table AS +SELECT * FROM source; +``` + +### Calculated Columns Reference + +**SAS allows referencing calculated columns:** +```sas +PROC SQL; + SELECT a, b, a+b AS total, CALCULATED total * 0.1 AS tax + FROM mytable; +QUIT; +``` + +**Snowflake requires subquery or CTE:** +```sql +WITH base AS ( + SELECT a, b, a + b AS total + FROM mytable +) +SELECT *, total * 0.1 AS tax +FROM base; +``` + +### Case Sensitivity + +**SAS:** Case-insensitive by default +**Snowflake:** Case-insensitive for unquoted identifiers, case-sensitive for quoted + +```sql +-- These are equivalent in Snowflake: +SELECT column1 FROM TABLE1; +SELECT COLUMN1 FROM table1; + +-- This preserves case: +SELECT "Column1" FROM "Table1"; +``` + +## Function Mappings + +> **Note:** Basic function mappings are in `common-patterns.md`. Below are comprehensive PROC SQL-specific mappings. + +### String Functions (Extended) + +| SAS PROC SQL | Snowflake | +|--------------|-----------| +| `PROPCASE(str)` | `INITCAP(str)` | +| `LENGTH(str)` | `LENGTH(str)` | +| `COMPRESS(str, 'ad')` | `REGEXP_REPLACE(str, '[^a-zA-Z0-9]', '')` | +| `CAT(a, b)` | `CONCAT(a, b)` | +| `CATS(a, b)` | `CONCAT(TRIM(a), TRIM(b))` | +| `TRANWRD(str, find, rep)` | `REPLACE(str, find, rep)` | + +### Numeric Functions + +| SAS PROC SQL | Snowflake | +|--------------|-----------| +| `ABS(x)` | `ABS(x)` | +| `CEIL(x)` | `CEIL(x)` | +| `FLOOR(x)` | `FLOOR(x)` | +| `ROUND(x, d)` | `ROUND(x, d)` | +| `INT(x)` | `TRUNC(x)` | +| `MOD(x, y)` | `MOD(x, y)` | +| `SQRT(x)` | `SQRT(x)` | +| `LOG(x)` | `LN(x)` | +| `LOG10(x)` | `LOG(10, x)` | +| `EXP(x)` | `EXP(x)` | +| `RANUNI(seed)` | `RANDOM()` / `UNIFORM(0::FLOAT, 1::FLOAT, RANDOM())` | + +### Date Functions + +| SAS PROC SQL | Snowflake | +|--------------|-----------| +| `TODAY()` | `CURRENT_DATE()` | +| `DATETIME()` | `CURRENT_TIMESTAMP()` | +| `DATEPART(datetime)` | `DATE(datetime)` or `datetime::DATE` | +| `TIMEPART(datetime)` | `TIME(datetime)` or `datetime::TIME` | +| `INTCK('unit', a, b)` | `DATEDIFF(unit, a, b)` | +| `INTNX('unit', date, n)` | `DATEADD(unit, n, date)` | +| `YEAR(date)` | `YEAR(date)` | +| `QTR(date)` | `QUARTER(date)` | +| `MONTH(date)` | `MONTH(date)` | +| `WEEK(date)` | `WEEKOFYEAR(date)` | +| `DAY(date)` | `DAY(date)` | +| `WEEKDAY(date)` | `DAYOFWEEK(date)` | + +### Aggregate Functions + +| SAS PROC SQL | Snowflake | +|--------------|-----------| +| `SUM(x)` | `SUM(x)` | +| `AVG(x)` | `AVG(x)` | +| `MIN(x)` | `MIN(x)` | +| `MAX(x)` | `MAX(x)` | +| `COUNT(*)` | `COUNT(*)` | +| `COUNT(DISTINCT x)` | `COUNT(DISTINCT x)` | +| `STD(x)` | `STDDEV(x)` | +| `VAR(x)` | `VARIANCE(x)` | +| `N(x)` | `COUNT(x)` | +| `NMISS(x)` | `COUNT(*) - COUNT(x)` | +| `USS(x)` | `SUM(x * x)` | +| `CSS(x)` | Use CTE: `WITH s AS (SELECT AVG(x) AS m FROM t) SELECT SUM(POWER(x-s.m,2)) FROM t,s` | + +### NULL/Missing Handling + +| SAS PROC SQL | Snowflake | +|--------------|-----------| +| `COALESCE(a, b, c)` | `COALESCE(a, b, c)` | +| `NULLIF(a, b)` | `NULLIF(a, b)` | +| `IFNULL(a, b)` | `IFNULL(a, b)` or `NVL(a, b)` | +| `MISSING(x)` | `x IS NULL` | + +## Join Syntax + +### Inner Join + +**SAS:** +```sas +PROC SQL; + SELECT a.*, b.col1 + FROM table_a a, table_b b + WHERE a.key = b.key; +QUIT; +``` + +**Snowflake (explicit JOIN preferred):** +```sql +SELECT a.*, b.col1 +FROM table_a a +INNER JOIN table_b b ON a.key = b.key; +``` + +### Left Join + +**SAS:** +```sas +PROC SQL; + SELECT a.*, b.col1 + FROM table_a a LEFT JOIN table_b b + ON a.key = b.key; +QUIT; +``` + +**Snowflake:** +```sql +SELECT a.*, b.col1 +FROM table_a a +LEFT JOIN table_b b ON a.key = b.key; +``` + +## Subqueries + +**SAS:** +```sas +PROC SQL; + SELECT * FROM orders + WHERE customer_id IN (SELECT customer_id FROM vip_customers); +QUIT; +``` + +**Snowflake (identical):** +```sql +SELECT * FROM orders +WHERE customer_id IN (SELECT customer_id FROM vip_customers); +``` + +## GROUP BY with HAVING + +**SAS:** +```sas +PROC SQL; + SELECT customer_id, SUM(amount) AS total + FROM orders + GROUP BY customer_id + HAVING CALCULATED total > 1000; +QUIT; +``` + +**Snowflake:** +```sql +SELECT customer_id, SUM(amount) AS total +FROM orders +GROUP BY customer_id +HAVING SUM(amount) > 1000; +``` + +## UNION Operations + +**SAS:** +```sas +PROC SQL; + SELECT * FROM table1 + UNION + SELECT * FROM table2; +QUIT; +``` + +**Snowflake (identical):** +```sql +SELECT * FROM table1 +UNION +SELECT * FROM table2; + +-- UNION ALL (keep duplicates) +SELECT * FROM table1 +UNION ALL +SELECT * FROM table2; +``` + +### Heterogeneous schemas → `UNION ALL BY NAME` + +Positional `UNION` / `UNION ALL` requires every branch to have the **same column count and order**. When a SAS `SET table1 table2 table3;` (or set-operation) stacks datasets with **different column sets** — common when the tables were built by separate `PROC IMPORT`/transformation paths or come from different vendors — use `UNION ALL BY NAME`, which aligns columns by name and fills missing columns with NULL. + +```sql +-- Datasets with different/extra columns: +SELECT * FROM table1 +UNION ALL BY NAME +SELECT * FROM table2 +UNION ALL BY NAME +SELECT * FROM table3; +``` + +- Use `... BY NAME` whenever the stacked sources may have differing columns (different vendors, different import blocks, different transformation paths). +- Keep plain positional `UNION ALL` only when all sources share an identical schema (same columns, same order — e.g., built from one DDL template). +- Do not silently deduplicate: SAS `SET` stacks rows (like `UNION ALL`), so preserve `ALL` unless the SAS logic explicitly removes duplicates. + +## INTO Clause (Creating Macro Variables) + +**SAS:** +```sas +PROC SQL NOPRINT; + SELECT COUNT(*) INTO :row_count FROM mytable; +QUIT; +``` + +**Snowflake (use variables in stored procedures):** +```sql +CREATE OR REPLACE PROCEDURE get_row_count(table_name STRING) +RETURNS INTEGER +LANGUAGE SQL +AS +$$ +DECLARE + row_count INTEGER; +BEGIN + SELECT COUNT(*) INTO :row_count FROM IDENTIFIER(:table_name); + RETURN row_count; +END; +$$; +``` + +## PROC SQL Options + +| SAS Option | Snowflake Equivalent | +|------------|---------------------| +| `NOPRINT` | Don't display (default in scripts) | +| `OUTOBS=n` | `LIMIT n` | +| `NUMBER` | `ROW_NUMBER() OVER()` | +| `DOUBLE` | N/A (formatting) | diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/proc-steps.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/proc-steps.md new file mode 100644 index 0000000..dc7b909 --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/proc-steps.md @@ -0,0 +1,723 @@ +# SAS PROC Steps to Snowflake Conversion + +## When to Load + +Load when SAS code contains: `PROC SORT`, `PROC MEANS`, `PROC SUMMARY`, `PROC FREQ`, `PROC TRANSPOSE`, `PROC RANK`, `PROC APPEND`, `PROC CONTENTS`, `PROC PRINT`, `PROC TABULATE`, `PROC UNIVARIATE`, `PROC COMPARE`, `PROC IMPORT`, `PROC EXPORT`, or `PROC FORMAT`. + +--- + +## PROC SORT + +**SAS:** +```sas +PROC SORT DATA=input OUT=sorted; + BY descending amount customer_id; +RUN; +``` + +**Snowflake:** +```sql +CREATE TABLE sorted AS +SELECT * FROM input +ORDER BY amount DESC, customer_id; +``` + +**With NODUPKEY (remove duplicates):** +```sas +PROC SORT DATA=input OUT=deduped NODUPKEY; + BY customer_id; +RUN; +``` + +**Snowflake (CRITICAL: use explicit column list, NOT SELECT *):** +```sql +-- CORRECT: explicit column list excludes helper column +CREATE TABLE deduped AS +SELECT col1, col2, col3, customer_id +FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY 1) AS rn + FROM input +) +WHERE rn = 1; + +-- WRONG: SELECT * leaks the rn column into output +-- SELECT * FROM (...) WHERE rn = 1; -- DO NOT DO THIS +``` + +**NODUPKEY with TEMPORARY source:** +If the source is a TEMPORARY TABLE, the deduplicated output MUST also be TEMPORARY: +```sql +CREATE OR REPLACE TEMPORARY TABLE deduped AS +SELECT col1, col2 FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY 1) AS rn + FROM temp_input +) WHERE rn = 1; +``` + +**PROC SORT stability:** SAS PROC SORT is stable (preserves original order for ties). If downstream logic depends on first/last row selection within ties, add deterministic ORDER BY columns. + +**NODUPKEY dedup rules:** +- Do NOT add deduplication unless SAS explicitly uses NODUPKEY or NODUPREC +- Preserve sort stability assumptions where later logic depends on first/last row selection + +**NODUPKEY BY _ALL_ → prefer SELECT DISTINCT:** +When SAS dedups on ALL columns (`PROC SORT NODUPKEY; BY _ALL_;`), every duplicate row is identical — there is no "which row to keep" decision. Use `SELECT DISTINCT`, which is simpler and faster than ROW_NUMBER(): +```sql +-- SAS: PROC SORT DATA=mydata NODUPKEY; BY _ALL_; RUN; +CREATE OR REPLACE TEMPORARY TABLE mydata AS SELECT DISTINCT * FROM mydata; +``` +Use ROW_NUMBER() (above) only when BY is a **subset** of columns, where you must choose which row survives each group. + +## SAS INDEX → CLUSTER BY + +SAS dataset indexes have no direct Snowflake equivalent. Convert index definitions to `CLUSTER BY` (or a documented comment), never silently drop them. + +```sql +-- SAS: PROC SQL; CREATE INDEX idx ON tbl(col1, col2); +-- SAS: DATA tbl(INDEX=(idx=(col1 col2))); +-- SAS: PROC DATASETS; MODIFY tbl; INDEX CREATE col1; +-- Snowflake: +ALTER TABLE tbl CLUSTER BY (col1, col2); +``` + +**Rules:** +- Single-column index → `CLUSTER BY (col)`; composite → `CLUSTER BY (col1, col2, ...)`. +- CLUSTER BY benefits large tables (~1TB+ or frequent range/filter queries). For small or TEMPORARY tables, emit a comment instead of clustering: + `-- CLUSTER BY (col) omitted — table is small/temporary; add if query performance requires it` +- UNIQUE index → clustering does NOT enforce uniqueness. Add a comment: + `-- NOTE: SAS had UNIQUE index on (col); consider ALTER TABLE t ADD CONSTRAINT uq UNIQUE (col) NOT ENFORCED;` + +## PROC MEANS / PROC SUMMARY + +**SAS:** +```sas +PROC MEANS DATA=sales N MEAN STD MIN MAX SUM; + CLASS region product; + VAR revenue quantity; + OUTPUT OUT=summary + N=n_revenue n_quantity + MEAN=avg_revenue avg_quantity + SUM=total_revenue total_quantity; +RUN; +``` + +**Snowflake (with GROUPING SETS for all CLASS combinations):** +```sql +CREATE TABLE summary AS +SELECT + region, + product, + COUNT(revenue) AS n_revenue, + COUNT(quantity) AS n_quantity, + AVG(revenue) AS avg_revenue, + AVG(quantity) AS avg_quantity, + SUM(revenue) AS total_revenue, + SUM(quantity) AS total_quantity, + STDDEV(revenue) AS std_revenue, + MIN(revenue) AS min_revenue, + MAX(revenue) AS max_revenue +FROM sales +GROUP BY GROUPING SETS ( + (region, product), + (region), + (product), + () +); +``` + +**With NWAY option (most detailed level only — no subtotals):** +```sas +PROC SUMMARY DATA=sales NWAY; + CLASS region product; + VAR revenue; + OUTPUT OUT=summary SUM=total; +RUN; +``` + +**Snowflake (NWAY = simple GROUP BY, no GROUPING SETS):** +```sql +CREATE TABLE summary AS +SELECT + region, + product, + SUM(revenue) AS total, + COUNT(*) AS _FREQ_ +FROM sales +GROUP BY region, product; +``` + +**CLASS vs BY semantics:** +- `CLASS` = creates all combinations including subtotals (use GROUPING SETS) +- `CLASS` with `NWAY` = only the most detailed level (use simple GROUP BY) +- `BY` = separate analysis per BY group (same as PARTITION BY in output) +- Preserve missing-group handling: SAS excludes missing CLASS values by default unless `MISSING` option is specified + +**Simple aggregation:** +```sas +PROC MEANS DATA=sales SUM; + VAR amount; +RUN; +``` + +**Snowflake:** +```sql +SELECT SUM(amount) AS amount_sum FROM sales; +``` + +## PROC FREQ + +**SAS:** +```sas +PROC FREQ DATA=customers; + TABLES region * status / NOCUM NOPERCENT; +RUN; +``` + +**Snowflake:** +```sql +SELECT + region, + status, + COUNT(*) AS frequency, + COUNT(*) * 100.0 / SUM(COUNT(*)) OVER() AS percent +FROM customers +GROUP BY region, status +ORDER BY region, status; +``` + +**One-way frequency:** +```sas +PROC FREQ DATA=customers; + TABLES status; +RUN; +``` + +**Snowflake:** +```sql +SELECT + status, + COUNT(*) AS frequency, + ROUND(COUNT(*) * 100.0 / SUM(COUNT(*)) OVER(), 2) AS percent, + SUM(COUNT(*)) OVER (ORDER BY status) AS cumulative_freq +FROM customers +GROUP BY status +ORDER BY status; +``` + +## PROC TRANSPOSE + +**SAS (long to wide):** +```sas +PROC TRANSPOSE DATA=long OUT=wide PREFIX=month_; + BY customer_id; + ID month; + VAR sales; +RUN; +``` + +**Snowflake (PIVOT):** +```sql +SELECT * FROM long +PIVOT ( + SUM(sales) FOR month IN ('JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN', + 'JUL', 'AUG', 'SEP', 'OCT', 'NOV', 'DEC') +) AS p (customer_id, month_JAN, month_FEB, month_MAR, month_APR, + month_MAY, month_JUN, month_JUL, month_AUG, month_SEP, + month_OCT, month_NOV, month_DEC); +``` + +**SAS (wide to long):** +```sas +PROC TRANSPOSE DATA=wide OUT=long NAME=quarter; + BY customer_id; + VAR q1 q2 q3 q4; +RUN; +``` + +**Snowflake (UNPIVOT):** +```sql +SELECT * FROM wide +UNPIVOT ( + value FOR quarter IN (q1, q2, q3, q4) +); +``` + +**PROC TRANSPOSE with PREFIX/SUFFIX/DELIMITER:** +- PREFIX= → column name prefix in output (use AS alias in PIVOT) +- SUFFIX= → column name suffix in output +- DELIMITER= → separator between ID value and prefix +- Preserve output column naming rules explicitly + +**Dynamic PROC TRANSPOSE (unknown ID values):** +When the ID column values are not known at conversion time, use a stored procedure with EXECUTE IMMEDIATE to build dynamic PIVOT SQL, or flag MANUAL_REVIEW_REQUIRED. + +**One-row-per-group vs multi-row-per-group:** +- If VAR lists multiple variables but no ID: produces one row per BY group with one column per VAR +- If VAR and ID both present: produces one row per BY group with columns named from ID values + +## PROC RANK + +**SAS:** +```sas +PROC RANK DATA=sales OUT=ranked GROUPS=10; + VAR revenue; + RANKS revenue_decile; +RUN; +``` + +**Snowflake:** +```sql +CREATE TABLE ranked AS +SELECT *, + NTILE(10) OVER (ORDER BY revenue) AS revenue_decile +FROM sales; +``` + +**Simple rank:** +```sas +PROC RANK DATA=sales OUT=ranked; + VAR revenue; + RANKS revenue_rank; +RUN; +``` + +**Snowflake:** +```sql +SELECT *, + RANK() OVER (ORDER BY revenue) AS revenue_rank +FROM sales; +``` + +## PROC APPEND + +**SAS:** +```sas +PROC APPEND BASE=master DATA=new_data FORCE; +RUN; +``` + +**Snowflake:** +```sql +INSERT INTO master +SELECT * FROM new_data; +``` + +## PROC DATASETS (DELETE) + +**SAS:** +```sas +PROC DATASETS LIBRARY=work NOLIST; + DELETE temp1 temp2 temp3; +QUIT; +``` + +**Snowflake:** +```sql +DROP TABLE IF EXISTS temp1; +DROP TABLE IF EXISTS temp2; +DROP TABLE IF EXISTS temp3; +``` + +## PROC CONTENTS + +**SAS:** +```sas +PROC CONTENTS DATA=mytable; +RUN; +``` + +**Snowflake:** +```sql +DESCRIBE TABLE mytable; +-- or +SELECT * FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_NAME = 'MYTABLE'; +``` + +## PROC PRINT + +**SAS:** +```sas +PROC PRINT DATA=sales (OBS=10); + VAR customer_id amount date; +RUN; +``` + +**Snowflake:** +```sql +SELECT customer_id, amount, date +FROM sales +LIMIT 10; +``` + +## PROC TABULATE + +**SAS:** +```sas +PROC TABULATE DATA=sales; + CLASS region year; + VAR revenue; + TABLE region, year * revenue * (SUM MEAN); +RUN; +``` + +**Snowflake:** +```sql +SELECT + region, + year, + SUM(revenue) AS sum_revenue, + AVG(revenue) AS mean_revenue +FROM sales +GROUP BY region, year +ORDER BY region, year; +``` + +## PROC UNIVARIATE + +**SAS:** +```sas +PROC UNIVARIATE DATA=sales; + VAR amount; + OUTPUT OUT=stats + N=n MEAN=mean STD=std + MIN=min MAX=max + P25=p25 MEDIAN=median P75=p75; +RUN; +``` + +**Snowflake:** +```sql +CREATE TABLE stats AS +SELECT + COUNT(amount) AS n, + AVG(amount) AS mean, + STDDEV(amount) AS std, + MIN(amount) AS min, + MAX(amount) AS max, + PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY amount) AS p25, + PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY amount) AS median, + PERCENTILE_CONT(0.75) WITHIN GROUP (ORDER BY amount) AS p75 +FROM sales; +``` + +## PROC COMPARE + +**SAS:** +```sas +PROC COMPARE BASE=old COMPARE=new; + ID customer_id; +RUN; +``` + +**Snowflake:** +```sql +-- Find differences +SELECT 'In OLD only' AS status, o.* +FROM old o +LEFT JOIN new n ON o.customer_id = n.customer_id +WHERE n.customer_id IS NULL + +UNION ALL + +SELECT 'In NEW only' AS status, n.* +FROM new n +LEFT JOIN old o ON n.customer_id = o.customer_id +WHERE o.customer_id IS NULL + +UNION ALL + +SELECT 'Different values' AS status, o.* +FROM old o +INNER JOIN new n ON o.customer_id = n.customer_id +WHERE o.amount != n.amount OR o.status != n.status; +``` + +## PROC IMPORT / PROC EXPORT + +**SAS (CSV import):** +```sas +PROC IMPORT DATAFILE='/path/to/file.csv' + OUT=mydata DBMS=CSV REPLACE; + GETNAMES=YES; +RUN; +``` + +**Snowflake:** +```sql +COPY INTO mydata +FROM @my_stage/file.csv +FILE_FORMAT = (TYPE = 'CSV' SKIP_HEADER = 1); +``` + +**SAS (Excel XLSX import):** +```sas +PROC IMPORT DATAFILE='/path/to/workbook.xlsx' + OUT=mydata DBMS=XLSX REPLACE; + SHEET='SheetName'; + GETNAMES=YES; +RUN; +``` + +**⚠️ Snowflake does NOT support native XLSX in `COPY INTO` / `INFER_SCHEMA`.** Do NOT generate `COPY INTO ... FILE_FORMAT = (TYPE = 'XLSX')` or `INFER_SCHEMA(... .xlsx)` — those fail at runtime. XLSX read/write requires Python (`openpyxl`/`pandas`). Per the SQL-first tiering, this is a **SQL-wrapped Python** case (NOT Tier 3): generate a Python stored procedure with a clean `CALL` interface so the user still executes only SQL. + +**Snowflake (XLSX import — stored procedure + CALL, fully parameterized, no hardcoded db/schema/stage):** +```sql +CREATE OR REPLACE PROCEDURE SP_IMPORT_XLSX( + STAGE_FILE VARCHAR, -- e.g. '@my_stage/workbook.xlsx' + SHEET_NAME VARCHAR, + TARGET_TABLE VARCHAR, -- bare name for temp, or DB.SCHEMA.TABLE for permanent + IS_TEMP BOOLEAN DEFAULT FALSE +) +RETURNS VARCHAR +LANGUAGE PYTHON +RUNTIME_VERSION = '3.11' +PACKAGES = ('snowflake-snowpark-python', 'openpyxl', 'pandas') +HANDLER = 'run' +EXECUTE AS CALLER +AS +$$ +def run(session, stage_file, sheet_name, target_table, is_temp): + import pandas as pd, os + filename = stage_file.split('/')[-1] + session.file.get(stage_file, '/tmp') + df = pd.read_excel(f'/tmp/{filename}', sheet_name=sheet_name, engine='openpyxl', dtype=str) + df.columns = [c.strip().upper().replace(' ', '_') for c in df.columns] + df = df.where(pd.notnull(df), other=None) + session.create_dataframe(df).write.save_as_table( + target_table, mode='overwrite', table_type=('temporary' if is_temp else None)) + n = session.sql(f"SELECT COUNT(*) FROM {target_table}").collect()[0][0] + if os.path.exists(f'/tmp/{filename}'): os.remove(f'/tmp/{filename}') + return f'Loaded {n} rows into {target_table} from sheet {sheet_name}' +$$; + +CALL SP_IMPORT_XLSX('@my_stage/workbook.xlsx', 'SheetName', 'MYDATA', FALSE); +``` + +**Excel import notes:** +- Stage the `.xlsx` file first: `PUT file:///path/to/workbook.xlsx @my_stage/` +- SAS `GETNAMES=YES` (default) → first row is headers (pandas default). +- SAS WORK target → pass `IS_TEMP = TRUE`. +- **Multi-sheet**: one procedure definition, one `CALL` per sheet with different `SHEET_NAME`/`TARGET_TABLE`. +- Flag `MANUAL_REVIEW_REQUIRED` only when merged cells / cell formulas affect business logic. + +**SAS (CSV export):** +```sas +PROC EXPORT DATA=mydata + OUTFILE='/path/to/output.csv' DBMS=CSV REPLACE; +RUN; +``` + +**Snowflake (CSV/pipe export stays native SQL — no Python needed):** +```sql +COPY INTO @my_stage/output.csv +FROM mydata +FILE_FORMAT = (TYPE = 'CSV' FIELD_DELIMITER = ',' HEADER = TRUE) +SINGLE = TRUE OVERWRITE = TRUE MAX_FILE_SIZE = 5368709120; +``` +- `REPLACE` → `OVERWRITE = TRUE`; `PUTNAMES=NO` → `HEADER = FALSE`; `DELIMITER='|'` → `FIELD_DELIMITER = '|'`. + +**SAS (XLSX export):** +```sas +PROC EXPORT DATA=mydata OUTFILE='/path/out.xlsx' DBMS=XLSX REPLACE; SHEET='Sheet1'; RUN; +``` + +**Snowflake (XLSX export — stored procedure + CALL; never raw `COPY INTO ... XLSX`):** +```sql +CREATE OR REPLACE PROCEDURE SP_EXPORT_XLSX( + SOURCE_QUERY VARCHAR, -- e.g. 'SELECT * FROM MYDATA WHERE region = ''EAST''' + FILE_NAME VARCHAR, + SHEET_NAME VARCHAR DEFAULT 'Sheet1', + STAGE_PATH VARCHAR DEFAULT '@~/' -- pass the target stage from conversion context +) +RETURNS VARCHAR +LANGUAGE PYTHON +RUNTIME_VERSION = '3.11' +PACKAGES = ('snowflake-snowpark-python', 'openpyxl', 'pandas') +HANDLER = 'run' +EXECUTE AS CALLER +AS +$$ +def run(session, source_query, file_name, sheet_name, stage_path): + import pandas as pd, os + df = session.sql(source_query).to_pandas() + local_path = f'/tmp/{file_name}' + if os.path.exists(local_path): + with pd.ExcelWriter(local_path, engine='openpyxl', mode='a', if_sheet_exists='replace') as w: + df.to_excel(w, sheet_name=sheet_name, index=False) + else: + df.to_excel(local_path, sheet_name=sheet_name, index=False, engine='openpyxl') + session.file.put(local_path, stage_path, auto_compress=False, overwrite=True) + os.remove(local_path) + return f'Exported {len(df)} rows to {stage_path}/{file_name} sheet {sheet_name}' +$$; + +CALL SP_EXPORT_XLSX('SELECT * FROM MYDATA', 'out.xlsx', 'Sheet1', '@my_stage'); +``` +- Multi-sheet to one workbook: call once per sheet against the SAME `FILE_NAME`; `if_sheet_exists='replace'` keeps re-runs idempotent. +- SAS `DATA=lib.tbl(WHERE=(...))` → fold the filter into `SOURCE_QUERY`. + +## PROC REPORT + +**SAS:** +```sas +PROC REPORT DATA=sales NOWD; + COLUMNS region product revenue; + DEFINE region / GROUP; + DEFINE product / GROUP; + DEFINE revenue / ANALYSIS SUM; + COMPUTE AFTER region; + LINE 'Subtotal for ' region $20.; + ENDCOMP; +RUN; +``` + +**Snowflake (GROUP BY with ROLLUP for subtotals):** +```sql +SELECT + region, + product, + SUM(revenue) AS revenue +FROM sales +GROUP BY ROLLUP(region, product) +ORDER BY region, product; +``` + +**PROC REPORT notes:** +- PROC REPORT is primarily a **reporting/display** procedure. If it does NOT create an output dataset (no `OUT=`), the conversion may be a no-op (display-only). +- If `OUT=` is present, convert to `CREATE TABLE AS SELECT` with appropriate GROUP BY. +- COMPUTE blocks with LINE statements are display formatting only — omit from SQL conversion. +- COMPUTE blocks with calculated columns → SQL expressions in SELECT. + +## Statistical PROCs (TIER 3) + +**The following PROCs require Python, not SQL. See `references/python-datascience.md` for conversion patterns:** + +| SAS PROC | Python Equivalent | Reference | +|----------|-------------------|-----------| +| PROC REG | `statsmodels.OLS()` | `references/python-datascience.md` | +| PROC GLM | `statsmodels.GLM()` | `references/python-datascience.md` | +| PROC LOGISTIC | `sklearn.LogisticRegression()` | `references/python-datascience.md` | +| PROC CLUSTER | `sklearn.AgglomerativeClustering()` | `references/python-datascience.md` | +| PROC FACTOR | `sklearn.FactorAnalysis()` | `references/python-datascience.md` | +| PROC PHREG | `lifelines.CoxPHFitter()` | Manual review required | +| PROC LIFETEST | `lifelines.KaplanMeierFitter()` | Manual review required | +| PROC MIXED | `statsmodels.MixedLM()` | Manual review required | +| PROC NLIN | `scipy.optimize.curve_fit()` | Manual review required | +| PROC SURVEYSELECT | `sklearn.model_selection` | Manual review required | + +Flag these as MANUAL_REVIEW_REQUIRED if no Python conversion is provided. + +## PROC FORMAT + +**SAS (simple value format):** +```sas +PROC FORMAT; + VALUE status_fmt + 1 = 'Active' + 2 = 'Inactive' + 3 = 'Pending'; +RUN; +``` + +**Snowflake (use CASE or lookup table):** +```sql +-- Inline CASE (for simple, few values) +SELECT *, + CASE status + WHEN 1 THEN 'Active' + WHEN 2 THEN 'Inactive' + WHEN 3 THEN 'Pending' + END AS status_desc +FROM mytable; + +-- Lookup table (for reusable or complex formats) +CREATE TABLE status_lookup (code INT, description STRING); +INSERT INTO status_lookup VALUES (1, 'Active'), (2, 'Inactive'), (3, 'Pending'); + +SELECT t.*, l.description AS status_desc +FROM mytable t +LEFT JOIN status_lookup l ON t.status = l.code; +``` + +> **Materialize as a queryable TEMPORARY table when a downstream block uses the format.** +> If any later block applies the format via a `PUT()`, `LEFT JOIN`, or CASE-based lookup, create the lookup as a **`CREATE OR REPLACE TEMPORARY TABLE`** (bare name, no schema prefix) — not an in-memory structure (a Python dict, when producing Python, is **not** queryable by downstream SQL). A session-scoped temp table is visible to all later blocks in the same session and can be joined directly. + +**SAS (range-based format):** +```sas +PROC FORMAT; + VALUE age_grp + LOW - 17 = 'Youth' + 18 - 64 = 'Adult' + 65 - HIGH = 'Senior'; +RUN; +``` + +**Snowflake:** +```sql +CASE + WHEN age <= 17 THEN 'Youth' + WHEN age BETWEEN 18 AND 64 THEN 'Adult' + WHEN age >= 65 THEN 'Senior' +END AS age_group +``` + +**SAS (CNTLIN= data-driven format):** +```sas +PROC FORMAT CNTLIN=format_dataset; +RUN; +``` + +**Snowflake (create lookup table from the format dataset):** +```sql +CREATE OR REPLACE TABLE LKP_MY_FORMAT AS +SELECT + START AS code_start, + END AS code_end, + LABEL AS description, + TYPE AS format_type +FROM format_dataset; + +-- Usage: JOIN with range conditions +SELECT t.*, l.description +FROM mytable t +LEFT JOIN LKP_MY_FORMAT l + ON t.value >= l.code_start AND t.value <= l.code_end; +``` + +**PROC FORMAT with OTHER= (default):** +- Always include an `ELSE` clause in CASE expressions to handle the OTHER= default +- Preserve inclusive/exclusive range semantics from SAS format definitions + +**SAS PUT() with format → Snowflake equivalent:** +```sql +-- SAS: new_col = PUT(status, status_fmt.); +-- Snowflake: +CASE status WHEN 1 THEN 'Active' WHEN 2 THEN 'Inactive' ELSE 'Unknown' END AS new_col +-- OR with lookup table: +l.description AS new_col +``` + +## Quick Reference Table + +| SAS PROC | Snowflake Equivalent | +|----------|---------------------| +| PROC SORT | ORDER BY / QUALIFY ROW_NUMBER() | +| PROC MEANS | GROUP BY with aggregates | +| PROC SUMMARY | GROUP BY with GROUPING SETS | +| PROC FREQ | GROUP BY with COUNT(*) | +| PROC TRANSPOSE | PIVOT / UNPIVOT | +| PROC RANK | RANK() / NTILE() window functions | +| PROC APPEND | INSERT INTO ... SELECT | +| PROC DATASETS DELETE | DROP TABLE | +| PROC CONTENTS | DESCRIBE TABLE | +| PROC PRINT | SELECT ... LIMIT | +| PROC UNIVARIATE | Aggregate + PERCENTILE_CONT | +| PROC COMPARE | EXCEPT / anti-join patterns | +| PROC IMPORT (CSV) | COPY INTO | +| PROC IMPORT (XLSX) | Python stored proc + CALL (openpyxl) — NOT COPY INTO | +| PROC EXPORT (CSV/pipe) | COPY INTO @stage (native SQL) | +| PROC EXPORT (XLSX) | Python stored proc + CALL (openpyxl) — NOT COPY INTO | +| PROC FORMAT | CASE expression / lookup table | +| PROC REPORT | GROUP BY + ROLLUP (or display-only no-op) | +| PROC REG/GLM/etc. | Python (see references/python-datascience.md) | diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/pyspark-fallback.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/pyspark-fallback.md new file mode 100644 index 0000000..d944b53 --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/pyspark-fallback.md @@ -0,0 +1,216 @@ +# PySpark Fallback - TIER 3 (LAST RESORT) + +⚠️ **CRITICAL**: Only use PySpark when SQL and Stored Procedures CANNOT accomplish the task. + +**⚠️ These patterns are SQL-translatable and MUST NOT use PySpark:** +- ARRAY iteration → SQL CASE expressions, GREATEST/LEAST +- RETAIN → SQL SUM() OVER window function +- FIRST./LAST. → SQL ROW_NUMBER(), QUALIFY +- KEY= lookup → SQL LEFT JOIN +- MERGE with IN= → SQL FULL OUTER JOIN + CASE +- %DO loops → SQL GENERATOR + CROSS JOIN +- Complex branching (>5 IF) → Stored Procedure + +See SKILL.md SQL-First Classification for these patterns. + +## When to Load + +Load ONLY when block is classified as TIER 3 (PySpark) after exhausting SQL and Stored Procedure options. + +--- + +## Before Using PySpark - Checklist + +**Ask yourself:** + +- [ ] Can this be done with window functions? (LAG, LEAD, SUM OVER, ROW_NUMBER) +- [ ] Can this be done with CTEs and CASE statements? +- [ ] Can this be done with a stored procedure using DECLARE/BEGIN/END? +- [ ] Is this truly a pattern that requires Python runtime? + +**If any checkbox is YES → Do NOT use PySpark. Use SQL or Stored Procedure.** + +--- + +## Valid PySpark Use Cases (ONLY These) + +| Pattern | Why PySpark Required | SQL/SP Alternative? | +|---------|---------------------|---------------------| +| HASH objects | In-memory key-value store not in Snowflake | ❌ No direct equivalent | +| DO UNTIL/WHILE with external state | Runtime condition evaluation | ❌ Cannot evaluate at runtime | +| CALL EXECUTE | Dynamic code generation | ❌ Limited dynamic SQL | +| External file I/O | Non-Snowflake files | ❌ COPY INTO limited | +| Complex ARRAY with carried state | Per-element iteration with memory | ❌ No array iteration | + +--- + +## Snowpark Connect (SCOS) Setup + +```python +# Cell 1: Setup (SCOS) +from snowflake.snowpark import Session +from snowflake.snowpark.context import get_active_session + +session = get_active_session() + +# Create Spark session via Snowpark Connect +from pyspark.sql import SparkSession +spark = SparkSession.builder.remote(session.connection).getOrCreate() + +print(f"Connected via Snowpark Connect") +TARGET_SCHEMA = "DATABASE.SCHEMA" +``` + +--- + +## Pattern: HASH Object Processing + +**SAS (No SQL equivalent):** +```sas +DATA enriched; + IF _N_ = 1 THEN DO; + DECLARE HASH lookup(dataset: 'codes'); + lookup.DEFINEKEY('code'); + lookup.DEFINEDATA('description'); + lookup.DEFINEDONE(); + END; + SET transactions; + rc = lookup.FIND(); + IF rc = 0 THEN code_desc = description; + ELSE code_desc = 'UNKNOWN'; +RUN; +``` + +**PySpark (broadcast join):** +```python +from pyspark.sql import functions as F +from pyspark.sql.functions import broadcast + +codes_df = spark.table(f"{TARGET_SCHEMA}.codes") +transactions_df = spark.table(f"{TARGET_SCHEMA}.transactions") + +enriched = transactions_df.join( + broadcast(codes_df), + transactions_df.code == codes_df.code, + "left" +).withColumn( + "code_desc", + F.coalesce(F.col("description"), F.lit("UNKNOWN")) +).drop(codes_df.code) + +enriched.write.mode("overwrite").saveAsTable(f"{TARGET_SCHEMA}.enriched") +``` + +--- + +## Pattern: DO UNTIL/WHILE with External State + +**SAS (Runtime iteration):** +```sas +DATA result; + SET input; + RETAIN running 0; + DO UNTIL (running > threshold OR _N_ > 1000); + running = running + increment; + /* Complex external condition check */ + END; +RUN; +``` + +**PySpark:** +```python +from pyspark.sql import functions as F +from pyspark.sql.window import Window + +df = spark.table(f"{TARGET_SCHEMA}.input") + +# Implement iterative logic +running = 0 +threshold = 1000 +results = [] + +for row in df.collect(): + running += row['increment'] + if running > threshold: + break + results.append(row) + +result_df = spark.createDataFrame(results) +result_df.write.mode("overwrite").saveAsTable(f"{TARGET_SCHEMA}.result") +``` + +--- + +## SAS Function Helpers (When PySpark Required) + +```python +from pyspark.sql import functions as F +from pyspark.sql.functions import col, lit, when, coalesce +from pyspark.sql.window import Window + +def intck(interval, start_date, end_date): + """SAS INTCK equivalent""" + interval_upper = interval.upper() + if interval_upper == 'YEAR': + return F.year(end_date) - F.year(start_date) + elif interval_upper == 'MONTH': + return F.months_between(end_date, start_date).cast('int') + elif interval_upper == 'DAY': + return F.datediff(end_date, start_date) + return F.datediff(end_date, start_date) + +def intnx(interval, start_date, n): + """SAS INTNX equivalent""" + interval_upper = interval.upper() + if interval_upper == 'YEAR': + return F.add_months(start_date, n * 12) + elif interval_upper == 'MONTH': + return F.add_months(start_date, n) + elif interval_upper == 'DAY': + return F.date_add(start_date, n) + return F.date_add(start_date, n) +``` + +--- + +## Confidence Levels for PySpark + +| Pattern | Confidence | Notes | +|---------|------------|-------| +| Broadcast join (HASH) | HIGH | Well-supported | +| Simple iteration | MEDIUM | Verify row ordering | +| CALL EXECUTE replacement | LOW | May need manual review | +| Complex state management | LOW | Test thoroughly | + +--- + +## Output Format + +When generating PySpark notebooks: + +```markdown +## Notebook: _pyspark.ipynb + +### Cell 1: Setup (SCOS Connection) +[Snowpark Connect setup] + +### Cell 2: Helper Functions +[Only if needed] + +### Cell 3: TIER 3 Block - +# Original SAS: +# Reason for PySpark: +[PySpark implementation] + +### Cell 4: Verification +[Query output table to verify] +``` + +--- + +## Remember + +1. **SQL first** - Window functions solve most SAS patterns +2. **Stored Procedures second** - Complex state management +3. **PySpark last** - Only for genuine edge cases +4. **Document why** - Always explain why PySpark was necessary diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/python-datascience.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/python-datascience.md new file mode 100644 index 0000000..4802cce --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/python-datascience.md @@ -0,0 +1,340 @@ +# Python Data Science Reference + +Translation patterns for SAS statistical/ML procedures to Python. + +--- + +## Snowpark Session Setup + +```python +# Cell 1: Setup +from snowflake.snowpark import Session +from snowflake.snowpark.context import get_active_session +import pandas as pd +import numpy as np + +session = get_active_session() +TARGET_SCHEMA = "DATABASE.SCHEMA" + +def read_table(table_name): + """Read Snowflake table to pandas DataFrame""" + return session.table(f"{TARGET_SCHEMA}.{table_name}").to_pandas() + +def write_table(df, table_name): + """Write pandas DataFrame to Snowflake""" + session.write_pandas(df, table_name, auto_create_table=True, overwrite=True) +``` + +--- + +## PROC MEANS → pandas + +### Basic Statistics + +```sas +/* SAS */ +PROC MEANS DATA=mydata N NMISS MEAN STD MIN MAX; + VAR score1 score2 score3; +RUN; +``` + +```python +# Python +df = read_table("mydata") +stats = df[['score1', 'score2', 'score3']].agg(['count', 'mean', 'std', 'min', 'max']) +print(stats) +``` + +### With OUTPUT Statement + +```sas +/* SAS */ +PROC MEANS DATA=mydata NOPRINT; + VAR sales; + BY region; + OUTPUT OUT=summary_stats MEAN=avg_sales SUM=total_sales N=count; +RUN; +``` + +```python +# Python +df = read_table("mydata") +summary = df.groupby('region')['sales'].agg( + avg_sales='mean', + total_sales='sum', + count='count' +).reset_index() +write_table(summary, "summary_stats") +``` + +--- + +## PROC UNIVARIATE → scipy.stats + +```sas +/* SAS */ +PROC UNIVARIATE DATA=mydata; + VAR score; + HISTOGRAM / NORMAL; + OUTPUT OUT=univ_stats MEAN=mean STD=std SKEWNESS=skew KURTOSIS=kurt; +RUN; +``` + +```python +# Python +from scipy import stats + +df = read_table("mydata") +score = df['score'].dropna() + +univ_stats = { + 'mean': score.mean(), + 'std': score.std(), + 'skew': stats.skew(score), + 'kurtosis': stats.kurtosis(score), + 'median': score.median(), + 'min': score.min(), + 'max': score.max(), + 'n': len(score), + 'nmiss': df['score'].isna().sum() +} + +# Normality test +shapiro_stat, shapiro_p = stats.shapiro(score[:5000]) # Shapiro limited to 5000 +univ_stats['shapiro_stat'] = shapiro_stat +univ_stats['shapiro_pvalue'] = shapiro_p + +print(pd.Series(univ_stats)) +write_table(pd.DataFrame([univ_stats]), "univ_stats") +``` + +--- + +## PROC CORR → pandas + +```sas +/* SAS */ +PROC CORR DATA=mydata; + VAR var1 var2 var3 var4; +RUN; +``` + +```python +# Python +df = read_table("mydata") +corr_matrix = df[['var1', 'var2', 'var3', 'var4']].corr() +print(corr_matrix) +write_table(corr_matrix.reset_index(), "correlation_matrix") +``` + +--- + +## PROC TTEST → scipy.stats + +```sas +/* SAS */ +PROC TTEST DATA=mydata; + CLASS group; + VAR score; +RUN; +``` + +```python +# Python +from scipy import stats + +df = read_table("mydata") +group_a = df[df['group'] == 'A']['score'] +group_b = df[df['group'] == 'B']['score'] + +# Independent t-test +t_stat, p_value = stats.ttest_ind(group_a, group_b) +print(f"t-statistic: {t_stat}, p-value: {p_value}") + +# Levene's test for equal variances +levene_stat, levene_p = stats.levene(group_a, group_b) +print(f"Levene's test: stat={levene_stat}, p={levene_p}") +``` + +--- + +## PROC REG → statsmodels + +```sas +/* SAS */ +PROC REG DATA=mydata; + MODEL y = x1 x2 x3; + OUTPUT OUT=reg_results P=predicted R=residual; +RUN; +``` + +```python +# Python +import statsmodels.api as sm + +df = read_table("mydata") +X = df[['x1', 'x2', 'x3']] +X = sm.add_constant(X) # Add intercept +y = df['y'] + +model = sm.OLS(y, X).fit() +print(model.summary()) + +# Output predictions and residuals +df['predicted'] = model.predict(X) +df['residual'] = model.resid +write_table(df, "reg_results") +``` + +--- + +## PROC LOGISTIC → sklearn + +```sas +/* SAS */ +PROC LOGISTIC DATA=mydata; + MODEL target(event='1') = x1 x2 x3; + OUTPUT OUT=logistic_results P=prob_1; +RUN; +``` + +```python +# Python +from sklearn.linear_model import LogisticRegression +from sklearn.preprocessing import StandardScaler + +df = read_table("mydata") +X = df[['x1', 'x2', 'x3']] +y = df['target'] + +# Scale features +scaler = StandardScaler() +X_scaled = scaler.fit_transform(X) + +# Fit model +model = LogisticRegression() +model.fit(X_scaled, y) + +# Predictions +df['prob_1'] = model.predict_proba(X_scaled)[:, 1] +df['predicted_class'] = model.predict(X_scaled) + +print(f"Coefficients: {dict(zip(['x1','x2','x3'], model.coef_[0]))}") +print(f"Intercept: {model.intercept_[0]}") + +write_table(df, "logistic_results") +``` + +--- + +## PROC CLUSTER → sklearn + +```sas +/* SAS */ +PROC CLUSTER DATA=mydata METHOD=WARD; + VAR x1 x2 x3; + ID customer_id; +RUN; +``` + +```python +# Python +from sklearn.cluster import AgglomerativeClustering +from scipy.cluster.hierarchy import dendrogram, linkage + +df = read_table("mydata") +X = df[['x1', 'x2', 'x3']] + +# Hierarchical clustering (Ward method) +clustering = AgglomerativeClustering(n_clusters=5, linkage='ward') +df['cluster'] = clustering.fit_predict(X) + +write_table(df, "clustered_data") +``` + +--- + +## PROC FACTOR → sklearn + +```sas +/* SAS */ +PROC FACTOR DATA=mydata NFACTORS=3 ROTATE=VARIMAX; + VAR x1 x2 x3 x4 x5; +RUN; +``` + +```python +# Python +from sklearn.decomposition import FactorAnalysis +from sklearn.preprocessing import StandardScaler + +df = read_table("mydata") +X = df[['x1', 'x2', 'x3', 'x4', 'x5']] + +# Standardize +scaler = StandardScaler() +X_scaled = scaler.fit_transform(X) + +# Factor analysis +fa = FactorAnalysis(n_components=3, rotation='varimax') +factors = fa.fit_transform(X_scaled) + +# Factor loadings +loadings = pd.DataFrame( + fa.components_.T, + columns=['Factor1', 'Factor2', 'Factor3'], + index=['x1', 'x2', 'x3', 'x4', 'x5'] +) +print("Factor Loadings:") +print(loadings) + +# Add factor scores to data +df['factor1'] = factors[:, 0] +df['factor2'] = factors[:, 1] +df['factor3'] = factors[:, 2] +write_table(df, "factor_results") +``` + +--- + +## DATA _NULL_ (Reporting) → Python print + +```sas +/* SAS */ +DATA _NULL_; + SET summary; + FILE PRINT; + PUT 'Total Sales: ' total_sales dollar12.2; + PUT 'Average: ' avg_sales 8.2; +RUN; +``` + +```python +# Python +df = read_table("summary") +row = df.iloc[0] + +print("=" * 40) +print(f"Total Sales: ${row['total_sales']:,.2f}") +print(f"Average: {row['avg_sales']:.2f}") +print("=" * 40) +``` + +--- + +## Library Mapping Reference + +| SAS Procedure | Python Library | Function/Class | +|---------------|----------------|----------------| +| PROC MEANS | pandas | `.describe()`, `.agg()` | +| PROC UNIVARIATE | scipy.stats | `describe()`, `skew()`, `kurtosis()` | +| PROC CORR | pandas | `.corr()` | +| PROC TTEST | scipy.stats | `ttest_ind()`, `ttest_rel()` | +| PROC ANOVA | scipy.stats | `f_oneway()` | +| PROC REG | statsmodels | `OLS()` | +| PROC GLM | statsmodels | `GLM()` | +| PROC LOGISTIC | sklearn | `LogisticRegression()` | +| PROC CLUSTER | sklearn | `AgglomerativeClustering()` | +| PROC FACTOR | sklearn | `FactorAnalysis()` | +| PROC PRINCOMP | sklearn | `PCA()` | +| PROC DISCRIM | sklearn | `LinearDiscriminantAnalysis()` | diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/schema-inference.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/schema-inference.md new file mode 100644 index 0000000..c81a9b3 --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/schema-inference.md @@ -0,0 +1,168 @@ +# Schema Inference Rules: SAS to Snowflake Type Mapping + +Rules for inferring Snowflake DDL from SAS code constructs. Used by Step 7a (Schema Inference) to auto-generate source table DDL when no real data exists in Snowflake. + +--- + +## SAS Format to Snowflake Type Mapping + +### Numeric Formats + +| SAS Format / Informat | Snowflake Type | Notes | +|----------------------|----------------|-------| +| `BEST.` / `BESTn.` | `NUMBER(n,0)` | Default integer; `BEST12.` → `NUMBER(12,0)` | +| `n.` (bare numeric, e.g., `8.`) | `NUMBER(n,0)` | SAS width = total digits | +| `n.d` (e.g., `12.2`) | `NUMBER(n,d)` | Width.Decimals → precision,scale | +| `COMMAn.d` | `NUMBER(n,d)` | Display format; same storage as n.d | +| `DOLLARn.d` | `NUMBER(n,d)` | Currency display; same storage | +| `PERCENTn.d` | `NUMBER(n,d)` | Percentage display; same storage | +| `Ew.d` (scientific) | `FLOAT` | Scientific notation → floating point | +| `(no format specified, numeric)` | `FLOAT` | Default for unformatted numerics | + +### Character Formats + +| SAS Format / Informat | Snowflake Type | Notes | +|----------------------|----------------|-------| +| `$n.` (e.g., `$50.`) | `VARCHAR(n)` | SAS char width → VARCHAR length | +| `$CHARn.` | `VARCHAR(n)` | Preserves leading/trailing blanks | +| `$VARYINGn.` | `VARCHAR(n)` | Variable-length character | +| `$UPCASEn.` | `VARCHAR(n)` | Uppercase format; apply UPPER() in transform | +| `$HEXn.` | `VARCHAR(n)` | Hex representation | +| `(no format specified, character)` | `VARCHAR(256)` | Default for unformatted characters | + +### Date / Time Formats + +| SAS Format / Informat | Snowflake Type | Notes | +|----------------------|----------------|-------| +| `MMDDYYn.` / `MMDDYY10.` | `DATE` | MM/DD/YYYY | +| `YYMMDDn.` / `YYMMDD10.` | `DATE` | YYYY-MM-DD | +| `DATEn.` / `DATE9.` | `DATE` | DDMonYYYY (e.g., 01JAN2024) | +| `DDMMYYn.` | `DATE` | DD/MM/YYYY | +| `MONYY.` / `MONYY7.` | `DATE` | MonYYYY (e.g., JAN2024) | +| `JULIAN.` / `JULIANn.` | `DATE` | Julian date | +| `DATETIME.` / `DATETIMEn.` | `TIMESTAMP_NTZ` | SAS datetime (seconds since 1960-01-01) | +| `TIME.` / `TIMEn.` | `TIME` | SAS time (seconds since midnight) | +| `DTDATE.` | `DATE` | Date portion of datetime | +| `ANYDTDTE.` / `ANYDTDTM.` | `DATE` / `TIMESTAMP_NTZ` | Auto-detect date/datetime | +| `YYMMN.` / `YYMMN6.` | `VARCHAR(6)` | Period key: YYYYMM (not a date — preserve as string) | +| `(no format, but used in date functions)` | `DATE` | If column appears in INTCK/INTNX/DATEPART | + +--- + +## PROC IMPORT Inference Rules + +When parsing `PROC IMPORT` statements: + +| PROC IMPORT Attribute | Inference Rule | +|----------------------|----------------| +| `DBMS=XLSX` / `DBMS=XLS` | Excel source; table name = SHEET= value or OUT= dataset name | +| `DBMS=CSV` | CSV source; all columns default to `VARCHAR(256)` unless refined by subsequent DATA step | +| `DBMS=DLM` | Delimited file; use DELIMITER= to identify separator | +| `GETNAMES=YES` | Column names come from first row (cannot infer types without data) | +| `GETNAMES=NO` | Columns are positional: VAR1, VAR2, ...; all VARCHAR(256) | +| `DATAROW=n` | Data starts at row n; header at row n-1 if GETNAMES=YES | +| `SHEET="name"` | Table name candidate = sanitized sheet name | +| `OUT=lib.dataset` | Output table name; lib maps to Snowflake schema | +| `RANGE="A1:Z100"` | Limits column/row range; infer column count from range | + +**When column types cannot be determined from PROC IMPORT alone**, check for subsequent DATA step or PROC SQL that reads the imported table — those often apply explicit FORMAT/INFORMAT statements or WHERE conditions that reveal types. + +--- + +## INFILE / INPUT Statement Inference Rules + +Parse the INPUT statement following INFILE to extract column definitions: + +### Column-Pointer INPUT + +```sas +INPUT @1 ACCT_NUM $20. @21 BALANCE 12.2 @33 OPEN_DATE MMDDYY10.; +``` + +| Component | Inference | +|-----------|-----------| +| `@n` | Column position (absolute pointer) — skip for DDL | +| `varname $n.` | `VARCHAR(n)` | +| `varname n.d` | `NUMBER(n,d)` | +| `varname informat.` | Apply Date/Time mapping table above | + +### List INPUT (space-delimited) + +```sas +INPUT ACCT_NUM $ BALANCE OPEN_DATE :MMDDYY10.; +``` + +| Component | Inference | +|-----------|-----------| +| `varname $` | `VARCHAR(256)` (no width → default) | +| `varname` (no $) | `FLOAT` (no format → numeric default) | +| `varname :informat.` | Apply format mapping with colon modifier | + +### Named INPUT + +```sas +INPUT ACCT_NUM= BALANCE= OPEN_DATE=; +``` + +All columns `VARCHAR(256)` unless subsequent FORMAT/INFORMAT statement clarifies types. + +--- + +## PROC SQL CREATE TABLE Inference + +Direct DDL extraction — these map cleanly: + +| SAS SQL Type | Snowflake Type | +|-------------|----------------| +| `CHAR(n)` / `CHARACTER(n)` | `VARCHAR(n)` | +| `VARCHAR(n)` | `VARCHAR(n)` | +| `INTEGER` / `INT` | `INTEGER` | +| `SMALLINT` | `SMALLINT` | +| `FLOAT` / `REAL` / `DOUBLE` | `FLOAT` | +| `NUMERIC(p,s)` / `DECIMAL(p,s)` | `NUMBER(p,s)` | +| `DATE` | `DATE` | +| `TIMESTAMP` | `TIMESTAMP_NTZ` | + +--- + +## Fallback Rules + +When type cannot be inferred from any SAS construct: + +| Scenario | Default Type | +|----------|-------------| +| Column name contains `_ID`, `_KEY`, `_NUM`, `_CODE` | `VARCHAR(50)` | +| Column name contains `_AMT`, `_BAL`, `_RATE`, `_PCT` | `NUMBER(18,4)` | +| Column name contains `_DT`, `_DATE` | `DATE` | +| Column name contains `_TS`, `_TIMESTAMP`, `_DTTM` | `TIMESTAMP_NTZ` | +| Column name contains `_DESC`, `_NAME`, `_LABEL` | `VARCHAR(500)` | +| Column name contains `_FLAG`, `_IND` | `VARCHAR(1)` | +| Column referenced only in WHERE with numeric comparison | `FLOAT` | +| Column referenced only in WHERE with string comparison | `VARCHAR(256)` | +| No heuristic matches | `VARIANT` | + +--- + +## Output Format + +Generate one DDL file per pipeline phase (or one consolidated file), with comments tracing each table back to its SAS source: + +```sql +-- ============================================================ +-- Auto-generated source table DDL +-- Inferred from SAS code: Phase 1 (01 through 50) +-- Generated by: /convert-sas-to-snowflake Step 7a +-- ============================================================ + +-- Source: 01_import_excel_data.sas (PROC IMPORT, SHEET="Customers") +CREATE TABLE IF NOT EXISTS TARGET_SCHEMA.WK_CUSTOMER_DIM ( + CUSTOMER_ID VARCHAR(50), -- inferred: _ID suffix + CUSTOMER_NAME VARCHAR(500), -- inferred: _NAME suffix + REGION VARCHAR(50), -- inferred: no format, char context + AMOUNT_A NUMBER(12,2), -- inferred: subsequent FORMAT 12.2 + AMOUNT_B NUMBER(12,2), -- inferred: subsequent FORMAT 12.2 + DISCOUNT_RATE NUMBER(18,4), -- inferred: _RATE suffix + UNIT_RATE NUMBER(18,4), -- inferred: _RATE suffix + EFFECTIVE_DATE DATE -- inferred: _DATE suffix +); +``` diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/snowflake-compile.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/snowflake-compile.md new file mode 100644 index 0000000..c412004 --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/snowflake-compile.md @@ -0,0 +1,213 @@ +# Snowflake Compilation Module + +## When to Load + +Load this file when Phase 3 (Snowflake Compilation) is selected in the Validation Pipeline. This module handles compile-only testing with auto-creation of temporary environments when the target schema does not exist. + +--- + +## Explicit Prompt (ALWAYS Required — Never Auto-Propagated) + +Before ANY Snowflake interaction, present: + +``` +VALIDATION PHASE 3: Snowflake Compilation (Credits Will Be Consumed) + +Action: Compile [N] .sql files with only_compile=true (dry-run syntax check) +Target schema: [TARGET_DB].[TARGET_SCHEMA] +Schema status: [EXISTS / DOES NOT EXIST] + +If schema does not exist: + → Will create temporary database: [TARGET_DB]_COMPILE_[YYYYMMDD] + → Will load source tables from source_table_ddl.sql + → Will populate with synthetic data for type-checking context + → Will drop temporary database after compilation + +Credit impact: MINIMAL (compilation metadata only, no data processing) + +Proceed? [Yes - Compile] / [No - Skip] / [Show compilation plan] +``` + +**This prompt fires EVERY time Phase 3 runs** — regardless of any prior validation menu selection. Selecting Phase 3 in the menu indicates intent; this prompt confirms the action. + +--- + +## Compilation Workflow + +### Step 1: Probe Compilation + +Compile ONE simple file (smallest Tier 1) to check environment readiness: + +```python +# Select the smallest .sql file as probe +probe_file = min(sql_files, key=lambda f: os.path.getsize(os.path.join(output_dir, f))) +``` + +Execute with `snowflake_sql_execute(sql=probe_content, only_compile=true)` + +**Interpret probe result:** +- SUCCESS → environment ready, proceed to Step 3 +- ERROR "Database does not exist" → proceed to Step 2 (Auto-Create) +- ERROR "Schema does not exist" → proceed to Step 2 (Auto-Create) +- ERROR "Object does not exist" (table-level) → proceed to Step 2 (load stubs) +- ERROR (other) → log error, attempt with next file + +### Step 2: Auto-Create Temporary Compilation Environment + +When the target schema does not exist, create a temporary environment: + +```sql +-- 2a. Create temporary database (timestamped to avoid conflicts) +CREATE DATABASE IF NOT EXISTS _COMPILE_; + +-- 2b. Set context +USE DATABASE _COMPILE_; + +-- 2c. Create target schema +CREATE SCHEMA IF NOT EXISTS ; + +-- 2d. Set schema context +USE SCHEMA _COMPILE_.; +``` + +Then load source tables and synthetic data: + +```sql +-- 2e. Execute source_table_ddl.sql (all CREATE TABLE IF NOT EXISTS) +-- 2f. Execute synthetic_data.sql (provides type context for compilation) +``` + +**Record in state:** +```json +{ + "compilation_env": { + "type": "temporary", + "database": "_COMPILE_", + "schema": "", + "created_at": "" + } +} +``` + +### Step 3: Compile All Files + +For each .sql file in the output directory: + +```python +results = [] +for sql_file in sorted(sql_files): + with open(os.path.join(output_dir, sql_file)) as f: + sql_content = f.read() + + # If using temp environment, rewrite schema references + if using_temp_env: + sql_content = sql_content.replace( + f"{target_db}.{target_schema}.", + f"{temp_db}.{target_schema}." + ) + + # Compile + result = snowflake_sql_execute(sql=sql_content, only_compile=True) + + if result.success: + results.append({"file": sql_file, "status": "PASS"}) + else: + results.append({ + "file": sql_file, + "status": "FAIL", + "error": result.error_message, + "error_line": result.error_line + }) +``` + +**Do NOT stop on first failure** — compile ALL files and report aggregate results. + +### Step 4: Fix-and-Retry Loop (Max 3 Attempts Per File) + +For each failed file: + +1. Parse error message +2. Map to known fix patterns: + +| Error Pattern | Auto-Fix | +|--------------|----------| +| Unknown function X | Check function-mappings.md, replace | +| Invalid identifier | Check for SAS macro variable or vendor syntax | +| Syntax error near X | Check Snowflake Scripting rules | +| Object does not exist | Add TEMPORARY or check dependency order | +| Type mismatch | Add explicit CAST | +| Unexpected keyword | Quote identifier with double quotes | +| Missing column | Verify against source DDL | + +3. Apply fix to .sql file on disk +4. Re-compile +5. If PASS: record success with fix details +6. If FAIL after 3 attempts: record as FAILED_AFTER_RETRIES + +### Step 5: Write Compilation Results + +```python +import json +from datetime import datetime + +compilation_results = { + "compiled_at": datetime.now().isoformat(), + "environment": "temporary" if using_temp_env else "existing", + "total_files": len(results), + "passed": sum(1 for r in results if r["status"] == "PASS"), + "failed": sum(1 for r in results if r["status"] != "PASS"), + "pass_rate": f"{sum(1 for r in results if r['status'] == 'PASS') / len(results) * 100:.1f}%", + "failures": [r for r in results if r["status"] != "PASS"], + "fixes_applied": [r for r in results if r.get("fix_applied")] +} + +with open(os.path.join(output_dir, "compilation_results.json"), "w") as f: + json.dump(compilation_results, f, indent=2) +``` + +### Step 6: Cleanup Temporary Environment + +If a temporary environment was created: + +``` +COMPILATION COMPLETE — Cleanup + +Results: [PASSED]/[TOTAL] files compiled successfully +Temporary database: [TEMP_DB_NAME] + +Drop temporary compilation database? [Yes - Drop] / [No - Keep for debugging] +``` + +If Yes: +```sql +DROP DATABASE IF EXISTS ; +``` + +Update state: `compilation_env.dropped: true` + +--- + +## Resume Behavior + +If `conversion_state.json` shows `compilation_env` is non-null and `dropped` is false: +- The temp environment already exists from a prior session +- Skip Steps 1-2, proceed directly to Step 3 (compile remaining files) +- Check per-file `compilation_status` to skip already-compiled files + +--- + +## HARD GATE (Phase 3) + +```python +import os, sys, json +output_dir = "" +results_path = os.path.join(output_dir, "compilation_results.json") +if not os.path.exists(results_path): + print("BLOCKED: compilation_results.json does not exist") + sys.exit(1) +with open(results_path) as f: + results = json.load(f) +print(f"Phase 3 GATE PASSED: {results['passed']}/{results['total_files']} compiled") +``` + +Update `conversion_state.json`: `gates.phase_3_snowflake_compile: "PASSED"` diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/snowflake-execution.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/snowflake-execution.md new file mode 100644 index 0000000..dc89dc0 --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/snowflake-execution.md @@ -0,0 +1,281 @@ +# Snowflake Execution Module (Tier 2/3 Only) + +## When to Load + +Load this file when Phase 4 (Snowflake Execution) is selected in the Validation Pipeline. Executes stored procedures and converted SQL against synthetic data on Snowflake. + +--- + +## Prerequisite Checks + +Before this phase can execute: +1. Phase 3 (Snowflake Compilation) must have PASSED for the target Tier 2/3 files +2. `source_table_ddl.sql` exists (from Phase 1) +3. `synthetic_data.sql` exists (from Phase 1) +4. Phase 2 expected baselines exist for the target files (recommended) + +--- + +## Explicit Prompt (ALWAYS Required — Never Auto-Propagated) + +``` +VALIDATION PHASE 4: Snowflake Execution — Tier 2/3 Only (Credits Will Be Consumed) + +Action: Execute [N] stored procedures / PySpark notebooks against synthetic data +Files: + [list each Tier 2/3 file with name and tier classification] + +Warehouse: [current warehouse name] +Credit impact: MODERATE (actual SQL execution, warehouse will be active) + +This will: + 1. Create temporary tables with synthetic data in [SCHEMA] + 2. Execute each stored procedure via CALL + 3. Capture output tables produced by each SP + 4. Compare output against Phase 2 expected baselines + 5. Drop ALL temporary objects after completion + +Proceed? [Yes - Execute on Snowflake] / [No - Skip Tier 2/3 testing] / [Show execution plan] +``` + +**This prompt fires EVERY time Phase 4 runs** — no consent propagation override exists for actual SQL execution. + +--- + +## Execution Workflow + +### Step 1: Prepare Temporary Environment + +If a compilation environment still exists from Phase 3 (not dropped), reuse it. +Otherwise, create fresh: + +```sql +-- Create or reuse temp schema +CREATE DATABASE IF NOT EXISTS _VALIDATE_; +USE DATABASE _VALIDATE_; +CREATE SCHEMA IF NOT EXISTS ; +USE SCHEMA _VALIDATE_.; +``` + +### Step 2: Load Source Tables and Synthetic Data + +```sql +-- Execute source_table_ddl.sql (creates all source tables) +-- Execute synthetic_data.sql (populates with test data) +``` + +### Step 3: Create Intermediate Tables (Dependency Chain) + +Tier 2/3 files often depend on output from earlier Tier 1 files. For each Tier 2/3 file: +1. Identify its input dependencies (tables it reads from) +2. If those tables are outputs of Tier 1 files: execute those Tier 1 files first (in dependency order) to populate the intermediate tables +3. This ensures the SP has realistic input data + +```python +# Build dependency chain for each Tier 2/3 file +for tier23_file in tier23_files: + dependencies = get_file_dependencies(tier23_file) + tier1_deps = [d for d in dependencies if d in tier1_files] + + # Execute Tier 1 dependencies first (in order) + for dep in topological_sort(tier1_deps): + execute_sql_file(dep) +``` + +### Step 4: Execute Tier 2 (Stored Procedures) + +For each Tier 2 file: + +```sql +-- The .sql file contains CREATE OR REPLACE PROCEDURE ... + CALL +-- Execute the entire file (creates + calls the SP) +``` + +After execution: +- Identify output tables (tables created/modified by the SP) +- SELECT * FROM each output table → write to `tests//actual/
    .csv` + +### Step 5: Execute Tier 3 (PySpark/SCOS Notebooks) + +For each Tier 3 file: +- If `.ipynb`: execute notebook cells against the Snowflake session +- If `.py`: execute the Python script with Snowpark Connect + +After execution: +- Identify output tables +- SELECT * FROM each → write to CSV + +### Step 6: Compare Against Expected Baselines + +```python +import pandas as pd, os, json + +comparison_results = [] + +for file_info in tier23_files: + file_stem = file_info["name"].replace(".sql", "") + expected_dir = os.path.join(output_dir, "tests", file_stem, "expected") + actual_dir = os.path.join(output_dir, "tests", file_stem, "actual") + + if not os.path.isdir(expected_dir): + comparison_results.append({ + "file": file_info["name"], + "status": "NO_BASELINE", + "note": "Phase 2 trace not run for this file" + }) + continue + + file_pass = True + mismatches = [] + + for expected_csv in os.listdir(expected_dir): + if not expected_csv.endswith(".csv"): + continue + table_name = expected_csv.replace("expected_", "").replace(".csv", "") + actual_csv_path = os.path.join(actual_dir, f"{table_name}.csv") + + if not os.path.exists(actual_csv_path): + mismatches.append(f"{table_name}: no actual output") + file_pass = False + continue + + expected_df = pd.read_csv(os.path.join(expected_dir, expected_csv)) + actual_df = pd.read_csv(actual_csv_path) + + # Schema comparison + if set(expected_df.columns) != set(actual_df.columns): + mismatches.append(f"{table_name}: column mismatch") + file_pass = False + + # Row count comparison + if len(expected_df) != len(actual_df): + mismatches.append(f"{table_name}: row count {len(actual_df)} vs expected {len(expected_df)}") + file_pass = False + + # Trivial-pass detection: empty expected AND empty actual is NOT a real pass. + # It usually means an upstream join produced zero rows (e.g. range/unit mismatch). + if len(expected_df) == 0 and len(actual_df) == 0: + mismatches.append(f"{table_name}: TRIVIAL_PASS — both expected and actual are empty (0 rows). " + f"Verify upstream joins actually produced data; check range-join/unit alignment.") + # Do not set file_pass=False, but surface as WARNING so it is never silently green. + + # Value comparison (with tolerance for numerics) + # Apply comparison rules from references/comparison-rules.md + + comparison_results.append({ + "file": file_info["name"], + "tier": file_info["tier"], + "status": "PASS" if file_pass else "FAIL", + "trivial_pass": any("TRIVIAL_PASS" in m for m in mismatches), + "mismatches": mismatches + }) +``` + +### Step 6b: Non-Empty Output Assertion + +For each file that creates a **terminal** or otherwise expected-to-be-populated output table, assert it produced rows. A SP that runs cleanly but writes 0 rows is a likely defect, not a pass. + +```python +# After execution, count rows in each output table +for tbl in output_tables_of(file_info): + n = run_scalar(f"SELECT COUNT(*) FROM {schema}.{tbl}") + if n == 0: + # Distinguish justified empties (genuinely empty source) from defects + empty_source = first_empty_upstream(tbl) # walk reads[] chain + comparison_results.append({ + "file": file_info["name"], "table": tbl, + "status": "EMPTY_JUSTIFIED" if empty_source else "EMPTY_DEFECT", + "empty_source": empty_source, + "note": f"output table {tbl} has 0 rows" + }) +``` + +`EMPTY_DEFECT` (empty despite populated sources) almost always indicates a join that did not overlap — re-check range-join / unit-consistency in the synthetic data (`references/synthetic-data-rules.md`). + +### Step 7: Write Results + +```python +sf_results = { + "executed_at": datetime.now().isoformat(), + "warehouse": "", + "total_files": len(tier23_files), + "passed": sum(1 for r in comparison_results if r["status"] == "PASS"), + "failed": sum(1 for r in comparison_results if r["status"] == "FAIL"), + "no_baseline": sum(1 for r in comparison_results if r["status"] == "NO_BASELINE"), + "results": comparison_results +} + +with open(os.path.join(output_dir, "snowflake_execution_results.json"), "w") as f: + json.dump(sf_results, f, indent=2) +``` + +### Step 8: Cleanup + +```sql +-- Drop all temporary objects +DROP DATABASE IF EXISTS _VALIDATE_; +``` + +Present cleanup confirmation: +``` +PHASE 4 COMPLETE — Cleanup + +Results: [PASSED]/[TOTAL] Tier 2/3 files validated +Temporary database: [TEMP_DB_NAME] + +Drop temporary execution database? [Yes - Drop] / [No - Keep for debugging] +``` + +--- + +## Modular Execution (Context-Aware — prevents timeouts) + +Executing dozens of files (deploy + CALL each) in one pass exhausts context and causes subagent timeouts. Process in layer-batches with checkpoint resume — the same pattern used for Phase 2 tracing. + +1. Order files by dependency layer (Layer 0 → terminal). Initialize `execution_progress` in `conversion_state.json`: + ```json + {"execution_progress": {"files_executed": [], "files_remaining": [""], "batch_size": 5}} + ``` +2. Execute in batches of ~5 files (respecting layer order — never run a file before its dependencies): + - Deploy + CALL each file in the batch + - Capture outputs, run comparison + non-empty assertion + - Move each file from `files_remaining` to `files_executed` + - Update `conversion_state.json` AND append a checkpoint line (`step:"phase_4" status:"INFO" notes:"batch N executed"`) +3. After each batch, check context budget: + - Sufficient → continue to next batch + - Nearing limit → write `snowflake_execution_results.json` (partial), save state, inform user, exit gracefully +4. Resume: read `execution_progress`, skip `files_executed`, continue from `files_remaining`. + +When delegating to subagents, give each subagent ONE layer-batch (not the whole pipeline) so no single agent runs more than ~5 files. + +--- + +## HARD GATE (Phase 4) + +```python +import os, sys, json +output_dir = "" +results_path = os.path.join(output_dir, "snowflake_execution_results.json") +if not os.path.exists(results_path): + print("BLOCKED: snowflake_execution_results.json does not exist") + sys.exit(1) +with open(results_path) as f: + results = json.load(f) +print(f"Phase 4 GATE PASSED: {results['passed']}/{results['total_files']} Tier 2/3 files validated") +``` + +Update `conversion_state.json`: `gates.phase_4_snowflake_execute: "PASSED"` + +--- + +## Files That Require Phase 4 + +These are identified during Step 3 (Classification) and recorded in `conversion_state.json`: + +| Classification | Reason | Phase 4 Required | +|---------------|--------|-----------------| +| Tier 2 SP | Stored procedures with DECLARE/BEGIN/END | Yes | +| Tier 3 SCOS | PySpark notebooks | Yes | +| Compilation FAILED | Files that failed compilation in Phase 3 | Yes | + +The validation summary from Phase 3 (`compilation_results.json`) identifies files that failed compilation — those may need manual review before Phase 4 execution. diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/state-tracker-schema.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/state-tracker-schema.md new file mode 100644 index 0000000..7217134 --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/state-tracker-schema.md @@ -0,0 +1,189 @@ +# Conversion State Tracker Schema (`conversion_state.json`) + +## When to Load + +Load on **resume** (when `conversion_state.json` exists and needs interpretation) or when writing state for the first time. Not needed in context for routine step execution — the step-specific instructions reference which fields to write. + +--- + +## Full Schema + +```json +{ + "metadata": { + "output_dir": "", + "target_schema": "DATABASE.SCHEMA", + "migration_mode": "1:1", + "created_at": "", + "last_updated": "", + "current_step": "5", + "total_files": 5, + "validation_scope": "full", + "pre_approved_compilation": false, + "pre_approved_validation": null, + "sp_permission": null + }, + "source_mappings": { + "LIBNAME_NAME": { + "engine": "sqlsvr", + "original_dsn": "DSN_NAME", + "original_schema": "dbo", + "target_database": "SNOWFLAKE_DB", + "target_schema": "SNOWFLAKE_SCHEMA", + "tables": ["table1", "table2"] + } + }, + "assessment": { + "consumed": true, + "source_path": "", + "generated_at": "", + "consumed_at": "", + "stale": false, + "reconciliation": { + "files_compared": 9, + "files_matched": 8, + "files_mismatched": 1, + "mismatches": [ + {"file": "b.sas", "field": "primary_tier", "assessment": "TIER_2_SP", "convert": "TIER_1_SQL"}, + {"file": "b.sas", "field": "blocks", "assessment": 22, "convert": 20} + ] + } + }, + "files": { + "script_name.sas": { + "status": "converted", + "tier": "1-SQL", + "output_file": "script_name.sql", + "blocks": 3, + "dependencies": {"creates": ["TABLE_A"], "reads": ["TABLE_B", "TABLE_C"]}, + "classification_complete": true, + "conversion_complete": true, + "self_check_passed": false, + "compilation_status": "PENDING", + "compilation_errors": [], + "compilation_fix_attempts": 0, + "validation_status": "PENDING", + "validation_mismatches": [] + } + }, + "gates": { + "step_2_5_source_resolution": "PENDING", + "step_6a_artifacts": "PENDING", + "step_7a_test_artifacts": "PENDING", + "phase_5_e2e_orchestration": "PENDING", + "step_10_final_artifacts": "PENDING" + }, + "trace_progress": { + "total_files_in_scope": 15, + "files_traced": [], + "files_remaining": [], + "current_batch": 0, + "batch_size": 5, + "concerns_found": [] + }, + "compilation_env": null, + "summary": { + "classified": 0, + "converted": 0, + "self_checked": 0, + "compiled_pass": 0, + "compiled_fail": 0, + "validated_pass": 0, + "validated_fail": 0, + "pending_validation": 0 + } +} +``` + +--- + +## `assessment` object + +Written in Step 3 to record whether a prior `assess-sas-migration` run was auto-discovered and +consumed as a baseline, and how this conversion's block counts/tiers reconcile against it. + +- **When found & consumed:** `consumed: true` with `source_path` (which `assessment.json` was used), + `generated_at`, `consumed_at`, `stale` (true if the assessment predates the newest source edit), + and a `reconciliation` object (`files_compared`, `files_matched`, `files_mismatched`, and a + `mismatches` list of `{file, field, assessment, convert}` where `field` ∈ `blocks` / `primary_tier` + / `tier_distribution`). Mismatches are informational — they never block conversion. +- **When none found:** `{"consumed": false, "reason": "not_found", "searched_paths": [...]}` + (or `reason: "unreadable"` if a candidate existed but failed to parse). Conversion proceeds with + fresh classification and no prompt. + +--- + +## Companion Audit Trail — `checkpoint_log.jsonl` + +`conversion_state.json` is the **current-state snapshot** (overwritten each transition). It is paired with `checkpoint_log.jsonl`, the **append-only audit trail** (one JSON line per checkpoint, never overwritten). BOTH are required artifacts. See `references/checkpoint-logging.md` for the line schema and append helper. + +At every transition in the table below, the workflow MUST: (1) update `conversion_state.json`, then (2) append one line to `checkpoint_log.jsonl` — before advancing. + +--- + +## Enforcement Rule — State Writes are BLOCKING PREREQUISITES + +`conversion_state.json` is NOT optional. It MUST be written/updated at each transition point below. The NEXT step CANNOT begin until the state file reflects the current step's completion. If the file is missing or stale, write it before proceeding. At each transition, also append a checkpoint line to `checkpoint_log.jsonl`. + +| Transition | Prerequisite (state file must show) | Checkpoint line appended | +|---|---|---| +| Step 1 → Step 2 | File exists with metadata (output_dir, target_schema, migration_mode, `current_step: "1"`) | `step:"1" status:"STARTED"` | +| Step 2.5 → Step 3 | `gates.step_2_5_source_resolution: "PASSED"` or `"SKIPPED"` (no external sources) — `source_mappings` populated if external sources found | `step:"2.5" status:"PASSED"\|"SKIPPED"` | +| Step 3 → Step 4 | All files have `status: "classified"`, dependencies populated; `assessment` object written (consumed+reconciliation, or `consumed: false`) | `step:"3" status:"PASSED"` | +| Step 5 → Step 6 | All files have `status: "converted"` | `step:"5" status:"PASSED"` | +| Step 6 → Step 6a | All files have `self_check_passed: true` | `step:"6" status:"PASSED"` | +| Step 6a → Step 7 | `gates.step_6a_artifacts: "PASSED"` | `step:"6a" gate:"step_6a_artifacts" status:"PASSED"` | +| Step 7a → Step 7b | `gates.step_7a_test_artifacts: "PASSED"` | `step:"phase_1" status:"PASSED"` | +| Step 7b → Step 7c | `trace_progress.files_remaining` is empty (all priority files traced) | `step:"phase_2" status:"PASSED"\|"SKIPPED"` | +| Step 7 → Step 8 | `current_step: "7"` | — | +| Step 8 → Step 8 compile | Target schema exists OR `compilation_env` is set (auto-created via Step 8-pre) | `step:"phase_3" status:"STARTED"` | +| Step 8 → Step 9 | All files have `compilation_status` set (SUCCESS, FAILED, or SKIPPED) | `step:"phase_3" gate:"phase_3_snowflake_compile" status:"PASSED"\|"SKIPPED"` | +| Step 9 → Phase 5 | Phase 4 gate shows PASSED or SKIPPED | `step:"phase_4" gate:"phase_4_snowflake_execute" status:"PASSED"\|"SKIPPED"` | +| Phase 5 → Step 10 | `gates.phase_5_e2e_orchestration: "PASSED"` or `"SKIPPED"` | `step:"phase_5" gate:"phase_5_e2e_orchestration" status:"PASSED"\|"SKIPPED"` | +| Step 10 → Complete | `gates.step_10_final_artifacts: "PASSED"`, `current_step: "complete"` | `step:"10" status:"PASSED"`, then `step:"complete" status:"PASSED"` | + +**If the state file does not satisfy the prerequisite for the next step:** +1. Write/update it immediately +2. Verify the write succeeded (re-read the file) +3. Append the corresponding checkpoint line to `checkpoint_log.jsonl` +4. Only then proceed to the next step + +This rule applies in BOTH interactive and batch modes. Context pressure does NOT exempt state writes. + +--- + +## Resume Behavior (Step 1 Pre-Check) + +- If `conversion_state.json` exists, read it and display: + ``` + Prior conversion found. Status: [summary counts]. Last step: [current_step]. + Resume from step [current_step + 1]? / Restart fresh (overwrites)? + ``` +- If resume: skip all completed files/steps, continue from `current_step` +- If restart: delete state file and begin from Step 1 + +--- + +## File Status Progression + +``` +classified → converted → self_checked → compiled → validated → complete +``` + +--- + +## Valid `current_step` Values + +| Value | Meaning | +|-------|---------| +| `"1"` | Step 1 complete (metadata captured) | +| `"2.5"` | External source resolution complete | +| `"3"` | Classification complete | +| `"5"` | Conversion in progress or complete | +| `"6"` | Self-check complete | +| `"7"` | Validation (tracing) complete | +| `"8"` | Compilation in progress | +| `"9"` | Snowflake execution in progress | +| `"10-in-progress"` | Report generation started | +| `"10"` | Report generation complete | +| `"complete"` | All gates passed, all artifacts verified | diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/synthetic-data-rules.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/synthetic-data-rules.md new file mode 100644 index 0000000..6ed1baa --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/synthetic-data-rules.md @@ -0,0 +1,219 @@ +# Synthetic Data Generation Rules for SAS Validation + +Rules for generating smart synthetic test data that catches common SAS-to-Snowflake translation errors. + +## General Rules + +| Rule | Details | +|------|---------| +| Rows per table | 5-8 rows (enough to test logic, small enough for LLM to trace) | +| Column coverage | Only columns referenced in the converted code (SELECT, WHERE, JOIN, GROUP BY) | +| Table naming | Use TEMPORARY tables -- bare names only (no schema prefix) | +| Data realism | Values should be plausible for the column name (e.g., `amount` = numbers, `name` = strings) | + +## Join-Aware Data Generation + +For every JOIN or MERGE in the converted code, the synthetic data MUST include these key distribution patterns: + +### Key Alignment Matrix + +| Scenario | Left Table | Right Table | Purpose | +|----------|-----------|-------------|---------| +| Match (both sides) | key = 1, 2, 3 | key = 1, 2, 3 | Tests normal join behavior | +| Left-only key | key = 4 | (no key 4) | Tests LEFT JOIN produces NULL for right-side columns | +| Right-only key | (no key 5) | key = 5 | Tests RIGHT/FULL join; inner join should exclude | +| Duplicate key (one side) | key = 1 (one row) | key = 1 (two rows) | Tests one-to-many join (row multiplication) | +| Duplicate key (both sides) | key = 2 (two rows) | key = 2 (two rows) | Tests many-to-many (SAS MERGE vs SQL JOIN difference) | + +### Example: Two-Table JOIN + +```sql +CREATE TEMPORARY TABLE customers (id NUMBER, name VARCHAR, region VARCHAR); +INSERT INTO customers VALUES + (1, 'Alice', 'EAST'), + (2, 'Bob', 'WEST'), + (2, 'Bob Jr', 'WEST'), -- duplicate key (many-to-many test) + (3, 'Carol', 'EAST'), + (4, 'Dave', 'SOUTH'), -- left-only key + (5, NULL, NULL); -- missing value row + +CREATE TEMPORARY TABLE orders (order_id NUMBER, customer_id NUMBER, amount NUMBER, order_date DATE); +INSERT INTO orders VALUES + (101, 1, 100.00, '2024-01-15'), + (102, 1, 250.50, '2024-02-20'), -- duplicate key (one-to-many) + (103, 2, -10.00, '2024-01-01'), -- negative amount + (104, 3, 0, '2024-06-30'), -- zero amount + mid-year date + (105, 3, NULL, '2024-12-31'), -- NULL amount + year-end date + (106, 6, 500.00, NULL), -- right-only key + NULL date + (107, 2, 75.00, '2024-01-01'); -- second row for key=2 +``` + +## Range-Join & Cross-Table Value Alignment (CRITICAL) + +The equality-join matrix above is necessary but NOT sufficient. The most common cause of silently-empty pipeline outputs is a **range join or unit mismatch** where rows exist on both sides but never overlap. Detect and handle these BEFORE generating data. + +### Detect non-equality join predicates + +Scan the converted SQL for join predicates beyond `a.key = b.key`: + +| Predicate pattern | Join type | Alignment requirement | +|-------------------|-----------|----------------------| +| `x BETWEEN t.top AND t.base` | Range / depth-interval | child `x` MUST fall inside ≥1 parent `[top, base]` interval | +| `a.depth >= t.top AND a.depth <= t.base` | Range (expanded) | same as BETWEEN | +| `a.d BETWEEN t.d - tol AND t.d + tol` | Tolerance / nearest | child `d` within `tol` of a parent `d` | +| `ROUND(a.x) = ROUND(b.x)` | Bucketed | both sides round to the same bucket | +| `CONTAINS(a.key, b.name)` / `LIKE` | Substring | child substring actually appears in parent key | + +### Range-join data rule + +For every range join, generate child-table values so that: +1. **At least one child row falls strictly inside a parent interval** — guarantees the join produces rows (positive test). Place it mid-interval, e.g. parent `[1375, 1420]` → child at `1395`. +2. **At least one child row falls outside all intervals** — negative test (verifies the join filters correctly). +3. **Boundary rows** at exactly `top` and exactly `base` — verifies inclusive/exclusive bound handling. + +```sql +-- Parent interval table +INSERT INTO zone_interval VALUES ('S1','ZONE_A', 1375.0, 1420.0); -- [top, base] in METERS +-- Child measurement table — depth values chosen to overlap the interval +INSERT INTO sensor_reading VALUES ('S1', 1380.0, 55.2); -- inside (positive) +INSERT INTO sensor_reading VALUES ('S1', 1410.0, 48.5); -- inside (positive) +INSERT INTO sensor_reading VALUES ('S1', 1375.0, 50.0); -- top boundary +INSERT INTO sensor_reading VALUES ('S1', 1600.0, 40.0); -- outside (negative test) +``` + +### Unit-consistency rule (the meters-vs-feet trap) + +When the SAME physical quantity appears under different column names or unit suffixes across tables that join on it, generate ALL of them in ONE canonical numeric range so cross-table joins overlap. + +**Worked example (canonical failure):** a pipeline joined `SENSOR_READING.DEPTH_M` (meters) to `ZONE_BOUNDARY.TOP_DEPTH`/`BASE_DEPTH`. The interval depths were seeded in feet (4500–5200) while the measurements were in meters (1175–1540). Every `BETWEEN` join returned zero rows, so all downstream tables were empty — yet every file "passed" compilation and isolated execution. + +Rule: +- Identify quantity families by name stem and unit suffix: `DEPTH`, `DEPTH_FT`, `DEPTH_M`, `TOP_DEPTH`, `BASE_DEPTH`, etc. +- Pick ONE canonical range for the family (e.g. depths 1170–1580). Seed every table in that family within the canonical range, regardless of the column's unit label. +- Do NOT mechanically convert (do not put 4500 in a `_FT` column and 1375 in a `_M` column). For synthetic *test* data, consistency-for-overlap beats physical realism. Note this in a comment. + +```sql +-- All depth-bearing tables seeded in the SAME 1170-1580 range so joins overlap: +INSERT INTO measurement_a (SITE_ID, DEPTH_FT, DEPTH_M) VALUES ('S1', 1380.0, 1380.0); -- both cols same range +INSERT INTO measurement_b (SITE_ID, DEPTH_M) VALUES ('S1', 1390.0); +INSERT INTO measurement_c (SITE_ID, DEPTH_M) VALUES ('S1', 1395.0); +``` + +### Multi-hop survival rule + +A seed row must survive ALL the way to the terminal output tables, not just the first join. For each terminal table in the cross-file DAG: +1. Walk the dependency chain backward to the source tables it ultimately derives from. +2. Ensure at least one fully-aligned "golden path" row exists at every hop: matching equality keys AND overlapping range values at each join along the path. +3. This guarantees the Phase 5 terminal-output assertion (non-empty) can pass for a correctly-converted pipeline. + +Verification heuristic: pick one entity (e.g. `S1`) and ensure it has aligned rows in EVERY source table on the golden path, so it flows through every layer to the terminal tables. + +## SAS Missing Value Test Rows + +Every table MUST include at least one row that exercises SAS missing value semantics: + +| Column Type | Test Value | What It Tests | +|-------------|-----------|---------------| +| NUMBER | NULL | SAS `.` missing numeric → Snowflake NULL | +| VARCHAR | NULL | SAS character missing → Snowflake NULL | +| VARCHAR | `''` (empty string) | SAS blank character → Snowflake empty string | +| DATE | NULL | Missing date handling | +| NUMBER | 0 | Zero is NOT missing in SAS (common confusion) | + +### Missing Value Row Pattern + +```sql +-- Every table should have one row like: +INSERT INTO table_name VALUES + (99, NULL, '', NULL, 0); -- id, numeric_col, char_col, date_col, zero_col +``` + +## Boundary Value Test Rows + +Include at least one row per table with boundary/edge-case values: + +| Data Type | Boundary Values | Purpose | +|-----------|----------------|---------| +| NUMBER | 0, -1, -999.99 | Zero and negative (SAS treats missing < negative) | +| NUMBER | 999999.99 | Large value (overflow test) | +| VARCHAR | `''` (empty) | Empty string vs NULL distinction | +| VARCHAR | `'A'` (single char) | Minimum-length string | +| DATE | `'2024-01-01'` | Year boundary | +| DATE | `'2024-12-31'` | Year-end boundary | +| DATE | `'2024-02-29'` | Leap year (if date logic involves month arithmetic) | + +## Duplicate Key Rows + +For GROUP BY, FIRST./LAST., PROC SORT NODUPKEY, or any aggregation: + +| Pattern | Minimum Requirement | +|---------|---------------------| +| GROUP BY key | At least one key value with 2+ rows | +| FIRST./LAST. processing | At least one BY-group with 3+ rows (to test first, middle, last) | +| PROC SORT NODUPKEY | Duplicate rows that differ only in non-key columns | +| SAS MERGE BY | Same key in both tables with different row counts | + +### Example: BY-Group Test Data + +```sql +CREATE TEMPORARY TABLE transactions ( + customer_id NUMBER, txn_date DATE, amount NUMBER, txn_type VARCHAR +); +INSERT INTO transactions VALUES + (1, '2024-01-01', 100, 'DEBIT'), -- customer 1, first + (1, '2024-01-15', 200, 'CREDIT'), -- customer 1, middle + (1, '2024-02-01', 50, 'DEBIT'), -- customer 1, last + (2, '2024-01-10', 300, 'CREDIT'), -- customer 2, only row (first=last) + (3, '2024-03-01', NULL, 'DEBIT'), -- customer 3, missing amount + (3, '2024-03-15', 0, NULL); -- customer 3, zero amount + missing type +``` + +## RETAIN / Window Function Test Data + +When the SAS code uses RETAIN, LAG, or running totals: + +| Requirement | Rationale | +|-------------|-----------| +| At least 4 rows per partition | Need enough rows to verify running totals, LAG(n), LEAD(n) | +| Include partition boundary | At least 2 distinct partition values to test reset behavior | +| Include NULL in retained column | SAS RETAIN ignores missing values (keeps previous) | + +## PROC FORMAT / Lookup Table Test Data + +When SAS uses PUT(value, format.) or PROC FORMAT: + +| Requirement | Test Case | +|-------------|-----------| +| Value in range | At least one value that matches a defined format range | +| Value NOT in range | At least one value that falls through to OTHER= default | +| Boundary value | Value exactly at range boundary (e.g., low <= val < high) | +| Missing value | NULL input to test format handling of missing | + +## Multi-Block Test Data + +When validating code with multiple blocks (stored procedures with multiple steps): + +| Rule | Details | +|------|---------| +| Shared input tables | Create once, used by multiple blocks | +| Intermediate tables | Do NOT pre-create -- these are outputs of earlier blocks, inputs to later blocks | +| Cross-block variables | If session variables are used, set initial values before execution | +| Execution order | Execute blocks sequentially (stored procedure handles this) | + +## Data Generation Checklist + +Before finalizing synthetic data, verify: + +- [ ] Every JOIN key has: matching rows, left-only key, right-only key +- [ ] Every table has at least one row with NULL in numeric columns +- [ ] Every table has at least one row with NULL or empty string in character columns +- [ ] At least one date boundary value exists (year-start, year-end, or leap day) +- [ ] At least one duplicate key exists for GROUP BY / aggregation testing +- [ ] Zero and negative numbers are present for numeric columns +- [ ] For BY-group processing, at least one group has 3+ rows +- [ ] For RETAIN/LAG code, at least 2 partitions with 4+ rows each +- [ ] INSERT VALUES match the CREATE TABLE column order exactly +- [ ] No schema prefix on TEMPORARY table names +- [ ] Every RANGE join (BETWEEN / >=...<=) has ≥1 child row inside a parent interval (positive test) and ≥1 outside (negative test) +- [ ] All columns in the same physical-quantity family (depths, dates) seeded in ONE canonical range so cross-table joins overlap (no meters-vs-feet mismatch) +- [ ] At least one "golden path" entity has aligned rows in EVERY source table along its dependency chain so it survives multi-hop joins to the terminal output tables diff --git a/plugin/skills/migration/sas/convert-sas-to-snowflake/references/validation-execution.md b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/validation-execution.md new file mode 100644 index 0000000..efe040d --- /dev/null +++ b/plugin/skills/migration/sas/convert-sas-to-snowflake/references/validation-execution.md @@ -0,0 +1,279 @@ +# Validation Execution: Phases 3A, 3B, and 4 + +## When to Load + +Load this file at Step 7 (Phase 2: SAS Logic Trace — runs as part of Local LLM Tracing Validation) and Step 9 (Phase 4: Snowflake Execution and comparison). Referenced from SKILL.md Step 7b and `workflows/steps-8-10-post-conversion.md` Step 9. + +--- + +### Phase 3A: Generate Expected Baseline (SAS Logic Trace) + +This is the core validation capability. Manually trace the original SAS code against the synthetic data to produce expected output. + +#### 3A.1 Read Inputs + +1. Read the original SAS code (block by block) +2. Read the synthetic data from the pandas DataFrames (or from CSV files): + +```python +customers_df = pd.read_csv('tests/