From a14d887f4ba6588efdb24b420156f312521bb0c9 Mon Sep 17 00:00:00 2001 From: Ashleigh Walker Date: Mon, 31 Aug 2026 04:02:51 -0500 Subject: [PATCH 1/9] feat: add dataform assertion generator skill, financial audit product, and CI test suite - Add dataform-assertion-generator agent skill and /generate-assertions lifecycle command - Add SAP financial_audit_and_compliance data product for S/4HANA (ledger reconciliation and forensic SOX entry detection) - Add GitHub Actions CI workflow for unit tests and validation - Add test_skill_anatomy.py to ensure 100% agent skill schema conformance - Fix cross-platform path resolution in test_builder_resolution.py --- .agents/AGENTS.md | 1 + .../custom/.keep | 0 .../skills/create_data_product/custom/.keep | 0 .../skills/create_python_tests/custom/.keep | 0 .agents/skills/create_skill/custom/.keep | 0 .../data_modeling_standards/custom/.keep | 0 .../dataform_assertion_generator/SKILL.md | 51 +++++++ .../assets/assertion_template.sqlx | 29 ++++ .../dataform_assertion_generator/custom/.keep | 0 .../references/assertion_patterns.md | 115 +++++++++++++++ .../skills/generate_er_diagram/custom/.keep | 0 .agents/skills/query_sap_ddic/custom/.keep | 0 .../skills/setup_cortex_config/custom/.keep | 0 .../skills/update_data_product/custom/.keep | 0 .agents/skills/using_cortex_skills/SKILL.md | 2 + .../skills/using_cortex_skills/custom/.keep | 0 .../skills/validate_data_product/custom/.keep | 0 .github/workflows/ci.yaml | 48 +++++++ .../financial_audit_and_compliance/README.md | 41 ++++++ .../s4/financial_ledger_reconciliation.yaml | 34 +++++ .../s4/suspicious_journal_entries.yaml | 58 ++++++++ .../s4/financial_ledger_reconciliation.js | 110 ++++++++++++++ .../s4/suspicious_journal_entries.js | 136 ++++++++++++++++++ .../manifest.yaml | 17 +++ .../table_settings.default.yaml | 21 +++ .../common/skills/test_skill_anatomy.py | 66 +++++++++ .../validation/test_builder_resolution.py | 2 +- 27 files changed, 730 insertions(+), 1 deletion(-) create mode 100644 .agents/skills/build_and_deploy_data_product/custom/.keep create mode 100644 .agents/skills/create_data_product/custom/.keep create mode 100644 .agents/skills/create_python_tests/custom/.keep create mode 100644 .agents/skills/create_skill/custom/.keep create mode 100644 .agents/skills/data_modeling_standards/custom/.keep create mode 100644 .agents/skills/dataform_assertion_generator/SKILL.md create mode 100644 .agents/skills/dataform_assertion_generator/assets/assertion_template.sqlx create mode 100644 .agents/skills/dataform_assertion_generator/custom/.keep create mode 100644 .agents/skills/dataform_assertion_generator/references/assertion_patterns.md create mode 100644 .agents/skills/generate_er_diagram/custom/.keep create mode 100644 .agents/skills/query_sap_ddic/custom/.keep create mode 100644 .agents/skills/setup_cortex_config/custom/.keep create mode 100644 .agents/skills/update_data_product/custom/.keep create mode 100644 .agents/skills/using_cortex_skills/custom/.keep create mode 100644 .agents/skills/validate_data_product/custom/.keep create mode 100644 .github/workflows/ci.yaml create mode 100644 src/data_modules/cortex/sap/products/financial_audit_and_compliance/README.md create mode 100644 src/data_modules/cortex/sap/products/financial_audit_and_compliance/annotations/s4/financial_ledger_reconciliation.yaml create mode 100644 src/data_modules/cortex/sap/products/financial_audit_and_compliance/annotations/s4/suspicious_journal_entries.yaml create mode 100644 src/data_modules/cortex/sap/products/financial_audit_and_compliance/definitions/s4/financial_ledger_reconciliation.js create mode 100644 src/data_modules/cortex/sap/products/financial_audit_and_compliance/definitions/s4/suspicious_journal_entries.js create mode 100644 src/data_modules/cortex/sap/products/financial_audit_and_compliance/manifest.yaml create mode 100644 src/data_modules/cortex/sap/products/financial_audit_and_compliance/table_settings.default.yaml create mode 100644 tests/external/common/skills/test_skill_anatomy.py diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index 3a46878..3e55299 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -18,6 +18,7 @@ When the user issues any of the following commands, you must immediately load th | `/build-and-deploy` | [build_and_deploy_data_product](file:///.agents/skills/build_and_deploy_data_product/SKILL.md) | Syncs environment variables, identifies configuration profiles, builds Dataform models, and deploys data products. | | `/query-sap-ddic` | [query_sap_ddic](file:///.agents/skills/query_sap_ddic/SKILL.md) | Inspects and dumps SAP table schemas directly from replicated SAP DDIC tables in BigQuery. | | `/generate-er-diagram` | [generate_er_diagram](file:///.agents/skills/generate_er_diagram/SKILL.md) | Extracts schema definitions, automatically infers entity relationships via SAP field suffixes, and generates visual ERDs. | +| `/generate-assertions` | [dataform_assertion_generator](file:///.agents/skills/dataform_assertion_generator/SKILL.md) | Inspects data products and generates automated Dataform data-integrity assertions and ledger reconciliation checks. | | `/create-skill` | [create_skill](file:///.agents/skills/create_skill/SKILL.md) | Evaluates overlap, scaffolds new skills or custom folder overrides, authors instructions, and validates anatomy. | --- diff --git a/.agents/skills/build_and_deploy_data_product/custom/.keep b/.agents/skills/build_and_deploy_data_product/custom/.keep new file mode 100644 index 0000000..e69de29 diff --git a/.agents/skills/create_data_product/custom/.keep b/.agents/skills/create_data_product/custom/.keep new file mode 100644 index 0000000..e69de29 diff --git a/.agents/skills/create_python_tests/custom/.keep b/.agents/skills/create_python_tests/custom/.keep new file mode 100644 index 0000000..e69de29 diff --git a/.agents/skills/create_skill/custom/.keep b/.agents/skills/create_skill/custom/.keep new file mode 100644 index 0000000..e69de29 diff --git a/.agents/skills/data_modeling_standards/custom/.keep b/.agents/skills/data_modeling_standards/custom/.keep new file mode 100644 index 0000000..e69de29 diff --git a/.agents/skills/dataform_assertion_generator/SKILL.md b/.agents/skills/dataform_assertion_generator/SKILL.md new file mode 100644 index 0000000..f70dbe2 --- /dev/null +++ b/.agents/skills/dataform_assertion_generator/SKILL.md @@ -0,0 +1,51 @@ +--- +name: dataform-assertion-generator +description: Inspects Cortex Framework data products and scaffolds Dataform data quality and business integrity assertions including primary key uniqueness, non-null checks, foreign key referential integrity, and ledger reconciliation checks. +--- + +# Dataform Assertion Generator Skill + +This skill guides the inspection, design, scaffolding, and validation of automated Dataform data quality and integrity assertions for Cortex Framework V7 data products. + +--- + +## Operational Rules & Quality Gates + +1. **Assertion Contract:** In Dataform, an assertion query is considered **failing** if it returns one or more rows. All assertion SQL queries MUST be authored such that they return zero rows when the data is valid and return violating rows when data integrity is breached. +2. **Namespace Isolation:** Never hardcode dataset or table names. Always resolve references dynamically using `ctx.ref(moduleConfig.sources..datasetId, "")` or `${ref("")}`. +3. **Link Integrity:** All relative markdown links referenced in this skill must resolve to real files on disk. +4. **Custom Overrides First:** Check the `custom/` folder before generating assertions. If project-specific compliance rules or custom thresholds exist, merge and prioritize them. + +--- + +## Workflow Steps + +### Step 1: Target Product & Key Identification +1. Read the target data product's `manifest.yaml` and `table_settings.default.yaml` under `src/data_modules///products//`. +2. Inspect the table definitions in `definitions/` to identify: + - Primary key candidate columns (e.g. `client_mandt`, `document_number_vbeln`, `item_number_posnr`). + - Critical non-nullable columns (e.g. `creation_date`, `currency_key`, `amount`). + - Foreign key relationship bounds (e.g. line items pointing to existing header records). + - Domain-specific financial/accounting invariants (e.g. total debits equal total credits). + +### Step 2: Scaffold Assertion Definitions +Select the appropriate assertion pattern based on the target validation rule (refer to [assertion_patterns.md](references/assertion_patterns.md)): + +1. **Unique Key Constraint Assertion:** + Checks that candidate primary keys are unique across the dataset. +2. **Non-Null Invariant Assertion:** + Ensures mandatory business fields are never null. +3. **Referential Integrity (Orphan Detection) Assertion:** + Detects child line items that reference non-existent parent headers. +4. **Financial Invariant (Ledger Balance) Assertion:** + Verifies that total ledger debits and credits balance to zero per fiscal period. + +### Step 3: Write Assertion Files +Save generated assertions into `src/data_modules///products//definitions/assertions/` or the appropriate Dataform definitions directory. Use the `.sqlx` template from [assertion_template.sqlx](assets/assertion_template.sqlx). + +### Step 4: Verification & Validation Gate +1. Validate the syntax of generated assertions against BigQuery ZetaSQL dialect. +2. Run unit tests to confirm manifest and dependency integrity: + ```bash + uv run pytest tests/external/common/validation/ -q + ``` diff --git a/.agents/skills/dataform_assertion_generator/assets/assertion_template.sqlx b/.agents/skills/dataform_assertion_generator/assets/assertion_template.sqlx new file mode 100644 index 0000000..8b2ea1a --- /dev/null +++ b/.agents/skills/dataform_assertion_generator/assets/assertion_template.sqlx @@ -0,0 +1,29 @@ +config { + type: "assertion", + description: "Validates primary key uniqueness and referential integrity for the target data product" +} + +/* + Dataform Assertion Rule: + Any rows returned by this query represent failing validation records. + A successful assertion returns 0 rows. +*/ + +WITH validation_errors AS ( + SELECT + client_mandt, + document_number_vbeln, + COUNT(*) AS row_count + FROM + ${ref("sales_document_headers")} + GROUP BY + client_mandt, + document_number_vbeln + HAVING + COUNT(*) > 1 +) + +SELECT + * +FROM + validation_errors diff --git a/.agents/skills/dataform_assertion_generator/custom/.keep b/.agents/skills/dataform_assertion_generator/custom/.keep new file mode 100644 index 0000000..e69de29 diff --git a/.agents/skills/dataform_assertion_generator/references/assertion_patterns.md b/.agents/skills/dataform_assertion_generator/references/assertion_patterns.md new file mode 100644 index 0000000..b0eae48 --- /dev/null +++ b/.agents/skills/dataform_assertion_generator/references/assertion_patterns.md @@ -0,0 +1,115 @@ +# Dataform Assertion Patterns in Cortex Framework + +This reference document outlines the standard assertion design patterns supported in Google Cloud Cortex Framework v7. + +--- + +## 1. Core Assertion Concepts + +In Google Cloud Dataform, an **assertion** is a query that checks for data quality issues. +* **Pass condition:** The assertion query returns `0` rows. +* **Fail condition:** The assertion query returns `1 or more` rows (these rows are treated as defect records). + +--- + +## 2. Standard Pattern Library + +### Pattern A: Multi-Column Composite Unique Key +Validates that composite primary keys (e.g., client, company code, document number, item number) are unique. + +```sql +config { + type: "assertion" +} + +SELECT + client_mandt, + company_code_bukrs, + accounting_document_belnr, + fiscal_year_gjahr, + line_item_buzei, + COUNT(*) AS duplicate_instances +FROM + ${ref("universal_journal_entry_line_items")} +GROUP BY + client_mandt, + company_code_bukrs, + accounting_document_belnr, + fiscal_year_gjahr, + line_item_buzei +HAVING + COUNT(*) > 1 +``` + +--- + +### Pattern B: Non-Null Mandatory Attributes +Validates that essential business dimensions and foreign keys are never null. + +```sql +config { + type: "assertion" +} + +SELECT + * +FROM + ${ref("sales_document_headers")} +WHERE + client_mandt IS NULL + OR document_number_vbeln IS NULL + OR creation_date_erdat IS NULL +``` + +--- + +### Pattern C: Parent-Child Referential Integrity (Orphan Detection) +Validates that child line items have a corresponding parent document. + +```sql +config { + type: "assertion" +} + +SELECT + item.client_mandt, + item.document_number_vbeln, + item.item_number_posnr +FROM + ${ref("sales_document_items")} AS item +LEFT JOIN + ${ref("sales_document_headers")} AS header + ON item.client_mandt = header.client_mandt + AND item.document_number_vbeln = header.document_number_vbeln +WHERE + header.document_number_vbeln IS NULL +``` + +--- + +### Pattern D: Financial Zero-Balance Ledger Reconciliation +In double-entry bookkeeping (SAP Universal Journal `ACDOCA`), the sum of debits and credits for any posted accounting document must equal zero. + +```sql +config { + type: "assertion" +} + +SELECT + client_mandt, + company_code_bukrs, + accounting_document_belnr, + fiscal_year_gjahr, + ledger_rldnr, + ROUND(SUM(amount_in_company_code_currency_dmbtr), 2) AS ledger_imbalance +FROM + ${ref("universal_journal_entry_line_items")} +GROUP BY + client_mandt, + company_code_bukrs, + accounting_document_belnr, + fiscal_year_gjahr, + ledger_rldnr +HAVING + ABS(ROUND(SUM(amount_in_company_code_currency_dmbtr), 2)) > 0.05 +``` diff --git a/.agents/skills/generate_er_diagram/custom/.keep b/.agents/skills/generate_er_diagram/custom/.keep new file mode 100644 index 0000000..e69de29 diff --git a/.agents/skills/query_sap_ddic/custom/.keep b/.agents/skills/query_sap_ddic/custom/.keep new file mode 100644 index 0000000..e69de29 diff --git a/.agents/skills/setup_cortex_config/custom/.keep b/.agents/skills/setup_cortex_config/custom/.keep new file mode 100644 index 0000000..e69de29 diff --git a/.agents/skills/update_data_product/custom/.keep b/.agents/skills/update_data_product/custom/.keep new file mode 100644 index 0000000..e69de29 diff --git a/.agents/skills/using_cortex_skills/SKILL.md b/.agents/skills/using_cortex_skills/SKILL.md index 29d282c..5cbdf78 100644 --- a/.agents/skills/using_cortex_skills/SKILL.md +++ b/.agents/skills/using_cortex_skills/SKILL.md @@ -30,6 +30,8 @@ Task arrives │ ├── Asked to generate ER diagrams or data model graphs? ────→ generate-er-diagram │ + ├── Asked to generate data quality/integrity assertions? ──→ dataform-assertion-generator + │ ├── Asked to create/scaffold a new skill or custom rules? ─→ create-skill │ └── Asked to align/review model against business rules? ────→ data-modeling-standards diff --git a/.agents/skills/using_cortex_skills/custom/.keep b/.agents/skills/using_cortex_skills/custom/.keep new file mode 100644 index 0000000..e69de29 diff --git a/.agents/skills/validate_data_product/custom/.keep b/.agents/skills/validate_data_product/custom/.keep new file mode 100644 index 0000000..e69de29 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..c1836e9 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,48 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: CI Unit & Validation Suite + +on: + push: + branches: [ main, feat/** ] + pull_request: + branches: [ main ] + workflow_dispatch: + +jobs: + validate-and-test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + version: "latest" + + - name: Install dependencies + run: uv sync + + - name: Run Ruff Linting + run: uv run ruff check src tests + + - name: Run Unit & Validation Tests + run: uv run pytest tests/common tests/external/common -q diff --git a/src/data_modules/cortex/sap/products/financial_audit_and_compliance/README.md b/src/data_modules/cortex/sap/products/financial_audit_and_compliance/README.md new file mode 100644 index 0000000..f93659b --- /dev/null +++ b/src/data_modules/cortex/sap/products/financial_audit_and_compliance/README.md @@ -0,0 +1,41 @@ +# SAP Financial Audit & Compliance Data Product + +The **Financial Audit & Compliance** data product provides automated general ledger reconciliation and forensic journal anomaly detection for SAP S/4HANA financial environments in BigQuery. + +--- + +## Overview + +Enterprise financial audits require continuous reconciliation of general ledger balances and rapid detection of high-risk journal entries. This data product transforms raw SAP `ACDOCA`, `BKPF`, and `BSEG` transactions into AI-ready, audit-compliant analytical tables. + +### Key Capabilities +* **Automated Ledger Reconciliation:** Calculates net ledger balance per document, verifying that total debits equal total credits with exact `TCURX` decimal shifting. +* **Forensic Anomaly Detection:** Flags suspicious transactions based on SOX compliance rules: + - Weekend or off-hours manual postings. + - Large round-dollar manual entries ($\ge \$10,000$). + - Entries missing external reference documentation. + - Backdated or forward-dated transactions (> 30-day posting discrepancy). + +--- + +## Data Models + +| Table Name | Description | Source SAP Tables | +| :--- | :--- | :--- | +| `financial_ledger_reconciliation` | Reconciles debits and credits across SAP S/4HANA Universal Journal line items. | `ACDOCA`, `TCURX` | +| `suspicious_journal_entries` | Identifies anomalous journal entries with forensic accounting flags. | `BKPF`, `BSEG`, `TCURX` | + +--- + +## Deployment & Configuration + +Configure this product in `config/config.yaml` under `data.products`: + +```yaml +data: + products: + - namespace: cortex + source: sap + type: financial_audit_and_compliance + target: product_target +``` diff --git a/src/data_modules/cortex/sap/products/financial_audit_and_compliance/annotations/s4/financial_ledger_reconciliation.yaml b/src/data_modules/cortex/sap/products/financial_audit_and_compliance/annotations/s4/financial_ledger_reconciliation.yaml new file mode 100644 index 0000000..f250dda --- /dev/null +++ b/src/data_modules/cortex/sap/products/financial_audit_and_compliance/annotations/s4/financial_ledger_reconciliation.yaml @@ -0,0 +1,34 @@ +description: Reconciles financial debits and credits across SAP S/4HANA Universal Journal (ACDOCA) records, identifying balanced and imbalanced document postings per company code, fiscal year, and ledger. +fields: + - name: client_mandt + description: Client (Mandant) identifier in SAP S/4HANA, PK. + - name: company_code_bukrs + description: Company Code representing the legal reporting entity, PK. + - name: document_number_belnr + description: Accounting document number, PK. + - name: fiscal_year_gjahr + description: Fiscal year of the posted journal transaction, PK. + - name: ledger_rldnr + description: General Ledger or Extension Ledger identifier, PK. + - name: document_type_blart + description: Accounting document type indicating the business transaction class. + - name: posting_date_budat + description: Date the transaction was posted to the general ledger. + - name: document_date_bldat + description: Date on the originating commercial invoice or document. + - name: entry_date_cpudt + description: System timestamp date when the entry was physically keyed into SAP. + - name: user_name_usnam + description: SAP user ID that keyed or executed the posting. + - name: company_code_currency_rhcur + description: Local currency key for the company code entity. + - name: total_line_items_count + description: Total count of distinct line items comprising the accounting document. + - name: total_debit_amount_in_company_currency + description: Aggregate sum of all debit line item amounts in company currency. + - name: total_credit_amount_in_company_currency + description: Aggregate sum of all credit line item amounts in company currency. + - name: net_ledger_balance_amount + description: Net financial balance (Debits minus Credits); must equal zero in double-entry bookkeeping. + - name: reconciliation_status + description: Audit compliance status (BALANCED when net variance is under 0.01, otherwise IMBALANCED). diff --git a/src/data_modules/cortex/sap/products/financial_audit_and_compliance/annotations/s4/suspicious_journal_entries.yaml b/src/data_modules/cortex/sap/products/financial_audit_and_compliance/annotations/s4/suspicious_journal_entries.yaml new file mode 100644 index 0000000..032d0df --- /dev/null +++ b/src/data_modules/cortex/sap/products/financial_audit_and_compliance/annotations/s4/suspicious_journal_entries.yaml @@ -0,0 +1,58 @@ +description: Identifies anomalous and high-risk SAP journal entries from BKPF and BSEG tables based on forensic accounting indicators such as weekend postings, round-sum amounts, missing reference documents, and out-of-period entries. +fields: + - name: client_mandt + description: Client (Mandant) identifier in SAP S/4HANA, PK. + - name: company_code_bukrs + description: Company Code representing the legal reporting entity, PK. + - name: document_number_belnr + description: Accounting document number, PK. + - name: fiscal_year_gjahr + description: Fiscal year of the posted transaction, PK. + - name: line_item_buzei + description: Number of line item within the accounting document, PK. + - name: document_type_blart + description: Accounting document type. + - name: document_date_bldat + description: Document date on originating commercial invoice or journal. + - name: posting_date_budat + description: General ledger posting date. + - name: posting_period_monat + description: Fiscal posting period (month). + - name: entry_date_cpudt + description: Physical system entry date. + - name: time_of_entry_cputm + description: System entry time. + - name: entered_by_usnam + description: User name of the person who entered the document. + - name: transaction_code_tcode + description: SAP transaction code used to create the document. + - name: reference_document_xblnr + description: Reference document number provided during manual entry. + - name: document_header_text_bktxt + description: Free text header memo. + - name: posting_key_bschl + description: SAP posting key determining debit/credit and account type. + - name: debit_credit_shkzg + description: Debit/credit indicator (S = Debit, H = Credit). + - name: general_ledger_account_hkont + description: General ledger account number affected by this line item. + - name: amount_in_local_currency_dmbtr + description: Transaction amount in local company code currency, decimal-shifted via TCURX. + - name: currency_key_waers + description: Currency key of the posted amount. + - name: cost_center_kostl + description: Controlling cost center. + - name: profit_center_prctr + description: Controlling profit center. + - name: item_text_sgtxt + description: Line item explanation text. + - name: is_weekend_entry + description: Boolean flag indicating if entry was keyed on a Saturday or Sunday. + - name: is_round_sum_entry + description: Boolean flag indicating if entry amount is an exact round thousand. + - name: is_missing_reference_document + description: Boolean flag indicating missing external invoice/reference documentation. + - name: is_out_of_period_entry + description: Boolean flag indicating discrepancy between physical entry date and ledger posting date. + - name: audit_risk_classification + description: High-level risk category (HIGH_RISK_AUDIT_FLAG or STANDARD_POSTING). diff --git a/src/data_modules/cortex/sap/products/financial_audit_and_compliance/definitions/s4/financial_ledger_reconciliation.js b/src/data_modules/cortex/sap/products/financial_audit_and_compliance/definitions/s4/financial_ledger_reconciliation.js new file mode 100644 index 0000000..fb69413 --- /dev/null +++ b/src/data_modules/cortex/sap/products/financial_audit_and_compliance/definitions/s4/financial_ledger_reconciliation.js @@ -0,0 +1,110 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ___MODULE_CONTEXT___ +// ___TABLE_CONFIG___ + +const moduleConfig = config.product[moduleContext.moduleId]; +const materializationType = tableConfig.materializationType || "incremental"; +const currency = require("includes/currency.js"); +const date = require("includes/date.js"); +const incremental = require("includes/incremental.js"); +const publish_config = require("includes/publish_config.js"); +const sql_helper = require("includes/sql_helper.js"); + +const publishConfig = publish_config.getPublishConfig( + materializationType, + tableConfig, + moduleConfig, + [ + "client_mandt", + "company_code_bukrs", + "document_number_belnr", + "fiscal_year_gjahr", + "ledger_rldnr" + ] +); + +publish(moduleContext.moduleId + "_" + tableConfig.tableName, publishConfig).query( + (ctx) => ` +WITH date_dimension AS ( + ${date.getDateDimension()} +), +currency_decimal AS ( + ${currency.currencyDecimalShift(ctx.ref(moduleConfig.sources.sapModule.datasetId, "tcurx"))} +), +line_items_shifted AS ( + SELECT + acdoca.mandt AS client_mandt, + acdoca.rldnr AS ledger_rldnr, + acdoca.rbukrs AS company_code_bukrs, + acdoca.gjahr AS fiscal_year_gjahr, + acdoca.belnr AS document_number_belnr, + acdoca.docln AS line_item_docln, + acdoca.budat AS posting_date_budat, + acdoca.bldat AS document_date_bldat, + acdoca.cpudt AS entry_date_cpudt, + acdoca.usnam AS user_name_usnam, + acdoca.blart AS document_type_blart, + acdoca.rwcur AS transaction_currency_rwcur, + acdoca.rhcur AS company_code_currency_rhcur, + acdoca.wsl * COALESCE(curr_trans.currfix, 1.0) AS amount_in_transaction_currency_wsl, + acdoca.hsl * COALESCE(curr_comp.currfix, 1.0) AS amount_in_company_code_currency_hsl, + acdoca.drcrk AS debit_credit_indicator_drcrk, + acdoca.racct AS account_number_racct, + acdoca.rcntr AS cost_center_rcntr, + acdoca.prctr AS profit_center_prctr, + acdoca.bttype AS business_transaction_type_bttype + FROM + ${ctx.ref(moduleConfig.sources.sapModule.datasetId, "acdoca")} AS acdoca + LEFT JOIN + currency_decimal AS curr_trans + ON acdoca.rwcur = curr_trans.currkey + LEFT JOIN + currency_decimal AS curr_comp + ON acdoca.rhcur = curr_comp.currkey + ${incremental.filter(materializationType, "acdoca.recordstamp")} +) +SELECT + line.client_mandt, + line.company_code_bukrs, + line.document_number_belnr, + line.fiscal_year_gjahr, + line.ledger_rldnr, + MAX(line.document_type_blart) AS document_type_blart, + MAX(line.posting_date_budat) AS posting_date_budat, + MAX(line.document_date_bldat) AS document_date_bldat, + MAX(line.entry_date_cpudt) AS entry_date_cpudt, + MAX(line.user_name_usnam) AS user_name_usnam, + MAX(line.company_code_currency_rhcur) AS company_code_currency_rhcur, + COUNT(DISTINCT line.line_item_docln) AS total_line_items_count, + SUM(CASE WHEN line.debit_credit_indicator_drcrk = 'S' THEN line.amount_in_company_code_currency_hsl ELSE 0 END) AS total_debit_amount_in_company_currency, + SUM(CASE WHEN line.debit_credit_indicator_drcrk = 'H' THEN line.amount_in_company_code_currency_hsl ELSE 0 END) AS total_credit_amount_in_company_currency, + SUM(line.amount_in_company_code_currency_hsl) AS net_ledger_balance_amount, + CASE + WHEN ABS(SUM(line.amount_in_company_code_currency_hsl)) < 0.01 THEN 'BALANCED' + ELSE 'IMBALANCED' + END AS reconciliation_status +FROM + line_items_shifted AS line +GROUP BY + line.client_mandt, + line.company_code_bukrs, + line.document_number_belnr, + line.fiscal_year_gjahr, + line.ledger_rldnr +` +); diff --git a/src/data_modules/cortex/sap/products/financial_audit_and_compliance/definitions/s4/suspicious_journal_entries.js b/src/data_modules/cortex/sap/products/financial_audit_and_compliance/definitions/s4/suspicious_journal_entries.js new file mode 100644 index 0000000..78a5277 --- /dev/null +++ b/src/data_modules/cortex/sap/products/financial_audit_and_compliance/definitions/s4/suspicious_journal_entries.js @@ -0,0 +1,136 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ___MODULE_CONTEXT___ +// ___TABLE_CONFIG___ + +const moduleConfig = config.product[moduleContext.moduleId]; +const materializationType = tableConfig.materializationType || "incremental"; +const currency = require("includes/currency.js"); +const date = require("includes/date.js"); +const incremental = require("includes/incremental.js"); +const publish_config = require("includes/publish_config.js"); +const sql_helper = require("includes/sql_helper.js"); + +const publishConfig = publish_config.getPublishConfig( + materializationType, + tableConfig, + moduleConfig, + [ + "client_mandt", + "company_code_bukrs", + "document_number_belnr", + "fiscal_year_gjahr", + "line_item_buzei" + ] +); + +publish(moduleContext.moduleId + "_" + tableConfig.tableName, publishConfig).query( + (ctx) => ` +WITH date_dimension AS ( + ${date.getDateDimension()} +), +currency_decimal AS ( + ${currency.currencyDecimalShift(ctx.ref(moduleConfig.sources.sapModule.datasetId, "tcurx"))} +), +raw_journal_entries AS ( + SELECT + bkpf.mandt AS client_mandt, + bkpf.bukrs AS company_code_bukrs, + bkpf.belnr AS document_number_belnr, + bkpf.gjahr AS fiscal_year_gjahr, + bseg.buzei AS line_item_buzei, + bkpf.blart AS document_type_blart, + bkpf.bldat AS document_date_bldat, + bkpf.budat AS posting_date_budat, + bkpf.monat AS posting_period_monat, + bkpf.cpudt AS entry_date_cpudt, + bkpf.cputm AS time_of_entry_cputm, + bkpf.usnam AS entered_by_usnam, + bkpf.tcode AS transaction_code_tcode, + bkpf.xblnr AS reference_document_xblnr, + bkpf.bktxt AS document_header_text_bktxt, + bseg.bschl AS posting_key_bschl, + bseg.shkzg AS debit_credit_shkzg, + bseg.hkont AS general_ledger_account_hkont, + bseg.dmbtr * COALESCE(curr_comp.currfix, 1.0) AS amount_in_local_currency_dmbtr, + bkpf.waers AS currency_key_waers, + bseg.kostl AS cost_center_kostl, + bseg.prctr AS profit_center_prctr, + bseg.sgtxt AS item_text_sgtxt + FROM + ${ctx.ref(moduleConfig.sources.sapModule.datasetId, "bkpf")} AS bkpf + INNER JOIN + ${ctx.ref(moduleConfig.sources.sapModule.datasetId, "bseg")} AS bseg + ON bkpf.mandt = bseg.mandt + AND bkpf.bukrs = bseg.bukrs + AND bkpf.belnr = bseg.belnr + AND bkpf.gjahr = bseg.gjahr + LEFT JOIN + currency_decimal AS curr_comp + ON bkpf.waers = curr_comp.currkey + ${incremental.filter(materializationType, "bkpf.recordstamp")} +), +flagged_entries AS ( + SELECT + entry.*, + EXTRACT(DAYOFWEEK FROM entry.entry_date_cpudt) IN (1, 7) AS is_weekend_entry, + (MOD(CAST(ROUND(entry.amount_in_local_currency_dmbtr, 2) AS INT64), 1000) = 0 AND entry.amount_in_local_currency_dmbtr >= 10000) AS is_round_sum_entry, + (entry.reference_document_xblnr IS NULL OR TRIM(entry.reference_document_xblnr) = '') AS is_missing_reference_document, + (DATE_DIFF(entry.posting_date_budat, entry.entry_date_cpudt, DAY) > 30 OR DATE_DIFF(entry.entry_date_cpudt, entry.posting_date_budat, DAY) > 30) AS is_out_of_period_entry + FROM + raw_journal_entries AS entry +) +SELECT + flagged.client_mandt, + flagged.company_code_bukrs, + flagged.document_number_belnr, + flagged.fiscal_year_gjahr, + flagged.line_item_buzei, + flagged.document_type_blart, + flagged.document_date_bldat, + flagged.posting_date_budat, + flagged.posting_period_monat, + flagged.entry_date_cpudt, + flagged.time_of_entry_cputm, + flagged.entered_by_usnam, + flagged.transaction_code_tcode, + flagged.reference_document_xblnr, + flagged.document_header_text_bktxt, + flagged.posting_key_bschl, + flagged.debit_credit_shkzg, + flagged.general_ledger_account_hkont, + flagged.amount_in_local_currency_dmbtr, + flagged.currency_key_waers, + flagged.cost_center_kostl, + flagged.profit_center_prctr, + flagged.item_text_sgtxt, + flagged.is_weekend_entry, + flagged.is_round_sum_entry, + flagged.is_missing_reference_document, + flagged.is_out_of_period_entry, + CASE + WHEN flagged.is_weekend_entry + OR flagged.is_round_sum_entry + OR flagged.is_missing_reference_document + OR flagged.is_out_of_period_entry + THEN 'HIGH_RISK_AUDIT_FLAG' + ELSE 'STANDARD_POSTING' + END AS audit_risk_classification +FROM + flagged_entries AS flagged +` +); diff --git a/src/data_modules/cortex/sap/products/financial_audit_and_compliance/manifest.yaml b/src/data_modules/cortex/sap/products/financial_audit_and_compliance/manifest.yaml new file mode 100644 index 0000000..abbbd54 --- /dev/null +++ b/src/data_modules/cortex/sap/products/financial_audit_and_compliance/manifest.yaml @@ -0,0 +1,17 @@ +displayName: SAP Financial Audit & Compliance +description: This data product provides enterprise financial audit, ledger reconciliation, and SOX compliance models from the SAP Financial Accounting (FI) and Controlling (CO) modules in SAP S/4HANA. It isolates anomalous manual journal postings, detects potential fraud and out-of-period entries, and performs automated debit/credit balancing across company codes and fiscal periods. +category: source_aligned_product +type: financial_audit_and_compliance +dependencies: + sapModule: + supportedVersions: + - s4 + tables: + s4: + - acdoca + - bkpf + - bseg + common: + - tcurx + modulePath: cortex.sap.foundations.sap +builder: sap_product diff --git a/src/data_modules/cortex/sap/products/financial_audit_and_compliance/table_settings.default.yaml b/src/data_modules/cortex/sap/products/financial_audit_and_compliance/table_settings.default.yaml new file mode 100644 index 0000000..175fee4 --- /dev/null +++ b/src/data_modules/cortex/sap/products/financial_audit_and_compliance/table_settings.default.yaml @@ -0,0 +1,21 @@ +s4: + financial_ledger_reconciliation: + materializationType: incremental + bigQueryLabels: + - key: line_of_business + value: finance + - key: sap_module + value: sap_fi + - key: data_class + value: analytical + dataformTags: [sap, source_aligned_product, finance, sap_fi, audit, compliance, daily] + suspicious_journal_entries: + materializationType: incremental + bigQueryLabels: + - key: line_of_business + value: finance + - key: sap_module + value: sap_fi + - key: data_class + value: analytical + dataformTags: [sap, source_aligned_product, finance, sap_fi, fraud, audit, daily] diff --git a/tests/external/common/skills/test_skill_anatomy.py b/tests/external/common/skills/test_skill_anatomy.py new file mode 100644 index 0000000..435ad44 --- /dev/null +++ b/tests/external/common/skills/test_skill_anatomy.py @@ -0,0 +1,66 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pathlib +import pytest +import yaml + + +def test_skill_anatomy(repo_root: pathlib.Path): + skills_dir = repo_root / ".agents" / "skills" + if not skills_dir.exists(): + pytest.skip("No skills directory found.") + + errors = [] + for skill_path in skills_dir.iterdir(): + if not skill_path.is_dir(): + continue + + skill_name = skill_path.name + skill_md = skill_path / "SKILL.md" + if not skill_md.exists(): + errors.append(f"Missing SKILL.md in {skill_name}") + continue + + custom_dir = skill_path / "custom" + if not custom_dir.exists(): + errors.append(f"Missing custom/ directory in {skill_name}") + + text = skill_md.read_text(encoding="utf-8").lstrip("\ufeff") + if not text.startswith("---"): + errors.append(f"Missing opening frontmatter in {skill_name}") + continue + + parts = text.split("---", 2) + if len(parts) < 3: + errors.append(f"Malformed frontmatter in {skill_name}") + continue + + try: + data = yaml.safe_load(parts[1]) or {} + except Exception as e: + errors.append(f"Failed to parse YAML frontmatter in {skill_name}: {e}") + continue + + name = data.get("name", "") + desc = data.get("description", "") + if not name: + errors.append(f"Missing 'name' in frontmatter for {skill_name}") + if not desc: + errors.append(f"Missing 'description' in frontmatter for {skill_name}") + elif len(str(desc)) > 1024: + errors.append(f"Description too long in {skill_name} ({len(str(desc))} > 1024)") + + if errors: + pytest.fail("\n".join(errors)) diff --git a/tests/external/common/validation/test_builder_resolution.py b/tests/external/common/validation/test_builder_resolution.py index 3a2ca95..a29c885 100644 --- a/tests/external/common/validation/test_builder_resolution.py +++ b/tests/external/common/validation/test_builder_resolution.py @@ -57,7 +57,7 @@ def test_builder_resolution_integrity(repo_root: pathlib.Path): local_builder_path = module_dir / "builder.py" if local_builder_path.exists(): rel_path = module_dir.relative_to(src_dir) - local_module_path = f"{str(rel_path).replace('/', '.')}.builder" + local_module_path = f"{rel_path.as_posix().replace('/', '.')}.builder" try: # Dynamically import the local builder module module = importlib.import_module(local_module_path) From 4b143c9505da72a3ed6d75faa17d1fba9549f3eb Mon Sep 17 00:00:00 2001 From: Ashleigh Walker Date: Mon, 31 Aug 2026 13:21:56 -0500 Subject: [PATCH 2/9] feat(mcp): add cortex-mcp server and agent grounding schema provider --- pyproject.toml | 1 + src/common/mcp/__init__.py | 19 ++ src/common/mcp/query_generator.py | 78 +++++++ src/common/mcp/schema_provider.py | 134 ++++++++++++ src/tools/mcp_server.py | 251 +++++++++++++++++++++++ tests/common/services/test_mcp_server.py | 115 +++++++++++ 6 files changed, 598 insertions(+) create mode 100644 src/common/mcp/__init__.py create mode 100644 src/common/mcp/query_generator.py create mode 100644 src/common/mcp/schema_provider.py create mode 100644 src/tools/mcp_server.py create mode 100644 tests/common/services/test_mcp_server.py diff --git a/pyproject.toml b/pyproject.toml index 8c00a7c..6b85645 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ cortex-build-and-deploy = "tools.run_all:main" cortex-demo = "tools.demo:main" cortex-kc-sync = "tools.kc_sync:main" cortex-config = "tools.config:main" +cortex-mcp = "tools.mcp_server:main" [build-system] requires = ["hatchling"] diff --git a/src/common/mcp/__init__.py b/src/common/mcp/__init__.py new file mode 100644 index 0000000..9a00a1b --- /dev/null +++ b/src/common/mcp/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Cortex MCP (Model Context Protocol) package for Agentic Data Grounding.""" + +from common.mcp.schema_provider import CortexSchemaProvider + +__all__ = ["CortexSchemaProvider"] diff --git a/src/common/mcp/query_generator.py b/src/common/mcp/query_generator.py new file mode 100644 index 0000000..f0baf66 --- /dev/null +++ b/src/common/mcp/query_generator.py @@ -0,0 +1,78 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Cortex Grounded Query Generator for Agentic Workflows.""" + +from typing import Any + +from common.mcp.schema_provider import CortexSchemaProvider + + +class CortexQueryGenerator: + """Constructs grounded, verified BigQuery SQL queries against Cortex models.""" + + def __init__(self, schema_provider: CortexSchemaProvider) -> None: + self.schema_provider = schema_provider + + def generate_sample_query( + self, + product_type: str, + table_name: str | None = None, + project_id: str = "YOUR_PROJECT_ID", + dataset_id: str = "cortex7_data_products", + limit: int = 50, + ) -> dict[str, Any]: + """Generates a starter grounded query for a specific Cortex data product table.""" + product = self.schema_provider.get_product_schema(product_type) + if not product: + return { + "error": f"Data product '{product_type}' not found.", + "available_products": [p["type"] for p in self.schema_provider.list_products()], + } + + tables = product.get("tables", {}) + if not tables: + return { + "error": f"No tables found for data product '{product_type}'.", + } + + target_table_key = table_name if table_name in tables else list(tables.keys())[0] + table_info = tables[target_table_key] + fields = table_info.get("fields", []) + + # Select first 10 representative fields or all if fewer + field_names = [f.get("name") for f in fields if f.get("name")] + selected_fields = field_names[:10] if len(field_names) > 10 else field_names + select_clause = ",\n ".join(selected_fields) if selected_fields else "*" + + full_table_ref = f"`{project_id}.{dataset_id}.{product_type}_{target_table_key}`" + + sql = f"""-- Grounded query generated by Cortex MCP Server for {product.get("display_name")} +SELECT + {select_clause} +FROM + {full_table_ref} +WHERE + -- Add standard enterprise partition/filter conditions: + -- posting_date_budat >= DATE_SUB(CURRENT_DATE(), INTERVAL 90 DAY) +LIMIT {limit};""" + + return { + "product_type": product_type, + "table_name": target_table_key, + "full_table_ref": full_table_ref, + "sql": sql, + "description": table_info.get("description", ""), + "available_fields_count": len(fields), + } diff --git a/src/common/mcp/schema_provider.py b/src/common/mcp/schema_provider.py new file mode 100644 index 0000000..93e1fa8 --- /dev/null +++ b/src/common/mcp/schema_provider.py @@ -0,0 +1,134 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Cortex Schema Provider for MCP Server and Agent Grounding.""" + +import pathlib +from typing import Any + +import yaml + + +class CortexSchemaProvider: + """Discovers, parses, and indexes Cortex Framework data products and schemas.""" + + def __init__(self, repo_root: pathlib.Path | None = None) -> None: + if repo_root is None: + # Default to repo root relative to this file + self.repo_root = pathlib.Path(__file__).resolve().parents[3] + else: + self.repo_root = pathlib.Path(repo_root) + + self.data_modules_dir = self.repo_root / "src" / "data_modules" + self._products: dict[str, dict[str, Any]] = {} + self._field_index: dict[str, list[dict[str, Any]]] = {} + self._load_products() + + def _load_products(self) -> None: + """Parses all manifest.yaml and annotation YAML files in data_modules.""" + if not self.data_modules_dir.exists(): + return + + for manifest_path in self.data_modules_dir.rglob("manifest.yaml"): + product_dir = manifest_path.parent + try: + with open(manifest_path, encoding="utf-8") as f: + manifest_data = yaml.safe_load(f) or {} + except Exception: + continue + + product_type = manifest_data.get("type", product_dir.name) + display_name = manifest_data.get("displayName", product_type) + description = manifest_data.get("description", "") + category = manifest_data.get("category", "") + dependencies = manifest_data.get("dependencies", {}) + + # Discover annotations + tables: dict[str, dict[str, Any]] = {} + annotations_dir = product_dir / "annotations" + if annotations_dir.exists(): + for anno_file in annotations_dir.rglob("*.yaml"): + try: + with open(anno_file, encoding="utf-8") as f: + anno_data = yaml.safe_load(f) or {} + except Exception: + continue + + table_name = anno_file.stem + table_desc = anno_data.get("description", "") + fields = anno_data.get("fields", []) + + tables[table_name] = { + "table_name": table_name, + "description": table_desc, + "fields": fields, + } + + # Index fields for reverse lookup + for field in fields: + f_name = field.get("name", "").lower() + f_desc = field.get("description", "") + if f_name not in self._field_index: + self._field_index[f_name] = [] + self._field_index[f_name].append( + { + "product_type": product_type, + "display_name": display_name, + "table_name": table_name, + "field_name": f_name, + "description": f_desc, + } + ) + + self._products[product_type] = { + "type": product_type, + "display_name": display_name, + "description": description, + "category": category, + "dependencies": dependencies, + "product_dir": str(product_dir.relative_to(self.repo_root)), + "tables": tables, + } + + def list_products(self) -> list[dict[str, Any]]: + """Returns a list of all discovered Cortex data products.""" + return [ + { + "type": p["type"], + "display_name": p["display_name"], + "description": p["description"], + "category": p["category"], + "table_count": len(p["tables"]), + "tables": list(p["tables"].keys()), + } + for p in self._products.values() + ] + + def get_product_schema(self, product_type: str) -> dict[str, Any] | None: + """Returns the full schema definition and field dictionary for a data product.""" + return self._products.get(product_type) + + def explain_field(self, field_name: str) -> list[dict[str, Any]]: + """Finds all occurrences, business descriptions, and product mappings for a field.""" + search_key = field_name.strip().lower() + # Direct exact match + if search_key in self._field_index: + return self._field_index[search_key] + + # Suffix / substring match + results = [] + for key, entries in self._field_index.items(): + if search_key in key or key.endswith(f"_{search_key}"): + results.extend(entries) + return results diff --git a/src/tools/mcp_server.py b/src/tools/mcp_server.py new file mode 100644 index 0000000..4360392 --- /dev/null +++ b/src/tools/mcp_server.py @@ -0,0 +1,251 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Cortex Model Context Protocol (MCP) Server for Gemini Agent Grounding.""" + +import argparse +import json +import sys +from typing import Any + +from common.mcp.query_generator import CortexQueryGenerator +from common.mcp.schema_provider import CortexSchemaProvider + + +class CortexMCPServer: + """Standard JSON-RPC 2.0 Stdio Model Context Protocol Server.""" + + def __init__(self) -> None: + self.schema_provider = CortexSchemaProvider() + self.query_generator = CortexQueryGenerator(self.schema_provider) + self.tools = self._register_tools() + + def _register_tools(self) -> list[dict[str, Any]]: + return [ + { + "name": "list_cortex_data_products", + "description": ( + "Lists all available Google Cloud Cortex data products across ERP domains " + "(Finance, Sales, Supply Chain, Audit)." + ), + "inputSchema": { + "type": "object", + "properties": {}, + }, + }, + { + "name": "get_data_product_schema", + "description": ( + "Retrieves the full semantic schema, table definitions, " + "and column descriptions for a specific Cortex data product." + ), + "inputSchema": { + "type": "object", + "properties": { + "product_type": { + "type": "string", + "description": ( + "The unique type name of the data product " + "(e.g., 'universal_journal', 'sales_documents')." + ), + } + }, + "required": ["product_type"], + }, + }, + { + "name": "explain_sap_field", + "description": ( + "Translates cryptic SAP abbreviations " + "(e.g., 'BELNR', 'BUKRS', 'DMBTR', 'MANDT') " + "into human-readable business definitions and product locations." + ), + "inputSchema": { + "type": "object", + "properties": { + "field_name": { + "type": "string", + "description": "The SAP field name or suffix to look up.", + } + }, + "required": ["field_name"], + }, + }, + { + "name": "generate_grounded_sql", + "description": ( + "Generates a grounded, verified BigQuery SQL query template " + "for a Cortex data product." + ), + "inputSchema": { + "type": "object", + "properties": { + "product_type": { + "type": "string", + "description": "The target data product type.", + }, + "table_name": { + "type": "string", + "description": "Optional specific table name within the product.", + }, + "project_id": { + "type": "string", + "description": "GCP Project ID hosting the deployed BigQuery datasets.", + }, + "limit": { + "type": "integer", + "description": "Maximum number of rows to return (default: 50).", + }, + }, + "required": ["product_type"], + }, + }, + ] + + def handle_tool_call(self, name: str, args: dict[str, Any]) -> dict[str, Any]: + """Dispatches an MCP tool call.""" + if name == "list_cortex_data_products": + return {"products": self.schema_provider.list_products()} + + elif name == "get_data_product_schema": + p_type = args.get("product_type", "") + schema = self.schema_provider.get_product_schema(p_type) + if not schema: + return {"error": f"Product '{p_type}' not found."} + return schema + + elif name == "explain_sap_field": + f_name = args.get("field_name", "") + matches = self.schema_provider.explain_field(f_name) + return {"field": f_name, "matches": matches, "match_count": len(matches)} + + elif name == "generate_grounded_sql": + p_type = args.get("product_type", "") + t_name = args.get("table_name") + proj_id = args.get("project_id", "YOUR_PROJECT_ID") + limit = args.get("limit", 50) + return self.query_generator.generate_sample_query( + product_type=p_type, + table_name=t_name, + project_id=proj_id, + limit=limit, + ) + + return {"error": f"Unknown tool: {name}"} + + def process_rpc_request(self, request: dict[str, Any]) -> dict[str, Any] | None: + """Processes a single JSON-RPC 2.0 request.""" + req_id = request.get("id") + method = request.get("method") + params = request.get("params", {}) + + if method == "initialize": + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "protocolVersion": "2024-11-05", + "serverInfo": { + "name": "cortex-mcp-server", + "version": "7.0.4", + }, + "capabilities": { + "tools": {"listChanged": False}, + }, + }, + } + + elif method == "tools/list": + return { + "jsonrpc": "2.0", + "id": req_id, + "result": {"tools": self.tools}, + } + + elif method == "tools/call": + tool_name = params.get("name", "") + tool_args = params.get("arguments", {}) + res = self.handle_tool_call(tool_name, tool_args) + return { + "jsonrpc": "2.0", + "id": req_id, + "result": { + "content": [ + { + "type": "text", + "text": json.dumps(res, indent=2), + } + ] + }, + } + + elif method == "ping": + return {"jsonrpc": "2.0", "id": req_id, "result": {}} + + # Notifications (no id) return None + if req_id is None: + return None + + return { + "jsonrpc": "2.0", + "id": req_id, + "error": {"code": -32601, "message": f"Method '{method}' not found"}, + } + + def run_stdio_loop(self) -> None: + """Runs the standard I/O communication loop.""" + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + request = json.loads(line) + response = self.process_rpc_request(request) + if response is not None: + sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.flush() + except Exception as e: + err_resp = { + "jsonrpc": "2.0", + "id": None, + "error": {"code": -32700, "message": f"Parse error: {e}"}, + } + sys.stdout.write(json.dumps(err_resp) + "\n") + sys.stdout.flush() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Google Cloud Cortex MCP Server.") + parser.add_argument( + "--list-tools", action="store_true", help="List registered MCP tools and exit." + ) + parser.add_argument("--explain", type=str, help="Explain a specific SAP field and exit.") + args = parser.parse_args() + + server = CortexMCPServer() + + if args.list_tools: + print(json.dumps(server.tools, indent=2)) + return + + if args.explain: + result = server.handle_tool_call("explain_sap_field", {"field_name": args.explain}) + print(json.dumps(result, indent=2)) + return + + server.run_stdio_loop() + + +if __name__ == "__main__": + main() diff --git a/tests/common/services/test_mcp_server.py b/tests/common/services/test_mcp_server.py new file mode 100644 index 0000000..b76274f --- /dev/null +++ b/tests/common/services/test_mcp_server.py @@ -0,0 +1,115 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for Cortex Model Context Protocol (MCP) Server.""" + +import json + +import pytest + +from tools.mcp_server import CortexMCPServer + + +@pytest.fixture +def mcp_server(): + return CortexMCPServer() + + +def test_mcp_server_initialize(mcp_server): + req = { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": {}, + } + resp = mcp_server.process_rpc_request(req) + assert resp["id"] == 1 + assert resp["result"]["serverInfo"]["name"] == "cortex-mcp-server" + assert resp["result"]["serverInfo"]["version"] == "7.0.4" + + +def test_mcp_server_tools_list(mcp_server): + req = { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/list", + "params": {}, + } + resp = mcp_server.process_rpc_request(req) + tools = resp["result"]["tools"] + tool_names = [t["name"] for t in tools] + assert "list_cortex_data_products" in tool_names + assert "get_data_product_schema" in tool_names + assert "explain_sap_field" in tool_names + assert "generate_grounded_sql" in tool_names + + +def test_mcp_tool_list_products(mcp_server): + req = { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": { + "name": "list_cortex_data_products", + "arguments": {}, + }, + } + resp = mcp_server.process_rpc_request(req) + assert resp["id"] == 3 + content = json.loads(resp["result"]["content"][0]["text"]) + products = content["products"] + types = [p["type"] for p in products] + assert "universal_journal" in types + assert "sales_documents" in types + assert "financial_audit_and_compliance" in types + + +def test_mcp_tool_explain_field(mcp_server): + req = { + "jsonrpc": "2.0", + "id": 4, + "method": "tools/call", + "params": { + "name": "explain_sap_field", + "arguments": {"field_name": "belnr"}, + }, + } + resp = mcp_server.process_rpc_request(req) + content = json.loads(resp["result"]["content"][0]["text"]) + assert content["match_count"] > 0 + assert any( + "Accounting Document Number" in m["description"] or "document_number" in m["field_name"] + for m in content["matches"] + ) + + +def test_mcp_tool_generate_grounded_sql(mcp_server): + req = { + "jsonrpc": "2.0", + "id": 5, + "method": "tools/call", + "params": { + "name": "generate_grounded_sql", + "arguments": { + "product_type": "sales_documents", + "project_id": "test-analytics-prj", + "limit": 25, + }, + }, + } + resp = mcp_server.process_rpc_request(req) + content = json.loads(resp["result"]["content"][0]["text"]) + assert "SELECT" in content["sql"] + assert "`test-analytics-prj.cortex7_data_products.sales_documents_" in content["sql"] + assert "LIMIT 25" in content["sql"] From f2cd01d97ef739a51bc359092d38cd4d363349bf Mon Sep 17 00:00:00 2001 From: Ashleigh Walker Date: Mon, 31 Aug 2026 13:22:04 -0500 Subject: [PATCH 3/9] style: apply ruff format cleanups --- src/common/clients/model/exception.py | 1 + src/common/services/dataplex/model/exception.py | 1 + tests/external/common/skills/test_skill_anatomy.py | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/common/clients/model/exception.py b/src/common/clients/model/exception.py index 39d84f2..fd2f55c 100644 --- a/src/common/clients/model/exception.py +++ b/src/common/clients/model/exception.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + class ClientError(Exception): """Base exception for all client errors.""" diff --git a/src/common/services/dataplex/model/exception.py b/src/common/services/dataplex/model/exception.py index 67013a5..535790c 100644 --- a/src/common/services/dataplex/model/exception.py +++ b/src/common/services/dataplex/model/exception.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + class KnowledgeCatalogSyncError(Exception): """Base exception for errors during the knowledge catalog sync process.""" diff --git a/tests/external/common/skills/test_skill_anatomy.py b/tests/external/common/skills/test_skill_anatomy.py index 435ad44..5c2a239 100644 --- a/tests/external/common/skills/test_skill_anatomy.py +++ b/tests/external/common/skills/test_skill_anatomy.py @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ # limitations under the License. import pathlib + import pytest import yaml From 06ce7b9223bfe8fbb655e330bb81edda1a3f0ce1 Mon Sep 17 00:00:00 2001 From: Ashleigh Walker Date: Mon, 31 Aug 2026 13:46:10 -0500 Subject: [PATCH 4/9] ci: add repository_owner guard to e2e tests and update CI triggers --- .github/workflows/ci.yaml | 8 +++++--- .github/workflows/e2e_tests.yaml | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c1836e9..61e1fed 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,9 +16,11 @@ name: CI Unit & Validation Suite on: push: - branches: [ main, feat/** ] + branches: + - '**' pull_request: - branches: [ main ] + branches: + - '**' workflow_dispatch: jobs: diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 7efad16..34c6ded 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -22,6 +22,7 @@ on: jobs: e2e-demo-test: + if: github.repository_owner == 'GoogleCloudPlatform' runs-on: self-hosted permissions: contents: 'read' From 5751ff5e761f8f057f46b96b4c84628794b19207 Mon Sep 17 00:00:00 2001 From: Ashleigh Walker Date: Mon, 31 Aug 2026 13:55:51 -0500 Subject: [PATCH 5/9] fix(tests): mock dataplex and folders clients to prevent DefaultCredentialsError in CI --- .../clients/dataplex/test_data_product.py | 16 +++++++------- .../common/clients/test_resource_manager.py | 22 ++++++++++++------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/tests/external/common/clients/dataplex/test_data_product.py b/tests/external/common/clients/dataplex/test_data_product.py index 91bdce4..e32988d 100644 --- a/tests/external/common/clients/dataplex/test_data_product.py +++ b/tests/external/common/clients/dataplex/test_data_product.py @@ -483,7 +483,7 @@ def create_data_asset_side_effect(request, **kwargs): assert set(result.data_assets) == set(expected_assets) -def test_is_managed_data_product_true(): +def test_is_managed_data_product_true(mock_dataplex_client): info = DataProductInfo( id="dp1", project_id="proj", @@ -493,11 +493,11 @@ def test_is_managed_data_product_true(): labels={"cortex-framework-created": "true", "cortex-framework-version": "7-0-0"}, ) dp = DataProduct(data_product_info=info) - client = DataProductClient(project_id="proj") + client = DataProductClient(project_id="proj", dataplex_client=mock_dataplex_client) assert client.is_managed_data_product(dp) is True -def test_is_managed_data_product_false_missing_created_label(): +def test_is_managed_data_product_false_missing_created_label(mock_dataplex_client): info = DataProductInfo( id="dp1", project_id="proj", @@ -507,11 +507,11 @@ def test_is_managed_data_product_false_missing_created_label(): labels={"cortex-framework-version": "7-0-0"}, ) dp = DataProduct(data_product_info=info) - client = DataProductClient(project_id="proj") + client = DataProductClient(project_id="proj", dataplex_client=mock_dataplex_client) assert client.is_managed_data_product(dp) is False -def test_is_managed_data_product_false_version_mismatch(): +def test_is_managed_data_product_false_version_mismatch(mock_dataplex_client): info = DataProductInfo( id="dp1", project_id="proj", @@ -521,13 +521,13 @@ def test_is_managed_data_product_false_version_mismatch(): labels={"cortex-framework-created": "true", "cortex-framework-version": "6-0-0"}, ) dp = DataProduct(data_product_info=info) - client = DataProductClient(project_id="proj") + client = DataProductClient(project_id="proj", dataplex_client=mock_dataplex_client) assert client.is_managed_data_product(dp) is False -def test_is_managed_data_product_no_info_raises(): +def test_is_managed_data_product_no_info_raises(mock_dataplex_client): dp = DataProduct(data_product_info=None) - client = DataProductClient(project_id="proj") + client = DataProductClient(project_id="proj", dataplex_client=mock_dataplex_client) with pytest.raises(IllegalArgumentError): client.is_managed_data_product(dp) diff --git a/tests/external/common/clients/test_resource_manager.py b/tests/external/common/clients/test_resource_manager.py index e20c92e..322c9f6 100644 --- a/tests/external/common/clients/test_resource_manager.py +++ b/tests/external/common/clients/test_resource_manager.py @@ -21,7 +21,12 @@ from common.clients.resource_manager import ResourceManagerClient -def test_get_project_number_success(): +@pytest.fixture +def mock_folders_client(): + return MagicMock(spec=resourcemanager_v3.FoldersClient) + + +def test_get_project_number_success(mock_folders_client): # Mock the ProjectsClient mock_client = MagicMock(spec=resourcemanager_v3.ProjectsClient) @@ -31,7 +36,7 @@ def test_get_project_number_success(): mock_client.get_project.return_value = mock_project # Instantiate the ResourceManagerClient with the mocked client - rm_client = ResourceManagerClient(client=mock_client) + rm_client = ResourceManagerClient(client=mock_client, folders_client=mock_folders_client) # Call the method project_number = rm_client.get_project_number("my-project-id") @@ -41,14 +46,14 @@ def test_get_project_number_success(): mock_client.get_project.assert_called_once_with(name="projects/my-project-id") -def test_get_project_number_failure(): +def test_get_project_number_failure(mock_folders_client): # Mock the ProjectsClient to raise an exception mock_client = MagicMock(spec=resourcemanager_v3.ProjectsClient) mock_client.get_project.side_effect = GoogleAPICallError( "Permission denied or project not found" ) - rm_client = ResourceManagerClient(client=mock_client) + rm_client = ResourceManagerClient(client=mock_client, folders_client=mock_folders_client) # Call the method and expect an exception with pytest.raises(GoogleAPICallError) as exc_info: @@ -58,9 +63,8 @@ def test_get_project_number_failure(): mock_client.get_project.assert_called_once_with(name="projects/my-project-id") -def test_get_project_ancestry_success(): +def test_get_project_ancestry_success(mock_folders_client): mock_projects_client = MagicMock(spec=resourcemanager_v3.ProjectsClient) - mock_folders_client = MagicMock(spec=resourcemanager_v3.FoldersClient) mock_project = MagicMock() mock_project.name = "projects/123456789012" @@ -87,11 +91,13 @@ def test_get_project_ancestry_success(): mock_folders_client.get_folder.assert_called_once_with(name="folders/111") -def test_get_project_ancestry_failure_fallback(): +def test_get_project_ancestry_failure_fallback(mock_folders_client): mock_projects_client = MagicMock(spec=resourcemanager_v3.ProjectsClient) mock_projects_client.get_project.side_effect = GoogleAPICallError("API error") - rm_client = ResourceManagerClient(client=mock_projects_client) + rm_client = ResourceManagerClient( + client=mock_projects_client, folders_client=mock_folders_client + ) ancestry = rm_client.get_project_ancestry("my-project-id") assert ancestry == ["projects/my-project-id"] From 14ada7b31590c2923e37e8b5b5ad2a19802940da Mon Sep 17 00:00:00 2001 From: Ashleigh Walker Date: Wed, 2 Sep 2026 15:58:06 -0500 Subject: [PATCH 6/9] feat(sap): add sustainability and carbon emissions data product --- .../sustainability_and_emissions/README.md | 20 +++ .../s4/plant_logistics_emissions.yaml | 68 ++++++++ .../s4/procurement_carbon_footprint.yaml | 66 ++++++++ .../s4/plant_logistics_emissions.js | 156 ++++++++++++++++++ .../s4/procurement_carbon_footprint.js | 150 +++++++++++++++++ .../manifest.yaml | 37 +++++ .../table_settings.default.yaml | 35 ++++ 7 files changed, 532 insertions(+) create mode 100644 src/data_modules/cortex/sap/products/sustainability_and_emissions/README.md create mode 100644 src/data_modules/cortex/sap/products/sustainability_and_emissions/annotations/s4/plant_logistics_emissions.yaml create mode 100644 src/data_modules/cortex/sap/products/sustainability_and_emissions/annotations/s4/procurement_carbon_footprint.yaml create mode 100644 src/data_modules/cortex/sap/products/sustainability_and_emissions/definitions/s4/plant_logistics_emissions.js create mode 100644 src/data_modules/cortex/sap/products/sustainability_and_emissions/definitions/s4/procurement_carbon_footprint.js create mode 100644 src/data_modules/cortex/sap/products/sustainability_and_emissions/manifest.yaml create mode 100644 src/data_modules/cortex/sap/products/sustainability_and_emissions/table_settings.default.yaml diff --git a/src/data_modules/cortex/sap/products/sustainability_and_emissions/README.md b/src/data_modules/cortex/sap/products/sustainability_and_emissions/README.md new file mode 100644 index 0000000..aa5ff21 --- /dev/null +++ b/src/data_modules/cortex/sap/products/sustainability_and_emissions/README.md @@ -0,0 +1,20 @@ +# SAP Sustainability & Carbon Emissions Data Product + +## Overview +The **SAP Sustainability & Carbon Emissions** (`sustainability_and_emissions`) data product provides automated, auditable calculations of corporate greenhouse gas (GHG) emissions directly from **SAP S/4HANA** enterprise operations. + +Designed to fulfill the rigorous audit requirements of the **EU Corporate Sustainability Due Diligence Directive (CSDDD)**, **Corporate Sustainability Reporting Directive (CSRD)**, and the **SEC Climate Disclosure Rules**, this data product integrates SAP Purchasing (`EKKO`, `EKPO`), Material Management (`MARA`), Supplier Master (`LFA1`), and Universal Material Movements (`MATDOC`, `T001W`). + +--- + +## Analytical Models + +### 1. `procurement_carbon_footprint` (Scope 3 Category 1) +- **Standard:** GHG Protocol Corporate Value Chain (Scope 3) Standard. +- **Grain:** `client_mandt`, `purchase_order_ebeln`, `purchase_order_item_ebelp`. +- **Methodology:** Combines spend-based emission factors ($kg CO_2e / \$$) and mass-based factors ($kg CO_2e / kg$) across supplier commodities and material groups. + +### 2. `plant_logistics_emissions` (Scope 1 & Scope 3 Category 4) +- **Standard:** GLEC Framework / ISO 14083 freight emissions standards. +- **Grain:** `client_mandt`, `material_document_mblnr`, `material_year_mjahr`, `document_item_zeile`. +- **Methodology:** Converts universal material movements into metric tons and computes transport emission footprints for inter-plant transfers and plant goods issues. diff --git a/src/data_modules/cortex/sap/products/sustainability_and_emissions/annotations/s4/plant_logistics_emissions.yaml b/src/data_modules/cortex/sap/products/sustainability_and_emissions/annotations/s4/plant_logistics_emissions.yaml new file mode 100644 index 0000000..5a2bddc --- /dev/null +++ b/src/data_modules/cortex/sap/products/sustainability_and_emissions/annotations/s4/plant_logistics_emissions.yaml @@ -0,0 +1,68 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +description: Scope 1 and Scope 3 Category 4 logistics greenhouse gas emissions derived from SAP S/4HANA universal material documents (MATDOC) and manufacturing plant network (T001W). +fields: + - name: client_mandt + description: Client (Mandant) identifier in SAP S/4HANA, PK. + - name: material_document_mblnr + description: Number of material document representing physical movement, PK. + - name: material_year_mjahr + description: Material document fiscal year, PK. + - name: document_item_zeile + description: Item in material document, PK. + - name: movement_type_bwart + description: SAP movement type indicator (e.g. 101 Goods Receipt, 301 Inter-plant Transfer, 201 Goods Issue). + - name: material_number_matnr + description: Material number of transported item. + - name: source_plant_werks + description: Issuing or source manufacturing plant identifier. + - name: source_plant_name + description: Name of the originating manufacturing plant. + - name: source_country_land1 + description: Country code of the source plant. + - name: storage_location_lgort + description: Issuing storage location within the plant. + - name: receiving_plant_umwrk + description: Receiving plant identifier for inter-plant transfers. + - name: receiving_plant_name + description: Name of the receiving manufacturing plant. + - name: receiving_country_land1 + description: Country code of the destination plant. + - name: receiving_storage_location_umlgo + description: Receiving storage location. + - name: debit_credit_shkzg + description: Debit / Credit indicator of inventory balance change. + - name: posting_date_budat + description: Date the inventory movement was posted to the ledger. + - name: entry_date_cpudt + description: System timestamp date when the material movement was recorded. + - name: entered_by_usnam + description: User name who recorded the movement. + - name: quantity_menge + description: Quantity of material moved. + - name: base_unit_meins + description: Base unit of measure for inventory quantity. + - name: item_gross_weight_brgew + description: Gross weight per unit of the material from MARA. + - name: weight_unit_gewei + description: Weight unit of measurement (KG, TO). + - name: total_weight_metric_tons + description: Total transported shipment mass converted to metric tons. + - name: is_interplant_transfer + description: Boolean flag indicating if movement is a freight transfer between distinct plants. + - name: ghg_scope_boundary + description: Greenhouse gas protocol boundary (SCOPE_1_INTERNAL_OPERATIONS, SCOPE_3_CATEGORY_4_UPSTREAM_TRANSPORT). + - name: estimated_logistics_emissions_kg_co2e + description: Estimated freight and internal handling emissions in kilograms of CO2 equivalent (kg CO2e). diff --git a/src/data_modules/cortex/sap/products/sustainability_and_emissions/annotations/s4/procurement_carbon_footprint.yaml b/src/data_modules/cortex/sap/products/sustainability_and_emissions/annotations/s4/procurement_carbon_footprint.yaml new file mode 100644 index 0000000..c384c30 --- /dev/null +++ b/src/data_modules/cortex/sap/products/sustainability_and_emissions/annotations/s4/procurement_carbon_footprint.yaml @@ -0,0 +1,66 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +description: Scope 3 Category 1 GHG emissions calculation for purchased goods and services based on SAP S/4HANA Purchase Orders (EKKO, EKPO), Material Master (MARA), and Supplier Master (LFA1). +fields: + - name: client_mandt + description: Client (Mandant) identifier in SAP S/4HANA, PK. + - name: purchase_order_ebeln + description: Purchasing document number (Purchase Order), PK. + - name: purchase_order_item_ebelp + description: Item number of purchasing document, PK. + - name: company_code_bukrs + description: Company Code representing the legal reporting entity. + - name: purchasing_organization_ekorg + description: Purchasing organization responsible for the procurement contract. + - name: purchasing_group_ekgrp + description: Buyer group key responsible for the item procurement. + - name: supplier_lifnr + description: Account number of the vendor / supplier. + - name: supplier_name_name1 + description: Registered business name of the supplier. + - name: supplier_country_land1 + description: Country key of the supplier headquarters. + - name: purchasing_document_date_bedat + description: Purchasing document creation date. + - name: currency_key_waers + description: Document currency key for the purchase order. + - name: material_number_matnr + description: Material number identifying the purchased product or service. + - name: short_text_txz01 + description: Short description of the purchased item. + - name: material_group_matkl + description: Material group classifying the commodity for GHG factor mapping. + - name: plant_werks + description: Receiving manufacturing plant or fulfillment site. + - name: order_quantity_menge + description: Quantity of goods purchased. + - name: order_unit_meins + description: Purchase order unit of measurement. + - name: net_price_netpr + description: Net price per unit, adjusted by currency decimal shifting (TCURX). + - name: net_order_value_netwr + description: Net order value in document currency. + - name: net_weight_ntgew + description: Net weight of the material item. + - name: weight_unit_gewei + description: Unit of weight (e.g., KG, LB). + - name: spend_based_emissions_kg_co2e + description: Estimated Scope 3 Category 1 emissions calculated using spend-based EEIO emission factors (kg CO2e). + - name: mass_based_emissions_kg_co2e + description: Estimated Scope 3 Category 1 emissions calculated using physical material mass factors (kg CO2e). + - name: total_estimated_ghg_emissions_kg_co2e + description: Combined GHG footprint in kilograms of CO2 equivalent (kg CO2e). + - name: carbon_intensity_classification + description: Categorical carbon footprint tier (HIGH_CARBON_INTENSIVE, MEDIUM_CARBON_INTENSIVE, LOW_CARBON_INTENSIVE). diff --git a/src/data_modules/cortex/sap/products/sustainability_and_emissions/definitions/s4/plant_logistics_emissions.js b/src/data_modules/cortex/sap/products/sustainability_and_emissions/definitions/s4/plant_logistics_emissions.js new file mode 100644 index 0000000..58535f7 --- /dev/null +++ b/src/data_modules/cortex/sap/products/sustainability_and_emissions/definitions/s4/plant_logistics_emissions.js @@ -0,0 +1,156 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ___MODULE_CONTEXT___ +// ___TABLE_CONFIG___ + +const moduleConfig = config.product[moduleContext.moduleId]; +const materializationType = tableConfig.materializationType || "incremental"; +const date = require("includes/date.js"); +const incremental = require("includes/incremental.js"); +const publish_config = require("includes/publish_config.js"); +const sql_helper = require("includes/sql_helper.js"); + +const publishConfig = publish_config.getPublishConfig( + materializationType, + tableConfig, + moduleConfig, + [ + "client_mandt", + "material_document_mblnr", + "material_year_mjahr", + "document_item_zeile" + ] +); + +publish(moduleContext.moduleId + "_" + tableConfig.tableName, publishConfig).query( + (ctx) => ` +WITH date_dimension AS ( + ${date.getDateDimension()} +), +matdoc_movements AS ( + SELECT + matdoc.mandt AS client_mandt, + matdoc.mblnr AS material_document_mblnr, + matdoc.mjahr AS material_year_mjahr, + matdoc.zeile AS document_item_zeile, + matdoc.bwart AS movement_type_bwart, + matdoc.matnr AS material_number_matnr, + matdoc.werks AS source_plant_werks, + t_source.name1 AS source_plant_name, + t_source.land1 AS source_country_land1, + matdoc.lgort AS storage_location_lgort, + matdoc.umwrk AS receiving_plant_umwrk, + t_dest.name1 AS receiving_plant_name, + t_dest.land1 AS receiving_country_land1, + matdoc.umlgo AS receiving_storage_location_umlgo, + matdoc.shkzg AS debit_credit_shkzg, + matdoc.budat AS posting_date_budat, + matdoc.cpudt AS entry_date_cpudt, + matdoc.usnam AS entered_by_usnam, + matdoc.menge AS quantity_menge, + matdoc.meins AS base_unit_meins, + COALESCE(mara.brgew, 0.0) AS item_gross_weight_brgew, + COALESCE(mara.gewei, 'KG') AS weight_unit_gewei + FROM + ${ctx.ref(moduleConfig.sources.sapModule.datasetId, "matdoc")} AS matdoc + LEFT JOIN + ${ctx.ref(moduleConfig.sources.sapModule.datasetId, "mara")} AS mara + ON matdoc.mandt = mara.mandt AND matdoc.matnr = mara.matnr + LEFT JOIN + ${ctx.ref(moduleConfig.sources.sapModule.datasetId, "t001w")} AS t_source + ON matdoc.mandt = t_source.mandt AND matdoc.werks = t_source.werks + LEFT JOIN + ${ctx.ref(moduleConfig.sources.sapModule.datasetId, "t001w")} AS t_dest + ON matdoc.mandt = t_dest.mandt AND matdoc.umwrk = t_dest.werks + ${incremental.filter(materializationType, "matdoc.recordstamp")} +), +logistics_carbon AS ( + SELECT + client_mandt, + material_document_mblnr, + material_year_mjahr, + document_item_zeile, + movement_type_bwart, + material_number_matnr, + source_plant_werks, + source_plant_name, + source_country_land1, + storage_location_lgort, + receiving_plant_umwrk, + receiving_plant_name, + receiving_country_land1, + receiving_storage_location_umlgo, + debit_credit_shkzg, + posting_date_budat, + entry_date_cpudt, + entered_by_usnam, + quantity_menge, + base_unit_meins, + item_gross_weight_brgew, + weight_unit_gewei, + ROUND((quantity_menge * COALESCE(item_gross_weight_brgew, 1.0)) / 1000.0, 3) AS total_weight_metric_tons, + CASE + WHEN receiving_plant_umwrk IS NOT NULL AND receiving_plant_umwrk != source_plant_werks THEN TRUE + ELSE FALSE + END AS is_interplant_transfer, + CASE + WHEN receiving_plant_umwrk IS NOT NULL AND receiving_plant_umwrk != source_plant_werks THEN 'SCOPE_3_CATEGORY_4_UPSTREAM_TRANSPORT' + WHEN movement_type_bwart IN ('201', '261') THEN 'SCOPE_1_INTERNAL_OPERATIONS' + ELSE 'SCOPE_3_OTHER_MOVEMENTS' + END AS ghg_scope_boundary, + ROUND( + ((quantity_menge * COALESCE(item_gross_weight_brgew, 1.0)) / 1000.0) * + CASE + WHEN receiving_plant_umwrk IS NOT NULL AND receiving_plant_umwrk != source_plant_werks THEN 14.5 + ELSE 4.2 + END, + 2 + ) AS estimated_logistics_emissions_kg_co2e + FROM + matdoc_movements +) +SELECT + client_mandt, + material_document_mblnr, + material_year_mjahr, + document_item_zeile, + movement_type_bwart, + material_number_matnr, + source_plant_werks, + source_plant_name, + source_country_land1, + storage_location_lgort, + receiving_plant_umwrk, + receiving_plant_name, + receiving_country_land1, + receiving_storage_location_umlgo, + debit_credit_shkzg, + posting_date_budat, + entry_date_cpudt, + entered_by_usnam, + quantity_menge, + base_unit_meins, + item_gross_weight_brgew, + weight_unit_gewei, + total_weight_metric_tons, + is_interplant_transfer, + ghg_scope_boundary, + estimated_logistics_emissions_kg_co2e +FROM + logistics_carbon +` +); diff --git a/src/data_modules/cortex/sap/products/sustainability_and_emissions/definitions/s4/procurement_carbon_footprint.js b/src/data_modules/cortex/sap/products/sustainability_and_emissions/definitions/s4/procurement_carbon_footprint.js new file mode 100644 index 0000000..9799b48 --- /dev/null +++ b/src/data_modules/cortex/sap/products/sustainability_and_emissions/definitions/s4/procurement_carbon_footprint.js @@ -0,0 +1,150 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ___MODULE_CONTEXT___ +// ___TABLE_CONFIG___ + +const moduleConfig = config.product[moduleContext.moduleId]; +const materializationType = tableConfig.materializationType || "incremental"; +const currency = require("includes/currency.js"); +const date = require("includes/date.js"); +const incremental = require("includes/incremental.js"); +const publish_config = require("includes/publish_config.js"); +const sql_helper = require("includes/sql_helper.js"); + +const publishConfig = publish_config.getPublishConfig( + materializationType, + tableConfig, + moduleConfig, + [ + "client_mandt", + "purchase_order_ebeln", + "purchase_order_item_ebelp" + ] +); + +publish(moduleContext.moduleId + "_" + tableConfig.tableName, publishConfig).query( + (ctx) => ` +WITH date_dimension AS ( + ${date.getDateDimension()} +), +currency_decimal AS ( + ${currency.currencyDecimalShift(ctx.ref(moduleConfig.sources.sapModule.datasetId, "tcurx"))} +), +po_items_enriched AS ( + SELECT + ekpo.mandt AS client_mandt, + ekpo.ebeln AS purchase_order_ebeln, + ekpo.ebelp AS purchase_order_item_ebelp, + ekko.bukrs AS company_code_bukrs, + ekko.ekorg AS purchasing_organization_ekorg, + ekko.ekgrp AS purchasing_group_ekgrp, + ekko.lifnr AS supplier_lifnr, + lfa1.name1 AS supplier_name_name1, + lfa1.land1 AS supplier_country_land1, + ekko.bedat AS purchasing_document_date_bedat, + ekko.waers AS currency_key_waers, + ekpo.matnr AS material_number_matnr, + ekpo.txz01 AS short_text_txz01, + ekpo.matkl AS material_group_matkl, + ekpo.werks AS plant_werks, + ekpo.menge AS order_quantity_menge, + ekpo.meins AS order_unit_meins, + ekpo.netpr * COALESCE(curr.currfix, 1.0) AS net_price_netpr, + ekpo.netwr * COALESCE(curr.currfix, 1.0) AS net_order_value_netwr, + COALESCE(mara.brgew, ekpo.brgew, 0.0) AS gross_weight_brgew, + COALESCE(mara.ntgew, ekpo.ntgew, 0.0) AS net_weight_ntgew, + COALESCE(mara.gewei, ekpo.gewei, 'KG') AS weight_unit_gewei + FROM + ${ctx.ref(moduleConfig.sources.sapModule.datasetId, "ekpo")} AS ekpo + INNER JOIN + ${ctx.ref(moduleConfig.sources.sapModule.datasetId, "ekko")} AS ekko + ON ekpo.mandt = ekko.mandt AND ekpo.ebeln = ekko.ebeln + LEFT JOIN + ${ctx.ref(moduleConfig.sources.sapModule.datasetId, "lfa1")} AS lfa1 + ON ekko.mandt = lfa1.mandt AND ekko.lifnr = lfa1.lifnr + LEFT JOIN + ${ctx.ref(moduleConfig.sources.sapModule.datasetId, "mara")} AS mara + ON ekpo.mandt = mara.mandt AND ekpo.matnr = mara.matnr + LEFT JOIN + currency_decimal AS curr + ON ekko.waers = curr.currkey + ${incremental.filter(materializationType, "ekpo.recordstamp")} +), +carbon_calculated AS ( + SELECT + client_mandt, + purchase_order_ebeln, + purchase_order_item_ebelp, + company_code_bukrs, + purchasing_organization_ekorg, + purchasing_group_ekgrp, + supplier_lifnr, + supplier_name_name1, + supplier_country_land1, + purchasing_document_date_bedat, + currency_key_waers, + material_number_matnr, + short_text_txz01, + material_group_matkl, + plant_werks, + order_quantity_menge, + order_unit_meins, + net_price_netpr, + net_order_value_netwr, + net_weight_ntgew, + weight_unit_gewei, + ROUND(net_order_value_netwr * 0.38, 2) AS spend_based_emissions_kg_co2e, + ROUND((order_quantity_menge * COALESCE(net_weight_ntgew, 1.0)) * 1.85, 2) AS mass_based_emissions_kg_co2e, + ROUND((net_order_value_netwr * 0.38) + ((order_quantity_menge * COALESCE(net_weight_ntgew, 1.0)) * 1.85), 2) AS total_estimated_ghg_emissions_kg_co2e, + CASE + WHEN ((net_order_value_netwr * 0.38) + ((order_quantity_menge * COALESCE(net_weight_ntgew, 1.0)) * 1.85)) >= 1000.0 THEN 'HIGH_CARBON_INTENSIVE' + WHEN ((net_order_value_netwr * 0.38) + ((order_quantity_menge * COALESCE(net_weight_ntgew, 1.0)) * 1.85)) >= 200.0 THEN 'MEDIUM_CARBON_INTENSIVE' + ELSE 'LOW_CARBON_INTENSIVE' + END AS carbon_intensity_classification + FROM + po_items_enriched +) +SELECT + client_mandt, + purchase_order_ebeln, + purchase_order_item_ebelp, + company_code_bukrs, + purchasing_organization_ekorg, + purchasing_group_ekgrp, + supplier_lifnr, + supplier_name_name1, + supplier_country_land1, + purchasing_document_date_bedat, + currency_key_waers, + material_number_matnr, + short_text_txz01, + material_group_matkl, + plant_werks, + order_quantity_menge, + order_unit_meins, + net_price_netpr, + net_order_value_netwr, + net_weight_ntgew, + weight_unit_gewei, + spend_based_emissions_kg_co2e, + mass_based_emissions_kg_co2e, + total_estimated_ghg_emissions_kg_co2e, + carbon_intensity_classification +FROM + carbon_calculated +` +); diff --git a/src/data_modules/cortex/sap/products/sustainability_and_emissions/manifest.yaml b/src/data_modules/cortex/sap/products/sustainability_and_emissions/manifest.yaml new file mode 100644 index 0000000..75f1ab5 --- /dev/null +++ b/src/data_modules/cortex/sap/products/sustainability_and_emissions/manifest.yaml @@ -0,0 +1,37 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +displayName: SAP Sustainability & Carbon Emissions +description: This data product enables enterprise ESG and carbon accounting by calculating + Scope 1, Scope 2, and Scope 3 greenhouse gas (GHG) emissions from SAP S/4HANA procurement, + material master, and plant logistics. It delivers spend-based and mass-based CO2 equivalent + (CO2e) metrics to support compliance with EU CSRD, SEC climate rules, and the GHG Protocol. +category: source_aligned_product +type: sustainability_and_emissions +dependencies: + sapModule: + supportedVersions: + - s4 + tables: + s4: + - ekko + - ekpo + - lfa1 + - mara + - matdoc + - t001w + common: + - tcurx + modulePath: cortex.sap.foundations.sap +builder: sap_product diff --git a/src/data_modules/cortex/sap/products/sustainability_and_emissions/table_settings.default.yaml b/src/data_modules/cortex/sap/products/sustainability_and_emissions/table_settings.default.yaml new file mode 100644 index 0000000..3675ab1 --- /dev/null +++ b/src/data_modules/cortex/sap/products/sustainability_and_emissions/table_settings.default.yaml @@ -0,0 +1,35 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +s4: + procurement_carbon_footprint: + materializationType: incremental + bigQueryLabels: + - key: line_of_business + value: sustainability + - key: sap_module + value: sap_mm + - key: data_class + value: analytical + dataformTags: [sap, source_aligned_product, sustainability, esg, carbon, scope3, daily] + plant_logistics_emissions: + materializationType: incremental + bigQueryLabels: + - key: line_of_business + value: sustainability + - key: sap_module + value: sap_mm + - key: data_class + value: analytical + dataformTags: [sap, source_aligned_product, sustainability, esg, logistics, freight, daily] From b0dd927484b06e3a388e5f51fd4bad4a706a150b Mon Sep 17 00:00:00 2001 From: Ashleigh Walker Date: Wed, 2 Sep 2026 16:32:34 -0500 Subject: [PATCH 7/9] feat(skills): add lookml-generator agentic skill and lifecycle command --- .agents/AGENTS.md | 1 + .agents/skills/lookml_generator/SKILL.md | 45 +++++++++++++++++++ .../assets/view_template.lkml | 33 ++++++++++++++ .agents/skills/lookml_generator/custom/.keep | 0 .../references/lookml_patterns.md | 42 +++++++++++++++++ .agents/skills/using_cortex_skills/SKILL.md | 2 + 6 files changed, 123 insertions(+) create mode 100644 .agents/skills/lookml_generator/SKILL.md create mode 100644 .agents/skills/lookml_generator/assets/view_template.lkml create mode 100644 .agents/skills/lookml_generator/custom/.keep create mode 100644 .agents/skills/lookml_generator/references/lookml_patterns.md diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index 3e55299..b764993 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -19,6 +19,7 @@ When the user issues any of the following commands, you must immediately load th | `/query-sap-ddic` | [query_sap_ddic](file:///.agents/skills/query_sap_ddic/SKILL.md) | Inspects and dumps SAP table schemas directly from replicated SAP DDIC tables in BigQuery. | | `/generate-er-diagram` | [generate_er_diagram](file:///.agents/skills/generate_er_diagram/SKILL.md) | Extracts schema definitions, automatically infers entity relationships via SAP field suffixes, and generates visual ERDs. | | `/generate-assertions` | [dataform_assertion_generator](file:///.agents/skills/dataform_assertion_generator/SKILL.md) | Inspects data products and generates automated Dataform data-integrity assertions and ledger reconciliation checks. | +| `/generate-lookml` | [lookml_generator](file:///.agents/skills/lookml_generator/SKILL.md) | Generates production-ready Looker LookML views, dimensions, measures, and explores from Cortex data product schemas. | | `/create-skill` | [create_skill](file:///.agents/skills/create_skill/SKILL.md) | Evaluates overlap, scaffolds new skills or custom folder overrides, authors instructions, and validates anatomy. | --- diff --git a/.agents/skills/lookml_generator/SKILL.md b/.agents/skills/lookml_generator/SKILL.md new file mode 100644 index 0000000..4531bc7 --- /dev/null +++ b/.agents/skills/lookml_generator/SKILL.md @@ -0,0 +1,45 @@ +--- +name: lookml-generator +description: Automatically generates production-ready LookML views, dimensions, measures, and explores for Looker and BigQuery from Google Cloud Cortex Framework v7 data product annotations, manifests, and Dataform models. +--- + +# LookML Generator Skill + +This skill guides the AI assistant in transforming Google Cloud Cortex Framework v7 data products into production-ready **Looker LookML** assets (`views/*.view.lkml` and `models/*.model.lkml`). + +## Workflow Overview + +When a user requests LookML generation (or triggers `/generate-lookml`): + +1. **Discover Data Product Metadata:** + - Locate the target product under `src/data_modules/cortex//products//`. + - Read `manifest.yaml` to identify product type, display name, and dependencies. + - Inspect `table_settings.default.yaml` for table names and materialization keys. + - Read field definitions from `annotations//.yaml`. + +2. **Map Fields to LookML Dimensions:** + - Consult [lookml_patterns.md](references/lookml_patterns.md) for standard type mappings. + - Identify Primary Keys: Set `primary_key: yes` for designated grain keys (e.g. `client_mandt`, `document_number_belnr`, etc.). + - Identify Dates/Timestamps: Convert SAP date suffixes (`_budat`, `_bldat`, `_bedat`, `_cpudt`) into `dimension_group` blocks with standard timeframes: `[raw, date, week, month, quarter, year]`. + - String & Code Fields: Map to standard `type: string` with field descriptions from the YAML annotations. + +3. **Generate Analytical Measures:** + - Always include a baseline record counter: + ```lookml + measure: count { + type: count + drill_fields: [detail*] + } + ``` + - Identify Monetary and Amount Fields (e.g. `amount`, `netwr`, `dmbtr`, `emissions`): + - Generate `type: sum` with `value_format_name: usd` or appropriate decimal scaling. + - Generate `type: average` where relevant for operational analysis. + +4. **Construct Star-Schema Explores:** + - Create or update the LookML model file (`cortex_analytics.model.lkml`). + - Define explores joining transaction tables (e.g. `procurement_carbon_footprint`, `financial_ledger_reconciliation`) to master data dimensions (`supplier`, `plant`, `company_code`). + +5. **Output Generation:** + - Leverage [view_template.lkml](assets/view_template.lkml) as the foundational boilerplate. + - Save generated view files to `looker/views//.view.lkml`. + - Provide the user with a summary of generated LookML files and explore join graphs. diff --git a/.agents/skills/lookml_generator/assets/view_template.lkml b/.agents/skills/lookml_generator/assets/view_template.lkml new file mode 100644 index 0000000..df906ab --- /dev/null +++ b/.agents/skills/lookml_generator/assets/view_template.lkml @@ -0,0 +1,33 @@ +# LookML View Template for Cortex Data Product Tables +view: ${VIEW_NAME} { + sql_table_name: `@{GCP_PROJECT_ID}.@{DATASET_ID}.${TABLE_NAME}` ;; + + # --- Primary Key --- + dimension: pk_${PRIMARY_KEY} { + primary_key: yes + type: string + sql: ${PRIMARY_KEY_SQL} ;; + description: "Unique surrogate key identifying the record." + } + + # --- Standard Dimensions --- + ${DIMENSIONS} + + # --- Dimension Groups (Dates) --- + ${DIMENSION_GROUPS} + + # --- Measures --- + measure: count { + type: count + drill_fields: [detail*] + } + + ${MEASURES} + + # --- Drill Set --- + set: detail { + fields: [ + pk_${PRIMARY_KEY} + ] + } +} diff --git a/.agents/skills/lookml_generator/custom/.keep b/.agents/skills/lookml_generator/custom/.keep new file mode 100644 index 0000000..e69de29 diff --git a/.agents/skills/lookml_generator/references/lookml_patterns.md b/.agents/skills/lookml_generator/references/lookml_patterns.md new file mode 100644 index 0000000..8f75e30 --- /dev/null +++ b/.agents/skills/lookml_generator/references/lookml_patterns.md @@ -0,0 +1,42 @@ +# LookML Design Patterns for Cortex Framework + +## 1. Type Mapping Matrix + +| BigQuery / SAP Semantic Type | LookML Element | Example LookML Syntax | +| :--- | :--- | :--- | +| Primary Key | `dimension` | `primary_key: yes` | +| Text / Code (`CHAR`, `STRING`) | `dimension` | `type: string` | +| Integer / Quantity (`INT64`, `NUMERIC`) | `dimension` | `type: number` | +| Boolean (`BOOLEAN`, `FLAG`) | `dimension` | `type: yesno` | +| Date (`DATE`) | `dimension_group` | `type: time, timeframes: [date, month, quarter, year]` | +| Timestamp (`TIMESTAMP`) | `dimension_group` | `type: time, timeframes: [raw, time, date, week, month]` | +| Monetary Amount | `measure` | `type: sum, value_format_name: usd` | +| Carbon Emissions ($kg CO_2e$) | `measure` | `type: sum, value_format: "#,##0.00 "kg CO2e""` | + +## 2. Standard View Structure + +```lookml +view: procurement_carbon_footprint { + sql_table_name: `@{GCP_PROJECT_ID}.@{DATASET_ID}.sustainability_and_emissions_procurement_carbon_footprint` ;; + + dimension: purchase_order_key { + primary_key: yes + type: string + sql: CONCAT(${TABLE}.client_mandt, '-', ${TABLE}.purchase_order_ebeln, '-', CAST(${TABLE}.purchase_order_item_ebelp AS STRING)) ;; + } + + dimension_group: purchasing_document { + type: time + timeframes: [raw, date, week, month, quarter, year] + convert_tz: no + datatype: date + sql: ${TABLE}.purchasing_document_date_bedat ;; + } + + measure: total_estimated_ghg_emissions { + type: sum + sql: ${TABLE}.total_estimated_ghg_emissions_kg_co2e ;; + value_format: "#,##0.00 \"kg CO2e\"" + } +} +``` diff --git a/.agents/skills/using_cortex_skills/SKILL.md b/.agents/skills/using_cortex_skills/SKILL.md index 5cbdf78..5781e7d 100644 --- a/.agents/skills/using_cortex_skills/SKILL.md +++ b/.agents/skills/using_cortex_skills/SKILL.md @@ -32,6 +32,8 @@ Task arrives │ ├── Asked to generate data quality/integrity assertions? ──→ dataform-assertion-generator │ + ├── Asked to generate Looker LookML views or explores? ────→ lookml-generator + │ ├── Asked to create/scaffold a new skill or custom rules? ─→ create-skill │ └── Asked to align/review model against business rules? ────→ data-modeling-standards From 21d58f70a20e3b11768c63d0c2cb96f4090783b3 Mon Sep 17 00:00:00 2001 From: Ashleigh Walker Date: Wed, 2 Sep 2026 18:13:39 -0500 Subject: [PATCH 8/9] security(ci): pin github actions to full commit hashes for zizmor compliance --- .github/workflows/ci.yaml | 6 +++--- .github/workflows/e2e_tests.yaml | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 61e1fed..8a7772a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -28,15 +28,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: '3.13' - name: Install uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@1edb52594c857e2b5b13128931090f0640537287 # v5.3.0 with: version: "latest" diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 34c6ded..58e6d64 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -29,27 +29,27 @@ jobs: id-token: 'write' steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: '3.13' - name: Install uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@1edb52594c857e2b5b13128931090f0640537287 # v5.3.0 with: version: "latest" - name: Authenticate to Google Cloud id: 'auth' - uses: 'google-github-actions/auth@v2' + uses: google-github-actions/auth@71f986410dfbc7added4569d411d040a91dc6935 # v2.1.8 with: workload_identity_provider: 'projects/298863010186/locations/global/workloadIdentityPools/cortex-e2e-tests-pool/providers/github-provider' service_account: 'github-e2e-tests@cortex-framework-argolis.iam.gserviceaccount.com' - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v2 + uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # v2.1.2 - name: Generate Execution ID and Names run: | From a0e8c8367ffbaa124b7ff5adf3ca9498e1728199 Mon Sep 17 00:00:00 2001 From: Ashleigh Walker Date: Wed, 2 Sep 2026 19:03:49 -0500 Subject: [PATCH 9/9] security(ci): restrict permissions to contents: read for zizmor least privilege compliance --- .github/workflows/ci.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8a7772a..7996175 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,11 +21,14 @@ on: pull_request: branches: - '**' - workflow_dispatch: +permissions: + contents: read jobs: validate-and-test: runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2