From ae3403a85378588205af43455706504623210480 Mon Sep 17 00:00:00 2001 From: Jesus Orosco Date: Mon, 12 Oct 2026 09:51:02 -0700 Subject: [PATCH 1/8] feat: initial attempt at lineage --- messages/lineage.md | 190 ++++++++++++ messages/scan.md | 31 ++ package.json | 8 +- src/base/lineageBase.ts | 214 ++++++++++++++ src/base/scanBase.ts | 64 ++++- .../script/lineage/generate.ts | 84 ++++++ .../data-code-extension/script/scan.ts | 8 + src/utils/lineageConfig.ts | 144 ++++++++++ src/utils/lineageRunner.ts | 166 +++++++++++ src/utils/nativeScan.ts | 35 ++- src/utils/pythonRunner.ts | 56 +++- .../data-code-extension/lineage.test.ts | 269 +++++++++++++++++ .../commands/data-code-extension/scan.test.ts | 107 ++++++- test/utils/lineageConfig.test.ts | 149 ++++++++++ test/utils/lineageRunner.test.ts | 271 ++++++++++++++++++ test/utils/nativeScan.test.ts | 66 +++++ 16 files changed, 1840 insertions(+), 22 deletions(-) create mode 100644 messages/lineage.md create mode 100644 src/base/lineageBase.ts create mode 100644 src/commands/data-code-extension/script/lineage/generate.ts create mode 100644 src/utils/lineageConfig.ts create mode 100644 src/utils/lineageRunner.ts create mode 100644 test/commands/data-code-extension/lineage.test.ts create mode 100644 test/utils/lineageConfig.test.ts create mode 100644 test/utils/lineageRunner.test.ts diff --git a/messages/lineage.md b/messages/lineage.md new file mode 100644 index 0000000..da3fe80 --- /dev/null +++ b/messages/lineage.md @@ -0,0 +1,190 @@ +# summary + +Generate field-level dataLineage for a Data Code Extension %s package. + +# description + +Runs the package's entrypoint against a Salesforce org with every write intercepted, traces each written field back to the fields it was computed from, and replaces the dataLineage section of the package's config.json with the result. Nothing is written to Data Cloud. Run 'scan' first so the read and write permissions in config.json are current. Fields whose lineage can't be traced are listed and get no lineage unless you add entries by hand. + +# examples.script + +- Generate lineage for a script package against the org with alias "myorg": + + <%= config.bin %> data-code-extension script lineage generate --entrypoint ./my-script-package/payload/entrypoint.py --target-org myorg + +- Preview the generated lineage without changing config.json: + + <%= config.bin %> data-code-extension script lineage generate --entrypoint ./my-script-package/payload/entrypoint.py --target-org myorg --dry-run + +- Replace existing lineage without prompting, and keep the full result for review: + + <%= config.bin %> data-code-extension script lineage generate --entrypoint ./my-script-package/payload/entrypoint.py --target-org myorg --no-prompt --output-file ./lineage-result.json + +# info.checkingPython + +Checking Python version... + +# info.pythonFound + +Python %s found at '%s' + +# info.checkingPackages + +Checking required Python packages... + +# info.packageFound + +Package '%s' version %s found + +# info.authenticating + +Authenticating with Salesforce org '%s'... + +# info.authenticated + +Successfully authenticated with org '%s' + +# info.generating + +Running package to trace lineage. Nothing is written to Data Cloud... + +# info.summary + +Generated %s dataLineage entries with %s (target, source) field pairs: +%s added, -%s removed, ~%s changed. + +# info.noSources + +%s target fields are computed from no source field (for example, literals) and get no lineage: +%s + +# info.outputWritten + +Wrote the full lineage result to '%s'. + +# info.dryRunNotice + +DRY RUN: config.json was not modified. Remove --dry-run to apply the generated lineage. + +# info.unchanged + +dataLineage in '%s' is already up to date. + +# info.written + +Updated dataLineage in '%s'. + +# info.notReplaced + +Kept the existing dataLineage; config.json was not modified. + +# info.complete + +Lineage generation completed successfully! + +# warning.unresolved + +%s target fields could not be traced and get no lineage unless you add entries by hand: +%s + +# warning.permissionGaps + +The generated lineage references objects that aren't in config.json permissions, so deploy would fail (1041/1042). Run 'data-code-extension script scan' first, then generate lineage again. +%s + +# warning.emptyResult + +No lineage could be generated, so config.json was not modified. The package follows the no-lineage path unless you add dataLineage entries by hand. + +# warning.replace + +This replaces the existing dataLineage section. Any entries you added or edited by hand will be lost. Lineage determines how governance tags propagate to the target; review the result before deploying. + +# prompt.replace + +Replace the existing dataLineage in config.json? + +# error.functionPackage + +Lineage can only be generated for script packages; '%s' is a function package. + +# error.replaceNotConfirmed + +config.json already has a different dataLineage section, and replacing it needs confirmation. + +# actions.replaceNotConfirmed + +- Re-run with --no-prompt to replace the existing dataLineage. +- Re-run with --dry-run to preview the generated lineage without changing config.json. + +# error.sdkTooOld + +The installed salesforce-data-customcode package doesn't support lineage generation. + +# actions.sdkTooOld + +- Upgrade the SDK with 'pip install -U salesforce-data-customcode', then try again. + +# error.resultMissing + +The lineage run finished but produced no result: %s + +# actions.resultMissing + +- Review the output above for errors from the entrypoint. +- Upgrade the SDK with 'pip install -U salesforce-data-customcode', then try again. + +# flags.entrypoint.summary + +Entrypoint file for the package to generate lineage for. + +# flags.entrypoint.description + +The path to the entrypoint file of your initialized Data Cloud custom code script package. The package's config.json, in the same directory, is updated with the generated lineage. + +# flags.targetOrg.summary + +Target Salesforce org to read data from. + +# flags.targetOrg.description + +The alias or username of the Salesforce org whose data the package reads while lineage is traced. The org must have Data Cloud enabled and appropriate permissions. Nothing is written to the org. + +# flags.configFile.summary + +Path to an SDK config file. + +# flags.configFile.description + +Optional path to an SDK config.yaml that overrides the reader and writer configuration for the run, for example the reader's default_row_limit (1000 rows by default). + +# flags.dependencies.summary + +Dependencies override for the run. + +# flags.dependencies.description + +Optional comma-separated list of Python package dependencies to use during the run, overriding those defined in the package's requirements.txt. + +# flags.dryRun.summary + +Print the generated lineage without modifying config.json. + +# flags.dryRun.description + +When set, generates and prints the lineage but doesn't modify config.json and never prompts. + +# flags.outputFile.summary + +Also write the full lineage result to this JSON file. + +# flags.outputFile.description + +Writes the full result, including unresolved fields, targets, sources, and the pair count, to the given path. Useful for review or CI. + +# flags.noPrompt.summary + +Replace existing dataLineage without prompting. + +# flags.noPrompt.description + +Skips the confirmation before replacing a dataLineage section that differs from the generated one. Required to replace existing lineage when running non-interactively or with --json. diff --git a/messages/scan.md b/messages/scan.md index 056dc61..a8402fd 100644 --- a/messages/scan.md +++ b/messages/scan.md @@ -101,3 +101,34 @@ Checking required Python packages... # info.packageFound Package '%s' version %s found + +# flags.noPrompt.summary + +Write config.json without prompting when dataLineage is stale. + +# flags.noPrompt.description + +Skips the confirmation scan asks for when the rebuilt permissions no longer cover objects that dataLineage references. Required to write config.json in that case when running non-interactively. + +# warning.staleLineage + +dataLineage references objects that are no longer in permissions (%s entries affected). The package will fail to deploy (1041/1042) until you rerun 'data-code-extension script lineage generate' or edit dataLineage. Scan doesn't change dataLineage. +%s + +# prompt.staleLineage + +Write the updated permissions to config.json anyway? + +# info.staleLineageNotWritten + +No files were modified. Update dataLineage, then scan again. + +# error.staleLineageNotConfirmed + +dataLineage references objects that are no longer in permissions, and writing config.json needs confirmation. + +# actions.staleLineageNotConfirmed + +- Re-run 'data-code-extension script lineage generate' to regenerate dataLineage, or edit it by hand. +- Re-run scan with --no-prompt to write the updated permissions anyway. +- Re-run scan with --dry-run to preview the changes. diff --git a/package.json b/package.json index 909eee4..d6aa5f2 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,13 @@ "description": "Commands for Data Cloud Code Extension.", "subtopics": { "script": { - "description": "Commands to initialize and manage script-based Data Cloud packages." + "description": "Commands to initialize and manage script-based Data Cloud packages.", + "subtopics": { + "lineage": { + "description": "Commands to generate field-level data lineage for script-based Data Cloud packages.", + "hidden": true + } + } }, "function": { "description": "Commands to initialize and manage function-based Data Cloud packages." diff --git a/src/base/lineageBase.ts b/src/base/lineageBase.ts new file mode 100644 index 0000000..af20b15 --- /dev/null +++ b/src/base/lineageBase.ts @@ -0,0 +1,214 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { SfCommand } from '@salesforce/sf-plugins-core'; +import { Messages, Org, SfError } from '@salesforce/core'; +import { checkEnvironment } from '../utils/environmentChecker.js'; +import { splitDependencies } from '../utils/pythonRunner.js'; +import { LineageRunner, type LineageReport } from '../utils/lineageRunner.js'; +import { findBaseDirectory, getPackageType, readConfigJson } from '../utils/nativeScan.js'; +import { + countPairs, + diffLineage, + findPermissionGaps, + isEmptyDiff, + readLineage, + replaceLineage, + writeConfigJson, + type LineageDiff, + type LineagePermissionGaps, +} from '../utils/lineageConfig.js'; +import { type SharedResultProps } from './types.js'; + +export type BaseLineageFlags = { + entrypoint: string; + 'target-org': Org; + 'config-file'?: string; + dependencies?: string; + 'dry-run': boolean; + 'output-file'?: string; + 'no-prompt': boolean; +}; + +export type LineageGenerateResult = SharedResultProps & { + targetOrg: string; + configPath: string; + dryRun: boolean; + /** True when config.json was updated. */ + written: boolean; + entryCount: number; + pairCount: number; + diff: LineageDiff; + permissionGaps: LineagePermissionGaps; + outputFile?: string; + lineage: LineageReport; +}; + +const CONFIG_FILE = 'config.json'; + +// eslint-disable-next-line sf-plugin/command-summary, sf-plugin/command-example +export abstract class LineageBase extends SfCommand { + public static enableJsonFlag = true; + + public async run(): Promise { + const { flags } = (await this.parse(this.constructor as typeof LineageBase)) as unknown as { + flags: BaseLineageFlags; + }; + const codeType = this.getCodeType(); + const messages = this.getMessages(); + + const entrypoint = path.resolve(flags.entrypoint); + const configPath = path.join(path.dirname(entrypoint), CONFIG_FILE); + + try { + if ((await getPackageType(findBaseDirectory(entrypoint))) === 'function') { + throw new SfError(messages.getMessage('error.functionPackage', [flags.entrypoint]), 'LineageFunctionPackage'); + } + // Read config.json up front so a missing or malformed file fails before the run. + const config = await readConfigJson(configPath); + + const { pythonInfo, packageInfo } = await checkEnvironment(this.spinner, this.log.bind(this), messages); + + const targetOrg = flags['target-org']; + const orgUsername = targetOrg.getUsername() ?? 'target org'; + this.spinner.start(messages.getMessage('info.authenticating', [orgUsername])); + await targetOrg.getConnection().refreshAuth(); + this.spinner.stop(); + this.log(messages.getMessage('info.authenticated', [orgUsername])); + + // Stream the entrypoint's output live; keep stdout clean for --json. + this.log(messages.getMessage('info.generating')); + const scriptOut = this.jsonEnabled() ? process.stderr : process.stdout; + const lineage = await LineageRunner.generate({ + pythonCommand: pythonInfo.command, + entrypoint, + configFile: flags['config-file'], + dependencies: splitDependencies(flags.dependencies), + sfCliOrg: orgUsername, + onStdout: (chunk) => scriptOut.write(chunk), + onStderr: (chunk) => process.stderr.write(chunk), + }); + + const outputFile = flags['output-file'] ? path.resolve(flags['output-file']) : undefined; + if (outputFile) { + await fs.writeFile(outputFile, `${JSON.stringify(lineage, null, 2)}\n`, 'utf8'); + this.log(messages.getMessage('info.outputWritten', [outputFile])); + } + + const entries = lineage.dataLineage; + const current = readLineage(config); + const diff = diffLineage(current, entries); + const permissionGaps = findPermissionGaps(config, lineage.targets, lineage.sources); + this.report(messages, lineage, diff, permissionGaps); + + const result: LineageGenerateResult = { + success: true, + pythonVersion: pythonInfo, + packageInfo, + codeType, + packageDir: path.dirname(entrypoint), + targetOrg: orgUsername, + configPath, + dryRun: flags['dry-run'], + written: false, + entryCount: entries.length, + pairCount: countPairs(entries), + diff, + permissionGaps, + outputFile, + lineage, + message: messages.getMessage('info.complete'), + }; + + if (entries.length === 0) { + this.warn(messages.getMessage('warning.emptyResult')); + return result; + } + + if (flags['dry-run']) { + this.log(JSON.stringify({ dataLineage: entries }, null, 2)); + this.log(messages.getMessage('info.dryRunNotice')); + return result; + } + + if (isEmptyDiff(diff)) { + this.log(messages.getMessage('info.unchanged', [configPath])); + return result; + } + + if (current.length > 0 && !(await this.confirmReplace(messages, flags['no-prompt']))) { + this.log(messages.getMessage('info.notReplaced')); + return result; + } + + await writeConfigJson(configPath, replaceLineage(config, entries)); + this.log(messages.getMessage('info.written', [configPath])); + return { ...result, written: true }; + } catch (error) { + this.spinner.stop(); + throw error; + } + } + + private report( + messages: Messages, + lineage: LineageReport, + diff: LineageDiff, + gaps: LineagePermissionGaps + ): void { + this.log( + messages.getMessage('info.summary', [ + lineage.dataLineage.length, + countPairs(lineage.dataLineage), + diff.added.length, + diff.removed.length, + diff.changed.length, + ]) + ); + if (lineage.noSources.length > 0) { + this.log(messages.getMessage('info.noSources', [lineage.noSources.length, bullets(lineage.noSources)])); + } + if (lineage.unresolved.length > 0) { + const lines = lineage.unresolved.map((field) => `${field.targetField} (${field.reason})`); + this.warn(messages.getMessage('warning.unresolved', [lineage.unresolved.length, bullets(lines)])); + } + const missing = [...gaps.targets.map((name) => `write: ${name}`), ...gaps.sources.map((name) => `read: ${name}`)]; + if (missing.length > 0) { + this.warn(messages.getMessage('warning.permissionGaps', [bullets(missing)])); + } + } + + private async confirmReplace(messages: Messages, noPrompt: boolean): Promise { + this.warn(messages.getMessage('warning.replace')); + if (noPrompt) return true; + if (this.jsonEnabled() || !process.stdin.isTTY) { + throw new SfError( + messages.getMessage('error.replaceNotConfirmed'), + 'LineageReplaceNotConfirmed', + messages.getMessages('actions.replaceNotConfirmed') + ); + } + return this.confirm({ message: messages.getMessage('prompt.replace'), defaultAnswer: false }); + } + + protected abstract getCodeType(): 'script'; + protected abstract getMessages(): Messages; +} + +function bullets(lines: string[]): string { + return lines.map((line) => ` - ${line}`).join('\n'); +} diff --git a/src/base/scanBase.ts b/src/base/scanBase.ts index c3f3c61..a88f3f1 100644 --- a/src/base/scanBase.ts +++ b/src/base/scanBase.ts @@ -15,15 +15,17 @@ */ import path from 'node:path'; import { SfCommand } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { Messages, SfError } from '@salesforce/core'; import { executeNativeScan, type NativeScanResult } from '../utils/nativeScan.js'; import { checkEnvironment, type EnvironmentCheckResult } from '../utils/environmentChecker.js'; +import { countEntriesReferencing, findStaleLineage, type LineagePermissionGaps } from '../utils/lineageConfig.js'; export type BaseScanFlags = { entrypoint?: string; 'config-file'?: string; 'dry-run': boolean; 'no-requirements': boolean; + 'no-prompt'?: boolean; }; export type ScanResult = { @@ -56,9 +58,20 @@ export abstract class ScanBase extends SfCommand { dryRun: flags['dry-run'], noRequirements: flags['no-requirements'], packageType: codeType, + // Functions carry no permissions or dataLineage, so only scripts can go stale. + confirmWrite: + codeType === 'script' + ? (updatedConfig): Promise => + this.confirmStaleLineage(messages, updatedConfig, flags['no-prompt'] ?? false) + : undefined, }); this.spinner.stop(); + if (codeType === 'script' && executionResult.dryRun) { + const stale = findStaleLineage(executionResult.config); + if (hasStaleLineage(stale)) this.warnStaleLineage(messages, executionResult.config, stale); + } + this.log(messages.getMessage('info.scanExecuted', [workingDir])); // Echo the resulting config — mirrors the Python CLI's behavior of printing the @@ -77,6 +90,10 @@ export abstract class ScanBase extends SfCommand { this.log(messages.getMessage('info.dryRunNotice')); } + if (executionResult.written === false) { + this.log(messages.getMessage('info.staleLineageNotWritten')); + } + return { success: true, pythonInfo, @@ -92,6 +109,51 @@ export abstract class ScanBase extends SfCommand { } } + /** + * Called just before scan writes config.json. When the rebuilt permissions no longer + * cover objects that dataLineage references, warn and ask before writing; scan never + * edits dataLineage itself. + */ + private async confirmStaleLineage( + messages: Messages, + updatedConfig: Record, + noPrompt: boolean + ): Promise { + const stale = findStaleLineage(updatedConfig); + if (!hasStaleLineage(stale)) return true; + + this.spinner.stop(); + this.warnStaleLineage(messages, updatedConfig, stale); + if (noPrompt) return true; + if (this.jsonEnabled() || !process.stdin.isTTY) { + throw new SfError( + messages.getMessage('error.staleLineageNotConfirmed'), + 'ScanStaleLineageNotConfirmed', + messages.getMessages('actions.staleLineageNotConfirmed') + ); + } + return this.confirm({ message: messages.getMessage('prompt.staleLineage'), defaultAnswer: false }); + } + + private warnStaleLineage( + messages: Messages, + config: Record, + stale: LineagePermissionGaps + ): void { + const objects = [...new Set([...stale.targets, ...stale.sources])]; + const lines = [...stale.targets.map((name) => `write: ${name}`), ...stale.sources.map((name) => `read: ${name}`)]; + this.warn( + messages.getMessage('warning.staleLineage', [ + countEntriesReferencing(config, objects), + lines.map((line) => ` - ${line}`).join('\n'), + ]) + ); + } + protected abstract getCodeType(): 'script' | 'function'; protected abstract getMessages(): Messages; } + +function hasStaleLineage(stale: LineagePermissionGaps): boolean { + return stale.targets.length > 0 || stale.sources.length > 0; +} diff --git a/src/commands/data-code-extension/script/lineage/generate.ts b/src/commands/data-code-extension/script/lineage/generate.ts new file mode 100644 index 0000000..dc8f0c6 --- /dev/null +++ b/src/commands/data-code-extension/script/lineage/generate.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Flags } from '@salesforce/sf-plugins-core'; +import { Messages } from '@salesforce/core'; +import { LineageBase } from '../../../../base/lineageBase.js'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-data-code-extension', 'lineage'); + +export default class Generate extends LineageBase { + public static readonly summary = messages.getMessage('summary', ['script']); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples.script'); + // Hidden until field-level lineage is generally available: the byoc-service and + // data-prep gates are tenant-whitelisted. + public static readonly hidden = true; + + public static readonly flags = { + entrypoint: Flags.file({ + char: 'e', + summary: messages.getMessage('flags.entrypoint.summary'), + description: messages.getMessage('flags.entrypoint.description'), + required: true, + exists: true, + }), + 'target-org': Flags.requiredOrg({ + char: 'o', + summary: messages.getMessage('flags.targetOrg.summary'), + description: messages.getMessage('flags.targetOrg.description'), + required: true, + }), + 'config-file': Flags.file({ + summary: messages.getMessage('flags.configFile.summary'), + description: messages.getMessage('flags.configFile.description'), + required: false, + exists: true, + }), + dependencies: Flags.string({ + summary: messages.getMessage('flags.dependencies.summary'), + description: messages.getMessage('flags.dependencies.description'), + required: false, + }), + 'dry-run': Flags.boolean({ + char: 'd', + summary: messages.getMessage('flags.dryRun.summary'), + description: messages.getMessage('flags.dryRun.description'), + default: false, + }), + 'output-file': Flags.string({ + summary: messages.getMessage('flags.outputFile.summary'), + description: messages.getMessage('flags.outputFile.description'), + required: false, + }), + 'no-prompt': Flags.boolean({ + char: 'p', + summary: messages.getMessage('flags.noPrompt.summary'), + description: messages.getMessage('flags.noPrompt.description'), + default: false, + }), + }; + + // eslint-disable-next-line class-methods-use-this + protected getCodeType(): 'script' { + return 'script'; + } + + // eslint-disable-next-line class-methods-use-this + protected getMessages(): Messages { + return messages; + } +} diff --git a/src/commands/data-code-extension/script/scan.ts b/src/commands/data-code-extension/script/scan.ts index b81acc9..d6afae1 100644 --- a/src/commands/data-code-extension/script/scan.ts +++ b/src/commands/data-code-extension/script/scan.ts @@ -50,6 +50,14 @@ export default class Scan extends ScanBase { description: messages.getMessage('flags.noRequirements.description'), default: false, }), + // Hidden alongside the lineage commands until field-level lineage is generally available. + 'no-prompt': Flags.boolean({ + char: 'p', + summary: messages.getMessage('flags.noPrompt.summary'), + description: messages.getMessage('flags.noPrompt.description'), + default: false, + hidden: true, + }), }; // eslint-disable-next-line class-methods-use-this diff --git a/src/utils/lineageConfig.ts b/src/utils/lineageConfig.ts new file mode 100644 index 0000000..4894541 --- /dev/null +++ b/src/utils/lineageConfig.ts @@ -0,0 +1,144 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { promises as fs } from 'node:fs'; +import { type LineageEntry } from './lineageRunner.js'; + +export type LineageDiff = { + added: string[]; + removed: string[]; + changed: string[]; +}; + +/** Objects referenced by lineage but missing from `permissions`, i.e. a 1041/1042 at deploy. */ +export type LineagePermissionGaps = { + targets: string[]; + sources: string[]; +}; + +/** The `dataLineage` entries in a config, or `[]` when the section is absent or malformed. */ +export function readLineage(config: Record): LineageEntry[] { + const section = config.dataLineage; + if (!Array.isArray(section)) return []; + return section.filter( + (entry): entry is LineageEntry => + typeof entry === 'object' && + entry !== null && + typeof (entry as LineageEntry).targetField === 'string' && + Array.isArray((entry as LineageEntry).sourceFields) + ); +} + +/** Per-target difference between the current `dataLineage` and a generated one. */ +export function diffLineage(current: LineageEntry[], generated: LineageEntry[]): LineageDiff { + const key = (entry: LineageEntry): string => [...entry.sourceFields].sort().join('\n'); + const before = new Map(current.map((entry) => [entry.targetField, key(entry)])); + const after = new Map(generated.map((entry) => [entry.targetField, key(entry)])); + const diff: LineageDiff = { added: [], removed: [], changed: [] }; + for (const [target, sources] of after) { + if (!before.has(target)) diff.added.push(target); + else if (before.get(target) !== sources) diff.changed.push(target); + } + for (const target of before.keys()) { + if (!after.has(target)) diff.removed.push(target); + } + return diff; +} + +export function isEmptyDiff(diff: LineageDiff): boolean { + return diff.added.length === 0 && diff.removed.length === 0 && diff.changed.length === 0; +} + +/** Number of (target, source) field pairs; Core stores one mapping row per pair. */ +export function countPairs(entries: LineageEntry[]): number { + return entries.reduce((total, entry) => total + entry.sourceFields.length, 0); +} + +/** `config` with `dataLineage` replaced wholesale; every other key is kept in place. */ +export function replaceLineage(config: Record, entries: LineageEntry[]): Record { + return { ...config, dataLineage: entries }; +} + +/** + * Objects that `targets` / `sources` name but `permissions.write.*` / `permissions.read.*` + * don't. Object names compare case-insensitively. + */ +export function findPermissionGaps( + config: Record, + targets: Iterable, + sources: Iterable +): LineagePermissionGaps { + const permissions = (config.permissions ?? {}) as Record; + const writable = permittedObjects(permissions.write); + const readable = permittedObjects(permissions.read); + return { + targets: missingFrom(writable, targets), + sources: missingFrom(readable, sources), + }; +} + +/** + * Objects referenced by the config's own `dataLineage` that its `permissions` no longer + * cover. Object-level only: a field that stopped being produced is not detected. + */ +export function findStaleLineage(config: Record): LineagePermissionGaps { + const entries = readLineage(config); + return findPermissionGaps( + config, + entries.map((entry) => objectOf(entry.targetField)), + entries.flatMap((entry) => entry.sourceFields.map(objectOf)) + ); +} + +/** Number of `dataLineage` entries that reference any of the given objects. */ +export function countEntriesReferencing(config: Record, objects: string[]): number { + const names = new Set(objects.map((name) => name.toLowerCase())); + return readLineage(config).filter( + (entry) => + names.has(objectOf(entry.targetField).toLowerCase()) || + entry.sourceFields.some((field) => names.has(objectOf(field).toLowerCase())) + ).length; +} + +/** Write config.json with a 2-space indent and a trailing newline. */ +export async function writeConfigJson(configPath: string, config: Record): Promise { + await fs.writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, 'utf8'); +} + +function objectOf(field: string): string { + const dot = field.indexOf('.'); + return dot === -1 ? field : field.slice(0, dot); +} + +function permittedObjects(section: unknown): Set { + const names = new Set(); + if (typeof section !== 'object' || section === null) return names; + for (const kind of ['dlo', 'dmo']) { + const list = (section as Record)[kind]; + if (!Array.isArray(list)) continue; + for (const name of list) { + if (typeof name === 'string') names.add(name.toLowerCase()); + } + } + return names; +} + +function missingFrom(permitted: Set, objects: Iterable): string[] { + const missing = new Set(); + for (const name of objects) { + if (!permitted.has(name.toLowerCase())) missing.add(name); + } + return [...missing].sort(); +} diff --git a/src/utils/lineageRunner.ts b/src/utils/lineageRunner.ts new file mode 100644 index 0000000..2b60741 --- /dev/null +++ b/src/utils/lineageRunner.ts @@ -0,0 +1,166 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Messages, SfError } from '@salesforce/core'; +import { DEFAULT_PROFILE, runPythonProgram, type PythonRunnerDeps } from './pythonRunner.js'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-data-code-extension', 'lineage'); + +/** Exit code the shim uses when the installed SDK has no `datacustomcode.lineage`. */ +export const LINEAGE_UNAVAILABLE_EXIT_CODE = 3; +/** Printed to stderr alongside {@link LINEAGE_UNAVAILABLE_EXIT_CODE}, so a script's own `sys.exit(3)` isn't mistaken for it. */ +export const LINEAGE_UNAVAILABLE_MARKER = 'DATACUSTOMCODE_LINEAGE_UNAVAILABLE'; + +/** + * The Python program executed via `python -c`. It calls the SDK *library's* + * `datacustomcode.lineage.generate_lineage`, which runs the entrypoint against the + * org with reads captured and writes traced (never written). + * + * Arguments arrive as a single JSON object in `sys.argv[1]`, never interpolated into + * this source. The result goes to `resultFile` rather than stdout, because the + * customer's script prints freely to stdout. + */ +export const GENERATE_LINEAGE_SHIM = [ + 'import importlib.util, json, sys', + 'if importlib.util.find_spec("datacustomcode.lineage") is None:', + ` sys.stderr.write("${LINEAGE_UNAVAILABLE_MARKER}\\n")`, + ` sys.exit(${LINEAGE_UNAVAILABLE_EXIT_CODE})`, + 'from datacustomcode.lineage import generate_lineage', + 'a = json.loads(sys.argv[1])', + 'result = generate_lineage(', + ' a["entrypoint"],', + ' a["configFile"],', + ' a["dependencies"],', + ' a["profile"],', + ' sf_cli_org=a["sfCliOrg"],', + ')', + 'with open(a["resultFile"], "w", encoding="utf-8") as f:', + ' json.dump(result.to_dict(), f)', +].join('\n'); + +/** JSON payload handed to {@link GENERATE_LINEAGE_SHIM}. */ +export type LineagePayload = { + entrypoint: string; + configFile: string | null; + dependencies: string[]; + profile: string; + sfCliOrg: string | null; + resultFile: string; +}; + +export type LineageEntry = { + targetField: string; + sourceFields: string[]; +}; + +export type UnresolvedField = { + targetField: string; + reason: string; +}; + +/** The SDK's `LineageResult.to_dict()`; `dataLineage` matches config.json. */ +export type LineageReport = { + dataLineage: LineageEntry[]; + unresolved: UnresolvedField[]; + noSources: string[]; + targets: string[]; + sources: string[]; + pairCount: number; +}; + +export type LineageRunOptions = { + pythonCommand: string; + entrypoint: string; + /** SDK config.yaml (reader/writer config, e.g. `default_row_limit`). */ + configFile?: string; + dependencies: string[]; + sfCliOrg?: string; + profile?: string; + timeoutMs?: number; + onStdout?: (chunk: string) => void; + onStderr?: (chunk: string) => void; +}; + +export function buildLineagePayload(opts: LineageRunOptions, resultFile: string): LineagePayload { + return { + entrypoint: opts.entrypoint, + configFile: opts.configFile ?? null, + dependencies: opts.dependencies, + profile: opts.profile ?? DEFAULT_PROFILE, + sfCliOrg: opts.sfCliOrg ?? null, + resultFile, + }; +} + +function mapLineageExitCode(exitCode: number, stderr: string): SfError | undefined { + if (exitCode === LINEAGE_UNAVAILABLE_EXIT_CODE && stderr.includes(LINEAGE_UNAVAILABLE_MARKER)) { + return new SfError( + messages.getMessage('error.sdkTooOld'), + 'LineageSdkTooOld', + messages.getMessages('actions.sdkTooOld') + ); + } + return undefined; +} + +/** + * Generate field-level lineage for a script package through the SDK's + * `generate_lineage`, streaming the run's output live. The result file lives in a + * private temp directory that is always removed. + */ +export async function generateLineage(opts: LineageRunOptions, deps?: PythonRunnerDeps): Promise { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dce-lineage-')); + const resultFile = path.join(tempDir, 'lineage.json'); + try { + const payload = buildLineagePayload(opts, resultFile); + await runPythonProgram( + { + pythonCommand: opts.pythonCommand, + args: ['-c', GENERATE_LINEAGE_SHIM, JSON.stringify(payload)], + sfCliOrg: opts.sfCliOrg, + timeoutMs: opts.timeoutMs, + onStdout: opts.onStdout, + onStderr: opts.onStderr, + mapExitCode: mapLineageExitCode, + }, + deps + ); + + let raw: string; + try { + raw = await fs.readFile(resultFile, 'utf8'); + } catch (err) { + throw new SfError( + messages.getMessage('error.resultMissing', [(err as Error).message]), + 'LineageResultMissing', + messages.getMessages('actions.resultMissing') + ); + } + return JSON.parse(raw) as LineageReport; + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +} + +/** Class wrapper around {@link generateLineage}, so command tests can stub the run. */ +export class LineageRunner { + public static async generate(opts: LineageRunOptions, deps?: PythonRunnerDeps): Promise { + return generateLineage(opts, deps); + } +} diff --git a/src/utils/nativeScan.ts b/src/utils/nativeScan.ts index 12a2ca6..1fd5214 100644 --- a/src/utils/nativeScan.ts +++ b/src/utils/nativeScan.ts @@ -44,6 +44,11 @@ export type NativeScanOptions = { noRequirements?: boolean; /** Override for the package type. If omitted, resolved via .datacustomcode_proj/sdk_config.json. */ packageType?: CodeType; + /** + * Called with the updated config just before config.json is written (never on dry run). + * Resolving `false` skips every write and marks the result `written: false`. + */ + confirmWrite?: (updatedConfig: Record) => Promise; }; export type NativeScanResult = { @@ -56,6 +61,8 @@ export type NativeScanResult = { requirements?: string[]; /** True when --dry-run prevented file writes. */ dryRun: boolean; + /** Set to false when `confirmWrite` declined the write. */ + written?: false; }; const ENTRYPOINT_FILE = 'entrypoint.py'; @@ -415,7 +422,21 @@ export async function updateConfig( configLocation?: string ): Promise> { const configPath = configLocation ?? path.join(path.dirname(entrypointPath), CONFIG_FILE); + const existing = await readConfigJson(configPath); + existing.entryPoint = path.basename(entrypointPath); + + if (packageType === 'script') { + existing.dataspace = resolveDataspace(existing); + const calls = await scanFile(entrypointPath); + existing.permissions = buildPermissions(calls); + } + + return existing; +} + +/** Read and parse a package's config.json, with the errors `updateConfig` raises. */ +export async function readConfigJson(configPath: string): Promise> { let raw: string; try { raw = await fs.readFile(configPath, 'utf8'); @@ -435,15 +456,6 @@ export async function updateConfig( } catch (err) { throw new SfError(`Failed to parse JSON from ${configPath}: ${(err as Error).message}`, 'InvalidConfig'); } - - existing.entryPoint = path.basename(entrypointPath); - - if (packageType === 'script') { - existing.dataspace = resolveDataspace(existing); - const calls = await scanFile(entrypointPath); - existing.permissions = buildPermissions(calls); - } - return existing; } @@ -515,6 +527,11 @@ export async function executeNativeScan(opts: NativeScanOptions): Promise void; + onStderr?: (chunk: string) => void; + /** Checked before the default failure mapping; return undefined to fall through to it. */ + mapExitCode?: (exitCode: number, stderr: string) => SfError | undefined; +}; + /** - * Run a package's entrypoint through the SDK's `run_entrypoint`, streaming output - * live. Resolves with the accumulated stdout/stderr on a clean exit; rejects with a - * friendly {@link SfError} on non-zero exit or a failure to start the interpreter. + * Spawn the interpreter with `opts.args`, streaming output live. Resolves with the + * accumulated stdout/stderr on a clean exit; rejects with a friendly {@link SfError} + * on non-zero exit or a failure to start the interpreter. */ -export async function runEntrypoint( - opts: PythonRunOptions, +export async function runPythonProgram( + opts: PythonProgramOptions, deps: PythonRunnerDeps = { spawn: defaultSpawn } ): Promise { - const payload = buildRunPayload(opts); - const args = ['-c', RUN_ENTRYPOINT_SHIM, JSON.stringify(payload)]; - debug('run spawn: %s %o', opts.pythonCommand, args); + debug('run spawn: %s %o', opts.pythonCommand, opts.args); return new Promise((resolve, reject) => { - const child = deps.spawn(opts.pythonCommand, args, { + const child = deps.spawn(opts.pythonCommand, opts.args, { timeout: opts.timeoutMs ?? RUN_TIMEOUT_MS, // PYTHONUNBUFFERED so the entrypoint's prints stream immediately instead of buffering. env: { ...process.env, PYTHONUNBUFFERED: '1' }, @@ -200,12 +212,36 @@ export async function runEntrypoint( if (exitCode === 0) { resolve({ stdout: stdout.trim(), stderr: stderr.trim(), exitCode }); } else { - reject(mapRunFailure(exitCode, stderr.trim(), opts.sfCliOrg)); + const trimmed = stderr.trim(); + reject(opts.mapExitCode?.(exitCode, trimmed) ?? mapRunFailure(exitCode, trimmed, opts.sfCliOrg)); } }); }); } +/** + * Run a package's entrypoint through the SDK's `run_entrypoint`, streaming output + * live. Resolves with the accumulated stdout/stderr on a clean exit; rejects with a + * friendly {@link SfError} on non-zero exit or a failure to start the interpreter. + */ +export async function runEntrypoint( + opts: PythonRunOptions, + deps: PythonRunnerDeps = { spawn: defaultSpawn } +): Promise { + const payload = buildRunPayload(opts); + return runPythonProgram( + { + pythonCommand: opts.pythonCommand, + args: ['-c', RUN_ENTRYPOINT_SHIM, JSON.stringify(payload)], + sfCliOrg: opts.sfCliOrg, + timeoutMs: opts.timeoutMs, + onStdout: opts.onStdout, + onStderr: opts.onStderr, + }, + deps + ); +} + /** * Class wrapper around {@link runEntrypoint}, exposed so command-level tests can * stub the entire local run the same way deploy tests stub `NativeDeployer.deploy`. diff --git a/test/commands/data-code-extension/lineage.test.ts b/test/commands/data-code-extension/lineage.test.ts new file mode 100644 index 0000000..c8b9775 --- /dev/null +++ b/test/commands/data-code-extension/lineage.test.ts @@ -0,0 +1,269 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { TestContext } from '@salesforce/core/testSetup'; +import { expect } from 'chai'; +import { SfCommand, stubSfCommandUx } from '@salesforce/sf-plugins-core'; +import { SfError, Org, Connection } from '@salesforce/core'; +import * as sinon from 'sinon'; +import LineageGenerate from '../../../src/commands/data-code-extension/script/lineage/generate.js'; +import { LineageRunner, type LineageReport } from '../../../src/utils/lineageRunner.js'; +import { PythonChecker } from '../../../src/utils/pythonChecker.js'; +import { PipChecker } from '../../../src/utils/pipChecker.js'; + +const PERMISSIONS = { read: { dlo: ['in__dll'] }, write: { dlo: ['out__dll'] } }; + +const REPORT: LineageReport = { + dataLineage: [ + { targetField: 'out__dll.id__c', sourceFields: ['in__dll.id'] }, + { targetField: 'out__dll.name__c', sourceFields: ['in__dll.first', 'in__dll.last'] }, + ], + unresolved: [{ targetField: 'out__dll.x__c', reason: 'untraceable source' }], + noSources: ['out__dll.const__c'], + targets: ['out__dll'], + sources: ['in__dll'], + pairCount: 3, +}; + +describe('data-code-extension script lineage generate', () => { + const $$ = new TestContext(); + let sfCommandStubs: ReturnType; + let generateStub: sinon.SinonStub; + let confirmStub: sinon.SinonStub; + let mockConnection: Connection; + let testDir: string; + let entrypointPath: string; + let configPath: string; + let originalIsTTY: boolean | undefined; + + const baseArgs = (): string[] => ['--entrypoint', entrypointPath, '--target-org', 'test@example.com']; + + function writeConfig(config: Record): void { + fs.writeFileSync(configPath, JSON.stringify(config)); + } + + function readConfig(): Record { + return JSON.parse(fs.readFileSync(configPath, 'utf8')) as Record; + } + + function warnings(): string[] { + return sfCommandStubs.warn.getCalls().map((call) => String(call.args[0])); + } + + beforeEach(() => { + testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-lineage-')); + fs.mkdirSync(path.join(testDir, '.datacustomcode_proj')); + fs.writeFileSync(path.join(testDir, '.datacustomcode_proj', 'sdk_config.json'), JSON.stringify({ type: 'script' })); + fs.mkdirSync(path.join(testDir, 'payload')); + entrypointPath = path.join(testDir, 'payload', 'entrypoint.py'); + fs.writeFileSync(entrypointPath, 'print("hi")\n'); + configPath = path.join(testDir, 'payload', 'config.json'); + writeConfig({ entryPoint: 'entrypoint.py', dataspace: 'default', permissions: PERMISSIONS }); + + sfCommandStubs = stubSfCommandUx($$.SANDBOX); + mockConnection = { refreshAuth: $$.SANDBOX.stub().resolves() } as unknown as Connection; + $$.SANDBOX.stub(Org, 'create').resolves({ + getUsername: () => 'test@example.com', + getConnection: () => mockConnection, + } as unknown as Org); + $$.SANDBOX.stub(PythonChecker, 'checkPython311').resolves({ + command: 'python3', + version: '3.11.5', + major: 3, + minor: 11, + patch: 5, + }); + $$.SANDBOX.stub(PipChecker, 'checkPackage').resolves({ + name: 'salesforce-data-customcode', + version: '1.0.0', + location: '/site-packages', + pipCommand: 'pip', + }); + $$.SANDBOX.stub(PipChecker, 'checkForUpdate').resolves(null); + generateStub = $$.SANDBOX.stub(LineageRunner, 'generate').resolves(structuredClone(REPORT)); + confirmStub = $$.SANDBOX.stub(SfCommand.prototype, 'confirm').resolves(true); + + originalIsTTY = process.stdin.isTTY; + process.stdin.isTTY = true; + }); + + afterEach(() => { + process.stdin.isTTY = originalIsTTY as boolean; + $$.restore(); + fs.rmSync(testDir, { recursive: true, force: true }); + }); + + it('is hidden and supports --json', () => { + expect(LineageGenerate.hidden).to.equal(true); + expect(LineageGenerate.enableJsonFlag).to.equal(true); + }); + + it('writes lineage into config.json, keeps other keys, and forwards run options', async () => { + const result = await LineageGenerate.run([...baseArgs(), '--dependencies', 'pandas, numpy']); + + const opts = generateStub.firstCall.args[0] as Record; + expect(opts.pythonCommand).to.equal('python3'); + expect(opts.entrypoint).to.equal(entrypointPath); + expect(opts.sfCliOrg).to.equal('test@example.com'); + expect(opts.dependencies).to.deep.equal(['pandas', 'numpy']); + expect((mockConnection.refreshAuth as sinon.SinonStub).calledOnce).to.be.true; + + const config = readConfig(); + expect(config.dataLineage).to.deep.equal(REPORT.dataLineage); + expect(config.permissions).to.deep.equal(PERMISSIONS); + expect(config.entryPoint).to.equal('entrypoint.py'); + expect(fs.readFileSync(configPath, 'utf8').endsWith('}\n')).to.be.true; + + expect(result.written).to.equal(true); + expect(result.entryCount).to.equal(2); + expect(result.pairCount).to.equal(3); + expect(result.diff.added).to.have.length(2); + expect(confirmStub.called).to.be.false; + expect(sfCommandStubs.log.calledWithMatch('Generated 2 dataLineage entries with 3')).to.be.true; + }); + + it('always lists unresolved fields', async () => { + await LineageGenerate.run(baseArgs()); + expect(warnings().some((w) => w.includes('out__dll.x__c (untraceable source)'))).to.be.true; + }); + + it('warns when lineage references objects missing from permissions', async () => { + writeConfig({ dataspace: 'default', permissions: { read: { dlo: [] }, write: { dlo: ['out__dll'] } } }); + const result = await LineageGenerate.run(baseArgs()); + expect(result.permissionGaps).to.deep.equal({ targets: [], sources: ['in__dll'] }); + expect(warnings().some((w) => w.includes('read: in__dll'))).to.be.true; + expect(result.written).to.equal(true); + }); + + it('does not write an empty result', async () => { + generateStub.resolves({ ...REPORT, dataLineage: [], pairCount: 0 }); + const result = await LineageGenerate.run(baseArgs()); + expect(result.written).to.equal(false); + expect(readConfig()).not.to.have.property('dataLineage'); + expect(warnings().some((w) => w.includes('No lineage could be generated'))).to.be.true; + }); + + it('--dry-run never writes or prompts', async () => { + writeConfig({ + dataspace: 'default', + permissions: PERMISSIONS, + dataLineage: [{ targetField: 'out__dll.old', sourceFields: ['in__dll.old'] }], + }); + const result = await LineageGenerate.run([...baseArgs(), '--dry-run']); + expect(result.written).to.equal(false); + expect(result.dryRun).to.equal(true); + expect(result.pairCount).to.equal(3); + expect(confirmStub.called).to.be.false; + expect((readConfig().dataLineage as unknown[])[0]).to.deep.equal({ + targetField: 'out__dll.old', + sourceFields: ['in__dll.old'], + }); + }); + + it('writes the full result to --output-file', async () => { + const outputFile = path.join(testDir, 'out', '..', 'lineage-result.json'); + const result = await LineageGenerate.run([...baseArgs(), '--dry-run', '--output-file', outputFile]); + expect(JSON.parse(fs.readFileSync(outputFile, 'utf8'))).to.deep.equal(REPORT); + expect(result.outputFile).to.equal(path.resolve(outputFile)); + }); + + it('rejects function packages before running', async () => { + fs.writeFileSync( + path.join(testDir, '.datacustomcode_proj', 'sdk_config.json'), + JSON.stringify({ type: 'function' }) + ); + try { + await LineageGenerate.run(baseArgs()); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageFunctionPackage'); + } + expect(generateStub.called).to.be.false; + }); + + it('fails with ConfigNotFound before running when config.json is missing', async () => { + fs.rmSync(configPath); + try { + await LineageGenerate.run(baseArgs()); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('ConfigNotFound'); + } + expect(generateStub.called).to.be.false; + }); + + describe('replacing existing lineage', () => { + const OLD = [{ targetField: 'out__dll.old', sourceFields: ['in__dll.old'] }]; + + beforeEach(() => { + writeConfig({ dataspace: 'default', permissions: PERMISSIONS, dataLineage: OLD }); + }); + + it('replaces after the customer confirms', async () => { + const result = await LineageGenerate.run(baseArgs()); + expect(confirmStub.calledOnce).to.be.true; + expect(confirmStub.firstCall.args[0]).to.have.property('defaultAnswer', false); + expect(warnings().some((w) => w.includes('This replaces the existing dataLineage section'))).to.be.true; + expect(result.written).to.equal(true); + expect(result.diff.removed).to.deep.equal(['out__dll.old']); + expect(readConfig().dataLineage).to.deep.equal(REPORT.dataLineage); + }); + + it('keeps existing lineage when the customer declines', async () => { + confirmStub.resolves(false); + const result = await LineageGenerate.run(baseArgs()); + expect(result.written).to.equal(false); + expect(readConfig().dataLineage).to.deep.equal(OLD); + }); + + it('--no-prompt replaces without asking', async () => { + const result = await LineageGenerate.run([...baseArgs(), '--no-prompt']); + expect(confirmStub.called).to.be.false; + expect(result.written).to.equal(true); + }); + + it('fails without --no-prompt when not interactive', async () => { + process.stdin.isTTY = false; + try { + await LineageGenerate.run(baseArgs()); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageReplaceNotConfirmed'); + } + expect(confirmStub.called).to.be.false; + expect(readConfig().dataLineage).to.deep.equal(OLD); + }); + + it('fails with --json without --no-prompt', async () => { + try { + await LineageGenerate.run([...baseArgs(), '--json']); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageReplaceNotConfirmed'); + } + expect(readConfig().dataLineage).to.deep.equal(OLD); + }); + + it('does not prompt when the generated lineage is identical', async () => { + writeConfig({ dataspace: 'default', permissions: PERMISSIONS, dataLineage: REPORT.dataLineage }); + const result = await LineageGenerate.run(baseArgs()); + expect(confirmStub.called).to.be.false; + expect(result.written).to.equal(false); + }); + }); +}); diff --git a/test/commands/data-code-extension/scan.test.ts b/test/commands/data-code-extension/scan.test.ts index 9d9b160..8af1aaa 100644 --- a/test/commands/data-code-extension/scan.test.ts +++ b/test/commands/data-code-extension/scan.test.ts @@ -18,7 +18,9 @@ import * as path from 'node:path'; import * as os from 'node:os'; import { TestContext } from '@salesforce/core/testSetup'; import { expect } from 'chai'; -import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; +import { SfCommand, stubSfCommandUx } from '@salesforce/sf-plugins-core'; +import { SfError } from '@salesforce/core'; +import * as sinon from 'sinon'; import { setPipreqsMock } from '../../../src/utils/nativeScan.js'; import { PythonChecker } from '../../../src/utils/pythonChecker.js'; import { PipChecker } from '../../../src/utils/pipChecker.js'; @@ -254,4 +256,107 @@ describe('data-code-extension scan commands', () => { expect(reqs).to.deep.equal(['pandas']); }); }); + + describe('script scan stale-lineage guard', () => { + const SCRIPT = + 'from datacustomcode.client import Client\n' + + 'def main():\n' + + ' c = Client()\n' + + ' c.read_dlo("Source__dll")\n' + + ' c.write_to_dlo("Dest__dll", df)\n'; + const STALE = [{ targetField: 'OldDest__dll.a', sourceFields: ['Source__dll.a'] }]; + let confirmStub: sinon.SinonStub; + let originalIsTTY: boolean | undefined; + + const configFile = (): string => path.join(testDir, 'payload', 'config.json'); + const readConfig = (): Record => + JSON.parse(fs.readFileSync(configFile(), 'utf8')) as Record; + const warnings = (): string[] => sfCommandStubs.warn.getCalls().map((call) => String(call.args[0])); + + beforeEach(() => { + confirmStub = $$.SANDBOX.stub(SfCommand.prototype, 'confirm').resolves(true); + originalIsTTY = process.stdin.isTTY; + process.stdin.isTTY = true; + }); + + afterEach(() => { + process.stdin.isTTY = originalIsTTY as boolean; + }); + + it('is unchanged without dataLineage: same config.json bytes, no prompt, no warning', async () => { + makeScriptPackage(SCRIPT); + await ScriptScan.run([]); + expect(fs.readFileSync(configFile(), 'utf8')).to.equal( + JSON.stringify( + { + dataspace: 'default', + entryPoint: 'entrypoint.py', + permissions: { read: { dlo: ['Source__dll'] }, write: { dlo: ['Dest__dll'] } }, + }, + null, + 2 + ) + ); + expect(confirmStub.called).to.be.false; + expect(sfCommandStubs.warn.called).to.be.false; + }); + + it('does not prompt when lineage is still covered by permissions', async () => { + const lineage = [{ targetField: 'Dest__dll.a', sourceFields: ['Source__dll.a'] }]; + makeScriptPackage(SCRIPT, { dataspace: 'default', dataLineage: lineage }); + await ScriptScan.run([]); + expect(confirmStub.called).to.be.false; + expect(readConfig().dataLineage).to.deep.equal(lineage); + }); + + it('writes permissions and leaves lineage untouched when confirmed', async () => { + makeScriptPackage(SCRIPT, { dataspace: 'default', dataLineage: STALE }); + const result = await ScriptScan.run([]); + expect(confirmStub.calledOnce).to.be.true; + expect(confirmStub.firstCall.args[0]).to.have.property('defaultAnswer', false); + expect(warnings().some((w) => w.includes('write: OldDest__dll') && w.includes('1 entries'))).to.be.true; + expect(result.executionResult.written).to.be.undefined; + const config = readConfig(); + expect(config.permissions).to.deep.equal({ read: { dlo: ['Source__dll'] }, write: { dlo: ['Dest__dll'] } }); + expect(config.dataLineage).to.deep.equal(STALE); + }); + + it('writes nothing when declined', async () => { + makeScriptPackage(SCRIPT, { dataspace: 'default', dataLineage: STALE }); + const before = fs.readFileSync(configFile(), 'utf8'); + confirmStub.resolves(false); + const result = await ScriptScan.run([]); + expect(result.success).to.be.true; + expect(result.executionResult.written).to.equal(false); + expect(fs.readFileSync(configFile(), 'utf8')).to.equal(before); + expect(fs.existsSync(path.join(testDir, 'requirements.txt'))).to.be.false; + }); + + it('--no-prompt writes without asking', async () => { + makeScriptPackage(SCRIPT, { dataspace: 'default', dataLineage: STALE }); + await ScriptScan.run(['--no-prompt']); + expect(confirmStub.called).to.be.false; + expect(readConfig().permissions).to.have.property('write'); + }); + + it('fails without --no-prompt when not interactive', async () => { + makeScriptPackage(SCRIPT, { dataspace: 'default', dataLineage: STALE }); + const before = fs.readFileSync(configFile(), 'utf8'); + process.stdin.isTTY = false; + try { + await ScriptScan.run([]); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('ScanStaleLineageNotConfirmed'); + } + expect(fs.readFileSync(configFile(), 'utf8')).to.equal(before); + }); + + it('--dry-run warns but never prompts', async () => { + makeScriptPackage(SCRIPT, { dataspace: 'default', dataLineage: STALE }); + await ScriptScan.run(['--dry-run']); + expect(confirmStub.called).to.be.false; + expect(warnings().some((w) => w.includes('write: OldDest__dll'))).to.be.true; + }); + }); }); diff --git a/test/utils/lineageConfig.test.ts b/test/utils/lineageConfig.test.ts new file mode 100644 index 0000000..c2896b4 --- /dev/null +++ b/test/utils/lineageConfig.test.ts @@ -0,0 +1,149 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { expect } from 'chai'; +import { + countEntriesReferencing, + countPairs, + diffLineage, + findPermissionGaps, + findStaleLineage, + isEmptyDiff, + readLineage, + replaceLineage, + writeConfigJson, +} from '../../src/utils/lineageConfig.js'; + +const CONFIG = { + entryPoint: 'entrypoint.py', + dataspace: 'default', + permissions: { read: { dlo: ['In__dll'] }, write: { dlo: ['out__dll'] } }, +}; + +describe('lineageConfig', () => { + describe('readLineage', () => { + it('returns [] when the section is absent or not an array', () => { + expect(readLineage({})).to.deep.equal([]); + expect(readLineage({ dataLineage: 'nope' })).to.deep.equal([]); + }); + + it('drops malformed entries', () => { + const good = { targetField: 'o.a', sourceFields: ['i.a'] }; + expect(readLineage({ dataLineage: [good, { targetField: 1 }, null] })).to.deep.equal([good]); + }); + }); + + describe('diffLineage', () => { + it('reports added, removed and changed targets', () => { + const diff = diffLineage( + [ + { targetField: 'o.keep', sourceFields: ['i.a', 'i.b'] }, + { targetField: 'o.change', sourceFields: ['i.a'] }, + { targetField: 'o.gone', sourceFields: ['i.a'] }, + ], + [ + { targetField: 'o.keep', sourceFields: ['i.b', 'i.a'] }, + { targetField: 'o.change', sourceFields: ['i.c'] }, + { targetField: 'o.new', sourceFields: ['i.a'] }, + ] + ); + expect(diff).to.deep.equal({ added: ['o.new'], removed: ['o.gone'], changed: ['o.change'] }); + expect(isEmptyDiff(diff)).to.equal(false); + }); + + it('is empty for identical lineage', () => { + const entries = [{ targetField: 'o.a', sourceFields: ['i.a'] }]; + expect(isEmptyDiff(diffLineage(entries, entries))).to.equal(true); + }); + }); + + it('countPairs sums source fields', () => { + expect( + countPairs([ + { targetField: 'o.a', sourceFields: ['i.a', 'i.b'] }, + { targetField: 'o.b', sourceFields: ['i.c'] }, + ]) + ).to.equal(3); + }); + + it('replaceLineage replaces wholesale and keeps other keys in order', () => { + const entries = [{ targetField: 'out__dll.a', sourceFields: ['In__dll.a'] }]; + const updated = replaceLineage( + { ...CONFIG, dataLineage: [{ targetField: 'out__dll.old', sourceFields: ['In__dll.old'] }], extra: 1 }, + entries + ); + expect(Object.keys(updated)).to.deep.equal(['entryPoint', 'dataspace', 'permissions', 'dataLineage', 'extra']); + expect(updated.dataLineage).to.deep.equal(entries); + }); + + describe('findPermissionGaps', () => { + it('reports objects missing from permissions, case-insensitively', () => { + expect(findPermissionGaps(CONFIG, ['out__dll', 'other__dll'], ['in__dll', 'x__dll', 'x__dll'])).to.deep.equal({ + targets: ['other__dll'], + sources: ['x__dll'], + }); + }); + + it('treats a missing permissions section as nothing permitted', () => { + expect(findPermissionGaps({}, ['o'], ['i'])).to.deep.equal({ targets: ['o'], sources: ['i'] }); + }); + }); + + describe('findStaleLineage', () => { + it('is empty without dataLineage', () => { + expect(findStaleLineage(CONFIG)).to.deep.equal({ targets: [], sources: [] }); + }); + + it('is empty when lineage matches permissions', () => { + const config = { ...CONFIG, dataLineage: [{ targetField: 'out__dll.a', sourceFields: ['In__dll.a'] }] }; + expect(findStaleLineage(config)).to.deep.equal({ targets: [], sources: [] }); + }); + + it('reports a stale target and a stale source', () => { + const config = { + ...CONFIG, + dataLineage: [ + { targetField: 'old_out__dll.a', sourceFields: ['In__dll.a'] }, + { targetField: 'out__dll.b', sourceFields: ['old_in__dll.b'] }, + ], + }; + expect(findStaleLineage(config)).to.deep.equal({ targets: ['old_out__dll'], sources: ['old_in__dll'] }); + expect(countEntriesReferencing(config, ['old_out__dll'])).to.equal(1); + expect(countEntriesReferencing(config, ['old_out__dll', 'OLD_IN__dll'])).to.equal(2); + }); + + it('reads DMO permissions too', () => { + const config = { + permissions: { read: { dmo: ['in__dlm'] }, write: { dmo: ['out__dlm'] } }, + dataLineage: [{ targetField: 'out__dlm.a', sourceFields: ['in__dlm.a', 'in__dll.a'] }], + }; + expect(findStaleLineage(config)).to.deep.equal({ targets: [], sources: ['in__dll'] }); + }); + }); + + it('writeConfigJson writes 2-space JSON with a trailing newline', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-lineage-config-')); + try { + const file = path.join(dir, 'config.json'); + await writeConfigJson(file, { a: [1] }); + expect(fs.readFileSync(file, 'utf8')).to.equal('{\n "a": [\n 1\n ]\n}\n'); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/utils/lineageRunner.test.ts b/test/utils/lineageRunner.test.ts new file mode 100644 index 0000000..40089fc --- /dev/null +++ b/test/utils/lineageRunner.test.ts @@ -0,0 +1,271 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { expect } from 'chai'; +import { SfError } from '@salesforce/core'; +import { + buildLineagePayload, + generateLineage, + GENERATE_LINEAGE_SHIM, + LINEAGE_UNAVAILABLE_EXIT_CODE, + LINEAGE_UNAVAILABLE_MARKER, + type LineagePayload, + type LineageReport, + type LineageRunOptions, +} from '../../src/utils/lineageRunner.js'; +import { RUN_TIMEOUT_MS, type RunnerChild, type RunnerStream, type SpawnFn } from '../../src/utils/pythonRunner.js'; + +async function expectReject(promise: Promise): Promise { + try { + await promise; + } catch (err) { + return err as SfError; + } + throw new Error('Expected promise to reject, but it resolved'); +} + +class FakeStream implements RunnerStream { + private dataListeners: Array<(chunk: string) => void> = []; + + public setEncoding(): this { + return this; + } + + public on(event: 'data', listener: (chunk: string) => void): this { + this.dataListeners.push(listener); + return this; + } + + public push(chunk: string): void { + for (const listener of this.dataListeners) listener(chunk); + } +} + +class FakeChild implements RunnerChild { + public stdout = new FakeStream(); + public stderr = new FakeStream(); + private closeListeners: Array<(code: number | null) => void> = []; + private errorListeners: Array<(err: Error) => void> = []; + + public on(event: 'close', listener: (code: number | null) => void): this; + public on(event: 'error', listener: (err: Error) => void): this; + public on(event: 'close' | 'error', listener: ((code: number | null) => void) | ((err: Error) => void)): this { + if (event === 'close') this.closeListeners.push(listener as (code: number | null) => void); + else this.errorListeners.push(listener as (err: Error) => void); + return this; + } + + public emitClose(code: number | null): void { + for (const listener of this.closeListeners) listener(code); + } + + public emitError(err: Error): void { + for (const listener of this.errorListeners) listener(err); + } +} + +type SpawnCall = { + command: string; + args: readonly string[]; + options: { env?: NodeJS.ProcessEnv; timeout?: number }; +}; + +const REPORT: LineageReport = { + dataLineage: [{ targetField: 'out__dll.id__c', sourceFields: ['in__dll.id'] }], + unresolved: [{ targetField: 'out__dll.x__c', reason: 'untraceable source' }], + noSources: [], + targets: ['out__dll'], + sources: ['in__dll'], + pairCount: 1, +}; + +function payloadOf(call: SpawnCall): LineagePayload { + return JSON.parse(call.args[2]) as LineagePayload; +} + +/** + * A spawn seam that, like the real shim, acts on the payload asynchronously: `behave` + * receives the child and payload once the runner has attached its listeners. + */ +function fakeSpawn(calls: SpawnCall[], behave: (child: FakeChild, payload: LineagePayload) => void): SpawnFn { + return (command, args, options) => { + const call = { command, args, options }; + calls.push(call); + const child = new FakeChild(); + setImmediate(() => behave(child, payloadOf(call))); + return child; + }; +} + +function succeed(report: LineageReport = REPORT): (child: FakeChild, payload: LineagePayload) => void { + return (child, payload) => { + fs.writeFileSync(payload.resultFile, JSON.stringify(report)); + child.emitClose(0); + }; +} + +function makeOpts(overrides: Partial = {}): LineageRunOptions { + return { + pythonCommand: 'python3', + entrypoint: '/pkg/payload/entrypoint.py', + dependencies: ['pandas'], + sfCliOrg: 'myorg', + ...overrides, + }; +} + +describe('lineageRunner', () => { + describe('buildLineagePayload', () => { + it('maps options to the shim payload with defaults', () => { + expect(buildLineagePayload(makeOpts(), '/tmp/x/lineage.json')).to.deep.equal({ + entrypoint: '/pkg/payload/entrypoint.py', + configFile: null, + dependencies: ['pandas'], + profile: 'default', + sfCliOrg: 'myorg', + resultFile: '/tmp/x/lineage.json', + }); + }); + + it('passes through config file and profile', () => { + const payload = buildLineagePayload(makeOpts({ configFile: '/c.yaml', profile: 'p' }), '/r.json'); + expect(payload.configFile).to.equal('/c.yaml'); + expect(payload.profile).to.equal('p'); + }); + }); + + describe('GENERATE_LINEAGE_SHIM', () => { + it('calls the SDK library, not the CLI', () => { + expect(GENERATE_LINEAGE_SHIM).to.include('from datacustomcode.lineage import generate_lineage'); + expect(GENERATE_LINEAGE_SHIM).to.include('json.dump(result.to_dict(), f)'); + expect(GENERATE_LINEAGE_SHIM).not.to.include('datacustomcode.cli'); + }); + + it('exits with the dedicated code and marker when the SDK lacks lineage', () => { + expect(GENERATE_LINEAGE_SHIM).to.include(`sys.exit(${LINEAGE_UNAVAILABLE_EXIT_CODE})`); + expect(GENERATE_LINEAGE_SHIM).to.include(LINEAGE_UNAVAILABLE_MARKER); + }); + }); + + describe('generateLineage', () => { + it('passes the payload as its own argv element and returns the result file', async () => { + const calls: SpawnCall[] = []; + const entrypoint = '/pkg/"; import os; os.system("x") #/entrypoint.py'; + const report = await generateLineage(makeOpts({ entrypoint }), { spawn: fakeSpawn(calls, succeed()) }); + + expect(report).to.deep.equal(REPORT); + expect(calls).to.have.length(1); + expect(calls[0].command).to.equal('python3'); + expect(calls[0].args[0]).to.equal('-c'); + expect(calls[0].args[1]).to.equal(GENERATE_LINEAGE_SHIM); + expect(payloadOf(calls[0]).entrypoint).to.equal(entrypoint); + expect(calls[0].options.timeout).to.equal(RUN_TIMEOUT_MS); + expect(calls[0].options.env?.PYTHONUNBUFFERED).to.equal('1'); + }); + + it('removes the temp directory after success', async () => { + const calls: SpawnCall[] = []; + await generateLineage(makeOpts(), { spawn: fakeSpawn(calls, succeed()) }); + expect(fs.existsSync(path.dirname(payloadOf(calls[0]).resultFile))).to.equal(false); + }); + + it('removes the temp directory after failure', async () => { + const calls: SpawnCall[] = []; + await expectReject( + generateLineage(makeOpts(), { + spawn: fakeSpawn(calls, (child) => { + child.stderr.push('boom'); + child.emitClose(1); + }), + }) + ); + expect(fs.existsSync(path.dirname(payloadOf(calls[0]).resultFile))).to.equal(false); + }); + + it('streams stdout and stderr chunks', async () => { + const out: string[] = []; + const err: string[] = []; + await generateLineage(makeOpts({ onStdout: (c) => out.push(c), onStderr: (c) => err.push(c) }), { + spawn: fakeSpawn([], (child, payload) => { + child.stdout.push('hello'); + child.stderr.push('spark log'); + succeed()(child, payload); + }), + }); + expect(out).to.deep.equal(['hello']); + expect(err).to.deep.equal(['spark log']); + }); + + it('maps the unavailable exit code to LineageSdkTooOld', async () => { + const err = await expectReject( + generateLineage(makeOpts(), { + spawn: fakeSpawn([], (child) => { + child.stderr.push(`${LINEAGE_UNAVAILABLE_MARKER}\n`); + child.emitClose(LINEAGE_UNAVAILABLE_EXIT_CODE); + }), + }) + ); + expect(err.name).to.equal('LineageSdkTooOld'); + expect(err.actions?.join(' ')).to.include('pip install -U salesforce-data-customcode'); + }); + + it("does not mistake a script's own exit code 3 for an old SDK", async () => { + const err = await expectReject( + generateLineage(makeOpts(), { + spawn: fakeSpawn([], (child) => { + child.stderr.push('SystemExit: 3'); + child.emitClose(LINEAGE_UNAVAILABLE_EXIT_CODE); + }), + }) + ); + expect(err.name).to.equal('RunExecutionFailed'); + }); + + it('maps authentication failures', async () => { + const err = await expectReject( + generateLineage(makeOpts(), { + spawn: fakeSpawn([], (child) => { + child.stderr.push('Authentication failed for org'); + child.emitClose(1); + }), + }) + ); + expect(err.name).to.equal('RunAuthenticationFailed'); + expect(err.message).to.include('myorg'); + }); + + it('maps a failure to start the interpreter', async () => { + const err = await expectReject( + generateLineage(makeOpts(), { spawn: fakeSpawn([], (child) => child.emitError(new Error('ENOENT'))) }) + ); + expect(err.name).to.equal('RunProcessStartFailed'); + }); + + it('honors a custom timeout', async () => { + const calls: SpawnCall[] = []; + await generateLineage(makeOpts({ timeoutMs: 1234 }), { spawn: fakeSpawn(calls, succeed()) }); + expect(calls[0].options.timeout).to.equal(1234); + }); + + it('fails with LineageResultMissing when the shim exits 0 without a result', async () => { + const err = await expectReject( + generateLineage(makeOpts(), { spawn: fakeSpawn([], (child) => child.emitClose(0)) }) + ); + expect(err.name).to.equal('LineageResultMissing'); + }); + }); +}); diff --git a/test/utils/nativeScan.test.ts b/test/utils/nativeScan.test.ts index ac73508..e1e1917 100644 --- a/test/utils/nativeScan.test.ts +++ b/test/utils/nativeScan.test.ts @@ -687,3 +687,69 @@ describe('nativeScan: executeNativeScan', () => { await fs.rm(dir, { recursive: true, force: true }); }); }); + +describe('nativeScan: executeNativeScan confirmWrite', () => { + const ENTRYPOINT = + 'from datacustomcode.client import Client\n' + + 'def main():\n' + + ' c.read_dlo("Source__dll")\n' + + ' c.write_to_dlo("Dest__dll", df)\n'; + + beforeEach(() => { + setPipreqsMock(async () => 'pandas\n'); + }); + + afterEach(() => { + setPipreqsMock(null); + }); + + it('omits written when confirmWrite is not supplied', async () => { + const dir = await makePackage({ entrypoint: ENTRYPOINT }); + const result = await executeNativeScan({ workingDir: dir, packageType: 'script' }); + expect(result).not.to.have.property('written'); + await fs.rm(dir, { recursive: true, force: true }); + }); + + it('writes when confirmWrite resolves true, passing the updated config', async () => { + const dir = await makePackage({ entrypoint: ENTRYPOINT }); + let seen: Record | undefined; + const result = await executeNativeScan({ + workingDir: dir, + packageType: 'script', + confirmWrite: async (config) => { + seen = config; + return true; + }, + }); + expect(seen).to.equal(result.config); + expect(result).not.to.have.property('written'); + expect(result.requirementsPath).to.equal(path.join(dir, 'requirements.txt')); + await fs.rm(dir, { recursive: true, force: true }); + }); + + it('writes nothing when confirmWrite resolves false', async () => { + const dir = await makePackage({ entrypoint: ENTRYPOINT }); + const before = await fs.readFile(path.join(dir, 'payload', 'config.json'), 'utf8'); + const result = await executeNativeScan({ workingDir: dir, packageType: 'script', confirmWrite: async () => false }); + expect(result.written).to.equal(false); + expect(result.requirementsPath).to.be.undefined; + expect(await fs.readFile(path.join(dir, 'payload', 'config.json'), 'utf8')).to.equal(before); + await fs.rm(dir, { recursive: true, force: true }); + }); + + it('never calls confirmWrite on dry run', async () => { + const dir = await makePackage({ entrypoint: ENTRYPOINT }); + let called = false; + await executeNativeScan({ + workingDir: dir, + packageType: 'script', + dryRun: true, + confirmWrite: async () => { + called = true; + return true; + }, + }); + expect(called).to.be.false; + await fs.rm(dir, { recursive: true, force: true }); + }); +}); From 8f5caa5f65fc82da72e81938143929b38b768961 Mon Sep 17 00:00:00 2001 From: Jesus Orosco Date: Mon, 12 Oct 2026 10:42:35 -0700 Subject: [PATCH 2/8] feat: addressing LLM complaints --- messages/lineage.md | 12 ++ src/base/lineageBase.ts | 94 ++++++++++++-- src/utils/lineageConfig.ts | 22 +++- .../data-code-extension/lineage.test.ts | 117 ++++++++++++++++++ test/utils/lineageConfig.test.ts | 40 ++++++ 5 files changed, 277 insertions(+), 8 deletions(-) diff --git a/messages/lineage.md b/messages/lineage.md index da3fe80..aeb62da 100644 --- a/messages/lineage.md +++ b/messages/lineage.md @@ -107,6 +107,18 @@ Replace the existing dataLineage in config.json? Lineage can only be generated for script packages; '%s' is a function package. +# error.outputFileCollision + +--output-file '%s' is inside the package's payload or project metadata, or is one of its config files. Choose a path outside the package's payload directory. + +# error.lineageChanged + +dataLineage in '%s' changed while waiting for confirmation, so it was not replaced. + +# actions.lineageChanged + +- Re-run the command to compare the generated lineage against the current dataLineage. + # error.replaceNotConfirmed config.json already has a different dataLineage section, and replacing it needs confirmation. diff --git a/src/base/lineageBase.ts b/src/base/lineageBase.ts index af20b15..9b0c150 100644 --- a/src/base/lineageBase.ts +++ b/src/base/lineageBase.ts @@ -13,18 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { promises as fs } from 'node:fs'; +import { promises as fs, type Stats } from 'node:fs'; import path from 'node:path'; import { SfCommand } from '@salesforce/sf-plugins-core'; import { Messages, Org, SfError } from '@salesforce/core'; import { checkEnvironment } from '../utils/environmentChecker.js'; import { splitDependencies } from '../utils/pythonRunner.js'; import { LineageRunner, type LineageReport } from '../utils/lineageRunner.js'; -import { findBaseDirectory, getPackageType, readConfigJson } from '../utils/nativeScan.js'; +import { findBaseDirectory, getPackageType, getSdkConfigPath, readConfigJson } from '../utils/nativeScan.js'; import { countPairs, diffLineage, findPermissionGaps, + hasInvalidLineage, + hasLineageSection, isEmptyDiff, readLineage, replaceLineage, @@ -59,6 +61,8 @@ export type LineageGenerateResult = SharedResultProps & { }; const CONFIG_FILE = 'config.json'; +/** Project config the SDK loads implicitly from the directory above `payload/`. */ +const PROJECT_CONFIG_YAML = 'config.yaml'; // eslint-disable-next-line sf-plugin/command-summary, sf-plugin/command-example export abstract class LineageBase extends SfCommand { @@ -79,7 +83,28 @@ export abstract class LineageBase extends SfCommand { throw new SfError(messages.getMessage('error.functionPackage', [flags.entrypoint]), 'LineageFunctionPackage'); } // Read config.json up front so a missing or malformed file fails before the run. - const config = await readConfigJson(configPath); + await readConfigJson(configPath); + + const outputFile = flags['output-file'] ? path.resolve(flags['output-file']) : undefined; + if (outputFile) { + // Everything under payload/ is run and deployed; .datacustomcode_proj/ holds package metadata. + const payloadDir = path.dirname(entrypoint); + const sdkConfigPath = getSdkConfigPath(findBaseDirectory(entrypoint)); + const inputDirs = [payloadDir, path.dirname(sdkConfigPath)]; + const inputFiles = [ + configPath, + entrypoint, + sdkConfigPath, + path.join(path.dirname(payloadDir), PROJECT_CONFIG_YAML), + ...(flags['config-file'] ? [path.resolve(flags['config-file'])] : []), + ]; + if (await isPackageInput(outputFile, inputDirs, inputFiles)) { + throw new SfError( + messages.getMessage('error.outputFileCollision', [outputFile]), + 'LineageOutputFileCollision' + ); + } + } const { pythonInfo, packageInfo } = await checkEnvironment(this.spinner, this.log.bind(this), messages); @@ -103,12 +128,14 @@ export abstract class LineageBase extends SfCommand { onStderr: (chunk) => process.stderr.write(chunk), }); - const outputFile = flags['output-file'] ? path.resolve(flags['output-file']) : undefined; if (outputFile) { await fs.writeFile(outputFile, `${JSON.stringify(lineage, null, 2)}\n`, 'utf8'); this.log(messages.getMessage('info.outputWritten', [outputFile])); } + // Re-read so edits made during the run aren't overwritten. + const config = await readConfigJson(configPath); + const shownLineage = JSON.stringify(config.dataLineage); const entries = lineage.dataLineage; const current = readLineage(config); const diff = diffLineage(current, entries); @@ -145,17 +172,27 @@ export abstract class LineageBase extends SfCommand { return result; } - if (isEmptyDiff(diff)) { + if (isEmptyDiff(diff) && !hasInvalidLineage(config)) { this.log(messages.getMessage('info.unchanged', [configPath])); return result; } - if (current.length > 0 && !(await this.confirmReplace(messages, flags['no-prompt']))) { + if (hasLineageSection(config) && !(await this.confirmReplace(messages, flags['no-prompt']))) { this.log(messages.getMessage('info.notReplaced')); return result; } - await writeConfigJson(configPath, replaceLineage(config, entries)); + // Re-read again: the prompt can wait indefinitely. Lineage edited meanwhile wasn't in the + // diff the customer confirmed, so don't replace it. + const latest = await readConfigJson(configPath); + if (JSON.stringify(latest.dataLineage) !== shownLineage) { + throw new SfError( + messages.getMessage('error.lineageChanged', [configPath]), + 'LineageChangedDuringConfirm', + messages.getMessages('actions.lineageChanged') + ); + } + await writeConfigJson(configPath, replaceLineage(latest, entries)); this.log(messages.getMessage('info.written', [configPath])); return { ...result, written: true }; } catch (error) { @@ -209,6 +246,49 @@ export abstract class LineageBase extends SfCommand { protected abstract getMessages(): Messages; } +/** + * True when `file` lies under one of `dirs` (as given or after resolving symlinks), or is one of + * `files` by path or by file identity, which also catches symlinks and hard links. + */ +async function isPackageInput(file: string, dirs: string[], files: string[]): Promise { + const [target, realDirs, targetStat, fileStats] = await Promise.all([ + realPath(file), + Promise.all(dirs.map(realPath)), + statOrUndefined(file), + Promise.all(files.map(statOrUndefined)), + ]); + const within = (child: string, dir: string): boolean => { + const relative = path.relative(dir, child); + return !path.isAbsolute(relative) && relative.split(path.sep)[0] !== '..'; + }; + return ( + dirs.some((dir) => within(file, dir)) || + realDirs.some((dir) => within(target, dir)) || + (targetStat !== undefined && + fileStats.some((stat) => stat !== undefined && stat.dev === targetStat.dev && stat.ino === targetStat.ino)) + ); +} + +async function statOrUndefined(file: string): Promise { + return fs.stat(file).catch(() => undefined); +} + +/** + * Canonical path of `file`, resolving symlinks (and case, on case-insensitive file systems). + * A file that doesn't exist yet resolves through its parent directory. + */ +async function realPath(file: string): Promise { + try { + return await fs.realpath(file); + } catch { + try { + return path.join(await fs.realpath(path.dirname(file)), path.basename(file)); + } catch { + return path.resolve(file); + } + } +} + function bullets(lines: string[]): string { return lines.map((line) => ` - ${line}`).join('\n'); } diff --git a/src/utils/lineageConfig.ts b/src/utils/lineageConfig.ts index 4894541..5c58a47 100644 --- a/src/utils/lineageConfig.ts +++ b/src/utils/lineageConfig.ts @@ -37,10 +37,30 @@ export function readLineage(config: Record): LineageEntry[] { typeof entry === 'object' && entry !== null && typeof (entry as LineageEntry).targetField === 'string' && - Array.isArray((entry as LineageEntry).sourceFields) + Array.isArray((entry as LineageEntry).sourceFields) && + (entry as LineageEntry).sourceFields.every((field) => typeof field === 'string') ); } +/** True when the config has a `dataLineage` section that isn't an empty array. */ +export function hasLineageSection(config: Record): boolean { + const section = config.dataLineage; + return section !== undefined && !(Array.isArray(section) && section.length === 0); +} + +/** + * True when the config's `dataLineage` has entries `readLineage` drops, or repeats a target + * field. Targets compare case-insensitively, as deploy validation does. + */ +export function hasInvalidLineage(config: Record): boolean { + const section = config.dataLineage; + if (section === undefined) return false; + if (!Array.isArray(section)) return true; + const entries = readLineage(config); + if (entries.length !== section.length) return true; + return new Set(entries.map((entry) => entry.targetField.toLowerCase())).size !== entries.length; +} + /** Per-target difference between the current `dataLineage` and a generated one. */ export function diffLineage(current: LineageEntry[], generated: LineageEntry[]): LineageDiff { const key = (entry: LineageEntry): string => [...entry.sourceFields].sort().join('\n'); diff --git a/test/commands/data-code-extension/lineage.test.ts b/test/commands/data-code-extension/lineage.test.ts index c8b9775..61ea767 100644 --- a/test/commands/data-code-extension/lineage.test.ts +++ b/test/commands/data-code-extension/lineage.test.ts @@ -182,6 +182,91 @@ describe('data-code-extension script lineage generate', () => { expect(result.outputFile).to.equal(path.resolve(outputFile)); }); + describe('--output-file collisions', () => { + async function expectCollision(outputFile: string): Promise { + const before = fs.readFileSync(configPath, 'utf8'); + try { + await LineageGenerate.run([...baseArgs(), '--dry-run', '--output-file', outputFile]); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageOutputFileCollision'); + } + expect(generateStub.called).to.be.false; + expect(fs.readFileSync(configPath, 'utf8')).to.equal(before); + } + + it('rejects the package config.json', async () => { + await expectCollision(configPath); + }); + + it('rejects the entrypoint', async () => { + await expectCollision(entrypointPath); + }); + + it('rejects a symlink to config.json', async () => { + const link = path.join(testDir, 'link.json'); + fs.symlinkSync(configPath, link); + await expectCollision(link); + }); + + it('rejects a new file inside payload/', async () => { + await expectCollision(path.join(testDir, 'payload', 'lineage-result.json')); + }); + + it('rejects SDK package metadata', async () => { + await expectCollision(path.join(testDir, '.datacustomcode_proj', 'sdk_config.json')); + }); + + it('rejects the project config.yaml', async () => { + fs.writeFileSync(path.join(testDir, 'config.yaml'), 'reader_config: {}\n'); + await expectCollision(path.join(testDir, 'config.yaml')); + }); + + it('rejects a symlinked directory that points into payload/', async () => { + const link = path.join(testDir, 'payload-link'); + fs.symlinkSync(path.join(testDir, 'payload'), link); + await expectCollision(path.join(link, 'lineage-result.json')); + }); + + it('rejects payload/config.json when it is a symlink to a config outside payload/', async () => { + const shared = path.join(testDir, 'shared-config.json'); + fs.renameSync(configPath, shared); + fs.symlinkSync(shared, configPath); + const before = fs.readFileSync(shared, 'utf8'); + await expectCollision(configPath); + expect(fs.readFileSync(shared, 'utf8')).to.equal(before); + }); + + it('rejects a hard link to config.json', async () => { + const link = path.join(testDir, 'hardlink.json'); + fs.linkSync(configPath, link); + await expectCollision(link); + }); + + it('rejects a hard link to the SDK metadata', async () => { + const link = path.join(testDir, 'sdk-hardlink.json'); + fs.linkSync(path.join(testDir, '.datacustomcode_proj', 'sdk_config.json'), link); + await expectCollision(link); + }); + + it('allows a new file at the project root', async () => { + const outputFile = path.join(testDir, 'lineage-result.json'); + await LineageGenerate.run([...baseArgs(), '--dry-run', '--output-file', outputFile]); + expect(fs.existsSync(outputFile)).to.be.true; + }); + }); + + it('keeps config.json edits made while lineage was generating', async () => { + generateStub.callsFake(async () => { + writeConfig({ ...readConfig(), dataspace: 'edited' }); + return structuredClone(REPORT); + }); + const result = await LineageGenerate.run(baseArgs()); + expect(result.written).to.equal(true); + expect(readConfig().dataspace).to.equal('edited'); + expect(readConfig().dataLineage).to.deep.equal(REPORT.dataLineage); + }); + it('rejects function packages before running', async () => { fs.writeFileSync( path.join(testDir, '.datacustomcode_proj', 'sdk_config.json'), @@ -237,6 +322,21 @@ describe('data-code-extension script lineage generate', () => { expect(result.written).to.equal(true); }); + it('does not replace lineage edited while the prompt was pending', async () => { + const edited = [{ targetField: 'out__dll.edited', sourceFields: ['in__dll.edited'] }]; + confirmStub.callsFake(async () => { + writeConfig({ ...readConfig(), dataLineage: edited }); + return true; + }); + try { + await LineageGenerate.run(baseArgs()); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageChangedDuringConfirm'); + } + expect(readConfig().dataLineage).to.deep.equal(edited); + }); + it('fails without --no-prompt when not interactive', async () => { process.stdin.isTTY = false; try { @@ -259,6 +359,23 @@ describe('data-code-extension script lineage generate', () => { expect(readConfig().dataLineage).to.deep.equal(OLD); }); + it('rewrites duplicate targets even when the diff is empty', async () => { + const duplicated = [...REPORT.dataLineage, REPORT.dataLineage[0]]; + writeConfig({ dataspace: 'default', permissions: PERMISSIONS, dataLineage: duplicated }); + const result = await LineageGenerate.run([...baseArgs(), '--no-prompt']); + expect(result.written).to.equal(true); + expect(readConfig().dataLineage).to.deep.equal(REPORT.dataLineage); + }); + + it('asks before replacing a section with only malformed entries', async () => { + const malformed = [{ targetField: 'out__dll.id__c', sourceFields: [null] }]; + writeConfig({ dataspace: 'default', permissions: PERMISSIONS, dataLineage: malformed }); + const result = await LineageGenerate.run(baseArgs()); + expect(confirmStub.calledOnce).to.be.true; + expect(result.written).to.equal(true); + expect(readConfig().dataLineage).to.deep.equal(REPORT.dataLineage); + }); + it('does not prompt when the generated lineage is identical', async () => { writeConfig({ dataspace: 'default', permissions: PERMISSIONS, dataLineage: REPORT.dataLineage }); const result = await LineageGenerate.run(baseArgs()); diff --git a/test/utils/lineageConfig.test.ts b/test/utils/lineageConfig.test.ts index c2896b4..c0306d6 100644 --- a/test/utils/lineageConfig.test.ts +++ b/test/utils/lineageConfig.test.ts @@ -23,6 +23,8 @@ import { diffLineage, findPermissionGaps, findStaleLineage, + hasInvalidLineage, + hasLineageSection, isEmptyDiff, readLineage, replaceLineage, @@ -46,6 +48,38 @@ describe('lineageConfig', () => { const good = { targetField: 'o.a', sourceFields: ['i.a'] }; expect(readLineage({ dataLineage: [good, { targetField: 1 }, null] })).to.deep.equal([good]); }); + + it('drops entries whose sourceFields are not all strings', () => { + const good = { targetField: 'o.a', sourceFields: ['i.a'] }; + expect(readLineage({ dataLineage: [good, { targetField: 'o.b', sourceFields: [null] }] })).to.deep.equal([good]); + }); + }); + + it('hasLineageSection ignores an absent or empty section', () => { + expect(hasLineageSection({})).to.equal(false); + expect(hasLineageSection({ dataLineage: [] })).to.equal(false); + expect(hasLineageSection({ dataLineage: [null] })).to.equal(true); + expect(hasLineageSection({ dataLineage: 'nope' })).to.equal(true); + }); + + describe('hasInvalidLineage', () => { + const entry = { targetField: 'o.a', sourceFields: ['i.a'] }; + + it('is false for absent or well-formed lineage', () => { + expect(hasInvalidLineage({})).to.equal(false); + expect(hasInvalidLineage({ dataLineage: [] })).to.equal(false); + expect(hasInvalidLineage({ dataLineage: [entry, { targetField: 'o.b', sourceFields: [] }] })).to.equal(false); + }); + + it('flags a non-array section and malformed entries', () => { + expect(hasInvalidLineage({ dataLineage: 'nope' })).to.equal(true); + expect(hasInvalidLineage({ dataLineage: [entry, { targetField: 'o.b', sourceFields: [null] }] })).to.equal(true); + }); + + it('flags duplicate targets case-insensitively', () => { + expect(hasInvalidLineage({ dataLineage: [entry, entry] })).to.equal(true); + expect(hasInvalidLineage({ dataLineage: [entry, { ...entry, targetField: 'O.A' }] })).to.equal(true); + }); }); describe('diffLineage', () => { @@ -127,6 +161,12 @@ describe('lineageConfig', () => { expect(countEntriesReferencing(config, ['old_out__dll', 'OLD_IN__dll'])).to.equal(2); }); + it('ignores malformed source fields instead of throwing', () => { + const config = { ...CONFIG, dataLineage: [{ targetField: 'out__dll.a', sourceFields: [null] }] }; + expect(findStaleLineage(config)).to.deep.equal({ targets: [], sources: [] }); + expect(countEntriesReferencing(config, ['in__dll'])).to.equal(0); + }); + it('reads DMO permissions too', () => { const config = { permissions: { read: { dmo: ['in__dlm'] }, write: { dmo: ['out__dlm'] } }, From 9783a65e0a1823df5936f64e06517d65bc479bc4 Mon Sep 17 00:00:00 2001 From: Jesus Orosco Date: Thu, 15 Oct 2026 15:16:15 -0700 Subject: [PATCH 3/8] feat: customer warning 10 mins, UDF warning --- messages/lineage.md | 46 +++++++- src/base/lineageBase.ts | 41 +++++-- src/base/scanBase.ts | 13 +- .../script/lineage/generate.ts | 5 + src/utils/lineageConfig.ts | 6 + src/utils/lineageRunner.ts | 2 + .../data-code-extension/lineage.test.ts | 111 ++++++++++++++++-- .../commands/data-code-extension/scan.test.ts | 2 +- test/utils/lineageRunner.test.ts | 1 + 9 files changed, 204 insertions(+), 23 deletions(-) diff --git a/messages/lineage.md b/messages/lineage.md index aeb62da..b93608f 100644 --- a/messages/lineage.md +++ b/messages/lineage.md @@ -6,6 +6,8 @@ Generate field-level dataLineage for a Data Code Extension %s package. Runs the package's entrypoint against a Salesforce org with every write intercepted, traces each written field back to the fields it was computed from, and replaces the dataLineage section of the package's config.json with the result. Nothing is written to Data Cloud. Run 'scan' first so the read and write permissions in config.json are current. Fields whose lineage can't be traced are listed and get no lineage unless you add entries by hand. +Generated lineage is a best-effort analysis of your script and may be incomplete or wrong (for example, for UDFs or other code the tool can't see into). It determines how governance tags propagate, which can restrict or open access to data. You're responsible for reviewing and maintaining it. Before config.json is written, you must accept this, either at the prompt or with --accept-lineage-ownership. + # examples.script - Generate lineage for a script package against the org with alias "myorg": @@ -16,9 +18,9 @@ Runs the package's entrypoint against a Salesforce org with every write intercep <%= config.bin %> data-code-extension script lineage generate --entrypoint ./my-script-package/payload/entrypoint.py --target-org myorg --dry-run -- Replace existing lineage without prompting, and keep the full result for review: +- Accept ownership of the generated lineage and replace existing lineage without prompting, keeping the full result for review: - <%= config.bin %> data-code-extension script lineage generate --entrypoint ./my-script-package/payload/entrypoint.py --target-org myorg --no-prompt --output-file ./lineage-result.json + <%= config.bin %> data-code-extension script lineage generate --entrypoint ./my-script-package/payload/entrypoint.py --target-org myorg --accept-lineage-ownership --no-prompt --output-file ./lineage-result.json # info.checkingPython @@ -73,9 +75,9 @@ dataLineage in '%s' is already up to date. Updated dataLineage in '%s'. -# info.notReplaced +# info.notWritten -Kept the existing dataLineage; config.json was not modified. +Generated lineage was not accepted; config.json was not modified. # info.complete @@ -86,6 +88,11 @@ Lineage generation completed successfully! %s target fields could not be traced and get no lineage unless you add entries by hand: %s +# warning.udf + +%s target fields are computed by user-defined functions (UDFs). Their lineage lists only the fields passed to the UDF; the tool can't see what else the UDF reads (for example, a lookup table, a broadcast variable, or a file). Review these fields and add any missing source fields by hand: +%s + # warning.permissionGaps The generated lineage references objects that aren't in config.json permissions, so deploy would fail (1041/1042). Run 'data-code-extension script scan' first, then generate lineage again. @@ -103,6 +110,18 @@ This replaces the existing dataLineage section. Any entries you added or edited Replace the existing dataLineage in config.json? +# warning.ownership + +Generated lineage is a best-effort analysis of your script and may be incomplete or wrong (for example, for UDFs or other code the tool can't see into). It determines how governance tags propagate, which can restrict or open access to data. You're responsible for reviewing and maintaining it. + +# prompt.acceptOwnership + +Accept and write dataLineage to config.json? + +# prompt.acceptOwnershipReplace + +Accept and replace the existing dataLineage in config.json? + # error.functionPackage Lineage can only be generated for script packages; '%s' is a function package. @@ -119,6 +138,15 @@ dataLineage in '%s' changed while waiting for confirmation, so it was not replac - Re-run the command to compare the generated lineage against the current dataLineage. +# error.ownershipNotAccepted + +Writing generated lineage to config.json requires accepting ownership of it. + +# actions.ownershipNotAccepted + +- Review the generated lineage, then re-run with --accept-lineage-ownership to accept responsibility for it and write it to config.json. +- Re-run with --dry-run to preview the generated lineage without changing config.json. + # error.replaceNotConfirmed config.json already has a different dataLineage section, and replacing it needs confirmation. @@ -199,4 +227,12 @@ Replace existing dataLineage without prompting. # flags.noPrompt.description -Skips the confirmation before replacing a dataLineage section that differs from the generated one. Required to replace existing lineage when running non-interactively or with --json. +Skips the confirmation before replacing a dataLineage section that differs from the generated one. Required to replace existing lineage when running non-interactively or with --json. Doesn't accept ownership of the generated lineage; use --accept-lineage-ownership for that. + +# flags.acceptLineageOwnership.summary + +Accept responsibility for the generated lineage without prompting. + +# flags.acceptLineageOwnership.description + +Confirms that you understand the generated lineage is best-effort and may be incomplete or wrong, that it determines how governance tags propagate, and that you're responsible for reviewing and maintaining it. Skips the ownership prompt. Required to write lineage when running non-interactively or with --json. diff --git a/src/base/lineageBase.ts b/src/base/lineageBase.ts index 9b0c150..40ef7dc 100644 --- a/src/base/lineageBase.ts +++ b/src/base/lineageBase.ts @@ -28,6 +28,7 @@ import { hasInvalidLineage, hasLineageSection, isEmptyDiff, + LINEAGE_CONFIRM_TIMEOUT_MS, readLineage, replaceLineage, writeConfigJson, @@ -44,6 +45,7 @@ export type BaseLineageFlags = { 'dry-run': boolean; 'output-file'?: string; 'no-prompt': boolean; + 'accept-lineage-ownership': boolean; }; export type LineageGenerateResult = SharedResultProps & { @@ -177,12 +179,12 @@ export abstract class LineageBase extends SfCommand { return result; } - if (hasLineageSection(config) && !(await this.confirmReplace(messages, flags['no-prompt']))) { - this.log(messages.getMessage('info.notReplaced')); + if (!(await this.confirmWrite(messages, flags, hasLineageSection(config)))) { + this.log(messages.getMessage('info.notWritten')); return result; } - // Re-read again: the prompt can wait indefinitely. Lineage edited meanwhile wasn't in the + // Re-read again: the prompt can wait up to LINEAGE_CONFIRM_TIMEOUT_MS. Lineage edited meanwhile wasn't in the // diff the customer confirmed, so don't replace it. const latest = await readConfigJson(configPath); if (JSON.stringify(latest.dataLineage) !== shownLineage) { @@ -223,23 +225,48 @@ export abstract class LineageBase extends SfCommand { const lines = lineage.unresolved.map((field) => `${field.targetField} (${field.reason})`); this.warn(messages.getMessage('warning.unresolved', [lineage.unresolved.length, bullets(lines)])); } + if (lineage.udfFields.length > 0) { + this.warn(messages.getMessage('warning.udf', [lineage.udfFields.length, bullets(lineage.udfFields)])); + } const missing = [...gaps.targets.map((name) => `write: ${name}`), ...gaps.sources.map((name) => `read: ${name}`)]; if (missing.length > 0) { this.warn(messages.getMessage('warning.permissionGaps', [bullets(missing)])); } } - private async confirmReplace(messages: Messages, noPrompt: boolean): Promise { - this.warn(messages.getMessage('warning.replace')); - if (noPrompt) return true; + // Every write needs the customer to accept ownership of the generated lineage: it is best-effort + // and drives governance tag propagation. Replacing an existing section also risks losing hand + // edits. Interactively that's one prompt for whatever the flags haven't already accepted. + private async confirmWrite( + messages: Messages, + flags: BaseLineageFlags, + replacing: boolean + ): Promise { + this.warn(messages.getMessage('warning.ownership')); + if (replacing) this.warn(messages.getMessage('warning.replace')); + const needsOwnership = !flags['accept-lineage-ownership']; + const needsReplace = replacing && !flags['no-prompt']; + if (!needsOwnership && !needsReplace) return true; if (this.jsonEnabled() || !process.stdin.isTTY) { + if (needsOwnership) { + throw new SfError( + messages.getMessage('error.ownershipNotAccepted'), + 'LineageOwnershipNotAccepted', + messages.getMessages('actions.ownershipNotAccepted') + ); + } throw new SfError( messages.getMessage('error.replaceNotConfirmed'), 'LineageReplaceNotConfirmed', messages.getMessages('actions.replaceNotConfirmed') ); } - return this.confirm({ message: messages.getMessage('prompt.replace'), defaultAnswer: false }); + const prompt = needsOwnership + ? replacing + ? 'prompt.acceptOwnershipReplace' + : 'prompt.acceptOwnership' + : 'prompt.replace'; + return this.confirm({ message: messages.getMessage(prompt), defaultAnswer: false, ms: LINEAGE_CONFIRM_TIMEOUT_MS }); } protected abstract getCodeType(): 'script'; diff --git a/src/base/scanBase.ts b/src/base/scanBase.ts index a88f3f1..298b963 100644 --- a/src/base/scanBase.ts +++ b/src/base/scanBase.ts @@ -18,7 +18,12 @@ import { SfCommand } from '@salesforce/sf-plugins-core'; import { Messages, SfError } from '@salesforce/core'; import { executeNativeScan, type NativeScanResult } from '../utils/nativeScan.js'; import { checkEnvironment, type EnvironmentCheckResult } from '../utils/environmentChecker.js'; -import { countEntriesReferencing, findStaleLineage, type LineagePermissionGaps } from '../utils/lineageConfig.js'; +import { + countEntriesReferencing, + findStaleLineage, + LINEAGE_CONFIRM_TIMEOUT_MS, + type LineagePermissionGaps, +} from '../utils/lineageConfig.js'; export type BaseScanFlags = { entrypoint?: string; @@ -132,7 +137,11 @@ export abstract class ScanBase extends SfCommand { messages.getMessages('actions.staleLineageNotConfirmed') ); } - return this.confirm({ message: messages.getMessage('prompt.staleLineage'), defaultAnswer: false }); + return this.confirm({ + message: messages.getMessage('prompt.staleLineage'), + defaultAnswer: false, + ms: LINEAGE_CONFIRM_TIMEOUT_MS, + }); } private warnStaleLineage( diff --git a/src/commands/data-code-extension/script/lineage/generate.ts b/src/commands/data-code-extension/script/lineage/generate.ts index dc8f0c6..3abb72c 100644 --- a/src/commands/data-code-extension/script/lineage/generate.ts +++ b/src/commands/data-code-extension/script/lineage/generate.ts @@ -70,6 +70,11 @@ export default class Generate extends LineageBase { description: messages.getMessage('flags.noPrompt.description'), default: false, }), + 'accept-lineage-ownership': Flags.boolean({ + summary: messages.getMessage('flags.acceptLineageOwnership.summary'), + description: messages.getMessage('flags.acceptLineageOwnership.description'), + default: false, + }), }; // eslint-disable-next-line class-methods-use-this diff --git a/src/utils/lineageConfig.ts b/src/utils/lineageConfig.ts index 5c58a47..9848120 100644 --- a/src/utils/lineageConfig.ts +++ b/src/utils/lineageConfig.ts @@ -22,6 +22,12 @@ export type LineageDiff = { changed: string[]; }; +/** + * How long lineage confirmation prompts wait before declining. The sf default of 10s is too short to + * review the warnings and diff. Must stay finite: Node fires timers over ~24.8 days after 1ms. + */ +export const LINEAGE_CONFIRM_TIMEOUT_MS = 10 * 60 * 1000; + /** Objects referenced by lineage but missing from `permissions`, i.e. a 1041/1042 at deploy. */ export type LineagePermissionGaps = { targets: string[]; diff --git a/src/utils/lineageRunner.ts b/src/utils/lineageRunner.ts index 2b60741..15f54b6 100644 --- a/src/utils/lineageRunner.ts +++ b/src/utils/lineageRunner.ts @@ -79,6 +79,8 @@ export type LineageReport = { dataLineage: LineageEntry[]; unresolved: UnresolvedField[]; noSources: string[]; + /** Fields in `dataLineage` or `noSources` computed by a UDF; their sources may be incomplete. */ + udfFields: string[]; targets: string[]; sources: string[]; pairCount: number; diff --git a/test/commands/data-code-extension/lineage.test.ts b/test/commands/data-code-extension/lineage.test.ts index 61ea767..2cf487b 100644 --- a/test/commands/data-code-extension/lineage.test.ts +++ b/test/commands/data-code-extension/lineage.test.ts @@ -35,6 +35,7 @@ const REPORT: LineageReport = { ], unresolved: [{ targetField: 'out__dll.x__c', reason: 'untraceable source' }], noSources: ['out__dll.const__c'], + udfFields: [], targets: ['out__dll'], sources: ['in__dll'], pairCount: 3, @@ -114,7 +115,12 @@ describe('data-code-extension script lineage generate', () => { }); it('writes lineage into config.json, keeps other keys, and forwards run options', async () => { - const result = await LineageGenerate.run([...baseArgs(), '--dependencies', 'pandas, numpy']); + const result = await LineageGenerate.run([ + ...baseArgs(), + '--dependencies', + 'pandas, numpy', + '--accept-lineage-ownership', + ]); const opts = generateStub.firstCall.args[0] as Record; expect(opts.pythonCommand).to.equal('python3'); @@ -142,6 +148,19 @@ describe('data-code-extension script lineage generate', () => { expect(warnings().some((w) => w.includes('out__dll.x__c (untraceable source)'))).to.be.true; }); + it('lists UDF-computed fields only when a UDF was detected', async () => { + await LineageGenerate.run([...baseArgs(), '--dry-run']); + expect(warnings().some((w) => w.includes('user-defined functions'))).to.be.false; + + sfCommandStubs.warn.resetHistory(); + generateStub.resolves({ ...REPORT, udfFields: ['out__dll.const__c', 'out__dll.name__c'] }); + await LineageGenerate.run([...baseArgs(), '--dry-run']); + const udfWarning = warnings().find((w) => w.includes('user-defined functions')); + expect(udfWarning).to.include('2 target fields'); + expect(udfWarning).to.include('out__dll.const__c'); + expect(udfWarning).to.include('out__dll.name__c'); + }); + it('warns when lineage references objects missing from permissions', async () => { writeConfig({ dataspace: 'default', permissions: { read: { dlo: [] }, write: { dlo: ['out__dll'] } } }); const result = await LineageGenerate.run(baseArgs()); @@ -292,6 +311,68 @@ describe('data-code-extension script lineage generate', () => { expect(generateStub.called).to.be.false; }); + describe('accepting lineage ownership', () => { + it('asks the customer to accept ownership before the first write', async () => { + const result = await LineageGenerate.run(baseArgs()); + expect(confirmStub.calledOnce).to.be.true; + expect(confirmStub.firstCall.args[0]).to.deep.include({ + message: 'Accept and write dataLineage to config.json?', + defaultAnswer: false, + }); + expect(warnings().some((w) => w.includes("You're responsible for reviewing and maintaining it"))).to.be.true; + expect(result.written).to.equal(true); + expect(readConfig().dataLineage).to.deep.equal(REPORT.dataLineage); + }); + + it('does not write when the customer declines', async () => { + confirmStub.resolves(false); + const result = await LineageGenerate.run(baseArgs()); + expect(result.written).to.equal(false); + expect(readConfig()).to.not.have.property('dataLineage'); + }); + + it('--accept-lineage-ownership writes without asking but still shows the warning', async () => { + const result = await LineageGenerate.run([...baseArgs(), '--accept-lineage-ownership']); + expect(confirmStub.called).to.be.false; + expect(warnings().some((w) => w.includes('best-effort analysis'))).to.be.true; + expect(result.written).to.equal(true); + }); + + it('--no-prompt does not accept ownership', async () => { + await LineageGenerate.run([...baseArgs(), '--no-prompt']); + expect(confirmStub.calledOnce).to.be.true; + }); + + it('fails without --accept-lineage-ownership when not interactive', async () => { + process.stdin.isTTY = false; + try { + await LineageGenerate.run([...baseArgs(), '--no-prompt']); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageOwnershipNotAccepted'); + } + expect(confirmStub.called).to.be.false; + expect(readConfig()).to.not.have.property('dataLineage'); + }); + + it('fails with --json without --accept-lineage-ownership', async () => { + try { + await LineageGenerate.run([...baseArgs(), '--json']); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageOwnershipNotAccepted'); + } + expect(readConfig()).to.not.have.property('dataLineage'); + }); + + it('--dry-run needs no acceptance', async () => { + process.stdin.isTTY = false; + const result = await LineageGenerate.run([...baseArgs(), '--dry-run']); + expect(confirmStub.called).to.be.false; + expect(result.written).to.equal(false); + }); + }); + describe('replacing existing lineage', () => { const OLD = [{ targetField: 'out__dll.old', sourceFields: ['in__dll.old'] }]; @@ -299,10 +380,15 @@ describe('data-code-extension script lineage generate', () => { writeConfig({ dataspace: 'default', permissions: PERMISSIONS, dataLineage: OLD }); }); - it('replaces after the customer confirms', async () => { + it('asks once to accept ownership and replace', async () => { const result = await LineageGenerate.run(baseArgs()); expect(confirmStub.calledOnce).to.be.true; - expect(confirmStub.firstCall.args[0]).to.have.property('defaultAnswer', false); + expect(confirmStub.firstCall.args[0]).to.deep.include({ + message: 'Accept and replace the existing dataLineage in config.json?', + defaultAnswer: false, + ms: 10 * 60 * 1000, + }); + expect(warnings().some((w) => w.includes('best-effort analysis'))).to.be.true; expect(warnings().some((w) => w.includes('This replaces the existing dataLineage section'))).to.be.true; expect(result.written).to.equal(true); expect(result.diff.removed).to.deep.equal(['out__dll.old']); @@ -316,8 +402,17 @@ describe('data-code-extension script lineage generate', () => { expect(readConfig().dataLineage).to.deep.equal(OLD); }); - it('--no-prompt replaces without asking', async () => { - const result = await LineageGenerate.run([...baseArgs(), '--no-prompt']); + it('asks only about replacing once ownership is accepted', async () => { + await LineageGenerate.run([...baseArgs(), '--accept-lineage-ownership']); + expect(confirmStub.calledOnce).to.be.true; + expect(confirmStub.firstCall.args[0]).to.have.property( + 'message', + 'Replace the existing dataLineage in config.json?' + ); + }); + + it('--accept-lineage-ownership --no-prompt replaces without asking', async () => { + const result = await LineageGenerate.run([...baseArgs(), '--accept-lineage-ownership', '--no-prompt']); expect(confirmStub.called).to.be.false; expect(result.written).to.equal(true); }); @@ -340,7 +435,7 @@ describe('data-code-extension script lineage generate', () => { it('fails without --no-prompt when not interactive', async () => { process.stdin.isTTY = false; try { - await LineageGenerate.run(baseArgs()); + await LineageGenerate.run([...baseArgs(), '--accept-lineage-ownership']); expect.fail('Should have thrown'); } catch (error) { expect((error as SfError).name).to.equal('LineageReplaceNotConfirmed'); @@ -351,7 +446,7 @@ describe('data-code-extension script lineage generate', () => { it('fails with --json without --no-prompt', async () => { try { - await LineageGenerate.run([...baseArgs(), '--json']); + await LineageGenerate.run([...baseArgs(), '--json', '--accept-lineage-ownership']); expect.fail('Should have thrown'); } catch (error) { expect((error as SfError).name).to.equal('LineageReplaceNotConfirmed'); @@ -362,7 +457,7 @@ describe('data-code-extension script lineage generate', () => { it('rewrites duplicate targets even when the diff is empty', async () => { const duplicated = [...REPORT.dataLineage, REPORT.dataLineage[0]]; writeConfig({ dataspace: 'default', permissions: PERMISSIONS, dataLineage: duplicated }); - const result = await LineageGenerate.run([...baseArgs(), '--no-prompt']); + const result = await LineageGenerate.run([...baseArgs(), '--accept-lineage-ownership', '--no-prompt']); expect(result.written).to.equal(true); expect(readConfig().dataLineage).to.deep.equal(REPORT.dataLineage); }); diff --git a/test/commands/data-code-extension/scan.test.ts b/test/commands/data-code-extension/scan.test.ts index 8af1aaa..72413d1 100644 --- a/test/commands/data-code-extension/scan.test.ts +++ b/test/commands/data-code-extension/scan.test.ts @@ -313,7 +313,7 @@ describe('data-code-extension scan commands', () => { makeScriptPackage(SCRIPT, { dataspace: 'default', dataLineage: STALE }); const result = await ScriptScan.run([]); expect(confirmStub.calledOnce).to.be.true; - expect(confirmStub.firstCall.args[0]).to.have.property('defaultAnswer', false); + expect(confirmStub.firstCall.args[0]).to.deep.include({ defaultAnswer: false, ms: 10 * 60 * 1000 }); expect(warnings().some((w) => w.includes('write: OldDest__dll') && w.includes('1 entries'))).to.be.true; expect(result.executionResult.written).to.be.undefined; const config = readConfig(); diff --git a/test/utils/lineageRunner.test.ts b/test/utils/lineageRunner.test.ts index 40089fc..36bf4d4 100644 --- a/test/utils/lineageRunner.test.ts +++ b/test/utils/lineageRunner.test.ts @@ -88,6 +88,7 @@ const REPORT: LineageReport = { dataLineage: [{ targetField: 'out__dll.id__c', sourceFields: ['in__dll.id'] }], unresolved: [{ targetField: 'out__dll.x__c', reason: 'untraceable source' }], noSources: [], + udfFields: [], targets: ['out__dll'], sources: ['in__dll'], pairCount: 1, From 01de939fffff2ccb267c380c68cefe8f17f906b5 Mon Sep 17 00:00:00 2001 From: Jesus Orosco Date: Thu, 15 Oct 2026 15:28:46 -0700 Subject: [PATCH 4/8] fix: guard scan writes and build-file outputs, fix Windows mount parsing in tests - scan: re-read config.json after confirmation and refuse to write if it changed while the prompt was pending (ScanConfigChangedDuringConfirm) - lineage generate: treat root-level requirements.txt, build_native_dependencies.sh and Dockerfile.dependencies as package inputs in the --output-file collision check - zipBuilder tests: strip only the trailing :/workspace from the docker mount so Windows drive-letter paths parse correctly --- messages/datacodeBinaryExecutor.md | 8 +++++ src/base/lineageBase.ts | 7 ++++- src/utils/nativeScan.ts | 9 ++++++ .../data-code-extension/lineage.test.ts | 9 ++++++ test/utils/nativeScan.test.ts | 29 +++++++++++++++++++ test/utils/zipBuilder.test.ts | 5 ++-- 6 files changed, 64 insertions(+), 3 deletions(-) diff --git a/messages/datacodeBinaryExecutor.md b/messages/datacodeBinaryExecutor.md index 3e1cf7a..d5f7ac1 100644 --- a/messages/datacodeBinaryExecutor.md +++ b/messages/datacodeBinaryExecutor.md @@ -18,6 +18,14 @@ Entrypoint file not found at '%s'. - Run 'init' first if the package has not been initialized - Use the default entrypoint path: payload/entrypoint.py +# error.scanConfigChanged + +'%s' changed while waiting for confirmation, so the scan results were not written. + +# actions.scanConfigChanged + +- Re-run the scan so it starts from the current config.json. + # error.zipExecutionFailed Failed to create archive for package at '%s': %s diff --git a/src/base/lineageBase.ts b/src/base/lineageBase.ts index 40ef7dc..0855cac 100644 --- a/src/base/lineageBase.ts +++ b/src/base/lineageBase.ts @@ -65,6 +65,8 @@ export type LineageGenerateResult = SharedResultProps & { const CONFIG_FILE = 'config.json'; /** Project config the SDK loads implicitly from the directory above `payload/`. */ const PROJECT_CONFIG_YAML = 'config.yaml'; +/** Base-directory files `deploy` packages alongside payload/ (see zipBuilder). */ +const PACKAGED_BUILD_FILES = ['requirements.txt', 'build_native_dependencies.sh', 'Dockerfile.dependencies']; // eslint-disable-next-line sf-plugin/command-summary, sf-plugin/command-example export abstract class LineageBase extends SfCommand { @@ -91,13 +93,16 @@ export abstract class LineageBase extends SfCommand { if (outputFile) { // Everything under payload/ is run and deployed; .datacustomcode_proj/ holds package metadata. const payloadDir = path.dirname(entrypoint); - const sdkConfigPath = getSdkConfigPath(findBaseDirectory(entrypoint)); + // Root-level requirements.txt and the native dependency build files are packaged too. + const baseDir = findBaseDirectory(entrypoint); + const sdkConfigPath = getSdkConfigPath(baseDir); const inputDirs = [payloadDir, path.dirname(sdkConfigPath)]; const inputFiles = [ configPath, entrypoint, sdkConfigPath, path.join(path.dirname(payloadDir), PROJECT_CONFIG_YAML), + ...PACKAGED_BUILD_FILES.map((name) => path.join(baseDir, name)), ...(flags['config-file'] ? [path.resolve(flags['config-file'])] : []), ]; if (await isPackageInput(outputFile, inputDirs, inputFiles)) { diff --git a/src/utils/nativeScan.ts b/src/utils/nativeScan.ts index 1fd5214..f1c0f66 100644 --- a/src/utils/nativeScan.ts +++ b/src/utils/nativeScan.ts @@ -512,6 +512,8 @@ export async function executeNativeScan(opts: NativeScanOptions): Promise { await expectCollision(path.join(testDir, 'config.yaml')); }); + for (const name of ['requirements.txt', 'build_native_dependencies.sh', 'Dockerfile.dependencies']) { + it(`rejects the packaged build file ${name}`, async () => { + const buildFile = path.join(testDir, name); + fs.writeFileSync(buildFile, 'original\n'); + await expectCollision(buildFile); + expect(fs.readFileSync(buildFile, 'utf8')).to.equal('original\n'); + }); + } + it('rejects a symlinked directory that points into payload/', async () => { const link = path.join(testDir, 'payload-link'); fs.symlinkSync(path.join(testDir, 'payload'), link); diff --git a/test/utils/nativeScan.test.ts b/test/utils/nativeScan.test.ts index e1e1917..7b7bfea 100644 --- a/test/utils/nativeScan.test.ts +++ b/test/utils/nativeScan.test.ts @@ -17,6 +17,7 @@ import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { expect } from 'chai'; +import { type SfError } from '@salesforce/core'; import { setPipreqsMock, executeNativeScan, @@ -737,6 +738,34 @@ describe('nativeScan: executeNativeScan confirmWrite', () => { await fs.rm(dir, { recursive: true, force: true }); }); + it('refuses to write when config.json changes while confirmWrite waits', async () => { + const dir = await makePackage({ entrypoint: ENTRYPOINT }); + const configPath = path.join(dir, 'payload', 'config.json'); + const edited = JSON.stringify({ dataspace: 'edited', dataLineage: { 'Dest.dll': { Id: ['Source.Id'] } } }); + try { + await executeNativeScan({ + workingDir: dir, + packageType: 'script', + confirmWrite: async () => { + await fs.writeFile(configPath, edited); + return true; + }, + }); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('ScanConfigChangedDuringConfirm'); + } + expect(await fs.readFile(configPath, 'utf8')).to.equal(edited); + let requirementsExists = true; + try { + await fs.access(path.join(dir, 'requirements.txt')); + } catch { + requirementsExists = false; + } + expect(requirementsExists).to.be.false; + await fs.rm(dir, { recursive: true, force: true }); + }); + it('never calls confirmWrite on dry run', async () => { const dir = await makePackage({ entrypoint: ENTRYPOINT }); let called = false; diff --git a/test/utils/zipBuilder.test.ts b/test/utils/zipBuilder.test.ts index 8f6cc7c..d9c5e77 100644 --- a/test/utils/zipBuilder.test.ts +++ b/test/utils/zipBuilder.test.ts @@ -334,9 +334,10 @@ describe('zipBuilder.prepareDependencyArchive', () => { }, async run(args) { calls.run.push({ args }); - // The dockerRunCmd shape is `['run', '--rm', '-v', ':/workspace', ...]` + // The dockerRunCmd shape is `['run', '--rm', '-v', ':/workspace', ...]`. Strip only the + // container suffix: on Windows the mount itself has a drive colon (`C:/...`). const mountArg = args[args.indexOf('-v') + 1]; - const [mountPath] = mountArg.split(':'); + const mountPath = mountArg.slice(0, mountArg.lastIndexOf(':')); opts.onRun?.(mountPath); }, }; From 4efd63568f06f47f91de3ff3ed03fb1efa51087d Mon Sep 17 00:00:00 2001 From: Jesus Orosco Date: Thu, 15 Oct 2026 15:41:30 -0700 Subject: [PATCH 5/8] fix: protect missing package inputs from --output-file by path - compare the output path against input files by resolved path as well as file identity, so build files that don't exist yet are still protected - follow dangling symlinks to the path a write would land on - compare case-insensitively on macOS and Windows --- src/base/lineageBase.ts | 35 ++++++++++++++++--- .../data-code-extension/lineage.test.ts | 26 ++++++++++++++ 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/base/lineageBase.ts b/src/base/lineageBase.ts index 0855cac..e05cb6c 100644 --- a/src/base/lineageBase.ts +++ b/src/base/lineageBase.ts @@ -280,39 +280,64 @@ export abstract class LineageBase extends SfCommand { /** * True when `file` lies under one of `dirs` (as given or after resolving symlinks), or is one of - * `files` by path or by file identity, which also catches symlinks and hard links. + * `files` by path or by file identity, which also catches symlinks and hard links. Paths are compared + * by path as well as identity so inputs that don't exist yet (no identity to match) are still caught. */ async function isPackageInput(file: string, dirs: string[], files: string[]): Promise { - const [target, realDirs, targetStat, fileStats] = await Promise.all([ + const [target, realDirs, realFiles, targetStat, fileStats] = await Promise.all([ realPath(file), Promise.all(dirs.map(realPath)), + Promise.all(files.map(realPath)), statOrUndefined(file), Promise.all(files.map(statOrUndefined)), ]); const within = (child: string, dir: string): boolean => { - const relative = path.relative(dir, child); + const relative = path.relative(pathKey(dir), pathKey(child)); return !path.isAbsolute(relative) && relative.split(path.sep)[0] !== '..'; }; + const sameFile = (a: string, b: string): boolean => pathKey(a) === pathKey(b); return ( dirs.some((dir) => within(file, dir)) || realDirs.some((dir) => within(target, dir)) || + files.some((input) => sameFile(input, file)) || + realFiles.some((input) => sameFile(input, target)) || (targetStat !== undefined && fileStats.some((stat) => stat !== undefined && stat.dev === targetStat.dev && stat.ino === targetStat.ino)) ); } +/** + * Absolute path folded to lower case where the default file system ignores case (macOS, Windows), so + * `REQUIREMENTS.TXT` matches a `requirements.txt` that doesn't exist yet. On a case-sensitive volume + * this only over-rejects names that differ from an input by case. + */ +function pathKey(file: string): string { + const resolved = path.resolve(file); + return process.platform === 'darwin' || process.platform === 'win32' ? resolved.toLowerCase() : resolved; +} + async function statOrUndefined(file: string): Promise { return fs.stat(file).catch(() => undefined); } +/** Symlink hops followed before giving up, matching the usual OS limit (ELOOP). */ +const MAX_SYMLINK_HOPS = 40; + /** * Canonical path of `file`, resolving symlinks (and case, on case-insensitive file systems). - * A file that doesn't exist yet resolves through its parent directory. + * A file that doesn't exist yet resolves through its parent directory, and a dangling symlink + * resolves to the path it points at, which is where a write through it would land. */ -async function realPath(file: string): Promise { +async function realPath(file: string, hops = 0): Promise { try { return await fs.realpath(file); } catch { + if (hops < MAX_SYMLINK_HOPS) { + const link = await fs.readlink(file).catch(() => undefined); + if (link !== undefined) { + return realPath(path.resolve(path.dirname(file), link), hops + 1); + } + } try { return path.join(await fs.realpath(path.dirname(file)), path.basename(file)); } catch { diff --git a/test/commands/data-code-extension/lineage.test.ts b/test/commands/data-code-extension/lineage.test.ts index 504f97d..3204466 100644 --- a/test/commands/data-code-extension/lineage.test.ts +++ b/test/commands/data-code-extension/lineage.test.ts @@ -248,6 +248,32 @@ describe('data-code-extension script lineage generate', () => { await expectCollision(buildFile); expect(fs.readFileSync(buildFile, 'utf8')).to.equal('original\n'); }); + + it(`rejects the packaged build file ${name} when it does not exist yet`, async () => { + const buildFile = path.join(testDir, name); + fs.rmSync(buildFile, { force: true }); + await expectCollision(buildFile); + expect(fs.existsSync(buildFile)).to.be.false; + }); + + it(`rejects a dangling symlink to the missing packaged build file ${name}`, async () => { + const buildFile = path.join(testDir, name); + fs.rmSync(buildFile, { force: true }); + const link = path.join(testDir, `link-${name}`); + fs.symlinkSync(buildFile, link); + await expectCollision(link); + expect(fs.existsSync(buildFile)).to.be.false; + }); + + it(`rejects the missing packaged build file ${name} spelled in another case`, async function () { + if (process.platform !== 'darwin' && process.platform !== 'win32') { + this.skip(); + } + const buildFile = path.join(testDir, name); + fs.rmSync(buildFile, { force: true }); + await expectCollision(path.join(testDir, name.toUpperCase())); + expect(fs.readdirSync(testDir).some((entry) => entry.toLowerCase() === name.toLowerCase())).to.be.false; + }); } it('rejects a symlinked directory that points into payload/', async () => { From 11282a3f54839883a59c63f384c650be7400c850 Mon Sep 17 00:00:00 2001 From: Jesus Orosco Date: Thu, 15 Oct 2026 15:52:29 -0700 Subject: [PATCH 6/8] fix: address review findings on lineage generate and scan - scan: a declined stale-lineage prompt no longer reports the scan as applied; it returns success: false and prints only the not-written notice - scan: always re-check config.json before writing, not only after a prompt - lineage generate: abort the write if any part of config.json (including permissions the gaps were checked against) changed during confirmation - lineage generate: fail fast when --output-file's directory isn't writable and when the target org has no username, instead of after the run or authenticating as a placeholder org - diffLineage: compare target and source fields case-insensitively, as deploy validation does - validate the SDK result shape (LineageResultInvalid) instead of casting --- messages/lineage.md | 20 +++++++++- src/base/lineageBase.ts | 22 ++++++++--- src/base/scanBase.ts | 17 ++++++-- src/utils/lineageConfig.ts | 32 ++++++++++----- src/utils/lineageRunner.ts | 30 +++++++++++++- src/utils/nativeScan.ts | 5 ++- .../data-code-extension/lineage.test.ts | 39 +++++++++++++++++++ .../commands/data-code-extension/scan.test.ts | 8 +++- test/utils/lineageConfig.test.ts | 14 +++++++ test/utils/lineageRunner.test.ts | 33 ++++++++++++++++ 10 files changed, 195 insertions(+), 25 deletions(-) diff --git a/messages/lineage.md b/messages/lineage.md index b93608f..a0ae82b 100644 --- a/messages/lineage.md +++ b/messages/lineage.md @@ -132,11 +132,19 @@ Lineage can only be generated for script packages; '%s' is a function package. # error.lineageChanged -dataLineage in '%s' changed while waiting for confirmation, so it was not replaced. +'%s' changed while waiting for confirmation, so dataLineage was not replaced. # actions.lineageChanged -- Re-run the command to compare the generated lineage against the current dataLineage. +- Re-run the command to compare the generated lineage against the current config.json. + +# error.outputFileUnwritable + +Can't write --output-file '%s': %s + +# error.noUsername + +The target org has no username to authenticate the lineage run with. # error.ownershipNotAccepted @@ -168,6 +176,14 @@ The installed salesforce-data-customcode package doesn't support lineage generat The lineage run finished but produced no result: %s +# error.resultInvalid + +The lineage run produced a result this CLI can't read: %s + +# actions.resultInvalid + +- Upgrade the SDK with 'pip install -U salesforce-data-customcode', then try again. + # actions.resultMissing - Review the output above for errors from the entrypoint. diff --git a/src/base/lineageBase.ts b/src/base/lineageBase.ts index e05cb6c..6ca06d2 100644 --- a/src/base/lineageBase.ts +++ b/src/base/lineageBase.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { promises as fs, type Stats } from 'node:fs'; +import { constants as fsConstants, promises as fs, type Stats } from 'node:fs'; import path from 'node:path'; import { SfCommand } from '@salesforce/sf-plugins-core'; import { Messages, Org, SfError } from '@salesforce/core'; @@ -111,12 +111,22 @@ export abstract class LineageBase extends SfCommand { 'LineageOutputFileCollision' ); } + // Fail before the run rather than lose its result to a bad path afterwards. + await fs.access(path.dirname(outputFile), fsConstants.W_OK).catch((err: Error) => { + throw new SfError( + messages.getMessage('error.outputFileUnwritable', [outputFile, err.message]), + 'LineageOutputFileUnwritable' + ); + }); } const { pythonInfo, packageInfo } = await checkEnvironment(this.spinner, this.log.bind(this), messages); const targetOrg = flags['target-org']; - const orgUsername = targetOrg.getUsername() ?? 'target org'; + const orgUsername = targetOrg.getUsername(); + if (!orgUsername) { + throw new SfError(messages.getMessage('error.noUsername'), 'LineageNoUsername'); + } this.spinner.start(messages.getMessage('info.authenticating', [orgUsername])); await targetOrg.getConnection().refreshAuth(); this.spinner.stop(); @@ -142,7 +152,7 @@ export abstract class LineageBase extends SfCommand { // Re-read so edits made during the run aren't overwritten. const config = await readConfigJson(configPath); - const shownLineage = JSON.stringify(config.dataLineage); + const shownConfig = JSON.stringify(config); const entries = lineage.dataLineage; const current = readLineage(config); const diff = diffLineage(current, entries); @@ -189,10 +199,10 @@ export abstract class LineageBase extends SfCommand { return result; } - // Re-read again: the prompt can wait up to LINEAGE_CONFIRM_TIMEOUT_MS. Lineage edited meanwhile wasn't in the - // diff the customer confirmed, so don't replace it. + // Re-read again: the prompt can wait up to LINEAGE_CONFIRM_TIMEOUT_MS. Edits made meanwhile (to dataLineage, or + // to permissions the gaps were checked against) weren't in what the customer confirmed, so don't write. const latest = await readConfigJson(configPath); - if (JSON.stringify(latest.dataLineage) !== shownLineage) { + if (JSON.stringify(latest) !== shownConfig) { throw new SfError( messages.getMessage('error.lineageChanged', [configPath]), 'LineageChangedDuringConfirm', diff --git a/src/base/scanBase.ts b/src/base/scanBase.ts index 298b963..c6e3ec6 100644 --- a/src/base/scanBase.ts +++ b/src/base/scanBase.ts @@ -77,6 +77,19 @@ export abstract class ScanBase extends SfCommand { if (hasStaleLineage(stale)) this.warnStaleLineage(messages, executionResult.config, stale); } + if (executionResult.written === false) { + this.log(messages.getMessage('info.staleLineageNotWritten')); + return { + success: false, + pythonInfo, + packageInfo, + codeType, + workingDirectory: workingDir, + executionResult, + message: messages.getMessage('info.staleLineageNotWritten'), + }; + } + this.log(messages.getMessage('info.scanExecuted', [workingDir])); // Echo the resulting config — mirrors the Python CLI's behavior of printing the @@ -95,10 +108,6 @@ export abstract class ScanBase extends SfCommand { this.log(messages.getMessage('info.dryRunNotice')); } - if (executionResult.written === false) { - this.log(messages.getMessage('info.staleLineageNotWritten')); - } - return { success: true, pythonInfo, diff --git a/src/utils/lineageConfig.ts b/src/utils/lineageConfig.ts index 9848120..63763f4 100644 --- a/src/utils/lineageConfig.ts +++ b/src/utils/lineageConfig.ts @@ -67,18 +67,32 @@ export function hasInvalidLineage(config: Record): boolean { return new Set(entries.map((entry) => entry.targetField.toLowerCase())).size !== entries.length; } -/** Per-target difference between the current `dataLineage` and a generated one. */ +/** + * Per-target difference between the current `dataLineage` and a generated one. Field names + * compare case-insensitively, as deploy validation does; results use each side's own spelling. + */ export function diffLineage(current: LineageEntry[], generated: LineageEntry[]): LineageDiff { - const key = (entry: LineageEntry): string => [...entry.sourceFields].sort().join('\n'); - const before = new Map(current.map((entry) => [entry.targetField, key(entry)])); - const after = new Map(generated.map((entry) => [entry.targetField, key(entry)])); + const sourcesKey = (entry: LineageEntry): string => + entry.sourceFields + .map((field) => field.toLowerCase()) + .sort() + .join('\n'); + const byTarget = (entries: LineageEntry[]): Map => + new Map( + entries.map((entry) => [ + entry.targetField.toLowerCase(), + { target: entry.targetField, sources: sourcesKey(entry) }, + ]) + ); + const before = byTarget(current); + const after = byTarget(generated); const diff: LineageDiff = { added: [], removed: [], changed: [] }; - for (const [target, sources] of after) { - if (!before.has(target)) diff.added.push(target); - else if (before.get(target) !== sources) diff.changed.push(target); + for (const [key, { target, sources }] of after) { + if (!before.has(key)) diff.added.push(target); + else if (before.get(key)!.sources !== sources) diff.changed.push(target); } - for (const target of before.keys()) { - if (!after.has(target)) diff.removed.push(target); + for (const [key, { target }] of before) { + if (!after.has(key)) diff.removed.push(target); } return diff; } diff --git a/src/utils/lineageRunner.ts b/src/utils/lineageRunner.ts index 15f54b6..f1653c6 100644 --- a/src/utils/lineageRunner.ts +++ b/src/utils/lineageRunner.ts @@ -154,12 +154,40 @@ export async function generateLineage(opts: LineageRunOptions, deps?: PythonRunn messages.getMessages('actions.resultMissing') ); } - return JSON.parse(raw) as LineageReport; + return parseLineageReport(raw); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } } +const REPORT_ARRAYS = ['dataLineage', 'unresolved', 'noSources', 'udfFields', 'targets', 'sources'] as const; + +/** Parse the SDK's result file, failing clearly when it isn't the shape this CLI reads. */ +export function parseLineageReport(raw: string): LineageReport { + let report: unknown; + try { + report = JSON.parse(raw); + } catch (err) { + throw invalidResult((err as Error).message); + } + if (typeof report !== 'object' || report === null || Array.isArray(report)) { + throw invalidResult('expected a JSON object'); + } + const missing = REPORT_ARRAYS.filter((key) => !Array.isArray((report as Record)[key])); + if (missing.length > 0) { + throw invalidResult(`missing or non-array ${missing.join(', ')}`); + } + return report as LineageReport; +} + +function invalidResult(detail: string): SfError { + return new SfError( + messages.getMessage('error.resultInvalid', [detail]), + 'LineageResultInvalid', + messages.getMessages('actions.resultInvalid') + ); +} + /** Class wrapper around {@link generateLineage}, so command tests can stub the run. */ export class LineageRunner { public static async generate(opts: LineageRunOptions, deps?: PythonRunnerDeps): Promise { diff --git a/src/utils/nativeScan.ts b/src/utils/nativeScan.ts index f1c0f66..f1b3f0b 100644 --- a/src/utils/nativeScan.ts +++ b/src/utils/nativeScan.ts @@ -512,7 +512,7 @@ export async function executeNativeScan(opts: NativeScanOptions): Promise { expect(result.outputFile).to.equal(path.resolve(outputFile)); }); + it('rejects an --output-file in a missing directory before the run', async () => { + const outputFile = path.join(testDir, 'missing', 'lineage-result.json'); + try { + await LineageGenerate.run([...baseArgs(), '--dry-run', '--output-file', outputFile]); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageOutputFileUnwritable'); + } + expect(generateStub.called).to.be.false; + }); + + it('fails clearly when the target org has no username', async () => { + (Org.create as sinon.SinonStub).resolves({ + getUsername: () => undefined, + getConnection: () => mockConnection, + } as unknown as Org); + try { + await LineageGenerate.run(baseArgs()); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageNoUsername'); + } + expect(generateStub.called).to.be.false; + }); + describe('--output-file collisions', () => { async function expectCollision(outputFile: string): Promise { const before = fs.readFileSync(configPath, 'utf8'); @@ -467,6 +492,20 @@ describe('data-code-extension script lineage generate', () => { expect(readConfig().dataLineage).to.deep.equal(edited); }); + it('does not write when permissions change while the prompt was pending', async () => { + confirmStub.callsFake(async () => { + writeConfig({ ...readConfig(), permissions: { read: {}, write: {} } }); + return true; + }); + try { + await LineageGenerate.run(baseArgs()); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageChangedDuringConfirm'); + } + expect(readConfig().permissions).to.deep.equal({ read: {}, write: {} }); + }); + it('fails without --no-prompt when not interactive', async () => { process.stdin.isTTY = false; try { diff --git a/test/commands/data-code-extension/scan.test.ts b/test/commands/data-code-extension/scan.test.ts index 72413d1..20f6ec6 100644 --- a/test/commands/data-code-extension/scan.test.ts +++ b/test/commands/data-code-extension/scan.test.ts @@ -326,8 +326,14 @@ describe('data-code-extension scan commands', () => { const before = fs.readFileSync(configFile(), 'utf8'); confirmStub.resolves(false); const result = await ScriptScan.run([]); - expect(result.success).to.be.true; + expect(result.success).to.be.false; expect(result.executionResult.written).to.equal(false); + const logged = sfCommandStubs.log + .getCalls() + .flatMap((call) => call.args) + .join('\n'); + expect(logged).to.not.include('Package scanned successfully'); + expect(logged).to.include('No files were modified'); expect(fs.readFileSync(configFile(), 'utf8')).to.equal(before); expect(fs.existsSync(path.join(testDir, 'requirements.txt'))).to.be.false; }); diff --git a/test/utils/lineageConfig.test.ts b/test/utils/lineageConfig.test.ts index c0306d6..6c2acdb 100644 --- a/test/utils/lineageConfig.test.ts +++ b/test/utils/lineageConfig.test.ts @@ -100,6 +100,20 @@ describe('lineageConfig', () => { expect(isEmptyDiff(diff)).to.equal(false); }); + it('matches targets and sources case-insensitively', () => { + const diff = diffLineage( + [ + { targetField: 'Out__dll.Name__c', sourceFields: ['In__dll.Name__c'] }, + { targetField: 'Out__dll.Gone__c', sourceFields: ['In__dll.A__c'] }, + ], + [ + { targetField: 'out__dll.name__c', sourceFields: ['in__dll.name__c'] }, + { targetField: 'out__dll.new__c', sourceFields: ['in__dll.a__c'] }, + ] + ); + expect(diff).to.deep.equal({ added: ['out__dll.new__c'], removed: ['Out__dll.Gone__c'], changed: [] }); + }); + it('is empty for identical lineage', () => { const entries = [{ targetField: 'o.a', sourceFields: ['i.a'] }]; expect(isEmptyDiff(diffLineage(entries, entries))).to.equal(true); diff --git a/test/utils/lineageRunner.test.ts b/test/utils/lineageRunner.test.ts index 36bf4d4..4c32eb6 100644 --- a/test/utils/lineageRunner.test.ts +++ b/test/utils/lineageRunner.test.ts @@ -23,6 +23,7 @@ import { GENERATE_LINEAGE_SHIM, LINEAGE_UNAVAILABLE_EXIT_CODE, LINEAGE_UNAVAILABLE_MARKER, + parseLineageReport, type LineagePayload, type LineageReport, type LineageRunOptions, @@ -162,6 +163,38 @@ describe('lineageRunner', () => { }); }); + describe('parseLineageReport', () => { + const valid = { + dataLineage: [], + unresolved: [], + noSources: [], + udfFields: [], + targets: [], + sources: [], + pairCount: 0, + }; + + it('returns a well-formed report', () => { + expect(parseLineageReport(JSON.stringify(valid))).to.deep.equal(valid); + }); + + for (const [name, raw, detail] of [ + ['malformed JSON', '{', 'JSON'], + ['a non-object', '[]', 'expected a JSON object'], + ['missing fields', JSON.stringify({ ...valid, udfFields: undefined, noSources: null }), 'noSources, udfFields'], + ]) { + it(`rejects ${name}`, () => { + try { + parseLineageReport(raw); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageResultInvalid'); + expect((error as SfError).message).to.include(detail); + } + }); + } + }); + describe('generateLineage', () => { it('passes the payload as its own argv element and returns the result file', async () => { const calls: SpawnCall[] = []; From 8269aeb4310243a8b5da968f2f866182a7860982 Mon Sep 17 00:00:00 2001 From: Jesus Orosco Date: Fri, 16 Oct 2026 10:17:18 -0700 Subject: [PATCH 7/8] fix: address various PR concerns and comments --- messages/lineage.md | 44 +++++-- messages/run.md | 13 +- messages/scan.md | 4 +- src/base/lineageBase.ts | 69 +++++++++- src/base/scanBase.ts | 16 +-- src/utils/lineageConfig.ts | 4 +- src/utils/lineageRunner.ts | 40 +++++- src/utils/pythonRunner.ts | 23 +++- .../data-code-extension/lineage.test.ts | 122 +++++++++++++++++- .../commands/data-code-extension/scan.test.ts | 12 +- test/utils/lineageConfig.test.ts | 4 +- test/utils/lineageRunner.test.ts | 55 +++++++- test/utils/pythonRunner.test.ts | 30 ++++- 13 files changed, 375 insertions(+), 61 deletions(-) diff --git a/messages/lineage.md b/messages/lineage.md index a0ae82b..0f9998f 100644 --- a/messages/lineage.md +++ b/messages/lineage.md @@ -4,7 +4,7 @@ Generate field-level dataLineage for a Data Code Extension %s package. # description -Runs the package's entrypoint against a Salesforce org with every write intercepted, traces each written field back to the fields it was computed from, and replaces the dataLineage section of the package's config.json with the result. Nothing is written to Data Cloud. Run 'scan' first so the read and write permissions in config.json are current. Fields whose lineage can't be traced are listed and get no lineage unless you add entries by hand. +Runs the package's entrypoint against a Salesforce org with its Data Cloud writes intercepted, traces each written field back to the fields it was computed from, and replaces the dataLineage section of the package's config.json with the result. Nothing is written to Data Cloud objects, but other calls the script makes still run for real with the org's credentials, including named credential requests, LLM gateway calls, and Einstein predictions. Streaming scripts aren't supported. Run 'scan' first so the read and write permissions in config.json are current. Fields whose lineage can't be traced are listed and get no lineage unless you add entries by hand. Generated lineage is a best-effort analysis of your script and may be incomplete or wrong (for example, for UDFs or other code the tool can't see into). It determines how governance tags propagate, which can restrict or open access to data. You're responsible for reviewing and maintaining it. Before config.json is written, you must accept this, either at the prompt or with --accept-lineage-ownership. @@ -48,12 +48,22 @@ Successfully authenticated with org '%s' # info.generating -Running package to trace lineage. Nothing is written to Data Cloud... +Running package to trace lineage. Data Cloud writes are intercepted; other external calls the script makes still run... # info.summary Generated %s dataLineage entries with %s (target, source) field pairs: +%s added, -%s removed, ~%s changed. +# info.removed + +Entries in config.json that the generated lineage drops: +%s + +# info.changed + +Entries in config.json whose source fields the generated lineage changes: +%s + # info.noSources %s target fields are computed from no source field (for example, literals) and get no lineage: @@ -95,13 +105,17 @@ Lineage generation completed successfully! # warning.permissionGaps -The generated lineage references objects that aren't in config.json permissions, so deploy would fail (1041/1042). Run 'data-code-extension script scan' first, then generate lineage again. +The generated lineage references objects that aren't in config.json permissions, so deploy would fail (1041/1042). Run 'data-code-extension script scan' first, then generate lineage again. Scan can't detect objects whose names your script builds at run time; add those to permissions in config.json by hand. %s # warning.emptyResult No lineage could be generated, so config.json was not modified. The package follows the no-lineage path unless you add dataLineage entries by hand. +# warning.emptyResultExisting + +No lineage could be generated, so config.json was not modified. Its existing %s dataLineage entries still apply and are deployed with the package. Review or remove them in '%s'. + # warning.replace This replaces the existing dataLineage section. Any entries you added or edited by hand will be lost. Lineage determines how governance tags propagate to the target; review the result before deploying. @@ -126,6 +140,14 @@ Accept and replace the existing dataLineage in config.json? Lineage can only be generated for script packages; '%s' is a function package. +# error.streamingPackage + +Lineage can't be generated for streaming scripts; '%s' has a streamingSource in config.json. + +# error.configUnwritable + +Can't write '%s': %s + # error.outputFileCollision --output-file '%s' is inside the package's payload or project metadata, or is one of its config files. Choose a path outside the package's payload directory. @@ -152,8 +174,8 @@ Writing generated lineage to config.json requires accepting ownership of it. # actions.ownershipNotAccepted -- Review the generated lineage, then re-run with --accept-lineage-ownership to accept responsibility for it and write it to config.json. -- Re-run with --dry-run to preview the generated lineage without changing config.json. +- Re-run with --accept-lineage-ownership to accept responsibility for the generated lineage and write it to config.json. +- Re-run with --dry-run to preview the generated lineage, or check it against config.json, without writing. # error.replaceNotConfirmed @@ -162,7 +184,7 @@ config.json already has a different dataLineage section, and replacing it needs # actions.replaceNotConfirmed - Re-run with --no-prompt to replace the existing dataLineage. -- Re-run with --dry-run to preview the generated lineage without changing config.json. +- Re-run with --dry-run to preview the generated lineage, or check it against config.json, without writing. # error.sdkTooOld @@ -203,7 +225,7 @@ Target Salesforce org to read data from. # flags.targetOrg.description -The alias or username of the Salesforce org whose data the package reads while lineage is traced. The org must have Data Cloud enabled and appropriate permissions. Nothing is written to the org. +The alias or username of the Salesforce org whose data the package reads while lineage is traced. The org must have Data Cloud enabled and appropriate permissions. Nothing is written to Data Cloud objects in the org, but other calls the script makes, such as named credential requests, LLM gateway calls, and Einstein predictions, still run against it. # flags.configFile.summary @@ -215,11 +237,11 @@ Optional path to an SDK config.yaml that overrides the reader and writer configu # flags.dependencies.summary -Dependencies override for the run. +Modules to import before the run. # flags.dependencies.description -Optional comma-separated list of Python package dependencies to use during the run, overriding those defined in the package's requirements.txt. +Optional comma-separated list of Python modules (or module.attribute) to import before the entrypoint runs. Use import names, such as 'yaml' rather than 'pyyaml'. Nothing is installed: each module must already be installed in the active Python environment. # flags.dryRun.summary @@ -243,7 +265,7 @@ Replace existing dataLineage without prompting. # flags.noPrompt.description -Skips the confirmation before replacing a dataLineage section that differs from the generated one. Required to replace existing lineage when running non-interactively or with --json. Doesn't accept ownership of the generated lineage; use --accept-lineage-ownership for that. +Skips the confirmation before replacing a dataLineage section that differs from the generated one. Required when running non-interactively or with --json and config.json already has dataLineage; checked before the run. Doesn't accept ownership of the generated lineage; use --accept-lineage-ownership for that. # flags.acceptLineageOwnership.summary @@ -251,4 +273,4 @@ Accept responsibility for the generated lineage without prompting. # flags.acceptLineageOwnership.description -Confirms that you understand the generated lineage is best-effort and may be incomplete or wrong, that it determines how governance tags propagate, and that you're responsible for reviewing and maintaining it. Skips the ownership prompt. Required to write lineage when running non-interactively or with --json. +Confirms that you understand the generated lineage is best-effort and may be incomplete or wrong, that it determines how governance tags propagate, and that you're responsible for reviewing and maintaining it. Skips the ownership prompt. Required when running non-interactively or with --json, unless you use --dry-run; checked before the run. diff --git a/messages/run.md b/messages/run.md index bf428fe..c3dcd66 100644 --- a/messages/run.md +++ b/messages/run.md @@ -98,6 +98,15 @@ Authentication with Salesforce org '%s' failed while running the package. - Confirm the org alias or username passed to --target-org is correct. - Verify the org has Data Cloud enabled and your user has the required permissions. +# error.runTerminated + +The package run was stopped by signal %s before it finished. Runs time out after %s seconds. + +# actions.runTerminated + +- If the run timed out, reduce the data it processes (for example, the reader's default_row_limit in an SDK config file) or check for a step that hangs, such as a network call. +- If the process ran out of memory or was stopped by another program, free resources and try again. + # error.runProcessStartFailed Could not start the Python interpreter '%s': %s @@ -149,8 +158,8 @@ Optional path to a JSON config file that provides input payload for the run. Def # flags.dependencies.summary -Dependencies override for the run. +Modules to import before the run. # flags.dependencies.description -Optional comma-separated list of Python package dependencies to use during the run, overriding those defined in the package's requirements.txt. +Optional comma-separated list of Python modules (or module.attribute) to import before the entrypoint runs. Use import names, such as 'yaml' rather than 'pyyaml'. Nothing is installed: each module must already be installed in the active Python environment. diff --git a/messages/scan.md b/messages/scan.md index a8402fd..0f833e5 100644 --- a/messages/scan.md +++ b/messages/scan.md @@ -119,9 +119,9 @@ dataLineage references objects that are no longer in permissions (%s entries aff Write the updated permissions to config.json anyway? -# info.staleLineageNotWritten +# error.staleLineageNotWritten -No files were modified. Update dataLineage, then scan again. +The updated permissions weren't confirmed, so no files were modified. Update dataLineage, then scan again. # error.staleLineageNotConfirmed diff --git a/src/base/lineageBase.ts b/src/base/lineageBase.ts index 6ca06d2..463eefd 100644 --- a/src/base/lineageBase.ts +++ b/src/base/lineageBase.ts @@ -87,7 +87,20 @@ export abstract class LineageBase extends SfCommand { throw new SfError(messages.getMessage('error.functionPackage', [flags.entrypoint]), 'LineageFunctionPackage'); } // Read config.json up front so a missing or malformed file fails before the run. - await readConfigJson(configPath); + const initialConfig = await readConfigJson(configPath); + if (initialConfig.streamingSource) { + throw new SfError(messages.getMessage('error.streamingPackage', [flags.entrypoint]), 'LineageStreamingPackage'); + } + // Fail before the run on everything that would only refuse the write afterwards. + if (!flags['dry-run']) { + this.checkNonInteractiveWrite(messages, flags, hasLineageSection(initialConfig)); + await fs.access(configPath, fsConstants.W_OK).catch((err: Error) => { + throw new SfError( + messages.getMessage('error.configUnwritable', [configPath, err.message]), + 'LineageConfigUnwritable' + ); + }); + } const outputFile = flags['output-file'] ? path.resolve(flags['output-file']) : undefined; if (outputFile) { @@ -112,7 +125,7 @@ export abstract class LineageBase extends SfCommand { ); } // Fail before the run rather than lose its result to a bad path afterwards. - await fs.access(path.dirname(outputFile), fsConstants.W_OK).catch((err: Error) => { + await checkOutputFileWritable(outputFile).catch((err: Error) => { throw new SfError( messages.getMessage('error.outputFileUnwritable', [outputFile, err.message]), 'LineageOutputFileUnwritable' @@ -179,7 +192,12 @@ export abstract class LineageBase extends SfCommand { }; if (entries.length === 0) { - this.warn(messages.getMessage('warning.emptyResult')); + // Existing entries still ship at deploy; say so rather than imply the no-lineage path. + this.warn( + current.length > 0 + ? messages.getMessage('warning.emptyResultExisting', [current.length, configPath]) + : messages.getMessage('warning.emptyResult') + ); return result; } @@ -233,6 +251,12 @@ export abstract class LineageBase extends SfCommand { diff.changed.length, ]) ); + if (diff.removed.length > 0) { + this.log(messages.getMessage('info.removed', [bullets(diff.removed)])); + } + if (diff.changed.length > 0) { + this.log(messages.getMessage('info.changed', [bullets(diff.changed)])); + } if (lineage.noSources.length > 0) { this.log(messages.getMessage('info.noSources', [lineage.noSources.length, bullets(lineage.noSources)])); } @@ -277,13 +301,36 @@ export abstract class LineageBase extends SfCommand { ); } const prompt = needsOwnership - ? replacing + ? needsReplace ? 'prompt.acceptOwnershipReplace' : 'prompt.acceptOwnership' : 'prompt.replace'; return this.confirm({ message: messages.getMessage(prompt), defaultAnswer: false, ms: LINEAGE_CONFIRM_TIMEOUT_MS }); } + /** + * Without a prompt to fall back on, a write needs its flags up front: fail before the run + * instead of after it. `--dry-run` is the way to check lineage without writing. + * confirmWrite repeats these checks in case config.json gains a dataLineage section during the run. + */ + private checkNonInteractiveWrite(messages: Messages, flags: BaseLineageFlags, replacing: boolean): void { + if (!this.jsonEnabled() && process.stdin.isTTY) return; + if (!flags['accept-lineage-ownership']) { + throw new SfError( + messages.getMessage('error.ownershipNotAccepted'), + 'LineageOwnershipNotAccepted', + messages.getMessages('actions.ownershipNotAccepted') + ); + } + if (replacing && !flags['no-prompt']) { + throw new SfError( + messages.getMessage('error.replaceNotConfirmed'), + 'LineageReplaceNotConfirmed', + messages.getMessages('actions.replaceNotConfirmed') + ); + } + } + protected abstract getCodeType(): 'script'; protected abstract getMessages(): Messages; } @@ -356,6 +403,20 @@ async function realPath(file: string, hops = 0): Promise { } } +/** + * Rejects an --output-file that `fs.writeFile` would fail on: an existing target must be a + * writable regular file, and a new one needs a writable parent directory. + */ +async function checkOutputFileWritable(outputFile: string): Promise { + const stat = await statOrUndefined(outputFile); + if (stat === undefined) { + await fs.access(path.dirname(outputFile), fsConstants.W_OK); + return; + } + if (!stat.isFile()) throw new Error('not a regular file'); + await fs.access(outputFile, fsConstants.W_OK); +} + function bullets(lines: string[]): string { return lines.map((line) => ` - ${line}`).join('\n'); } diff --git a/src/base/scanBase.ts b/src/base/scanBase.ts index c6e3ec6..c8f19fb 100644 --- a/src/base/scanBase.ts +++ b/src/base/scanBase.ts @@ -77,17 +77,13 @@ export abstract class ScanBase extends SfCommand { if (hasStaleLineage(stale)) this.warnStaleLineage(messages, executionResult.config, stale); } + // Exit non-zero so a following deploy doesn't ship the old permissions. if (executionResult.written === false) { - this.log(messages.getMessage('info.staleLineageNotWritten')); - return { - success: false, - pythonInfo, - packageInfo, - codeType, - workingDirectory: workingDir, - executionResult, - message: messages.getMessage('info.staleLineageNotWritten'), - }; + throw new SfError( + messages.getMessage('error.staleLineageNotWritten'), + 'ScanStaleLineageNotWritten', + messages.getMessages('actions.staleLineageNotConfirmed') + ); } this.log(messages.getMessage('info.scanExecuted', [workingDir])); diff --git a/src/utils/lineageConfig.ts b/src/utils/lineageConfig.ts index 63763f4..5782a9d 100644 --- a/src/utils/lineageConfig.ts +++ b/src/utils/lineageConfig.ts @@ -152,9 +152,9 @@ export function countEntriesReferencing(config: Record, objects ).length; } -/** Write config.json with a 2-space indent and a trailing newline. */ +/** Write config.json with a 2-space indent and no trailing newline, as scan and init do. */ export async function writeConfigJson(configPath: string, config: Record): Promise { - await fs.writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, 'utf8'); + await fs.writeFile(configPath, JSON.stringify(config, null, 2), 'utf8'); } function objectOf(field: string): string { diff --git a/src/utils/lineageRunner.ts b/src/utils/lineageRunner.ts index f1653c6..41bb780 100644 --- a/src/utils/lineageRunner.ts +++ b/src/utils/lineageRunner.ts @@ -16,8 +16,17 @@ import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { debuglog } from 'node:util'; import { Messages, SfError } from '@salesforce/core'; -import { DEFAULT_PROFILE, runPythonProgram, type PythonRunnerDeps } from './pythonRunner.js'; +import { + DEFAULT_PROFILE, + mapRunTerminated, + RUN_TIMEOUT_MS, + runPythonProgram, + type PythonRunnerDeps, +} from './pythonRunner.js'; + +const debug = debuglog('datacustomcode'); Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-data-code-extension', 'lineage'); @@ -27,6 +36,11 @@ export const LINEAGE_UNAVAILABLE_EXIT_CODE = 3; /** Printed to stderr alongside {@link LINEAGE_UNAVAILABLE_EXIT_CODE}, so a script's own `sys.exit(3)` isn't mistaken for it. */ export const LINEAGE_UNAVAILABLE_MARKER = 'DATACUSTOMCODE_LINEAGE_UNAVAILABLE'; +/** Exit code the shim uses when SIGTERM (the run timeout) stopped it: 128 + SIGTERM. */ +export const LINEAGE_TERMINATED_EXIT_CODE = 143; +/** Printed to stderr alongside {@link LINEAGE_TERMINATED_EXIT_CODE}. */ +export const LINEAGE_TERMINATED_MARKER = 'DATACUSTOMCODE_LINEAGE_TERMINATED'; + /** * The Python program executed via `python -c`. It calls the SDK *library's* * `datacustomcode.lineage.generate_lineage`, which runs the entrypoint against the @@ -35,9 +49,17 @@ export const LINEAGE_UNAVAILABLE_MARKER = 'DATACUSTOMCODE_LINEAGE_UNAVAILABLE'; * Arguments arrive as a single JSON object in `sys.argv[1]`, never interpolated into * this source. The result goes to `resultFile` rather than stdout, because the * customer's script prints freely to stdout. + * + * The SDK runs the entrypoint in a worker process that shares this one's stdio, so + * SIGTERM (the run timeout) must stop it too, or the CLI waits on its open pipes. + * Raising SystemExit inside `subprocess.run` makes it kill the worker. */ export const GENERATE_LINEAGE_SHIM = [ - 'import importlib.util, json, sys', + 'import importlib.util, json, signal, sys', + 'def _terminated(*_):', + ` sys.stderr.write("${LINEAGE_TERMINATED_MARKER}\\n")`, + ` sys.exit(${LINEAGE_TERMINATED_EXIT_CODE})`, + 'signal.signal(signal.SIGTERM, _terminated)', 'if importlib.util.find_spec("datacustomcode.lineage") is None:', ` sys.stderr.write("${LINEAGE_UNAVAILABLE_MARKER}\\n")`, ` sys.exit(${LINEAGE_UNAVAILABLE_EXIT_CODE})`, @@ -110,7 +132,10 @@ export function buildLineagePayload(opts: LineageRunOptions, resultFile: string) }; } -function mapLineageExitCode(exitCode: number, stderr: string): SfError | undefined { +function mapLineageExitCode(exitCode: number, stderr: string, timeoutMs: number): SfError | undefined { + if (exitCode === LINEAGE_TERMINATED_EXIT_CODE && stderr.includes(LINEAGE_TERMINATED_MARKER)) { + return mapRunTerminated('SIGTERM', timeoutMs); + } if (exitCode === LINEAGE_UNAVAILABLE_EXIT_CODE && stderr.includes(LINEAGE_UNAVAILABLE_MARKER)) { return new SfError( messages.getMessage('error.sdkTooOld'), @@ -139,7 +164,7 @@ export async function generateLineage(opts: LineageRunOptions, deps?: PythonRunn timeoutMs: opts.timeoutMs, onStdout: opts.onStdout, onStderr: opts.onStderr, - mapExitCode: mapLineageExitCode, + mapExitCode: (exitCode, stderr) => mapLineageExitCode(exitCode, stderr, opts.timeoutMs ?? RUN_TIMEOUT_MS), }, deps ); @@ -156,7 +181,12 @@ export async function generateLineage(opts: LineageRunOptions, deps?: PythonRunn } return parseLineageReport(raw); } finally { - await fs.rm(tempDir, { recursive: true, force: true }); + // A file another program briefly holds open (e.g. antivirus on Windows) must not hide the result. + try { + await fs.rm(tempDir, { recursive: true, force: true }); + } catch (err) { + debug('temp dir cleanup error (ignored): %o', err); + } } } diff --git a/src/utils/pythonRunner.ts b/src/utils/pythonRunner.ts index b91c641..45861f4 100644 --- a/src/utils/pythonRunner.ts +++ b/src/utils/pythonRunner.ts @@ -92,7 +92,7 @@ export type RunnerStream = { export type RunnerChild = { stdout: RunnerStream | null; stderr: RunnerStream | null; - on(event: 'close', listener: (code: number | null) => void): unknown; + on(event: 'close', listener: (code: number | null, signal: NodeJS.Signals | null) => void): unknown; on(event: 'error', listener: (err: Error) => void): unknown; }; export type SpawnFn = ( @@ -147,6 +147,14 @@ function mapRunFailure(exitCode: number, stderr: string, sfCliOrg?: string): SfE ); } +export function mapRunTerminated(signal: NodeJS.Signals | null, timeoutMs: number): SfError { + return new SfError( + messages.getMessage('error.runTerminated', [signal ?? 'unknown', Math.round(timeoutMs / 1000)]), + 'RunTerminated', + messages.getMessages('actions.runTerminated') + ); +} + function mapProcessStartError(err: Error, pythonCommand: string): SfError { return new SfError( messages.getMessage('error.runProcessStartFailed', [pythonCommand, err.message]), @@ -180,9 +188,10 @@ export async function runPythonProgram( ): Promise { debug('run spawn: %s %o', opts.pythonCommand, opts.args); + const timeoutMs = opts.timeoutMs ?? RUN_TIMEOUT_MS; return new Promise((resolve, reject) => { const child = deps.spawn(opts.pythonCommand, opts.args, { - timeout: opts.timeoutMs ?? RUN_TIMEOUT_MS, + timeout: timeoutMs, // PYTHONUNBUFFERED so the entrypoint's prints stream immediately instead of buffering. env: { ...process.env, PYTHONUNBUFFERED: '1' }, }); @@ -206,8 +215,14 @@ export async function runPythonProgram( reject(mapProcessStartError(err, opts.pythonCommand)); }); - child.on('close', (code: number | null) => { - const exitCode = code ?? 0; + child.on('close', (code: number | null, signal: NodeJS.Signals | null) => { + // No exit code means a signal stopped the run: the timeout, an OOM kill, or the user. + if (code === null) { + debug('run killed by signal: %s', signal); + reject(mapRunTerminated(signal, timeoutMs)); + return; + } + const exitCode = code; debug('run exit code: %d', exitCode); if (exitCode === 0) { resolve({ stdout: stdout.trim(), stderr: stderr.trim(), exitCode }); diff --git a/test/commands/data-code-extension/lineage.test.ts b/test/commands/data-code-extension/lineage.test.ts index 6fc7cfe..6c37904 100644 --- a/test/commands/data-code-extension/lineage.test.ts +++ b/test/commands/data-code-extension/lineage.test.ts @@ -133,7 +133,7 @@ describe('data-code-extension script lineage generate', () => { expect(config.dataLineage).to.deep.equal(REPORT.dataLineage); expect(config.permissions).to.deep.equal(PERMISSIONS); expect(config.entryPoint).to.equal('entrypoint.py'); - expect(fs.readFileSync(configPath, 'utf8').endsWith('}\n')).to.be.true; + expect(fs.readFileSync(configPath, 'utf8')).to.equal(JSON.stringify(config, null, 2)); expect(result.written).to.equal(true); expect(result.entryCount).to.equal(2); @@ -177,6 +177,72 @@ describe('data-code-extension script lineage generate', () => { expect(warnings().some((w) => w.includes('No lineage could be generated'))).to.be.true; }); + it('warns that existing lineage still ships when the result is empty', async () => { + const existing = [{ targetField: 'out__dll.old', sourceFields: ['in__dll.old'] }]; + writeConfig({ dataspace: 'default', permissions: PERMISSIONS, dataLineage: existing }); + generateStub.resolves({ ...REPORT, dataLineage: [], pairCount: 0 }); + const result = await LineageGenerate.run(baseArgs()); + expect(result.written).to.equal(false); + expect(readConfig().dataLineage).to.deep.equal(existing); + const warning = warnings().find((w) => w.includes('No lineage could be generated')); + expect(warning).to.include('existing 1 dataLineage entries still apply'); + expect(warning).to.not.include('no-lineage path'); + }); + + it('lists the entries that replacing drops or changes', async () => { + writeConfig({ + dataspace: 'default', + permissions: PERMISSIONS, + dataLineage: [ + { targetField: 'out__dll.old', sourceFields: ['in__dll.old'] }, + { targetField: 'out__dll.id__c', sourceFields: ['in__dll.other'] }, + ], + }); + await LineageGenerate.run([...baseArgs(), '--dry-run']); + const logged = sfCommandStubs.log + .getCalls() + .map((call) => String(call.args[0])) + .join('\n'); + expect(logged).to.match(/drops:\n {2}- out__dll\.old/); + expect(logged).to.match(/changes:\n {2}- out__dll\.id__c/); + }); + + it('rejects streaming scripts before the run', async () => { + writeConfig({ dataspace: 'default', permissions: PERMISSIONS, streamingSource: { name: 'in__dll' } }); + try { + await LineageGenerate.run(baseArgs()); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageStreamingPackage'); + } + expect(generateStub.called).to.be.false; + }); + + it('rejects a read-only config.json before the run', async function () { + if (process.platform === 'win32' || process.getuid?.() === 0) this.skip(); + fs.chmodSync(configPath, 0o444); + try { + await LineageGenerate.run([...baseArgs(), '--accept-lineage-ownership']); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageConfigUnwritable'); + } finally { + fs.chmodSync(configPath, 0o644); + } + expect(generateStub.called).to.be.false; + }); + + it('--dry-run does not need a writable config.json', async function () { + if (process.platform === 'win32' || process.getuid?.() === 0) this.skip(); + fs.chmodSync(configPath, 0o444); + try { + const result = await LineageGenerate.run([...baseArgs(), '--dry-run']); + expect(result.written).to.equal(false); + } finally { + fs.chmodSync(configPath, 0o644); + } + }); + it('--dry-run never writes or prompts', async () => { writeConfig({ dataspace: 'default', @@ -212,6 +278,37 @@ describe('data-code-extension script lineage generate', () => { expect(generateStub.called).to.be.false; }); + it('rejects an --output-file that is a directory before the run', async () => { + const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-lineage-out-')); + try { + await LineageGenerate.run([...baseArgs(), '--dry-run', '--output-file', outputDir]); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageOutputFileUnwritable'); + expect((error as SfError).message).to.include('not a regular file'); + } finally { + fs.rmSync(outputDir, { recursive: true, force: true }); + } + expect(generateStub.called).to.be.false; + }); + + it('rejects a read-only existing --output-file before the run', async function () { + if (process.platform === 'win32' || process.getuid?.() === 0) this.skip(); + const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-lineage-out-')); + const outputFile = path.join(outputDir, 'lineage-result.json'); + fs.writeFileSync(outputFile, '{}'); + fs.chmodSync(outputFile, 0o444); + try { + await LineageGenerate.run([...baseArgs(), '--dry-run', '--output-file', outputFile]); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageOutputFileUnwritable'); + } finally { + fs.rmSync(outputDir, { recursive: true, force: true }); + } + expect(generateStub.called).to.be.false; + }); + it('fails clearly when the target org has no username', async () => { (Org.create as sinon.SinonStub).resolves({ getUsername: () => undefined, @@ -411,10 +508,24 @@ describe('data-code-extension script lineage generate', () => { } catch (error) { expect((error as SfError).name).to.equal('LineageOwnershipNotAccepted'); } + expect(generateStub.called).to.be.false; expect(confirmStub.called).to.be.false; expect(readConfig()).to.not.have.property('dataLineage'); }); + it('fails before the run even when the result would be empty', async () => { + process.stdin.isTTY = false; + generateStub.resolves({ ...REPORT, dataLineage: [], pairCount: 0 }); + try { + await LineageGenerate.run(baseArgs()); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageOwnershipNotAccepted'); + expect((error as SfError).actions?.join(' ')).to.include('--dry-run'); + } + expect(generateStub.called).to.be.false; + }); + it('fails with --json without --accept-lineage-ownership', async () => { try { await LineageGenerate.run([...baseArgs(), '--json']); @@ -422,6 +533,7 @@ describe('data-code-extension script lineage generate', () => { } catch (error) { expect((error as SfError).name).to.equal('LineageOwnershipNotAccepted'); } + expect(generateStub.called).to.be.false; expect(readConfig()).to.not.have.property('dataLineage'); }); @@ -471,6 +583,12 @@ describe('data-code-extension script lineage generate', () => { ); }); + it('--no-prompt alone asks only to accept ownership', async () => { + await LineageGenerate.run([...baseArgs(), '--no-prompt']); + expect(confirmStub.calledOnce).to.be.true; + expect(confirmStub.firstCall.args[0]).to.have.property('message', 'Accept and write dataLineage to config.json?'); + }); + it('--accept-lineage-ownership --no-prompt replaces without asking', async () => { const result = await LineageGenerate.run([...baseArgs(), '--accept-lineage-ownership', '--no-prompt']); expect(confirmStub.called).to.be.false; @@ -514,6 +632,7 @@ describe('data-code-extension script lineage generate', () => { } catch (error) { expect((error as SfError).name).to.equal('LineageReplaceNotConfirmed'); } + expect(generateStub.called).to.be.false; expect(confirmStub.called).to.be.false; expect(readConfig().dataLineage).to.deep.equal(OLD); }); @@ -525,6 +644,7 @@ describe('data-code-extension script lineage generate', () => { } catch (error) { expect((error as SfError).name).to.equal('LineageReplaceNotConfirmed'); } + expect(generateStub.called).to.be.false; expect(readConfig().dataLineage).to.deep.equal(OLD); }); diff --git a/test/commands/data-code-extension/scan.test.ts b/test/commands/data-code-extension/scan.test.ts index 20f6ec6..78247cd 100644 --- a/test/commands/data-code-extension/scan.test.ts +++ b/test/commands/data-code-extension/scan.test.ts @@ -325,15 +325,19 @@ describe('data-code-extension scan commands', () => { makeScriptPackage(SCRIPT, { dataspace: 'default', dataLineage: STALE }); const before = fs.readFileSync(configFile(), 'utf8'); confirmStub.resolves(false); - const result = await ScriptScan.run([]); - expect(result.success).to.be.false; - expect(result.executionResult.written).to.equal(false); + try { + await ScriptScan.run([]); + expect.fail('Should have thrown'); + } catch (error) { + // Non-zero exit, so a following `&& deploy` doesn't ship the old permissions. + expect((error as SfError).name).to.equal('ScanStaleLineageNotWritten'); + expect((error as SfError).message).to.include('no files were modified'); + } const logged = sfCommandStubs.log .getCalls() .flatMap((call) => call.args) .join('\n'); expect(logged).to.not.include('Package scanned successfully'); - expect(logged).to.include('No files were modified'); expect(fs.readFileSync(configFile(), 'utf8')).to.equal(before); expect(fs.existsSync(path.join(testDir, 'requirements.txt'))).to.be.false; }); diff --git a/test/utils/lineageConfig.test.ts b/test/utils/lineageConfig.test.ts index 6c2acdb..7126a5e 100644 --- a/test/utils/lineageConfig.test.ts +++ b/test/utils/lineageConfig.test.ts @@ -190,12 +190,12 @@ describe('lineageConfig', () => { }); }); - it('writeConfigJson writes 2-space JSON with a trailing newline', async () => { + it('writeConfigJson writes 2-space JSON without a trailing newline, as scan does', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-lineage-config-')); try { const file = path.join(dir, 'config.json'); await writeConfigJson(file, { a: [1] }); - expect(fs.readFileSync(file, 'utf8')).to.equal('{\n "a": [\n 1\n ]\n}\n'); + expect(fs.readFileSync(file, 'utf8')).to.equal('{\n "a": [\n 1\n ]\n}'); } finally { fs.rmSync(dir, { recursive: true, force: true }); } diff --git a/test/utils/lineageRunner.test.ts b/test/utils/lineageRunner.test.ts index 4c32eb6..ffa314a 100644 --- a/test/utils/lineageRunner.test.ts +++ b/test/utils/lineageRunner.test.ts @@ -21,6 +21,8 @@ import { buildLineagePayload, generateLineage, GENERATE_LINEAGE_SHIM, + LINEAGE_TERMINATED_EXIT_CODE, + LINEAGE_TERMINATED_MARKER, LINEAGE_UNAVAILABLE_EXIT_CODE, LINEAGE_UNAVAILABLE_MARKER, parseLineageReport, @@ -59,19 +61,23 @@ class FakeStream implements RunnerStream { class FakeChild implements RunnerChild { public stdout = new FakeStream(); public stderr = new FakeStream(); - private closeListeners: Array<(code: number | null) => void> = []; + private closeListeners: Array<(code: number | null, signal: NodeJS.Signals | null) => void> = []; private errorListeners: Array<(err: Error) => void> = []; - public on(event: 'close', listener: (code: number | null) => void): this; + public on(event: 'close', listener: (code: number | null, signal: NodeJS.Signals | null) => void): this; public on(event: 'error', listener: (err: Error) => void): this; - public on(event: 'close' | 'error', listener: ((code: number | null) => void) | ((err: Error) => void)): this { - if (event === 'close') this.closeListeners.push(listener as (code: number | null) => void); - else this.errorListeners.push(listener as (err: Error) => void); + public on( + event: 'close' | 'error', + listener: ((code: number | null, signal: NodeJS.Signals | null) => void) | ((err: Error) => void) + ): this { + if (event === 'close') { + this.closeListeners.push(listener as (code: number | null, signal: NodeJS.Signals | null) => void); + } else this.errorListeners.push(listener as (err: Error) => void); return this; } - public emitClose(code: number | null): void { - for (const listener of this.closeListeners) listener(code); + public emitClose(code: number | null, signal: NodeJS.Signals | null = null): void { + for (const listener of this.closeListeners) listener(code, signal); } public emitError(err: Error): void { @@ -161,6 +167,16 @@ describe('lineageRunner', () => { expect(GENERATE_LINEAGE_SHIM).to.include(`sys.exit(${LINEAGE_UNAVAILABLE_EXIT_CODE})`); expect(GENERATE_LINEAGE_SHIM).to.include(LINEAGE_UNAVAILABLE_MARKER); }); + + it('turns SIGTERM into SystemExit so subprocess.run stops the SDK worker', () => { + expect(GENERATE_LINEAGE_SHIM).to.include('signal.signal(signal.SIGTERM, _terminated)'); + expect(GENERATE_LINEAGE_SHIM).to.include(`sys.exit(${LINEAGE_TERMINATED_EXIT_CODE})`); + expect(GENERATE_LINEAGE_SHIM).to.include(LINEAGE_TERMINATED_MARKER); + // The handler must be installed before the SDK (and its worker) starts. + expect(GENERATE_LINEAGE_SHIM.indexOf('signal.signal(')).to.be.lessThan( + GENERATE_LINEAGE_SHIM.indexOf('generate_lineage(') + ); + }); }); describe('parseLineageReport', () => { @@ -269,6 +285,31 @@ describe('lineageRunner', () => { expect(err.name).to.equal('RunExecutionFailed'); }); + it('maps the terminated exit code to RunTerminated with the timeout', async () => { + const err = await expectReject( + generateLineage(makeOpts({ timeoutMs: 60_000 }), { + spawn: fakeSpawn([], (child) => { + child.stderr.push(`${LINEAGE_TERMINATED_MARKER}\n`); + child.emitClose(LINEAGE_TERMINATED_EXIT_CODE); + }), + }) + ); + expect(err.name).to.equal('RunTerminated'); + expect(err.message).to.include('60 seconds'); + }); + + it("does not mistake a script's own exit code 143 for the timeout", async () => { + const err = await expectReject( + generateLineage(makeOpts(), { + spawn: fakeSpawn([], (child) => { + child.stderr.push('SystemExit: 143'); + child.emitClose(LINEAGE_TERMINATED_EXIT_CODE); + }), + }) + ); + expect(err.name).to.equal('RunExecutionFailed'); + }); + it('maps authentication failures', async () => { const err = await expectReject( generateLineage(makeOpts(), { diff --git a/test/utils/pythonRunner.test.ts b/test/utils/pythonRunner.test.ts index 2c27136..a5e7f21 100644 --- a/test/utils/pythonRunner.test.ts +++ b/test/utils/pythonRunner.test.ts @@ -65,20 +65,24 @@ class FakeStream implements RunnerStream { class FakeChild implements RunnerChild { public stdout: FakeStream | null = new FakeStream(); public stderr: FakeStream | null = new FakeStream(); - private closeListeners: Array<(code: number | null) => void> = []; + private closeListeners: Array<(code: number | null, signal: NodeJS.Signals | null) => void> = []; private errorListeners: Array<(err: Error) => void> = []; - public on(event: 'close', listener: (code: number | null) => void): this; + public on(event: 'close', listener: (code: number | null, signal: NodeJS.Signals | null) => void): this; public on(event: 'error', listener: (err: Error) => void): this; - public on(event: 'close' | 'error', listener: ((code: number | null) => void) | ((err: Error) => void)): this { - if (event === 'close') this.closeListeners.push(listener as (code: number | null) => void); - else this.errorListeners.push(listener as (err: Error) => void); + public on( + event: 'close' | 'error', + listener: ((code: number | null, signal: NodeJS.Signals | null) => void) | ((err: Error) => void) + ): this { + if (event === 'close') { + this.closeListeners.push(listener as (code: number | null, signal: NodeJS.Signals | null) => void); + } else this.errorListeners.push(listener as (err: Error) => void); return this; } /** Test helper: fire the process 'close' event. */ - public emitClose(code: number | null): void { - for (const listener of this.closeListeners) listener(code); + public emitClose(code: number | null, signal: NodeJS.Signals | null = null): void { + for (const listener of this.closeListeners) listener(code, signal); } /** Test helper: fire the process 'error' event (failure to spawn). */ @@ -296,6 +300,18 @@ describe('pythonRunner', () => { expect(error.message).to.include('3'); }); + it('rejects with RunTerminated when a signal stopped the process (e.g. the timeout)', async () => { + const child = new FakeChild(); + const promise = runEntrypoint(makeOpts({ timeoutMs: 300_000 }), { spawn: fakeSpawn(child, []) }); + + child.emitClose(null, 'SIGTERM'); + const error = await expectReject(promise); + + expect(error.name).to.equal('RunTerminated'); + expect(error.message).to.include('SIGTERM'); + expect(error.message).to.include('300 seconds'); + }); + it('rejects with RunAuthenticationFailed when stderr signals an auth problem', async () => { const child = new FakeChild(); const calls: SpawnCall[] = []; From 7afa713e3eaf6898db02b27fc719bc253acf5c94 Mon Sep 17 00:00:00 2001 From: Jesus Orosco Date: Fri, 16 Oct 2026 10:28:26 -0700 Subject: [PATCH 8/8] fix: failing windows unit test --- test/commands/data-code-extension/lineage.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/commands/data-code-extension/lineage.test.ts b/test/commands/data-code-extension/lineage.test.ts index 6c37904..60ccc58 100644 --- a/test/commands/data-code-extension/lineage.test.ts +++ b/test/commands/data-code-extension/lineage.test.ts @@ -203,8 +203,8 @@ describe('data-code-extension script lineage generate', () => { .getCalls() .map((call) => String(call.args[0])) .join('\n'); - expect(logged).to.match(/drops:\n {2}- out__dll\.old/); - expect(logged).to.match(/changes:\n {2}- out__dll\.id__c/); + expect(logged).to.match(/drops:\r?\n {2}- out__dll\.old/); + expect(logged).to.match(/changes:\r?\n {2}- out__dll\.id__c/); }); it('rejects streaming scripts before the run', async () => {