diff --git a/messages/datacodeBinaryExecutor.md b/messages/datacodeBinaryExecutor.md index 3e1cf7a..d5f7ac1 100644 --- a/messages/datacodeBinaryExecutor.md +++ b/messages/datacodeBinaryExecutor.md @@ -18,6 +18,14 @@ Entrypoint file not found at '%s'. - Run 'init' first if the package has not been initialized - Use the default entrypoint path: payload/entrypoint.py +# error.scanConfigChanged + +'%s' changed while waiting for confirmation, so the scan results were not written. + +# actions.scanConfigChanged + +- Re-run the scan so it starts from the current config.json. + # error.zipExecutionFailed Failed to create archive for package at '%s': %s diff --git a/messages/lineage.md b/messages/lineage.md new file mode 100644 index 0000000..0f9998f --- /dev/null +++ b/messages/lineage.md @@ -0,0 +1,276 @@ +# summary + +Generate field-level dataLineage for a Data Code Extension %s package. + +# description + +Runs the package's entrypoint against a Salesforce org with its Data Cloud writes intercepted, traces each written field back to the fields it was computed from, and replaces the dataLineage section of the package's config.json with the result. Nothing is written to Data Cloud objects, but other calls the script makes still run for real with the org's credentials, including named credential requests, LLM gateway calls, and Einstein predictions. Streaming scripts aren't supported. Run 'scan' first so the read and write permissions in config.json are current. Fields whose lineage can't be traced are listed and get no lineage unless you add entries by hand. + +Generated lineage is a best-effort analysis of your script and may be incomplete or wrong (for example, for UDFs or other code the tool can't see into). It determines how governance tags propagate, which can restrict or open access to data. You're responsible for reviewing and maintaining it. Before config.json is written, you must accept this, either at the prompt or with --accept-lineage-ownership. + +# examples.script + +- Generate lineage for a script package against the org with alias "myorg": + + <%= config.bin %> data-code-extension script lineage generate --entrypoint ./my-script-package/payload/entrypoint.py --target-org myorg + +- Preview the generated lineage without changing config.json: + + <%= config.bin %> data-code-extension script lineage generate --entrypoint ./my-script-package/payload/entrypoint.py --target-org myorg --dry-run + +- Accept ownership of the generated lineage and replace existing lineage without prompting, keeping the full result for review: + + <%= config.bin %> data-code-extension script lineage generate --entrypoint ./my-script-package/payload/entrypoint.py --target-org myorg --accept-lineage-ownership --no-prompt --output-file ./lineage-result.json + +# info.checkingPython + +Checking Python version... + +# info.pythonFound + +Python %s found at '%s' + +# info.checkingPackages + +Checking required Python packages... + +# info.packageFound + +Package '%s' version %s found + +# info.authenticating + +Authenticating with Salesforce org '%s'... + +# info.authenticated + +Successfully authenticated with org '%s' + +# info.generating + +Running package to trace lineage. Data Cloud writes are intercepted; other external calls the script makes still run... + +# info.summary + +Generated %s dataLineage entries with %s (target, source) field pairs: +%s added, -%s removed, ~%s changed. + +# info.removed + +Entries in config.json that the generated lineage drops: +%s + +# info.changed + +Entries in config.json whose source fields the generated lineage changes: +%s + +# info.noSources + +%s target fields are computed from no source field (for example, literals) and get no lineage: +%s + +# info.outputWritten + +Wrote the full lineage result to '%s'. + +# info.dryRunNotice + +DRY RUN: config.json was not modified. Remove --dry-run to apply the generated lineage. + +# info.unchanged + +dataLineage in '%s' is already up to date. + +# info.written + +Updated dataLineage in '%s'. + +# info.notWritten + +Generated lineage was not accepted; config.json was not modified. + +# info.complete + +Lineage generation completed successfully! + +# warning.unresolved + +%s target fields could not be traced and get no lineage unless you add entries by hand: +%s + +# warning.udf + +%s target fields are computed by user-defined functions (UDFs). Their lineage lists only the fields passed to the UDF; the tool can't see what else the UDF reads (for example, a lookup table, a broadcast variable, or a file). Review these fields and add any missing source fields by hand: +%s + +# warning.permissionGaps + +The generated lineage references objects that aren't in config.json permissions, so deploy would fail (1041/1042). Run 'data-code-extension script scan' first, then generate lineage again. Scan can't detect objects whose names your script builds at run time; add those to permissions in config.json by hand. +%s + +# warning.emptyResult + +No lineage could be generated, so config.json was not modified. The package follows the no-lineage path unless you add dataLineage entries by hand. + +# warning.emptyResultExisting + +No lineage could be generated, so config.json was not modified. Its existing %s dataLineage entries still apply and are deployed with the package. Review or remove them in '%s'. + +# warning.replace + +This replaces the existing dataLineage section. Any entries you added or edited by hand will be lost. Lineage determines how governance tags propagate to the target; review the result before deploying. + +# prompt.replace + +Replace the existing dataLineage in config.json? + +# warning.ownership + +Generated lineage is a best-effort analysis of your script and may be incomplete or wrong (for example, for UDFs or other code the tool can't see into). It determines how governance tags propagate, which can restrict or open access to data. You're responsible for reviewing and maintaining it. + +# prompt.acceptOwnership + +Accept and write dataLineage to config.json? + +# prompt.acceptOwnershipReplace + +Accept and replace the existing dataLineage in config.json? + +# error.functionPackage + +Lineage can only be generated for script packages; '%s' is a function package. + +# error.streamingPackage + +Lineage can't be generated for streaming scripts; '%s' has a streamingSource in config.json. + +# error.configUnwritable + +Can't write '%s': %s + +# error.outputFileCollision + +--output-file '%s' is inside the package's payload or project metadata, or is one of its config files. Choose a path outside the package's payload directory. + +# error.lineageChanged + +'%s' changed while waiting for confirmation, so dataLineage was not replaced. + +# actions.lineageChanged + +- Re-run the command to compare the generated lineage against the current config.json. + +# error.outputFileUnwritable + +Can't write --output-file '%s': %s + +# error.noUsername + +The target org has no username to authenticate the lineage run with. + +# error.ownershipNotAccepted + +Writing generated lineage to config.json requires accepting ownership of it. + +# actions.ownershipNotAccepted + +- Re-run with --accept-lineage-ownership to accept responsibility for the generated lineage and write it to config.json. +- Re-run with --dry-run to preview the generated lineage, or check it against config.json, without writing. + +# error.replaceNotConfirmed + +config.json already has a different dataLineage section, and replacing it needs confirmation. + +# actions.replaceNotConfirmed + +- Re-run with --no-prompt to replace the existing dataLineage. +- Re-run with --dry-run to preview the generated lineage, or check it against config.json, without writing. + +# error.sdkTooOld + +The installed salesforce-data-customcode package doesn't support lineage generation. + +# actions.sdkTooOld + +- Upgrade the SDK with 'pip install -U salesforce-data-customcode', then try again. + +# error.resultMissing + +The lineage run finished but produced no result: %s + +# error.resultInvalid + +The lineage run produced a result this CLI can't read: %s + +# actions.resultInvalid + +- Upgrade the SDK with 'pip install -U salesforce-data-customcode', then try again. + +# actions.resultMissing + +- Review the output above for errors from the entrypoint. +- Upgrade the SDK with 'pip install -U salesforce-data-customcode', then try again. + +# flags.entrypoint.summary + +Entrypoint file for the package to generate lineage for. + +# flags.entrypoint.description + +The path to the entrypoint file of your initialized Data Cloud custom code script package. The package's config.json, in the same directory, is updated with the generated lineage. + +# flags.targetOrg.summary + +Target Salesforce org to read data from. + +# flags.targetOrg.description + +The alias or username of the Salesforce org whose data the package reads while lineage is traced. The org must have Data Cloud enabled and appropriate permissions. Nothing is written to Data Cloud objects in the org, but other calls the script makes, such as named credential requests, LLM gateway calls, and Einstein predictions, still run against it. + +# flags.configFile.summary + +Path to an SDK config file. + +# flags.configFile.description + +Optional path to an SDK config.yaml that overrides the reader and writer configuration for the run, for example the reader's default_row_limit (1000 rows by default). + +# flags.dependencies.summary + +Modules to import before the run. + +# flags.dependencies.description + +Optional comma-separated list of Python modules (or module.attribute) to import before the entrypoint runs. Use import names, such as 'yaml' rather than 'pyyaml'. Nothing is installed: each module must already be installed in the active Python environment. + +# flags.dryRun.summary + +Print the generated lineage without modifying config.json. + +# flags.dryRun.description + +When set, generates and prints the lineage but doesn't modify config.json and never prompts. + +# flags.outputFile.summary + +Also write the full lineage result to this JSON file. + +# flags.outputFile.description + +Writes the full result, including unresolved fields, targets, sources, and the pair count, to the given path. Useful for review or CI. + +# flags.noPrompt.summary + +Replace existing dataLineage without prompting. + +# flags.noPrompt.description + +Skips the confirmation before replacing a dataLineage section that differs from the generated one. Required when running non-interactively or with --json and config.json already has dataLineage; checked before the run. Doesn't accept ownership of the generated lineage; use --accept-lineage-ownership for that. + +# flags.acceptLineageOwnership.summary + +Accept responsibility for the generated lineage without prompting. + +# flags.acceptLineageOwnership.description + +Confirms that you understand the generated lineage is best-effort and may be incomplete or wrong, that it determines how governance tags propagate, and that you're responsible for reviewing and maintaining it. Skips the ownership prompt. Required when running non-interactively or with --json, unless you use --dry-run; checked before the run. diff --git a/messages/run.md b/messages/run.md index bf428fe..c3dcd66 100644 --- a/messages/run.md +++ b/messages/run.md @@ -98,6 +98,15 @@ Authentication with Salesforce org '%s' failed while running the package. - Confirm the org alias or username passed to --target-org is correct. - Verify the org has Data Cloud enabled and your user has the required permissions. +# error.runTerminated + +The package run was stopped by signal %s before it finished. Runs time out after %s seconds. + +# actions.runTerminated + +- If the run timed out, reduce the data it processes (for example, the reader's default_row_limit in an SDK config file) or check for a step that hangs, such as a network call. +- If the process ran out of memory or was stopped by another program, free resources and try again. + # error.runProcessStartFailed Could not start the Python interpreter '%s': %s @@ -149,8 +158,8 @@ Optional path to a JSON config file that provides input payload for the run. Def # flags.dependencies.summary -Dependencies override for the run. +Modules to import before the run. # flags.dependencies.description -Optional comma-separated list of Python package dependencies to use during the run, overriding those defined in the package's requirements.txt. +Optional comma-separated list of Python modules (or module.attribute) to import before the entrypoint runs. Use import names, such as 'yaml' rather than 'pyyaml'. Nothing is installed: each module must already be installed in the active Python environment. diff --git a/messages/scan.md b/messages/scan.md index 056dc61..0f833e5 100644 --- a/messages/scan.md +++ b/messages/scan.md @@ -101,3 +101,34 @@ Checking required Python packages... # info.packageFound Package '%s' version %s found + +# flags.noPrompt.summary + +Write config.json without prompting when dataLineage is stale. + +# flags.noPrompt.description + +Skips the confirmation scan asks for when the rebuilt permissions no longer cover objects that dataLineage references. Required to write config.json in that case when running non-interactively. + +# warning.staleLineage + +dataLineage references objects that are no longer in permissions (%s entries affected). The package will fail to deploy (1041/1042) until you rerun 'data-code-extension script lineage generate' or edit dataLineage. Scan doesn't change dataLineage. +%s + +# prompt.staleLineage + +Write the updated permissions to config.json anyway? + +# error.staleLineageNotWritten + +The updated permissions weren't confirmed, so no files were modified. Update dataLineage, then scan again. + +# error.staleLineageNotConfirmed + +dataLineage references objects that are no longer in permissions, and writing config.json needs confirmation. + +# actions.staleLineageNotConfirmed + +- Re-run 'data-code-extension script lineage generate' to regenerate dataLineage, or edit it by hand. +- Re-run scan with --no-prompt to write the updated permissions anyway. +- Re-run scan with --dry-run to preview the changes. diff --git a/package.json b/package.json index 909eee4..d6aa5f2 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,13 @@ "description": "Commands for Data Cloud Code Extension.", "subtopics": { "script": { - "description": "Commands to initialize and manage script-based Data Cloud packages." + "description": "Commands to initialize and manage script-based Data Cloud packages.", + "subtopics": { + "lineage": { + "description": "Commands to generate field-level data lineage for script-based Data Cloud packages.", + "hidden": true + } + } }, "function": { "description": "Commands to initialize and manage function-based Data Cloud packages." diff --git a/src/base/lineageBase.ts b/src/base/lineageBase.ts new file mode 100644 index 0000000..463eefd --- /dev/null +++ b/src/base/lineageBase.ts @@ -0,0 +1,422 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { constants as fsConstants, promises as fs, type Stats } from 'node:fs'; +import path from 'node:path'; +import { SfCommand } from '@salesforce/sf-plugins-core'; +import { Messages, Org, SfError } from '@salesforce/core'; +import { checkEnvironment } from '../utils/environmentChecker.js'; +import { splitDependencies } from '../utils/pythonRunner.js'; +import { LineageRunner, type LineageReport } from '../utils/lineageRunner.js'; +import { findBaseDirectory, getPackageType, getSdkConfigPath, readConfigJson } from '../utils/nativeScan.js'; +import { + countPairs, + diffLineage, + findPermissionGaps, + hasInvalidLineage, + hasLineageSection, + isEmptyDiff, + LINEAGE_CONFIRM_TIMEOUT_MS, + readLineage, + replaceLineage, + writeConfigJson, + type LineageDiff, + type LineagePermissionGaps, +} from '../utils/lineageConfig.js'; +import { type SharedResultProps } from './types.js'; + +export type BaseLineageFlags = { + entrypoint: string; + 'target-org': Org; + 'config-file'?: string; + dependencies?: string; + 'dry-run': boolean; + 'output-file'?: string; + 'no-prompt': boolean; + 'accept-lineage-ownership': boolean; +}; + +export type LineageGenerateResult = SharedResultProps & { + targetOrg: string; + configPath: string; + dryRun: boolean; + /** True when config.json was updated. */ + written: boolean; + entryCount: number; + pairCount: number; + diff: LineageDiff; + permissionGaps: LineagePermissionGaps; + outputFile?: string; + lineage: LineageReport; +}; + +const CONFIG_FILE = 'config.json'; +/** Project config the SDK loads implicitly from the directory above `payload/`. */ +const PROJECT_CONFIG_YAML = 'config.yaml'; +/** Base-directory files `deploy` packages alongside payload/ (see zipBuilder). */ +const PACKAGED_BUILD_FILES = ['requirements.txt', 'build_native_dependencies.sh', 'Dockerfile.dependencies']; + +// eslint-disable-next-line sf-plugin/command-summary, sf-plugin/command-example +export abstract class LineageBase extends SfCommand { + public static enableJsonFlag = true; + + public async run(): Promise { + const { flags } = (await this.parse(this.constructor as typeof LineageBase)) as unknown as { + flags: BaseLineageFlags; + }; + const codeType = this.getCodeType(); + const messages = this.getMessages(); + + const entrypoint = path.resolve(flags.entrypoint); + const configPath = path.join(path.dirname(entrypoint), CONFIG_FILE); + + try { + if ((await getPackageType(findBaseDirectory(entrypoint))) === 'function') { + throw new SfError(messages.getMessage('error.functionPackage', [flags.entrypoint]), 'LineageFunctionPackage'); + } + // Read config.json up front so a missing or malformed file fails before the run. + const initialConfig = await readConfigJson(configPath); + if (initialConfig.streamingSource) { + throw new SfError(messages.getMessage('error.streamingPackage', [flags.entrypoint]), 'LineageStreamingPackage'); + } + // Fail before the run on everything that would only refuse the write afterwards. + if (!flags['dry-run']) { + this.checkNonInteractiveWrite(messages, flags, hasLineageSection(initialConfig)); + await fs.access(configPath, fsConstants.W_OK).catch((err: Error) => { + throw new SfError( + messages.getMessage('error.configUnwritable', [configPath, err.message]), + 'LineageConfigUnwritable' + ); + }); + } + + const outputFile = flags['output-file'] ? path.resolve(flags['output-file']) : undefined; + if (outputFile) { + // Everything under payload/ is run and deployed; .datacustomcode_proj/ holds package metadata. + const payloadDir = path.dirname(entrypoint); + // Root-level requirements.txt and the native dependency build files are packaged too. + const baseDir = findBaseDirectory(entrypoint); + const sdkConfigPath = getSdkConfigPath(baseDir); + const inputDirs = [payloadDir, path.dirname(sdkConfigPath)]; + const inputFiles = [ + configPath, + entrypoint, + sdkConfigPath, + path.join(path.dirname(payloadDir), PROJECT_CONFIG_YAML), + ...PACKAGED_BUILD_FILES.map((name) => path.join(baseDir, name)), + ...(flags['config-file'] ? [path.resolve(flags['config-file'])] : []), + ]; + if (await isPackageInput(outputFile, inputDirs, inputFiles)) { + throw new SfError( + messages.getMessage('error.outputFileCollision', [outputFile]), + 'LineageOutputFileCollision' + ); + } + // Fail before the run rather than lose its result to a bad path afterwards. + await checkOutputFileWritable(outputFile).catch((err: Error) => { + throw new SfError( + messages.getMessage('error.outputFileUnwritable', [outputFile, err.message]), + 'LineageOutputFileUnwritable' + ); + }); + } + + const { pythonInfo, packageInfo } = await checkEnvironment(this.spinner, this.log.bind(this), messages); + + const targetOrg = flags['target-org']; + const orgUsername = targetOrg.getUsername(); + if (!orgUsername) { + throw new SfError(messages.getMessage('error.noUsername'), 'LineageNoUsername'); + } + this.spinner.start(messages.getMessage('info.authenticating', [orgUsername])); + await targetOrg.getConnection().refreshAuth(); + this.spinner.stop(); + this.log(messages.getMessage('info.authenticated', [orgUsername])); + + // Stream the entrypoint's output live; keep stdout clean for --json. + this.log(messages.getMessage('info.generating')); + const scriptOut = this.jsonEnabled() ? process.stderr : process.stdout; + const lineage = await LineageRunner.generate({ + pythonCommand: pythonInfo.command, + entrypoint, + configFile: flags['config-file'], + dependencies: splitDependencies(flags.dependencies), + sfCliOrg: orgUsername, + onStdout: (chunk) => scriptOut.write(chunk), + onStderr: (chunk) => process.stderr.write(chunk), + }); + + if (outputFile) { + await fs.writeFile(outputFile, `${JSON.stringify(lineage, null, 2)}\n`, 'utf8'); + this.log(messages.getMessage('info.outputWritten', [outputFile])); + } + + // Re-read so edits made during the run aren't overwritten. + const config = await readConfigJson(configPath); + const shownConfig = JSON.stringify(config); + const entries = lineage.dataLineage; + const current = readLineage(config); + const diff = diffLineage(current, entries); + const permissionGaps = findPermissionGaps(config, lineage.targets, lineage.sources); + this.report(messages, lineage, diff, permissionGaps); + + const result: LineageGenerateResult = { + success: true, + pythonVersion: pythonInfo, + packageInfo, + codeType, + packageDir: path.dirname(entrypoint), + targetOrg: orgUsername, + configPath, + dryRun: flags['dry-run'], + written: false, + entryCount: entries.length, + pairCount: countPairs(entries), + diff, + permissionGaps, + outputFile, + lineage, + message: messages.getMessage('info.complete'), + }; + + if (entries.length === 0) { + // Existing entries still ship at deploy; say so rather than imply the no-lineage path. + this.warn( + current.length > 0 + ? messages.getMessage('warning.emptyResultExisting', [current.length, configPath]) + : messages.getMessage('warning.emptyResult') + ); + return result; + } + + if (flags['dry-run']) { + this.log(JSON.stringify({ dataLineage: entries }, null, 2)); + this.log(messages.getMessage('info.dryRunNotice')); + return result; + } + + if (isEmptyDiff(diff) && !hasInvalidLineage(config)) { + this.log(messages.getMessage('info.unchanged', [configPath])); + return result; + } + + if (!(await this.confirmWrite(messages, flags, hasLineageSection(config)))) { + this.log(messages.getMessage('info.notWritten')); + return result; + } + + // Re-read again: the prompt can wait up to LINEAGE_CONFIRM_TIMEOUT_MS. Edits made meanwhile (to dataLineage, or + // to permissions the gaps were checked against) weren't in what the customer confirmed, so don't write. + const latest = await readConfigJson(configPath); + if (JSON.stringify(latest) !== shownConfig) { + throw new SfError( + messages.getMessage('error.lineageChanged', [configPath]), + 'LineageChangedDuringConfirm', + messages.getMessages('actions.lineageChanged') + ); + } + await writeConfigJson(configPath, replaceLineage(latest, entries)); + this.log(messages.getMessage('info.written', [configPath])); + return { ...result, written: true }; + } catch (error) { + this.spinner.stop(); + throw error; + } + } + + private report( + messages: Messages, + lineage: LineageReport, + diff: LineageDiff, + gaps: LineagePermissionGaps + ): void { + this.log( + messages.getMessage('info.summary', [ + lineage.dataLineage.length, + countPairs(lineage.dataLineage), + diff.added.length, + diff.removed.length, + diff.changed.length, + ]) + ); + if (diff.removed.length > 0) { + this.log(messages.getMessage('info.removed', [bullets(diff.removed)])); + } + if (diff.changed.length > 0) { + this.log(messages.getMessage('info.changed', [bullets(diff.changed)])); + } + if (lineage.noSources.length > 0) { + this.log(messages.getMessage('info.noSources', [lineage.noSources.length, bullets(lineage.noSources)])); + } + if (lineage.unresolved.length > 0) { + const lines = lineage.unresolved.map((field) => `${field.targetField} (${field.reason})`); + this.warn(messages.getMessage('warning.unresolved', [lineage.unresolved.length, bullets(lines)])); + } + if (lineage.udfFields.length > 0) { + this.warn(messages.getMessage('warning.udf', [lineage.udfFields.length, bullets(lineage.udfFields)])); + } + const missing = [...gaps.targets.map((name) => `write: ${name}`), ...gaps.sources.map((name) => `read: ${name}`)]; + if (missing.length > 0) { + this.warn(messages.getMessage('warning.permissionGaps', [bullets(missing)])); + } + } + + // Every write needs the customer to accept ownership of the generated lineage: it is best-effort + // and drives governance tag propagation. Replacing an existing section also risks losing hand + // edits. Interactively that's one prompt for whatever the flags haven't already accepted. + private async confirmWrite( + messages: Messages, + flags: BaseLineageFlags, + replacing: boolean + ): Promise { + this.warn(messages.getMessage('warning.ownership')); + if (replacing) this.warn(messages.getMessage('warning.replace')); + const needsOwnership = !flags['accept-lineage-ownership']; + const needsReplace = replacing && !flags['no-prompt']; + if (!needsOwnership && !needsReplace) return true; + if (this.jsonEnabled() || !process.stdin.isTTY) { + if (needsOwnership) { + throw new SfError( + messages.getMessage('error.ownershipNotAccepted'), + 'LineageOwnershipNotAccepted', + messages.getMessages('actions.ownershipNotAccepted') + ); + } + throw new SfError( + messages.getMessage('error.replaceNotConfirmed'), + 'LineageReplaceNotConfirmed', + messages.getMessages('actions.replaceNotConfirmed') + ); + } + const prompt = needsOwnership + ? needsReplace + ? 'prompt.acceptOwnershipReplace' + : 'prompt.acceptOwnership' + : 'prompt.replace'; + return this.confirm({ message: messages.getMessage(prompt), defaultAnswer: false, ms: LINEAGE_CONFIRM_TIMEOUT_MS }); + } + + /** + * Without a prompt to fall back on, a write needs its flags up front: fail before the run + * instead of after it. `--dry-run` is the way to check lineage without writing. + * confirmWrite repeats these checks in case config.json gains a dataLineage section during the run. + */ + private checkNonInteractiveWrite(messages: Messages, flags: BaseLineageFlags, replacing: boolean): void { + if (!this.jsonEnabled() && process.stdin.isTTY) return; + if (!flags['accept-lineage-ownership']) { + throw new SfError( + messages.getMessage('error.ownershipNotAccepted'), + 'LineageOwnershipNotAccepted', + messages.getMessages('actions.ownershipNotAccepted') + ); + } + if (replacing && !flags['no-prompt']) { + throw new SfError( + messages.getMessage('error.replaceNotConfirmed'), + 'LineageReplaceNotConfirmed', + messages.getMessages('actions.replaceNotConfirmed') + ); + } + } + + protected abstract getCodeType(): 'script'; + protected abstract getMessages(): Messages; +} + +/** + * True when `file` lies under one of `dirs` (as given or after resolving symlinks), or is one of + * `files` by path or by file identity, which also catches symlinks and hard links. Paths are compared + * by path as well as identity so inputs that don't exist yet (no identity to match) are still caught. + */ +async function isPackageInput(file: string, dirs: string[], files: string[]): Promise { + const [target, realDirs, realFiles, targetStat, fileStats] = await Promise.all([ + realPath(file), + Promise.all(dirs.map(realPath)), + Promise.all(files.map(realPath)), + statOrUndefined(file), + Promise.all(files.map(statOrUndefined)), + ]); + const within = (child: string, dir: string): boolean => { + const relative = path.relative(pathKey(dir), pathKey(child)); + return !path.isAbsolute(relative) && relative.split(path.sep)[0] !== '..'; + }; + const sameFile = (a: string, b: string): boolean => pathKey(a) === pathKey(b); + return ( + dirs.some((dir) => within(file, dir)) || + realDirs.some((dir) => within(target, dir)) || + files.some((input) => sameFile(input, file)) || + realFiles.some((input) => sameFile(input, target)) || + (targetStat !== undefined && + fileStats.some((stat) => stat !== undefined && stat.dev === targetStat.dev && stat.ino === targetStat.ino)) + ); +} + +/** + * Absolute path folded to lower case where the default file system ignores case (macOS, Windows), so + * `REQUIREMENTS.TXT` matches a `requirements.txt` that doesn't exist yet. On a case-sensitive volume + * this only over-rejects names that differ from an input by case. + */ +function pathKey(file: string): string { + const resolved = path.resolve(file); + return process.platform === 'darwin' || process.platform === 'win32' ? resolved.toLowerCase() : resolved; +} + +async function statOrUndefined(file: string): Promise { + return fs.stat(file).catch(() => undefined); +} + +/** Symlink hops followed before giving up, matching the usual OS limit (ELOOP). */ +const MAX_SYMLINK_HOPS = 40; + +/** + * Canonical path of `file`, resolving symlinks (and case, on case-insensitive file systems). + * A file that doesn't exist yet resolves through its parent directory, and a dangling symlink + * resolves to the path it points at, which is where a write through it would land. + */ +async function realPath(file: string, hops = 0): Promise { + try { + return await fs.realpath(file); + } catch { + if (hops < MAX_SYMLINK_HOPS) { + const link = await fs.readlink(file).catch(() => undefined); + if (link !== undefined) { + return realPath(path.resolve(path.dirname(file), link), hops + 1); + } + } + try { + return path.join(await fs.realpath(path.dirname(file)), path.basename(file)); + } catch { + return path.resolve(file); + } + } +} + +/** + * Rejects an --output-file that `fs.writeFile` would fail on: an existing target must be a + * writable regular file, and a new one needs a writable parent directory. + */ +async function checkOutputFileWritable(outputFile: string): Promise { + const stat = await statOrUndefined(outputFile); + if (stat === undefined) { + await fs.access(path.dirname(outputFile), fsConstants.W_OK); + return; + } + if (!stat.isFile()) throw new Error('not a regular file'); + await fs.access(outputFile, fsConstants.W_OK); +} + +function bullets(lines: string[]): string { + return lines.map((line) => ` - ${line}`).join('\n'); +} diff --git a/src/base/scanBase.ts b/src/base/scanBase.ts index c3f3c61..c8f19fb 100644 --- a/src/base/scanBase.ts +++ b/src/base/scanBase.ts @@ -15,15 +15,22 @@ */ import path from 'node:path'; import { SfCommand } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { Messages, SfError } from '@salesforce/core'; import { executeNativeScan, type NativeScanResult } from '../utils/nativeScan.js'; import { checkEnvironment, type EnvironmentCheckResult } from '../utils/environmentChecker.js'; +import { + countEntriesReferencing, + findStaleLineage, + LINEAGE_CONFIRM_TIMEOUT_MS, + type LineagePermissionGaps, +} from '../utils/lineageConfig.js'; export type BaseScanFlags = { entrypoint?: string; 'config-file'?: string; 'dry-run': boolean; 'no-requirements': boolean; + 'no-prompt'?: boolean; }; export type ScanResult = { @@ -56,9 +63,29 @@ export abstract class ScanBase extends SfCommand { dryRun: flags['dry-run'], noRequirements: flags['no-requirements'], packageType: codeType, + // Functions carry no permissions or dataLineage, so only scripts can go stale. + confirmWrite: + codeType === 'script' + ? (updatedConfig): Promise => + this.confirmStaleLineage(messages, updatedConfig, flags['no-prompt'] ?? false) + : undefined, }); this.spinner.stop(); + if (codeType === 'script' && executionResult.dryRun) { + const stale = findStaleLineage(executionResult.config); + if (hasStaleLineage(stale)) this.warnStaleLineage(messages, executionResult.config, stale); + } + + // Exit non-zero so a following deploy doesn't ship the old permissions. + if (executionResult.written === false) { + throw new SfError( + messages.getMessage('error.staleLineageNotWritten'), + 'ScanStaleLineageNotWritten', + messages.getMessages('actions.staleLineageNotConfirmed') + ); + } + this.log(messages.getMessage('info.scanExecuted', [workingDir])); // Echo the resulting config — mirrors the Python CLI's behavior of printing the @@ -92,6 +119,55 @@ export abstract class ScanBase extends SfCommand { } } + /** + * Called just before scan writes config.json. When the rebuilt permissions no longer + * cover objects that dataLineage references, warn and ask before writing; scan never + * edits dataLineage itself. + */ + private async confirmStaleLineage( + messages: Messages, + updatedConfig: Record, + noPrompt: boolean + ): Promise { + const stale = findStaleLineage(updatedConfig); + if (!hasStaleLineage(stale)) return true; + + this.spinner.stop(); + this.warnStaleLineage(messages, updatedConfig, stale); + if (noPrompt) return true; + if (this.jsonEnabled() || !process.stdin.isTTY) { + throw new SfError( + messages.getMessage('error.staleLineageNotConfirmed'), + 'ScanStaleLineageNotConfirmed', + messages.getMessages('actions.staleLineageNotConfirmed') + ); + } + return this.confirm({ + message: messages.getMessage('prompt.staleLineage'), + defaultAnswer: false, + ms: LINEAGE_CONFIRM_TIMEOUT_MS, + }); + } + + private warnStaleLineage( + messages: Messages, + config: Record, + stale: LineagePermissionGaps + ): void { + const objects = [...new Set([...stale.targets, ...stale.sources])]; + const lines = [...stale.targets.map((name) => `write: ${name}`), ...stale.sources.map((name) => `read: ${name}`)]; + this.warn( + messages.getMessage('warning.staleLineage', [ + countEntriesReferencing(config, objects), + lines.map((line) => ` - ${line}`).join('\n'), + ]) + ); + } + protected abstract getCodeType(): 'script' | 'function'; protected abstract getMessages(): Messages; } + +function hasStaleLineage(stale: LineagePermissionGaps): boolean { + return stale.targets.length > 0 || stale.sources.length > 0; +} diff --git a/src/commands/data-code-extension/script/lineage/generate.ts b/src/commands/data-code-extension/script/lineage/generate.ts new file mode 100644 index 0000000..3abb72c --- /dev/null +++ b/src/commands/data-code-extension/script/lineage/generate.ts @@ -0,0 +1,89 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Flags } from '@salesforce/sf-plugins-core'; +import { Messages } from '@salesforce/core'; +import { LineageBase } from '../../../../base/lineageBase.js'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-data-code-extension', 'lineage'); + +export default class Generate extends LineageBase { + public static readonly summary = messages.getMessage('summary', ['script']); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples.script'); + // Hidden until field-level lineage is generally available: the byoc-service and + // data-prep gates are tenant-whitelisted. + public static readonly hidden = true; + + public static readonly flags = { + entrypoint: Flags.file({ + char: 'e', + summary: messages.getMessage('flags.entrypoint.summary'), + description: messages.getMessage('flags.entrypoint.description'), + required: true, + exists: true, + }), + 'target-org': Flags.requiredOrg({ + char: 'o', + summary: messages.getMessage('flags.targetOrg.summary'), + description: messages.getMessage('flags.targetOrg.description'), + required: true, + }), + 'config-file': Flags.file({ + summary: messages.getMessage('flags.configFile.summary'), + description: messages.getMessage('flags.configFile.description'), + required: false, + exists: true, + }), + dependencies: Flags.string({ + summary: messages.getMessage('flags.dependencies.summary'), + description: messages.getMessage('flags.dependencies.description'), + required: false, + }), + 'dry-run': Flags.boolean({ + char: 'd', + summary: messages.getMessage('flags.dryRun.summary'), + description: messages.getMessage('flags.dryRun.description'), + default: false, + }), + 'output-file': Flags.string({ + summary: messages.getMessage('flags.outputFile.summary'), + description: messages.getMessage('flags.outputFile.description'), + required: false, + }), + 'no-prompt': Flags.boolean({ + char: 'p', + summary: messages.getMessage('flags.noPrompt.summary'), + description: messages.getMessage('flags.noPrompt.description'), + default: false, + }), + 'accept-lineage-ownership': Flags.boolean({ + summary: messages.getMessage('flags.acceptLineageOwnership.summary'), + description: messages.getMessage('flags.acceptLineageOwnership.description'), + default: false, + }), + }; + + // eslint-disable-next-line class-methods-use-this + protected getCodeType(): 'script' { + return 'script'; + } + + // eslint-disable-next-line class-methods-use-this + protected getMessages(): Messages { + return messages; + } +} diff --git a/src/commands/data-code-extension/script/scan.ts b/src/commands/data-code-extension/script/scan.ts index b81acc9..d6afae1 100644 --- a/src/commands/data-code-extension/script/scan.ts +++ b/src/commands/data-code-extension/script/scan.ts @@ -50,6 +50,14 @@ export default class Scan extends ScanBase { description: messages.getMessage('flags.noRequirements.description'), default: false, }), + // Hidden alongside the lineage commands until field-level lineage is generally available. + 'no-prompt': Flags.boolean({ + char: 'p', + summary: messages.getMessage('flags.noPrompt.summary'), + description: messages.getMessage('flags.noPrompt.description'), + default: false, + hidden: true, + }), }; // eslint-disable-next-line class-methods-use-this diff --git a/src/utils/lineageConfig.ts b/src/utils/lineageConfig.ts new file mode 100644 index 0000000..5782a9d --- /dev/null +++ b/src/utils/lineageConfig.ts @@ -0,0 +1,184 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { promises as fs } from 'node:fs'; +import { type LineageEntry } from './lineageRunner.js'; + +export type LineageDiff = { + added: string[]; + removed: string[]; + changed: string[]; +}; + +/** + * How long lineage confirmation prompts wait before declining. The sf default of 10s is too short to + * review the warnings and diff. Must stay finite: Node fires timers over ~24.8 days after 1ms. + */ +export const LINEAGE_CONFIRM_TIMEOUT_MS = 10 * 60 * 1000; + +/** Objects referenced by lineage but missing from `permissions`, i.e. a 1041/1042 at deploy. */ +export type LineagePermissionGaps = { + targets: string[]; + sources: string[]; +}; + +/** The `dataLineage` entries in a config, or `[]` when the section is absent or malformed. */ +export function readLineage(config: Record): LineageEntry[] { + const section = config.dataLineage; + if (!Array.isArray(section)) return []; + return section.filter( + (entry): entry is LineageEntry => + typeof entry === 'object' && + entry !== null && + typeof (entry as LineageEntry).targetField === 'string' && + Array.isArray((entry as LineageEntry).sourceFields) && + (entry as LineageEntry).sourceFields.every((field) => typeof field === 'string') + ); +} + +/** True when the config has a `dataLineage` section that isn't an empty array. */ +export function hasLineageSection(config: Record): boolean { + const section = config.dataLineage; + return section !== undefined && !(Array.isArray(section) && section.length === 0); +} + +/** + * True when the config's `dataLineage` has entries `readLineage` drops, or repeats a target + * field. Targets compare case-insensitively, as deploy validation does. + */ +export function hasInvalidLineage(config: Record): boolean { + const section = config.dataLineage; + if (section === undefined) return false; + if (!Array.isArray(section)) return true; + const entries = readLineage(config); + if (entries.length !== section.length) return true; + return new Set(entries.map((entry) => entry.targetField.toLowerCase())).size !== entries.length; +} + +/** + * Per-target difference between the current `dataLineage` and a generated one. Field names + * compare case-insensitively, as deploy validation does; results use each side's own spelling. + */ +export function diffLineage(current: LineageEntry[], generated: LineageEntry[]): LineageDiff { + const sourcesKey = (entry: LineageEntry): string => + entry.sourceFields + .map((field) => field.toLowerCase()) + .sort() + .join('\n'); + const byTarget = (entries: LineageEntry[]): Map => + new Map( + entries.map((entry) => [ + entry.targetField.toLowerCase(), + { target: entry.targetField, sources: sourcesKey(entry) }, + ]) + ); + const before = byTarget(current); + const after = byTarget(generated); + const diff: LineageDiff = { added: [], removed: [], changed: [] }; + for (const [key, { target, sources }] of after) { + if (!before.has(key)) diff.added.push(target); + else if (before.get(key)!.sources !== sources) diff.changed.push(target); + } + for (const [key, { target }] of before) { + if (!after.has(key)) diff.removed.push(target); + } + return diff; +} + +export function isEmptyDiff(diff: LineageDiff): boolean { + return diff.added.length === 0 && diff.removed.length === 0 && diff.changed.length === 0; +} + +/** Number of (target, source) field pairs; Core stores one mapping row per pair. */ +export function countPairs(entries: LineageEntry[]): number { + return entries.reduce((total, entry) => total + entry.sourceFields.length, 0); +} + +/** `config` with `dataLineage` replaced wholesale; every other key is kept in place. */ +export function replaceLineage(config: Record, entries: LineageEntry[]): Record { + return { ...config, dataLineage: entries }; +} + +/** + * Objects that `targets` / `sources` name but `permissions.write.*` / `permissions.read.*` + * don't. Object names compare case-insensitively. + */ +export function findPermissionGaps( + config: Record, + targets: Iterable, + sources: Iterable +): LineagePermissionGaps { + const permissions = (config.permissions ?? {}) as Record; + const writable = permittedObjects(permissions.write); + const readable = permittedObjects(permissions.read); + return { + targets: missingFrom(writable, targets), + sources: missingFrom(readable, sources), + }; +} + +/** + * Objects referenced by the config's own `dataLineage` that its `permissions` no longer + * cover. Object-level only: a field that stopped being produced is not detected. + */ +export function findStaleLineage(config: Record): LineagePermissionGaps { + const entries = readLineage(config); + return findPermissionGaps( + config, + entries.map((entry) => objectOf(entry.targetField)), + entries.flatMap((entry) => entry.sourceFields.map(objectOf)) + ); +} + +/** Number of `dataLineage` entries that reference any of the given objects. */ +export function countEntriesReferencing(config: Record, objects: string[]): number { + const names = new Set(objects.map((name) => name.toLowerCase())); + return readLineage(config).filter( + (entry) => + names.has(objectOf(entry.targetField).toLowerCase()) || + entry.sourceFields.some((field) => names.has(objectOf(field).toLowerCase())) + ).length; +} + +/** Write config.json with a 2-space indent and no trailing newline, as scan and init do. */ +export async function writeConfigJson(configPath: string, config: Record): Promise { + await fs.writeFile(configPath, JSON.stringify(config, null, 2), 'utf8'); +} + +function objectOf(field: string): string { + const dot = field.indexOf('.'); + return dot === -1 ? field : field.slice(0, dot); +} + +function permittedObjects(section: unknown): Set { + const names = new Set(); + if (typeof section !== 'object' || section === null) return names; + for (const kind of ['dlo', 'dmo']) { + const list = (section as Record)[kind]; + if (!Array.isArray(list)) continue; + for (const name of list) { + if (typeof name === 'string') names.add(name.toLowerCase()); + } + } + return names; +} + +function missingFrom(permitted: Set, objects: Iterable): string[] { + const missing = new Set(); + for (const name of objects) { + if (!permitted.has(name.toLowerCase())) missing.add(name); + } + return [...missing].sort(); +} diff --git a/src/utils/lineageRunner.ts b/src/utils/lineageRunner.ts new file mode 100644 index 0000000..41bb780 --- /dev/null +++ b/src/utils/lineageRunner.ts @@ -0,0 +1,226 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { debuglog } from 'node:util'; +import { Messages, SfError } from '@salesforce/core'; +import { + DEFAULT_PROFILE, + mapRunTerminated, + RUN_TIMEOUT_MS, + runPythonProgram, + type PythonRunnerDeps, +} from './pythonRunner.js'; + +const debug = debuglog('datacustomcode'); + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-data-code-extension', 'lineage'); + +/** Exit code the shim uses when the installed SDK has no `datacustomcode.lineage`. */ +export const LINEAGE_UNAVAILABLE_EXIT_CODE = 3; +/** Printed to stderr alongside {@link LINEAGE_UNAVAILABLE_EXIT_CODE}, so a script's own `sys.exit(3)` isn't mistaken for it. */ +export const LINEAGE_UNAVAILABLE_MARKER = 'DATACUSTOMCODE_LINEAGE_UNAVAILABLE'; + +/** Exit code the shim uses when SIGTERM (the run timeout) stopped it: 128 + SIGTERM. */ +export const LINEAGE_TERMINATED_EXIT_CODE = 143; +/** Printed to stderr alongside {@link LINEAGE_TERMINATED_EXIT_CODE}. */ +export const LINEAGE_TERMINATED_MARKER = 'DATACUSTOMCODE_LINEAGE_TERMINATED'; + +/** + * The Python program executed via `python -c`. It calls the SDK *library's* + * `datacustomcode.lineage.generate_lineage`, which runs the entrypoint against the + * org with reads captured and writes traced (never written). + * + * Arguments arrive as a single JSON object in `sys.argv[1]`, never interpolated into + * this source. The result goes to `resultFile` rather than stdout, because the + * customer's script prints freely to stdout. + * + * The SDK runs the entrypoint in a worker process that shares this one's stdio, so + * SIGTERM (the run timeout) must stop it too, or the CLI waits on its open pipes. + * Raising SystemExit inside `subprocess.run` makes it kill the worker. + */ +export const GENERATE_LINEAGE_SHIM = [ + 'import importlib.util, json, signal, sys', + 'def _terminated(*_):', + ` sys.stderr.write("${LINEAGE_TERMINATED_MARKER}\\n")`, + ` sys.exit(${LINEAGE_TERMINATED_EXIT_CODE})`, + 'signal.signal(signal.SIGTERM, _terminated)', + 'if importlib.util.find_spec("datacustomcode.lineage") is None:', + ` sys.stderr.write("${LINEAGE_UNAVAILABLE_MARKER}\\n")`, + ` sys.exit(${LINEAGE_UNAVAILABLE_EXIT_CODE})`, + 'from datacustomcode.lineage import generate_lineage', + 'a = json.loads(sys.argv[1])', + 'result = generate_lineage(', + ' a["entrypoint"],', + ' a["configFile"],', + ' a["dependencies"],', + ' a["profile"],', + ' sf_cli_org=a["sfCliOrg"],', + ')', + 'with open(a["resultFile"], "w", encoding="utf-8") as f:', + ' json.dump(result.to_dict(), f)', +].join('\n'); + +/** JSON payload handed to {@link GENERATE_LINEAGE_SHIM}. */ +export type LineagePayload = { + entrypoint: string; + configFile: string | null; + dependencies: string[]; + profile: string; + sfCliOrg: string | null; + resultFile: string; +}; + +export type LineageEntry = { + targetField: string; + sourceFields: string[]; +}; + +export type UnresolvedField = { + targetField: string; + reason: string; +}; + +/** The SDK's `LineageResult.to_dict()`; `dataLineage` matches config.json. */ +export type LineageReport = { + dataLineage: LineageEntry[]; + unresolved: UnresolvedField[]; + noSources: string[]; + /** Fields in `dataLineage` or `noSources` computed by a UDF; their sources may be incomplete. */ + udfFields: string[]; + targets: string[]; + sources: string[]; + pairCount: number; +}; + +export type LineageRunOptions = { + pythonCommand: string; + entrypoint: string; + /** SDK config.yaml (reader/writer config, e.g. `default_row_limit`). */ + configFile?: string; + dependencies: string[]; + sfCliOrg?: string; + profile?: string; + timeoutMs?: number; + onStdout?: (chunk: string) => void; + onStderr?: (chunk: string) => void; +}; + +export function buildLineagePayload(opts: LineageRunOptions, resultFile: string): LineagePayload { + return { + entrypoint: opts.entrypoint, + configFile: opts.configFile ?? null, + dependencies: opts.dependencies, + profile: opts.profile ?? DEFAULT_PROFILE, + sfCliOrg: opts.sfCliOrg ?? null, + resultFile, + }; +} + +function mapLineageExitCode(exitCode: number, stderr: string, timeoutMs: number): SfError | undefined { + if (exitCode === LINEAGE_TERMINATED_EXIT_CODE && stderr.includes(LINEAGE_TERMINATED_MARKER)) { + return mapRunTerminated('SIGTERM', timeoutMs); + } + if (exitCode === LINEAGE_UNAVAILABLE_EXIT_CODE && stderr.includes(LINEAGE_UNAVAILABLE_MARKER)) { + return new SfError( + messages.getMessage('error.sdkTooOld'), + 'LineageSdkTooOld', + messages.getMessages('actions.sdkTooOld') + ); + } + return undefined; +} + +/** + * Generate field-level lineage for a script package through the SDK's + * `generate_lineage`, streaming the run's output live. The result file lives in a + * private temp directory that is always removed. + */ +export async function generateLineage(opts: LineageRunOptions, deps?: PythonRunnerDeps): Promise { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dce-lineage-')); + const resultFile = path.join(tempDir, 'lineage.json'); + try { + const payload = buildLineagePayload(opts, resultFile); + await runPythonProgram( + { + pythonCommand: opts.pythonCommand, + args: ['-c', GENERATE_LINEAGE_SHIM, JSON.stringify(payload)], + sfCliOrg: opts.sfCliOrg, + timeoutMs: opts.timeoutMs, + onStdout: opts.onStdout, + onStderr: opts.onStderr, + mapExitCode: (exitCode, stderr) => mapLineageExitCode(exitCode, stderr, opts.timeoutMs ?? RUN_TIMEOUT_MS), + }, + deps + ); + + let raw: string; + try { + raw = await fs.readFile(resultFile, 'utf8'); + } catch (err) { + throw new SfError( + messages.getMessage('error.resultMissing', [(err as Error).message]), + 'LineageResultMissing', + messages.getMessages('actions.resultMissing') + ); + } + return parseLineageReport(raw); + } finally { + // A file another program briefly holds open (e.g. antivirus on Windows) must not hide the result. + try { + await fs.rm(tempDir, { recursive: true, force: true }); + } catch (err) { + debug('temp dir cleanup error (ignored): %o', err); + } + } +} + +const REPORT_ARRAYS = ['dataLineage', 'unresolved', 'noSources', 'udfFields', 'targets', 'sources'] as const; + +/** Parse the SDK's result file, failing clearly when it isn't the shape this CLI reads. */ +export function parseLineageReport(raw: string): LineageReport { + let report: unknown; + try { + report = JSON.parse(raw); + } catch (err) { + throw invalidResult((err as Error).message); + } + if (typeof report !== 'object' || report === null || Array.isArray(report)) { + throw invalidResult('expected a JSON object'); + } + const missing = REPORT_ARRAYS.filter((key) => !Array.isArray((report as Record)[key])); + if (missing.length > 0) { + throw invalidResult(`missing or non-array ${missing.join(', ')}`); + } + return report as LineageReport; +} + +function invalidResult(detail: string): SfError { + return new SfError( + messages.getMessage('error.resultInvalid', [detail]), + 'LineageResultInvalid', + messages.getMessages('actions.resultInvalid') + ); +} + +/** Class wrapper around {@link generateLineage}, so command tests can stub the run. */ +export class LineageRunner { + public static async generate(opts: LineageRunOptions, deps?: PythonRunnerDeps): Promise { + return generateLineage(opts, deps); + } +} diff --git a/src/utils/nativeScan.ts b/src/utils/nativeScan.ts index 12a2ca6..f1b3f0b 100644 --- a/src/utils/nativeScan.ts +++ b/src/utils/nativeScan.ts @@ -44,6 +44,11 @@ export type NativeScanOptions = { noRequirements?: boolean; /** Override for the package type. If omitted, resolved via .datacustomcode_proj/sdk_config.json. */ packageType?: CodeType; + /** + * Called with the updated config just before config.json is written (never on dry run). + * Resolving `false` skips every write and marks the result `written: false`. + */ + confirmWrite?: (updatedConfig: Record) => Promise; }; export type NativeScanResult = { @@ -56,6 +61,8 @@ export type NativeScanResult = { requirements?: string[]; /** True when --dry-run prevented file writes. */ dryRun: boolean; + /** Set to false when `confirmWrite` declined the write. */ + written?: false; }; const ENTRYPOINT_FILE = 'entrypoint.py'; @@ -415,7 +422,21 @@ export async function updateConfig( configLocation?: string ): Promise> { const configPath = configLocation ?? path.join(path.dirname(entrypointPath), CONFIG_FILE); + const existing = await readConfigJson(configPath); + existing.entryPoint = path.basename(entrypointPath); + + if (packageType === 'script') { + existing.dataspace = resolveDataspace(existing); + const calls = await scanFile(entrypointPath); + existing.permissions = buildPermissions(calls); + } + + return existing; +} + +/** Read and parse a package's config.json, with the errors `updateConfig` raises. */ +export async function readConfigJson(configPath: string): Promise> { let raw: string; try { raw = await fs.readFile(configPath, 'utf8'); @@ -435,15 +456,6 @@ export async function updateConfig( } catch (err) { throw new SfError(`Failed to parse JSON from ${configPath}: ${(err as Error).message}`, 'InvalidConfig'); } - - existing.entryPoint = path.basename(entrypointPath); - - if (packageType === 'script') { - existing.dataspace = resolveDataspace(existing); - const calls = await scanFile(entrypointPath); - existing.permissions = buildPermissions(calls); - } - return existing; } @@ -500,6 +512,8 @@ export async function executeNativeScan(opts: NativeScanOptions): Promise void): unknown; + on(event: 'close', listener: (code: number | null, signal: NodeJS.Signals | null) => void): unknown; on(event: 'error', listener: (err: Error) => void): unknown; }; export type SpawnFn = ( @@ -147,6 +147,14 @@ function mapRunFailure(exitCode: number, stderr: string, sfCliOrg?: string): SfE ); } +export function mapRunTerminated(signal: NodeJS.Signals | null, timeoutMs: number): SfError { + return new SfError( + messages.getMessage('error.runTerminated', [signal ?? 'unknown', Math.round(timeoutMs / 1000)]), + 'RunTerminated', + messages.getMessages('actions.runTerminated') + ); +} + function mapProcessStartError(err: Error, pythonCommand: string): SfError { return new SfError( messages.getMessage('error.runProcessStartFailed', [pythonCommand, err.message]), @@ -155,22 +163,35 @@ function mapProcessStartError(err: Error, pythonCommand: string): SfError { ); } +/** Options for {@link runPythonProgram}: everything but the program itself is shared with `run`. */ +export type PythonProgramOptions = { + pythonCommand: string; + /** Interpreter arguments, e.g. `['-c', SHIM, payloadJson]`. */ + args: readonly string[]; + /** SF CLI org username/alias, named in the authentication-failure message. */ + sfCliOrg?: string; + timeoutMs?: number; + onStdout?: (chunk: string) => void; + onStderr?: (chunk: string) => void; + /** Checked before the default failure mapping; return undefined to fall through to it. */ + mapExitCode?: (exitCode: number, stderr: string) => SfError | undefined; +}; + /** - * Run a package's entrypoint through the SDK's `run_entrypoint`, streaming output - * live. Resolves with the accumulated stdout/stderr on a clean exit; rejects with a - * friendly {@link SfError} on non-zero exit or a failure to start the interpreter. + * Spawn the interpreter with `opts.args`, streaming output live. Resolves with the + * accumulated stdout/stderr on a clean exit; rejects with a friendly {@link SfError} + * on non-zero exit or a failure to start the interpreter. */ -export async function runEntrypoint( - opts: PythonRunOptions, +export async function runPythonProgram( + opts: PythonProgramOptions, deps: PythonRunnerDeps = { spawn: defaultSpawn } ): Promise { - const payload = buildRunPayload(opts); - const args = ['-c', RUN_ENTRYPOINT_SHIM, JSON.stringify(payload)]; - debug('run spawn: %s %o', opts.pythonCommand, args); + debug('run spawn: %s %o', opts.pythonCommand, opts.args); + const timeoutMs = opts.timeoutMs ?? RUN_TIMEOUT_MS; return new Promise((resolve, reject) => { - const child = deps.spawn(opts.pythonCommand, args, { - timeout: opts.timeoutMs ?? RUN_TIMEOUT_MS, + const child = deps.spawn(opts.pythonCommand, opts.args, { + timeout: timeoutMs, // PYTHONUNBUFFERED so the entrypoint's prints stream immediately instead of buffering. env: { ...process.env, PYTHONUNBUFFERED: '1' }, }); @@ -194,18 +215,48 @@ export async function runEntrypoint( reject(mapProcessStartError(err, opts.pythonCommand)); }); - child.on('close', (code: number | null) => { - const exitCode = code ?? 0; + child.on('close', (code: number | null, signal: NodeJS.Signals | null) => { + // No exit code means a signal stopped the run: the timeout, an OOM kill, or the user. + if (code === null) { + debug('run killed by signal: %s', signal); + reject(mapRunTerminated(signal, timeoutMs)); + return; + } + const exitCode = code; debug('run exit code: %d', exitCode); if (exitCode === 0) { resolve({ stdout: stdout.trim(), stderr: stderr.trim(), exitCode }); } else { - reject(mapRunFailure(exitCode, stderr.trim(), opts.sfCliOrg)); + const trimmed = stderr.trim(); + reject(opts.mapExitCode?.(exitCode, trimmed) ?? mapRunFailure(exitCode, trimmed, opts.sfCliOrg)); } }); }); } +/** + * Run a package's entrypoint through the SDK's `run_entrypoint`, streaming output + * live. Resolves with the accumulated stdout/stderr on a clean exit; rejects with a + * friendly {@link SfError} on non-zero exit or a failure to start the interpreter. + */ +export async function runEntrypoint( + opts: PythonRunOptions, + deps: PythonRunnerDeps = { spawn: defaultSpawn } +): Promise { + const payload = buildRunPayload(opts); + return runPythonProgram( + { + pythonCommand: opts.pythonCommand, + args: ['-c', RUN_ENTRYPOINT_SHIM, JSON.stringify(payload)], + sfCliOrg: opts.sfCliOrg, + timeoutMs: opts.timeoutMs, + onStdout: opts.onStdout, + onStderr: opts.onStderr, + }, + deps + ); +} + /** * Class wrapper around {@link runEntrypoint}, exposed so command-level tests can * stub the entire local run the same way deploy tests stub `NativeDeployer.deploy`. diff --git a/test/commands/data-code-extension/lineage.test.ts b/test/commands/data-code-extension/lineage.test.ts new file mode 100644 index 0000000..60ccc58 --- /dev/null +++ b/test/commands/data-code-extension/lineage.test.ts @@ -0,0 +1,675 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { TestContext } from '@salesforce/core/testSetup'; +import { expect } from 'chai'; +import { SfCommand, stubSfCommandUx } from '@salesforce/sf-plugins-core'; +import { SfError, Org, Connection } from '@salesforce/core'; +import * as sinon from 'sinon'; +import LineageGenerate from '../../../src/commands/data-code-extension/script/lineage/generate.js'; +import { LineageRunner, type LineageReport } from '../../../src/utils/lineageRunner.js'; +import { PythonChecker } from '../../../src/utils/pythonChecker.js'; +import { PipChecker } from '../../../src/utils/pipChecker.js'; + +const PERMISSIONS = { read: { dlo: ['in__dll'] }, write: { dlo: ['out__dll'] } }; + +const REPORT: LineageReport = { + dataLineage: [ + { targetField: 'out__dll.id__c', sourceFields: ['in__dll.id'] }, + { targetField: 'out__dll.name__c', sourceFields: ['in__dll.first', 'in__dll.last'] }, + ], + unresolved: [{ targetField: 'out__dll.x__c', reason: 'untraceable source' }], + noSources: ['out__dll.const__c'], + udfFields: [], + targets: ['out__dll'], + sources: ['in__dll'], + pairCount: 3, +}; + +describe('data-code-extension script lineage generate', () => { + const $$ = new TestContext(); + let sfCommandStubs: ReturnType; + let generateStub: sinon.SinonStub; + let confirmStub: sinon.SinonStub; + let mockConnection: Connection; + let testDir: string; + let entrypointPath: string; + let configPath: string; + let originalIsTTY: boolean | undefined; + + const baseArgs = (): string[] => ['--entrypoint', entrypointPath, '--target-org', 'test@example.com']; + + function writeConfig(config: Record): void { + fs.writeFileSync(configPath, JSON.stringify(config)); + } + + function readConfig(): Record { + return JSON.parse(fs.readFileSync(configPath, 'utf8')) as Record; + } + + function warnings(): string[] { + return sfCommandStubs.warn.getCalls().map((call) => String(call.args[0])); + } + + beforeEach(() => { + testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-lineage-')); + fs.mkdirSync(path.join(testDir, '.datacustomcode_proj')); + fs.writeFileSync(path.join(testDir, '.datacustomcode_proj', 'sdk_config.json'), JSON.stringify({ type: 'script' })); + fs.mkdirSync(path.join(testDir, 'payload')); + entrypointPath = path.join(testDir, 'payload', 'entrypoint.py'); + fs.writeFileSync(entrypointPath, 'print("hi")\n'); + configPath = path.join(testDir, 'payload', 'config.json'); + writeConfig({ entryPoint: 'entrypoint.py', dataspace: 'default', permissions: PERMISSIONS }); + + sfCommandStubs = stubSfCommandUx($$.SANDBOX); + mockConnection = { refreshAuth: $$.SANDBOX.stub().resolves() } as unknown as Connection; + $$.SANDBOX.stub(Org, 'create').resolves({ + getUsername: () => 'test@example.com', + getConnection: () => mockConnection, + } as unknown as Org); + $$.SANDBOX.stub(PythonChecker, 'checkPython311').resolves({ + command: 'python3', + version: '3.11.5', + major: 3, + minor: 11, + patch: 5, + }); + $$.SANDBOX.stub(PipChecker, 'checkPackage').resolves({ + name: 'salesforce-data-customcode', + version: '1.0.0', + location: '/site-packages', + pipCommand: 'pip', + }); + $$.SANDBOX.stub(PipChecker, 'checkForUpdate').resolves(null); + generateStub = $$.SANDBOX.stub(LineageRunner, 'generate').resolves(structuredClone(REPORT)); + confirmStub = $$.SANDBOX.stub(SfCommand.prototype, 'confirm').resolves(true); + + originalIsTTY = process.stdin.isTTY; + process.stdin.isTTY = true; + }); + + afterEach(() => { + process.stdin.isTTY = originalIsTTY as boolean; + $$.restore(); + fs.rmSync(testDir, { recursive: true, force: true }); + }); + + it('is hidden and supports --json', () => { + expect(LineageGenerate.hidden).to.equal(true); + expect(LineageGenerate.enableJsonFlag).to.equal(true); + }); + + it('writes lineage into config.json, keeps other keys, and forwards run options', async () => { + const result = await LineageGenerate.run([ + ...baseArgs(), + '--dependencies', + 'pandas, numpy', + '--accept-lineage-ownership', + ]); + + const opts = generateStub.firstCall.args[0] as Record; + expect(opts.pythonCommand).to.equal('python3'); + expect(opts.entrypoint).to.equal(entrypointPath); + expect(opts.sfCliOrg).to.equal('test@example.com'); + expect(opts.dependencies).to.deep.equal(['pandas', 'numpy']); + expect((mockConnection.refreshAuth as sinon.SinonStub).calledOnce).to.be.true; + + const config = readConfig(); + expect(config.dataLineage).to.deep.equal(REPORT.dataLineage); + expect(config.permissions).to.deep.equal(PERMISSIONS); + expect(config.entryPoint).to.equal('entrypoint.py'); + expect(fs.readFileSync(configPath, 'utf8')).to.equal(JSON.stringify(config, null, 2)); + + expect(result.written).to.equal(true); + expect(result.entryCount).to.equal(2); + expect(result.pairCount).to.equal(3); + expect(result.diff.added).to.have.length(2); + expect(confirmStub.called).to.be.false; + expect(sfCommandStubs.log.calledWithMatch('Generated 2 dataLineage entries with 3')).to.be.true; + }); + + it('always lists unresolved fields', async () => { + await LineageGenerate.run(baseArgs()); + expect(warnings().some((w) => w.includes('out__dll.x__c (untraceable source)'))).to.be.true; + }); + + it('lists UDF-computed fields only when a UDF was detected', async () => { + await LineageGenerate.run([...baseArgs(), '--dry-run']); + expect(warnings().some((w) => w.includes('user-defined functions'))).to.be.false; + + sfCommandStubs.warn.resetHistory(); + generateStub.resolves({ ...REPORT, udfFields: ['out__dll.const__c', 'out__dll.name__c'] }); + await LineageGenerate.run([...baseArgs(), '--dry-run']); + const udfWarning = warnings().find((w) => w.includes('user-defined functions')); + expect(udfWarning).to.include('2 target fields'); + expect(udfWarning).to.include('out__dll.const__c'); + expect(udfWarning).to.include('out__dll.name__c'); + }); + + it('warns when lineage references objects missing from permissions', async () => { + writeConfig({ dataspace: 'default', permissions: { read: { dlo: [] }, write: { dlo: ['out__dll'] } } }); + const result = await LineageGenerate.run(baseArgs()); + expect(result.permissionGaps).to.deep.equal({ targets: [], sources: ['in__dll'] }); + expect(warnings().some((w) => w.includes('read: in__dll'))).to.be.true; + expect(result.written).to.equal(true); + }); + + it('does not write an empty result', async () => { + generateStub.resolves({ ...REPORT, dataLineage: [], pairCount: 0 }); + const result = await LineageGenerate.run(baseArgs()); + expect(result.written).to.equal(false); + expect(readConfig()).not.to.have.property('dataLineage'); + expect(warnings().some((w) => w.includes('No lineage could be generated'))).to.be.true; + }); + + it('warns that existing lineage still ships when the result is empty', async () => { + const existing = [{ targetField: 'out__dll.old', sourceFields: ['in__dll.old'] }]; + writeConfig({ dataspace: 'default', permissions: PERMISSIONS, dataLineage: existing }); + generateStub.resolves({ ...REPORT, dataLineage: [], pairCount: 0 }); + const result = await LineageGenerate.run(baseArgs()); + expect(result.written).to.equal(false); + expect(readConfig().dataLineage).to.deep.equal(existing); + const warning = warnings().find((w) => w.includes('No lineage could be generated')); + expect(warning).to.include('existing 1 dataLineage entries still apply'); + expect(warning).to.not.include('no-lineage path'); + }); + + it('lists the entries that replacing drops or changes', async () => { + writeConfig({ + dataspace: 'default', + permissions: PERMISSIONS, + dataLineage: [ + { targetField: 'out__dll.old', sourceFields: ['in__dll.old'] }, + { targetField: 'out__dll.id__c', sourceFields: ['in__dll.other'] }, + ], + }); + await LineageGenerate.run([...baseArgs(), '--dry-run']); + const logged = sfCommandStubs.log + .getCalls() + .map((call) => String(call.args[0])) + .join('\n'); + expect(logged).to.match(/drops:\r?\n {2}- out__dll\.old/); + expect(logged).to.match(/changes:\r?\n {2}- out__dll\.id__c/); + }); + + it('rejects streaming scripts before the run', async () => { + writeConfig({ dataspace: 'default', permissions: PERMISSIONS, streamingSource: { name: 'in__dll' } }); + try { + await LineageGenerate.run(baseArgs()); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageStreamingPackage'); + } + expect(generateStub.called).to.be.false; + }); + + it('rejects a read-only config.json before the run', async function () { + if (process.platform === 'win32' || process.getuid?.() === 0) this.skip(); + fs.chmodSync(configPath, 0o444); + try { + await LineageGenerate.run([...baseArgs(), '--accept-lineage-ownership']); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageConfigUnwritable'); + } finally { + fs.chmodSync(configPath, 0o644); + } + expect(generateStub.called).to.be.false; + }); + + it('--dry-run does not need a writable config.json', async function () { + if (process.platform === 'win32' || process.getuid?.() === 0) this.skip(); + fs.chmodSync(configPath, 0o444); + try { + const result = await LineageGenerate.run([...baseArgs(), '--dry-run']); + expect(result.written).to.equal(false); + } finally { + fs.chmodSync(configPath, 0o644); + } + }); + + it('--dry-run never writes or prompts', async () => { + writeConfig({ + dataspace: 'default', + permissions: PERMISSIONS, + dataLineage: [{ targetField: 'out__dll.old', sourceFields: ['in__dll.old'] }], + }); + const result = await LineageGenerate.run([...baseArgs(), '--dry-run']); + expect(result.written).to.equal(false); + expect(result.dryRun).to.equal(true); + expect(result.pairCount).to.equal(3); + expect(confirmStub.called).to.be.false; + expect((readConfig().dataLineage as unknown[])[0]).to.deep.equal({ + targetField: 'out__dll.old', + sourceFields: ['in__dll.old'], + }); + }); + + it('writes the full result to --output-file', async () => { + const outputFile = path.join(testDir, 'out', '..', 'lineage-result.json'); + const result = await LineageGenerate.run([...baseArgs(), '--dry-run', '--output-file', outputFile]); + expect(JSON.parse(fs.readFileSync(outputFile, 'utf8'))).to.deep.equal(REPORT); + expect(result.outputFile).to.equal(path.resolve(outputFile)); + }); + + it('rejects an --output-file in a missing directory before the run', async () => { + const outputFile = path.join(testDir, 'missing', 'lineage-result.json'); + try { + await LineageGenerate.run([...baseArgs(), '--dry-run', '--output-file', outputFile]); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageOutputFileUnwritable'); + } + expect(generateStub.called).to.be.false; + }); + + it('rejects an --output-file that is a directory before the run', async () => { + const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-lineage-out-')); + try { + await LineageGenerate.run([...baseArgs(), '--dry-run', '--output-file', outputDir]); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageOutputFileUnwritable'); + expect((error as SfError).message).to.include('not a regular file'); + } finally { + fs.rmSync(outputDir, { recursive: true, force: true }); + } + expect(generateStub.called).to.be.false; + }); + + it('rejects a read-only existing --output-file before the run', async function () { + if (process.platform === 'win32' || process.getuid?.() === 0) this.skip(); + const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-lineage-out-')); + const outputFile = path.join(outputDir, 'lineage-result.json'); + fs.writeFileSync(outputFile, '{}'); + fs.chmodSync(outputFile, 0o444); + try { + await LineageGenerate.run([...baseArgs(), '--dry-run', '--output-file', outputFile]); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageOutputFileUnwritable'); + } finally { + fs.rmSync(outputDir, { recursive: true, force: true }); + } + expect(generateStub.called).to.be.false; + }); + + it('fails clearly when the target org has no username', async () => { + (Org.create as sinon.SinonStub).resolves({ + getUsername: () => undefined, + getConnection: () => mockConnection, + } as unknown as Org); + try { + await LineageGenerate.run(baseArgs()); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageNoUsername'); + } + expect(generateStub.called).to.be.false; + }); + + describe('--output-file collisions', () => { + async function expectCollision(outputFile: string): Promise { + const before = fs.readFileSync(configPath, 'utf8'); + try { + await LineageGenerate.run([...baseArgs(), '--dry-run', '--output-file', outputFile]); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageOutputFileCollision'); + } + expect(generateStub.called).to.be.false; + expect(fs.readFileSync(configPath, 'utf8')).to.equal(before); + } + + it('rejects the package config.json', async () => { + await expectCollision(configPath); + }); + + it('rejects the entrypoint', async () => { + await expectCollision(entrypointPath); + }); + + it('rejects a symlink to config.json', async () => { + const link = path.join(testDir, 'link.json'); + fs.symlinkSync(configPath, link); + await expectCollision(link); + }); + + it('rejects a new file inside payload/', async () => { + await expectCollision(path.join(testDir, 'payload', 'lineage-result.json')); + }); + + it('rejects SDK package metadata', async () => { + await expectCollision(path.join(testDir, '.datacustomcode_proj', 'sdk_config.json')); + }); + + it('rejects the project config.yaml', async () => { + fs.writeFileSync(path.join(testDir, 'config.yaml'), 'reader_config: {}\n'); + await expectCollision(path.join(testDir, 'config.yaml')); + }); + + for (const name of ['requirements.txt', 'build_native_dependencies.sh', 'Dockerfile.dependencies']) { + it(`rejects the packaged build file ${name}`, async () => { + const buildFile = path.join(testDir, name); + fs.writeFileSync(buildFile, 'original\n'); + await expectCollision(buildFile); + expect(fs.readFileSync(buildFile, 'utf8')).to.equal('original\n'); + }); + + it(`rejects the packaged build file ${name} when it does not exist yet`, async () => { + const buildFile = path.join(testDir, name); + fs.rmSync(buildFile, { force: true }); + await expectCollision(buildFile); + expect(fs.existsSync(buildFile)).to.be.false; + }); + + it(`rejects a dangling symlink to the missing packaged build file ${name}`, async () => { + const buildFile = path.join(testDir, name); + fs.rmSync(buildFile, { force: true }); + const link = path.join(testDir, `link-${name}`); + fs.symlinkSync(buildFile, link); + await expectCollision(link); + expect(fs.existsSync(buildFile)).to.be.false; + }); + + it(`rejects the missing packaged build file ${name} spelled in another case`, async function () { + if (process.platform !== 'darwin' && process.platform !== 'win32') { + this.skip(); + } + const buildFile = path.join(testDir, name); + fs.rmSync(buildFile, { force: true }); + await expectCollision(path.join(testDir, name.toUpperCase())); + expect(fs.readdirSync(testDir).some((entry) => entry.toLowerCase() === name.toLowerCase())).to.be.false; + }); + } + + it('rejects a symlinked directory that points into payload/', async () => { + const link = path.join(testDir, 'payload-link'); + fs.symlinkSync(path.join(testDir, 'payload'), link); + await expectCollision(path.join(link, 'lineage-result.json')); + }); + + it('rejects payload/config.json when it is a symlink to a config outside payload/', async () => { + const shared = path.join(testDir, 'shared-config.json'); + fs.renameSync(configPath, shared); + fs.symlinkSync(shared, configPath); + const before = fs.readFileSync(shared, 'utf8'); + await expectCollision(configPath); + expect(fs.readFileSync(shared, 'utf8')).to.equal(before); + }); + + it('rejects a hard link to config.json', async () => { + const link = path.join(testDir, 'hardlink.json'); + fs.linkSync(configPath, link); + await expectCollision(link); + }); + + it('rejects a hard link to the SDK metadata', async () => { + const link = path.join(testDir, 'sdk-hardlink.json'); + fs.linkSync(path.join(testDir, '.datacustomcode_proj', 'sdk_config.json'), link); + await expectCollision(link); + }); + + it('allows a new file at the project root', async () => { + const outputFile = path.join(testDir, 'lineage-result.json'); + await LineageGenerate.run([...baseArgs(), '--dry-run', '--output-file', outputFile]); + expect(fs.existsSync(outputFile)).to.be.true; + }); + }); + + it('keeps config.json edits made while lineage was generating', async () => { + generateStub.callsFake(async () => { + writeConfig({ ...readConfig(), dataspace: 'edited' }); + return structuredClone(REPORT); + }); + const result = await LineageGenerate.run(baseArgs()); + expect(result.written).to.equal(true); + expect(readConfig().dataspace).to.equal('edited'); + expect(readConfig().dataLineage).to.deep.equal(REPORT.dataLineage); + }); + + it('rejects function packages before running', async () => { + fs.writeFileSync( + path.join(testDir, '.datacustomcode_proj', 'sdk_config.json'), + JSON.stringify({ type: 'function' }) + ); + try { + await LineageGenerate.run(baseArgs()); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageFunctionPackage'); + } + expect(generateStub.called).to.be.false; + }); + + it('fails with ConfigNotFound before running when config.json is missing', async () => { + fs.rmSync(configPath); + try { + await LineageGenerate.run(baseArgs()); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('ConfigNotFound'); + } + expect(generateStub.called).to.be.false; + }); + + describe('accepting lineage ownership', () => { + it('asks the customer to accept ownership before the first write', async () => { + const result = await LineageGenerate.run(baseArgs()); + expect(confirmStub.calledOnce).to.be.true; + expect(confirmStub.firstCall.args[0]).to.deep.include({ + message: 'Accept and write dataLineage to config.json?', + defaultAnswer: false, + }); + expect(warnings().some((w) => w.includes("You're responsible for reviewing and maintaining it"))).to.be.true; + expect(result.written).to.equal(true); + expect(readConfig().dataLineage).to.deep.equal(REPORT.dataLineage); + }); + + it('does not write when the customer declines', async () => { + confirmStub.resolves(false); + const result = await LineageGenerate.run(baseArgs()); + expect(result.written).to.equal(false); + expect(readConfig()).to.not.have.property('dataLineage'); + }); + + it('--accept-lineage-ownership writes without asking but still shows the warning', async () => { + const result = await LineageGenerate.run([...baseArgs(), '--accept-lineage-ownership']); + expect(confirmStub.called).to.be.false; + expect(warnings().some((w) => w.includes('best-effort analysis'))).to.be.true; + expect(result.written).to.equal(true); + }); + + it('--no-prompt does not accept ownership', async () => { + await LineageGenerate.run([...baseArgs(), '--no-prompt']); + expect(confirmStub.calledOnce).to.be.true; + }); + + it('fails without --accept-lineage-ownership when not interactive', async () => { + process.stdin.isTTY = false; + try { + await LineageGenerate.run([...baseArgs(), '--no-prompt']); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageOwnershipNotAccepted'); + } + expect(generateStub.called).to.be.false; + expect(confirmStub.called).to.be.false; + expect(readConfig()).to.not.have.property('dataLineage'); + }); + + it('fails before the run even when the result would be empty', async () => { + process.stdin.isTTY = false; + generateStub.resolves({ ...REPORT, dataLineage: [], pairCount: 0 }); + try { + await LineageGenerate.run(baseArgs()); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageOwnershipNotAccepted'); + expect((error as SfError).actions?.join(' ')).to.include('--dry-run'); + } + expect(generateStub.called).to.be.false; + }); + + it('fails with --json without --accept-lineage-ownership', async () => { + try { + await LineageGenerate.run([...baseArgs(), '--json']); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageOwnershipNotAccepted'); + } + expect(generateStub.called).to.be.false; + expect(readConfig()).to.not.have.property('dataLineage'); + }); + + it('--dry-run needs no acceptance', async () => { + process.stdin.isTTY = false; + const result = await LineageGenerate.run([...baseArgs(), '--dry-run']); + expect(confirmStub.called).to.be.false; + expect(result.written).to.equal(false); + }); + }); + + describe('replacing existing lineage', () => { + const OLD = [{ targetField: 'out__dll.old', sourceFields: ['in__dll.old'] }]; + + beforeEach(() => { + writeConfig({ dataspace: 'default', permissions: PERMISSIONS, dataLineage: OLD }); + }); + + it('asks once to accept ownership and replace', async () => { + const result = await LineageGenerate.run(baseArgs()); + expect(confirmStub.calledOnce).to.be.true; + expect(confirmStub.firstCall.args[0]).to.deep.include({ + message: 'Accept and replace the existing dataLineage in config.json?', + defaultAnswer: false, + ms: 10 * 60 * 1000, + }); + expect(warnings().some((w) => w.includes('best-effort analysis'))).to.be.true; + expect(warnings().some((w) => w.includes('This replaces the existing dataLineage section'))).to.be.true; + expect(result.written).to.equal(true); + expect(result.diff.removed).to.deep.equal(['out__dll.old']); + expect(readConfig().dataLineage).to.deep.equal(REPORT.dataLineage); + }); + + it('keeps existing lineage when the customer declines', async () => { + confirmStub.resolves(false); + const result = await LineageGenerate.run(baseArgs()); + expect(result.written).to.equal(false); + expect(readConfig().dataLineage).to.deep.equal(OLD); + }); + + it('asks only about replacing once ownership is accepted', async () => { + await LineageGenerate.run([...baseArgs(), '--accept-lineage-ownership']); + expect(confirmStub.calledOnce).to.be.true; + expect(confirmStub.firstCall.args[0]).to.have.property( + 'message', + 'Replace the existing dataLineage in config.json?' + ); + }); + + it('--no-prompt alone asks only to accept ownership', async () => { + await LineageGenerate.run([...baseArgs(), '--no-prompt']); + expect(confirmStub.calledOnce).to.be.true; + expect(confirmStub.firstCall.args[0]).to.have.property('message', 'Accept and write dataLineage to config.json?'); + }); + + it('--accept-lineage-ownership --no-prompt replaces without asking', async () => { + const result = await LineageGenerate.run([...baseArgs(), '--accept-lineage-ownership', '--no-prompt']); + expect(confirmStub.called).to.be.false; + expect(result.written).to.equal(true); + }); + + it('does not replace lineage edited while the prompt was pending', async () => { + const edited = [{ targetField: 'out__dll.edited', sourceFields: ['in__dll.edited'] }]; + confirmStub.callsFake(async () => { + writeConfig({ ...readConfig(), dataLineage: edited }); + return true; + }); + try { + await LineageGenerate.run(baseArgs()); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageChangedDuringConfirm'); + } + expect(readConfig().dataLineage).to.deep.equal(edited); + }); + + it('does not write when permissions change while the prompt was pending', async () => { + confirmStub.callsFake(async () => { + writeConfig({ ...readConfig(), permissions: { read: {}, write: {} } }); + return true; + }); + try { + await LineageGenerate.run(baseArgs()); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageChangedDuringConfirm'); + } + expect(readConfig().permissions).to.deep.equal({ read: {}, write: {} }); + }); + + it('fails without --no-prompt when not interactive', async () => { + process.stdin.isTTY = false; + try { + await LineageGenerate.run([...baseArgs(), '--accept-lineage-ownership']); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageReplaceNotConfirmed'); + } + expect(generateStub.called).to.be.false; + expect(confirmStub.called).to.be.false; + expect(readConfig().dataLineage).to.deep.equal(OLD); + }); + + it('fails with --json without --no-prompt', async () => { + try { + await LineageGenerate.run([...baseArgs(), '--json', '--accept-lineage-ownership']); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageReplaceNotConfirmed'); + } + expect(generateStub.called).to.be.false; + expect(readConfig().dataLineage).to.deep.equal(OLD); + }); + + it('rewrites duplicate targets even when the diff is empty', async () => { + const duplicated = [...REPORT.dataLineage, REPORT.dataLineage[0]]; + writeConfig({ dataspace: 'default', permissions: PERMISSIONS, dataLineage: duplicated }); + const result = await LineageGenerate.run([...baseArgs(), '--accept-lineage-ownership', '--no-prompt']); + expect(result.written).to.equal(true); + expect(readConfig().dataLineage).to.deep.equal(REPORT.dataLineage); + }); + + it('asks before replacing a section with only malformed entries', async () => { + const malformed = [{ targetField: 'out__dll.id__c', sourceFields: [null] }]; + writeConfig({ dataspace: 'default', permissions: PERMISSIONS, dataLineage: malformed }); + const result = await LineageGenerate.run(baseArgs()); + expect(confirmStub.calledOnce).to.be.true; + expect(result.written).to.equal(true); + expect(readConfig().dataLineage).to.deep.equal(REPORT.dataLineage); + }); + + it('does not prompt when the generated lineage is identical', async () => { + writeConfig({ dataspace: 'default', permissions: PERMISSIONS, dataLineage: REPORT.dataLineage }); + const result = await LineageGenerate.run(baseArgs()); + expect(confirmStub.called).to.be.false; + expect(result.written).to.equal(false); + }); + }); +}); diff --git a/test/commands/data-code-extension/scan.test.ts b/test/commands/data-code-extension/scan.test.ts index 9d9b160..78247cd 100644 --- a/test/commands/data-code-extension/scan.test.ts +++ b/test/commands/data-code-extension/scan.test.ts @@ -18,7 +18,9 @@ import * as path from 'node:path'; import * as os from 'node:os'; import { TestContext } from '@salesforce/core/testSetup'; import { expect } from 'chai'; -import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; +import { SfCommand, stubSfCommandUx } from '@salesforce/sf-plugins-core'; +import { SfError } from '@salesforce/core'; +import * as sinon from 'sinon'; import { setPipreqsMock } from '../../../src/utils/nativeScan.js'; import { PythonChecker } from '../../../src/utils/pythonChecker.js'; import { PipChecker } from '../../../src/utils/pipChecker.js'; @@ -254,4 +256,117 @@ describe('data-code-extension scan commands', () => { expect(reqs).to.deep.equal(['pandas']); }); }); + + describe('script scan stale-lineage guard', () => { + const SCRIPT = + 'from datacustomcode.client import Client\n' + + 'def main():\n' + + ' c = Client()\n' + + ' c.read_dlo("Source__dll")\n' + + ' c.write_to_dlo("Dest__dll", df)\n'; + const STALE = [{ targetField: 'OldDest__dll.a', sourceFields: ['Source__dll.a'] }]; + let confirmStub: sinon.SinonStub; + let originalIsTTY: boolean | undefined; + + const configFile = (): string => path.join(testDir, 'payload', 'config.json'); + const readConfig = (): Record => + JSON.parse(fs.readFileSync(configFile(), 'utf8')) as Record; + const warnings = (): string[] => sfCommandStubs.warn.getCalls().map((call) => String(call.args[0])); + + beforeEach(() => { + confirmStub = $$.SANDBOX.stub(SfCommand.prototype, 'confirm').resolves(true); + originalIsTTY = process.stdin.isTTY; + process.stdin.isTTY = true; + }); + + afterEach(() => { + process.stdin.isTTY = originalIsTTY as boolean; + }); + + it('is unchanged without dataLineage: same config.json bytes, no prompt, no warning', async () => { + makeScriptPackage(SCRIPT); + await ScriptScan.run([]); + expect(fs.readFileSync(configFile(), 'utf8')).to.equal( + JSON.stringify( + { + dataspace: 'default', + entryPoint: 'entrypoint.py', + permissions: { read: { dlo: ['Source__dll'] }, write: { dlo: ['Dest__dll'] } }, + }, + null, + 2 + ) + ); + expect(confirmStub.called).to.be.false; + expect(sfCommandStubs.warn.called).to.be.false; + }); + + it('does not prompt when lineage is still covered by permissions', async () => { + const lineage = [{ targetField: 'Dest__dll.a', sourceFields: ['Source__dll.a'] }]; + makeScriptPackage(SCRIPT, { dataspace: 'default', dataLineage: lineage }); + await ScriptScan.run([]); + expect(confirmStub.called).to.be.false; + expect(readConfig().dataLineage).to.deep.equal(lineage); + }); + + it('writes permissions and leaves lineage untouched when confirmed', async () => { + makeScriptPackage(SCRIPT, { dataspace: 'default', dataLineage: STALE }); + const result = await ScriptScan.run([]); + expect(confirmStub.calledOnce).to.be.true; + expect(confirmStub.firstCall.args[0]).to.deep.include({ defaultAnswer: false, ms: 10 * 60 * 1000 }); + expect(warnings().some((w) => w.includes('write: OldDest__dll') && w.includes('1 entries'))).to.be.true; + expect(result.executionResult.written).to.be.undefined; + const config = readConfig(); + expect(config.permissions).to.deep.equal({ read: { dlo: ['Source__dll'] }, write: { dlo: ['Dest__dll'] } }); + expect(config.dataLineage).to.deep.equal(STALE); + }); + + it('writes nothing when declined', async () => { + makeScriptPackage(SCRIPT, { dataspace: 'default', dataLineage: STALE }); + const before = fs.readFileSync(configFile(), 'utf8'); + confirmStub.resolves(false); + try { + await ScriptScan.run([]); + expect.fail('Should have thrown'); + } catch (error) { + // Non-zero exit, so a following `&& deploy` doesn't ship the old permissions. + expect((error as SfError).name).to.equal('ScanStaleLineageNotWritten'); + expect((error as SfError).message).to.include('no files were modified'); + } + const logged = sfCommandStubs.log + .getCalls() + .flatMap((call) => call.args) + .join('\n'); + expect(logged).to.not.include('Package scanned successfully'); + expect(fs.readFileSync(configFile(), 'utf8')).to.equal(before); + expect(fs.existsSync(path.join(testDir, 'requirements.txt'))).to.be.false; + }); + + it('--no-prompt writes without asking', async () => { + makeScriptPackage(SCRIPT, { dataspace: 'default', dataLineage: STALE }); + await ScriptScan.run(['--no-prompt']); + expect(confirmStub.called).to.be.false; + expect(readConfig().permissions).to.have.property('write'); + }); + + it('fails without --no-prompt when not interactive', async () => { + makeScriptPackage(SCRIPT, { dataspace: 'default', dataLineage: STALE }); + const before = fs.readFileSync(configFile(), 'utf8'); + process.stdin.isTTY = false; + try { + await ScriptScan.run([]); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('ScanStaleLineageNotConfirmed'); + } + expect(fs.readFileSync(configFile(), 'utf8')).to.equal(before); + }); + + it('--dry-run warns but never prompts', async () => { + makeScriptPackage(SCRIPT, { dataspace: 'default', dataLineage: STALE }); + await ScriptScan.run(['--dry-run']); + expect(confirmStub.called).to.be.false; + expect(warnings().some((w) => w.includes('write: OldDest__dll'))).to.be.true; + }); + }); }); diff --git a/test/utils/lineageConfig.test.ts b/test/utils/lineageConfig.test.ts new file mode 100644 index 0000000..7126a5e --- /dev/null +++ b/test/utils/lineageConfig.test.ts @@ -0,0 +1,203 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { expect } from 'chai'; +import { + countEntriesReferencing, + countPairs, + diffLineage, + findPermissionGaps, + findStaleLineage, + hasInvalidLineage, + hasLineageSection, + isEmptyDiff, + readLineage, + replaceLineage, + writeConfigJson, +} from '../../src/utils/lineageConfig.js'; + +const CONFIG = { + entryPoint: 'entrypoint.py', + dataspace: 'default', + permissions: { read: { dlo: ['In__dll'] }, write: { dlo: ['out__dll'] } }, +}; + +describe('lineageConfig', () => { + describe('readLineage', () => { + it('returns [] when the section is absent or not an array', () => { + expect(readLineage({})).to.deep.equal([]); + expect(readLineage({ dataLineage: 'nope' })).to.deep.equal([]); + }); + + it('drops malformed entries', () => { + const good = { targetField: 'o.a', sourceFields: ['i.a'] }; + expect(readLineage({ dataLineage: [good, { targetField: 1 }, null] })).to.deep.equal([good]); + }); + + it('drops entries whose sourceFields are not all strings', () => { + const good = { targetField: 'o.a', sourceFields: ['i.a'] }; + expect(readLineage({ dataLineage: [good, { targetField: 'o.b', sourceFields: [null] }] })).to.deep.equal([good]); + }); + }); + + it('hasLineageSection ignores an absent or empty section', () => { + expect(hasLineageSection({})).to.equal(false); + expect(hasLineageSection({ dataLineage: [] })).to.equal(false); + expect(hasLineageSection({ dataLineage: [null] })).to.equal(true); + expect(hasLineageSection({ dataLineage: 'nope' })).to.equal(true); + }); + + describe('hasInvalidLineage', () => { + const entry = { targetField: 'o.a', sourceFields: ['i.a'] }; + + it('is false for absent or well-formed lineage', () => { + expect(hasInvalidLineage({})).to.equal(false); + expect(hasInvalidLineage({ dataLineage: [] })).to.equal(false); + expect(hasInvalidLineage({ dataLineage: [entry, { targetField: 'o.b', sourceFields: [] }] })).to.equal(false); + }); + + it('flags a non-array section and malformed entries', () => { + expect(hasInvalidLineage({ dataLineage: 'nope' })).to.equal(true); + expect(hasInvalidLineage({ dataLineage: [entry, { targetField: 'o.b', sourceFields: [null] }] })).to.equal(true); + }); + + it('flags duplicate targets case-insensitively', () => { + expect(hasInvalidLineage({ dataLineage: [entry, entry] })).to.equal(true); + expect(hasInvalidLineage({ dataLineage: [entry, { ...entry, targetField: 'O.A' }] })).to.equal(true); + }); + }); + + describe('diffLineage', () => { + it('reports added, removed and changed targets', () => { + const diff = diffLineage( + [ + { targetField: 'o.keep', sourceFields: ['i.a', 'i.b'] }, + { targetField: 'o.change', sourceFields: ['i.a'] }, + { targetField: 'o.gone', sourceFields: ['i.a'] }, + ], + [ + { targetField: 'o.keep', sourceFields: ['i.b', 'i.a'] }, + { targetField: 'o.change', sourceFields: ['i.c'] }, + { targetField: 'o.new', sourceFields: ['i.a'] }, + ] + ); + expect(diff).to.deep.equal({ added: ['o.new'], removed: ['o.gone'], changed: ['o.change'] }); + expect(isEmptyDiff(diff)).to.equal(false); + }); + + it('matches targets and sources case-insensitively', () => { + const diff = diffLineage( + [ + { targetField: 'Out__dll.Name__c', sourceFields: ['In__dll.Name__c'] }, + { targetField: 'Out__dll.Gone__c', sourceFields: ['In__dll.A__c'] }, + ], + [ + { targetField: 'out__dll.name__c', sourceFields: ['in__dll.name__c'] }, + { targetField: 'out__dll.new__c', sourceFields: ['in__dll.a__c'] }, + ] + ); + expect(diff).to.deep.equal({ added: ['out__dll.new__c'], removed: ['Out__dll.Gone__c'], changed: [] }); + }); + + it('is empty for identical lineage', () => { + const entries = [{ targetField: 'o.a', sourceFields: ['i.a'] }]; + expect(isEmptyDiff(diffLineage(entries, entries))).to.equal(true); + }); + }); + + it('countPairs sums source fields', () => { + expect( + countPairs([ + { targetField: 'o.a', sourceFields: ['i.a', 'i.b'] }, + { targetField: 'o.b', sourceFields: ['i.c'] }, + ]) + ).to.equal(3); + }); + + it('replaceLineage replaces wholesale and keeps other keys in order', () => { + const entries = [{ targetField: 'out__dll.a', sourceFields: ['In__dll.a'] }]; + const updated = replaceLineage( + { ...CONFIG, dataLineage: [{ targetField: 'out__dll.old', sourceFields: ['In__dll.old'] }], extra: 1 }, + entries + ); + expect(Object.keys(updated)).to.deep.equal(['entryPoint', 'dataspace', 'permissions', 'dataLineage', 'extra']); + expect(updated.dataLineage).to.deep.equal(entries); + }); + + describe('findPermissionGaps', () => { + it('reports objects missing from permissions, case-insensitively', () => { + expect(findPermissionGaps(CONFIG, ['out__dll', 'other__dll'], ['in__dll', 'x__dll', 'x__dll'])).to.deep.equal({ + targets: ['other__dll'], + sources: ['x__dll'], + }); + }); + + it('treats a missing permissions section as nothing permitted', () => { + expect(findPermissionGaps({}, ['o'], ['i'])).to.deep.equal({ targets: ['o'], sources: ['i'] }); + }); + }); + + describe('findStaleLineage', () => { + it('is empty without dataLineage', () => { + expect(findStaleLineage(CONFIG)).to.deep.equal({ targets: [], sources: [] }); + }); + + it('is empty when lineage matches permissions', () => { + const config = { ...CONFIG, dataLineage: [{ targetField: 'out__dll.a', sourceFields: ['In__dll.a'] }] }; + expect(findStaleLineage(config)).to.deep.equal({ targets: [], sources: [] }); + }); + + it('reports a stale target and a stale source', () => { + const config = { + ...CONFIG, + dataLineage: [ + { targetField: 'old_out__dll.a', sourceFields: ['In__dll.a'] }, + { targetField: 'out__dll.b', sourceFields: ['old_in__dll.b'] }, + ], + }; + expect(findStaleLineage(config)).to.deep.equal({ targets: ['old_out__dll'], sources: ['old_in__dll'] }); + expect(countEntriesReferencing(config, ['old_out__dll'])).to.equal(1); + expect(countEntriesReferencing(config, ['old_out__dll', 'OLD_IN__dll'])).to.equal(2); + }); + + it('ignores malformed source fields instead of throwing', () => { + const config = { ...CONFIG, dataLineage: [{ targetField: 'out__dll.a', sourceFields: [null] }] }; + expect(findStaleLineage(config)).to.deep.equal({ targets: [], sources: [] }); + expect(countEntriesReferencing(config, ['in__dll'])).to.equal(0); + }); + + it('reads DMO permissions too', () => { + const config = { + permissions: { read: { dmo: ['in__dlm'] }, write: { dmo: ['out__dlm'] } }, + dataLineage: [{ targetField: 'out__dlm.a', sourceFields: ['in__dlm.a', 'in__dll.a'] }], + }; + expect(findStaleLineage(config)).to.deep.equal({ targets: [], sources: ['in__dll'] }); + }); + }); + + it('writeConfigJson writes 2-space JSON without a trailing newline, as scan does', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-lineage-config-')); + try { + const file = path.join(dir, 'config.json'); + await writeConfigJson(file, { a: [1] }); + expect(fs.readFileSync(file, 'utf8')).to.equal('{\n "a": [\n 1\n ]\n}'); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/utils/lineageRunner.test.ts b/test/utils/lineageRunner.test.ts new file mode 100644 index 0000000..ffa314a --- /dev/null +++ b/test/utils/lineageRunner.test.ts @@ -0,0 +1,346 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { expect } from 'chai'; +import { SfError } from '@salesforce/core'; +import { + buildLineagePayload, + generateLineage, + GENERATE_LINEAGE_SHIM, + LINEAGE_TERMINATED_EXIT_CODE, + LINEAGE_TERMINATED_MARKER, + LINEAGE_UNAVAILABLE_EXIT_CODE, + LINEAGE_UNAVAILABLE_MARKER, + parseLineageReport, + type LineagePayload, + type LineageReport, + type LineageRunOptions, +} from '../../src/utils/lineageRunner.js'; +import { RUN_TIMEOUT_MS, type RunnerChild, type RunnerStream, type SpawnFn } from '../../src/utils/pythonRunner.js'; + +async function expectReject(promise: Promise): Promise { + try { + await promise; + } catch (err) { + return err as SfError; + } + throw new Error('Expected promise to reject, but it resolved'); +} + +class FakeStream implements RunnerStream { + private dataListeners: Array<(chunk: string) => void> = []; + + public setEncoding(): this { + return this; + } + + public on(event: 'data', listener: (chunk: string) => void): this { + this.dataListeners.push(listener); + return this; + } + + public push(chunk: string): void { + for (const listener of this.dataListeners) listener(chunk); + } +} + +class FakeChild implements RunnerChild { + public stdout = new FakeStream(); + public stderr = new FakeStream(); + private closeListeners: Array<(code: number | null, signal: NodeJS.Signals | null) => void> = []; + private errorListeners: Array<(err: Error) => void> = []; + + public on(event: 'close', listener: (code: number | null, signal: NodeJS.Signals | null) => void): this; + public on(event: 'error', listener: (err: Error) => void): this; + public on( + event: 'close' | 'error', + listener: ((code: number | null, signal: NodeJS.Signals | null) => void) | ((err: Error) => void) + ): this { + if (event === 'close') { + this.closeListeners.push(listener as (code: number | null, signal: NodeJS.Signals | null) => void); + } else this.errorListeners.push(listener as (err: Error) => void); + return this; + } + + public emitClose(code: number | null, signal: NodeJS.Signals | null = null): void { + for (const listener of this.closeListeners) listener(code, signal); + } + + public emitError(err: Error): void { + for (const listener of this.errorListeners) listener(err); + } +} + +type SpawnCall = { + command: string; + args: readonly string[]; + options: { env?: NodeJS.ProcessEnv; timeout?: number }; +}; + +const REPORT: LineageReport = { + dataLineage: [{ targetField: 'out__dll.id__c', sourceFields: ['in__dll.id'] }], + unresolved: [{ targetField: 'out__dll.x__c', reason: 'untraceable source' }], + noSources: [], + udfFields: [], + targets: ['out__dll'], + sources: ['in__dll'], + pairCount: 1, +}; + +function payloadOf(call: SpawnCall): LineagePayload { + return JSON.parse(call.args[2]) as LineagePayload; +} + +/** + * A spawn seam that, like the real shim, acts on the payload asynchronously: `behave` + * receives the child and payload once the runner has attached its listeners. + */ +function fakeSpawn(calls: SpawnCall[], behave: (child: FakeChild, payload: LineagePayload) => void): SpawnFn { + return (command, args, options) => { + const call = { command, args, options }; + calls.push(call); + const child = new FakeChild(); + setImmediate(() => behave(child, payloadOf(call))); + return child; + }; +} + +function succeed(report: LineageReport = REPORT): (child: FakeChild, payload: LineagePayload) => void { + return (child, payload) => { + fs.writeFileSync(payload.resultFile, JSON.stringify(report)); + child.emitClose(0); + }; +} + +function makeOpts(overrides: Partial = {}): LineageRunOptions { + return { + pythonCommand: 'python3', + entrypoint: '/pkg/payload/entrypoint.py', + dependencies: ['pandas'], + sfCliOrg: 'myorg', + ...overrides, + }; +} + +describe('lineageRunner', () => { + describe('buildLineagePayload', () => { + it('maps options to the shim payload with defaults', () => { + expect(buildLineagePayload(makeOpts(), '/tmp/x/lineage.json')).to.deep.equal({ + entrypoint: '/pkg/payload/entrypoint.py', + configFile: null, + dependencies: ['pandas'], + profile: 'default', + sfCliOrg: 'myorg', + resultFile: '/tmp/x/lineage.json', + }); + }); + + it('passes through config file and profile', () => { + const payload = buildLineagePayload(makeOpts({ configFile: '/c.yaml', profile: 'p' }), '/r.json'); + expect(payload.configFile).to.equal('/c.yaml'); + expect(payload.profile).to.equal('p'); + }); + }); + + describe('GENERATE_LINEAGE_SHIM', () => { + it('calls the SDK library, not the CLI', () => { + expect(GENERATE_LINEAGE_SHIM).to.include('from datacustomcode.lineage import generate_lineage'); + expect(GENERATE_LINEAGE_SHIM).to.include('json.dump(result.to_dict(), f)'); + expect(GENERATE_LINEAGE_SHIM).not.to.include('datacustomcode.cli'); + }); + + it('exits with the dedicated code and marker when the SDK lacks lineage', () => { + expect(GENERATE_LINEAGE_SHIM).to.include(`sys.exit(${LINEAGE_UNAVAILABLE_EXIT_CODE})`); + expect(GENERATE_LINEAGE_SHIM).to.include(LINEAGE_UNAVAILABLE_MARKER); + }); + + it('turns SIGTERM into SystemExit so subprocess.run stops the SDK worker', () => { + expect(GENERATE_LINEAGE_SHIM).to.include('signal.signal(signal.SIGTERM, _terminated)'); + expect(GENERATE_LINEAGE_SHIM).to.include(`sys.exit(${LINEAGE_TERMINATED_EXIT_CODE})`); + expect(GENERATE_LINEAGE_SHIM).to.include(LINEAGE_TERMINATED_MARKER); + // The handler must be installed before the SDK (and its worker) starts. + expect(GENERATE_LINEAGE_SHIM.indexOf('signal.signal(')).to.be.lessThan( + GENERATE_LINEAGE_SHIM.indexOf('generate_lineage(') + ); + }); + }); + + describe('parseLineageReport', () => { + const valid = { + dataLineage: [], + unresolved: [], + noSources: [], + udfFields: [], + targets: [], + sources: [], + pairCount: 0, + }; + + it('returns a well-formed report', () => { + expect(parseLineageReport(JSON.stringify(valid))).to.deep.equal(valid); + }); + + for (const [name, raw, detail] of [ + ['malformed JSON', '{', 'JSON'], + ['a non-object', '[]', 'expected a JSON object'], + ['missing fields', JSON.stringify({ ...valid, udfFields: undefined, noSources: null }), 'noSources, udfFields'], + ]) { + it(`rejects ${name}`, () => { + try { + parseLineageReport(raw); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('LineageResultInvalid'); + expect((error as SfError).message).to.include(detail); + } + }); + } + }); + + describe('generateLineage', () => { + it('passes the payload as its own argv element and returns the result file', async () => { + const calls: SpawnCall[] = []; + const entrypoint = '/pkg/"; import os; os.system("x") #/entrypoint.py'; + const report = await generateLineage(makeOpts({ entrypoint }), { spawn: fakeSpawn(calls, succeed()) }); + + expect(report).to.deep.equal(REPORT); + expect(calls).to.have.length(1); + expect(calls[0].command).to.equal('python3'); + expect(calls[0].args[0]).to.equal('-c'); + expect(calls[0].args[1]).to.equal(GENERATE_LINEAGE_SHIM); + expect(payloadOf(calls[0]).entrypoint).to.equal(entrypoint); + expect(calls[0].options.timeout).to.equal(RUN_TIMEOUT_MS); + expect(calls[0].options.env?.PYTHONUNBUFFERED).to.equal('1'); + }); + + it('removes the temp directory after success', async () => { + const calls: SpawnCall[] = []; + await generateLineage(makeOpts(), { spawn: fakeSpawn(calls, succeed()) }); + expect(fs.existsSync(path.dirname(payloadOf(calls[0]).resultFile))).to.equal(false); + }); + + it('removes the temp directory after failure', async () => { + const calls: SpawnCall[] = []; + await expectReject( + generateLineage(makeOpts(), { + spawn: fakeSpawn(calls, (child) => { + child.stderr.push('boom'); + child.emitClose(1); + }), + }) + ); + expect(fs.existsSync(path.dirname(payloadOf(calls[0]).resultFile))).to.equal(false); + }); + + it('streams stdout and stderr chunks', async () => { + const out: string[] = []; + const err: string[] = []; + await generateLineage(makeOpts({ onStdout: (c) => out.push(c), onStderr: (c) => err.push(c) }), { + spawn: fakeSpawn([], (child, payload) => { + child.stdout.push('hello'); + child.stderr.push('spark log'); + succeed()(child, payload); + }), + }); + expect(out).to.deep.equal(['hello']); + expect(err).to.deep.equal(['spark log']); + }); + + it('maps the unavailable exit code to LineageSdkTooOld', async () => { + const err = await expectReject( + generateLineage(makeOpts(), { + spawn: fakeSpawn([], (child) => { + child.stderr.push(`${LINEAGE_UNAVAILABLE_MARKER}\n`); + child.emitClose(LINEAGE_UNAVAILABLE_EXIT_CODE); + }), + }) + ); + expect(err.name).to.equal('LineageSdkTooOld'); + expect(err.actions?.join(' ')).to.include('pip install -U salesforce-data-customcode'); + }); + + it("does not mistake a script's own exit code 3 for an old SDK", async () => { + const err = await expectReject( + generateLineage(makeOpts(), { + spawn: fakeSpawn([], (child) => { + child.stderr.push('SystemExit: 3'); + child.emitClose(LINEAGE_UNAVAILABLE_EXIT_CODE); + }), + }) + ); + expect(err.name).to.equal('RunExecutionFailed'); + }); + + it('maps the terminated exit code to RunTerminated with the timeout', async () => { + const err = await expectReject( + generateLineage(makeOpts({ timeoutMs: 60_000 }), { + spawn: fakeSpawn([], (child) => { + child.stderr.push(`${LINEAGE_TERMINATED_MARKER}\n`); + child.emitClose(LINEAGE_TERMINATED_EXIT_CODE); + }), + }) + ); + expect(err.name).to.equal('RunTerminated'); + expect(err.message).to.include('60 seconds'); + }); + + it("does not mistake a script's own exit code 143 for the timeout", async () => { + const err = await expectReject( + generateLineage(makeOpts(), { + spawn: fakeSpawn([], (child) => { + child.stderr.push('SystemExit: 143'); + child.emitClose(LINEAGE_TERMINATED_EXIT_CODE); + }), + }) + ); + expect(err.name).to.equal('RunExecutionFailed'); + }); + + it('maps authentication failures', async () => { + const err = await expectReject( + generateLineage(makeOpts(), { + spawn: fakeSpawn([], (child) => { + child.stderr.push('Authentication failed for org'); + child.emitClose(1); + }), + }) + ); + expect(err.name).to.equal('RunAuthenticationFailed'); + expect(err.message).to.include('myorg'); + }); + + it('maps a failure to start the interpreter', async () => { + const err = await expectReject( + generateLineage(makeOpts(), { spawn: fakeSpawn([], (child) => child.emitError(new Error('ENOENT'))) }) + ); + expect(err.name).to.equal('RunProcessStartFailed'); + }); + + it('honors a custom timeout', async () => { + const calls: SpawnCall[] = []; + await generateLineage(makeOpts({ timeoutMs: 1234 }), { spawn: fakeSpawn(calls, succeed()) }); + expect(calls[0].options.timeout).to.equal(1234); + }); + + it('fails with LineageResultMissing when the shim exits 0 without a result', async () => { + const err = await expectReject( + generateLineage(makeOpts(), { spawn: fakeSpawn([], (child) => child.emitClose(0)) }) + ); + expect(err.name).to.equal('LineageResultMissing'); + }); + }); +}); diff --git a/test/utils/nativeScan.test.ts b/test/utils/nativeScan.test.ts index ac73508..7b7bfea 100644 --- a/test/utils/nativeScan.test.ts +++ b/test/utils/nativeScan.test.ts @@ -17,6 +17,7 @@ import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { expect } from 'chai'; +import { type SfError } from '@salesforce/core'; import { setPipreqsMock, executeNativeScan, @@ -687,3 +688,97 @@ describe('nativeScan: executeNativeScan', () => { await fs.rm(dir, { recursive: true, force: true }); }); }); + +describe('nativeScan: executeNativeScan confirmWrite', () => { + const ENTRYPOINT = + 'from datacustomcode.client import Client\n' + + 'def main():\n' + + ' c.read_dlo("Source__dll")\n' + + ' c.write_to_dlo("Dest__dll", df)\n'; + + beforeEach(() => { + setPipreqsMock(async () => 'pandas\n'); + }); + + afterEach(() => { + setPipreqsMock(null); + }); + + it('omits written when confirmWrite is not supplied', async () => { + const dir = await makePackage({ entrypoint: ENTRYPOINT }); + const result = await executeNativeScan({ workingDir: dir, packageType: 'script' }); + expect(result).not.to.have.property('written'); + await fs.rm(dir, { recursive: true, force: true }); + }); + + it('writes when confirmWrite resolves true, passing the updated config', async () => { + const dir = await makePackage({ entrypoint: ENTRYPOINT }); + let seen: Record | undefined; + const result = await executeNativeScan({ + workingDir: dir, + packageType: 'script', + confirmWrite: async (config) => { + seen = config; + return true; + }, + }); + expect(seen).to.equal(result.config); + expect(result).not.to.have.property('written'); + expect(result.requirementsPath).to.equal(path.join(dir, 'requirements.txt')); + await fs.rm(dir, { recursive: true, force: true }); + }); + + it('writes nothing when confirmWrite resolves false', async () => { + const dir = await makePackage({ entrypoint: ENTRYPOINT }); + const before = await fs.readFile(path.join(dir, 'payload', 'config.json'), 'utf8'); + const result = await executeNativeScan({ workingDir: dir, packageType: 'script', confirmWrite: async () => false }); + expect(result.written).to.equal(false); + expect(result.requirementsPath).to.be.undefined; + expect(await fs.readFile(path.join(dir, 'payload', 'config.json'), 'utf8')).to.equal(before); + await fs.rm(dir, { recursive: true, force: true }); + }); + + it('refuses to write when config.json changes while confirmWrite waits', async () => { + const dir = await makePackage({ entrypoint: ENTRYPOINT }); + const configPath = path.join(dir, 'payload', 'config.json'); + const edited = JSON.stringify({ dataspace: 'edited', dataLineage: { 'Dest.dll': { Id: ['Source.Id'] } } }); + try { + await executeNativeScan({ + workingDir: dir, + packageType: 'script', + confirmWrite: async () => { + await fs.writeFile(configPath, edited); + return true; + }, + }); + expect.fail('Should have thrown'); + } catch (error) { + expect((error as SfError).name).to.equal('ScanConfigChangedDuringConfirm'); + } + expect(await fs.readFile(configPath, 'utf8')).to.equal(edited); + let requirementsExists = true; + try { + await fs.access(path.join(dir, 'requirements.txt')); + } catch { + requirementsExists = false; + } + expect(requirementsExists).to.be.false; + await fs.rm(dir, { recursive: true, force: true }); + }); + + it('never calls confirmWrite on dry run', async () => { + const dir = await makePackage({ entrypoint: ENTRYPOINT }); + let called = false; + await executeNativeScan({ + workingDir: dir, + packageType: 'script', + dryRun: true, + confirmWrite: async () => { + called = true; + return true; + }, + }); + expect(called).to.be.false; + await fs.rm(dir, { recursive: true, force: true }); + }); +}); diff --git a/test/utils/pythonRunner.test.ts b/test/utils/pythonRunner.test.ts index 2c27136..a5e7f21 100644 --- a/test/utils/pythonRunner.test.ts +++ b/test/utils/pythonRunner.test.ts @@ -65,20 +65,24 @@ class FakeStream implements RunnerStream { class FakeChild implements RunnerChild { public stdout: FakeStream | null = new FakeStream(); public stderr: FakeStream | null = new FakeStream(); - private closeListeners: Array<(code: number | null) => void> = []; + private closeListeners: Array<(code: number | null, signal: NodeJS.Signals | null) => void> = []; private errorListeners: Array<(err: Error) => void> = []; - public on(event: 'close', listener: (code: number | null) => void): this; + public on(event: 'close', listener: (code: number | null, signal: NodeJS.Signals | null) => void): this; public on(event: 'error', listener: (err: Error) => void): this; - public on(event: 'close' | 'error', listener: ((code: number | null) => void) | ((err: Error) => void)): this { - if (event === 'close') this.closeListeners.push(listener as (code: number | null) => void); - else this.errorListeners.push(listener as (err: Error) => void); + public on( + event: 'close' | 'error', + listener: ((code: number | null, signal: NodeJS.Signals | null) => void) | ((err: Error) => void) + ): this { + if (event === 'close') { + this.closeListeners.push(listener as (code: number | null, signal: NodeJS.Signals | null) => void); + } else this.errorListeners.push(listener as (err: Error) => void); return this; } /** Test helper: fire the process 'close' event. */ - public emitClose(code: number | null): void { - for (const listener of this.closeListeners) listener(code); + public emitClose(code: number | null, signal: NodeJS.Signals | null = null): void { + for (const listener of this.closeListeners) listener(code, signal); } /** Test helper: fire the process 'error' event (failure to spawn). */ @@ -296,6 +300,18 @@ describe('pythonRunner', () => { expect(error.message).to.include('3'); }); + it('rejects with RunTerminated when a signal stopped the process (e.g. the timeout)', async () => { + const child = new FakeChild(); + const promise = runEntrypoint(makeOpts({ timeoutMs: 300_000 }), { spawn: fakeSpawn(child, []) }); + + child.emitClose(null, 'SIGTERM'); + const error = await expectReject(promise); + + expect(error.name).to.equal('RunTerminated'); + expect(error.message).to.include('SIGTERM'); + expect(error.message).to.include('300 seconds'); + }); + it('rejects with RunAuthenticationFailed when stderr signals an auth problem', async () => { const child = new FakeChild(); const calls: SpawnCall[] = []; diff --git a/test/utils/zipBuilder.test.ts b/test/utils/zipBuilder.test.ts index 8f6cc7c..d9c5e77 100644 --- a/test/utils/zipBuilder.test.ts +++ b/test/utils/zipBuilder.test.ts @@ -334,9 +334,10 @@ describe('zipBuilder.prepareDependencyArchive', () => { }, async run(args) { calls.run.push({ args }); - // The dockerRunCmd shape is `['run', '--rm', '-v', ':/workspace', ...]` + // The dockerRunCmd shape is `['run', '--rm', '-v', ':/workspace', ...]`. Strip only the + // container suffix: on Windows the mount itself has a drive colon (`C:/...`). const mountArg = args[args.indexOf('-v') + 1]; - const [mountPath] = mountArg.split(':'); + const mountPath = mountArg.slice(0, mountArg.lastIndexOf(':')); opts.onRun?.(mountPath); }, };