From 0dbe7db27e35c3916b37b54dbbe5adbca9a2dd75 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 13:04:27 -0700 Subject: [PATCH 1/7] fix(code-placeholders): harden shell placeholder interpolation in arithmetic contexts --- .../code-placeholders/compiler.test.ts | 162 +++ .../lib/execution/code-placeholders/shared.ts | 8 +- .../lib/execution/code-placeholders/shell.ts | 980 ++++++++++++++---- 3 files changed, 959 insertions(+), 191 deletions(-) diff --git a/apps/sim/lib/execution/code-placeholders/compiler.test.ts b/apps/sim/lib/execution/code-placeholders/compiler.test.ts index 96cc224341e..e1135e5eea6 100644 --- a/apps/sim/lib/execution/code-placeholders/compiler.test.ts +++ b/apps/sim/lib/execution/code-placeholders/compiler.test.ts @@ -1053,6 +1053,78 @@ describe('code placeholder compiler', () => { 'total=$(( $(( 1 + 1 )) + {{KEY}} ))', 'cat </dev/null let x="{{KEY}}"', + '2>/dev/null let x="{{KEY}}"', + 'builtin let x="{{KEY}}"', + 'command let x="{{KEY}}"', + 'declare "-i" n="{{KEY}}"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + '[[ ${missing:-{{KEY}}} -eq 0 ]]', + 'declare -i n; echo "$(declare +i n)"; n="{{KEY}}"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 'printf "%s" "${missing:-{text}"; let x="{{KEY}}"', + 'declare -i n; n="$(printf "%s" "{{KEY}}")"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 'declare -i n; n="${missing:-{{KEY}}}"', + 'declare -i n; declare n="{{KEY}}"', + '> /dev/null let x="{{KEY}}"', + 'let x="$(cat </dev/null {{KEY}}', + 'a=([{{KEY}}]=text)', + "unset 'a[{{KEY}}]'", + 'le\\\nt "x={{KEY}}"', + 'local -A a; a[{{KEY}}]=1', + 'declare -A m; unset m; m[{{KEY}}]=1', + 'declare -A m; unset m; declare -a m; m[{{KEY}}]=1', + '(declare -A m); m[{{KEY}}]=1', + 'a[{{KEY}}]=1', + 'a[{{KEY}}]+=1', + 'declare -i n; n="{{KEY}}"', + 'declare -i n\nn="{{KEY}}"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 'a=(1 2); echo "${a[{{KEY}}]}"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 'a=(1 2); echo "${a[0]:{{KEY}}}"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 'a=(1 2); echo "${a[0]:0:{{KEY}}}"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 's=abc; echo "${s:{{KEY}}}"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 's=abc; echo "${s:0:{{KEY}}}"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 's=abc; echo "${s: -1:{{KEY}}}"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 'set -- a b; echo "${@:{{KEY}}}"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 's=abc; printf "%s\\n" "${missing:-"${s:{{KEY}}}"}"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 'cat < { await expect( compileCodePlaceholders({ @@ -1087,6 +1159,96 @@ describe('code placeholder compiler', () => { ) }) + it.each([ + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 'printf "%s\\n" "${missing:-https://{{KEY}}}"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 'printf "%s\\n" "${missing:-items[{{KEY}}]}"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 's=abc; printf "%s\\n" "${s#[{{KEY}}]}"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 's=abc; printf "%s\\n" "${s/[{{KEY}}]/x}"', + 'f() { local -i n; }; n="{{KEY}}"', + 'PREFIX="$(printf "%s" "{{KEY}}")" let total=2', + 'printf "%s" a[{{KEY}}]=1', + 'declare -i n; printf "%s" n={{KEY}}', + 'declare -i n; declare +i n; n="{{KEY}}"', + 'declare -i n; n=1 printf "%s" "{{KEY}}"', + 'declare -i n; n=5 text="{{KEY}}"', + 'declare -i +i n="{{KEY}}"', + 'declare -i n; echo "$(declare +i n; n=\'{{KEY}}\'; printf "%s" "$n")"', + '$(let x=1 <"{{KEY}}"', + 'let x=1 > {{KEY}}', + 'declare -i n=1 >"{{KEY}}"', + '[[ "{{KEY}}" == "-eq" ]]', + ])('compiles parameter and prefix text that is not an arithmetic operand: %s', async (code) => { + // These forms use the value as pattern, default, or a shielded prefix — never arithmetic — so a + // conservative scanner must not reject them. Each would fail compilation if wrongly rejected. + await expect( + compileCodePlaceholders({ + code, + language: CodeLanguage.Shell, + environmentVariables: { KEY: 'values[$(printf injected >&2)]' }, + }) + ).resolves.toBeDefined() + }) + + it('keeps literal single quotes inside a double-quoted default expansion', async () => { + const compiled = await compileCodePlaceholders({ + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + code: 'printf "%s\\n" "${missing:-\'{{KEY}}\'}"', + language: CodeLanguage.Shell, + environmentVariables: { KEY: 'hello world' }, + }) + expect(executeShell(compiled.code, compiled.bindings)).toBe("'hello world'\n") + }) + + it('compiles shell placeholders beside arithmetic that never evaluates them', async () => { + const value = 'values[$(printf injected >&2)]' + const compiled = await compileCodePlaceholders({ + code: [ + '[ "{{KEY}}" -eq 0 ] 2>/dev/null || printf "%s\\n" not-zero', + '[[ "{{KEY}}" == values* && 1 -eq 1 ]] && printf "%s\\n" matched', + 'let total=1+1; printf "%s\\n" "{{KEY}}"', + 'f() { local copy="{{KEY}}"; printf "%s\\n" "$copy"; }; f', + 'declare copy="{{KEY}}"; printf "%s\\n" "$copy"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 'printf "%s\\n" "${missing:-{{KEY}}}"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 'a=(x y); printf "%s\\n" "${a[1]}{{KEY}}"', + 'printf "%s\\n" "let {{KEY}}"', + 'printf "%s\\n" "items[{{KEY}}]"', + 'PREFIX="{{KEY}}" let total=2', + '# declare -i n', + 'n="{{KEY}}"; printf "%s\\n" "$n"', + 'let total=1 <&1`, compiled.bindings)).toBe( + `not-zero\nmatched\n${value}\n${value}\n${value}\n${value}\ny${value}\n` + + `let ${value}\nitems[${value}]\n${value}\n1\n${value} 2\n` + ) + }) + it('discovers shell arithmetic placeholders without compiling missing values', async () => { const code = 'total=$(( {{MISSING}} + {{KEY}} ))' await expect(analyzeCodePlaceholders(code, CodeLanguage.Shell)).resolves.toEqual([ diff --git a/apps/sim/lib/execution/code-placeholders/shared.ts b/apps/sim/lib/execution/code-placeholders/shared.ts index 177e7d21d5b..523a44e0e62 100644 --- a/apps/sim/lib/execution/code-placeholders/shared.ts +++ b/apps/sim/lib/execution/code-placeholders/shared.ts @@ -327,12 +327,12 @@ export function createOffsetRangeLookup( * The placeholders lying wholly inside `[start, end]`. Occurrences are ordered and never * overlap, so the matches are one contiguous run found by bisection. */ -export function occurrencesWithin( - occurrences: readonly CodePlaceholderOccurrence[], +export function occurrencesWithin>( + occurrences: readonly T[], start: number, end: number -): CodePlaceholderOccurrence[] { - const matches: CodePlaceholderOccurrence[] = [] +): T[] { + const matches: T[] = [] for ( let index = partitionPoint(occurrences, (occurrence) => occurrence.start < start); index < occurrences.length && occurrences[index].end <= end; diff --git a/apps/sim/lib/execution/code-placeholders/shell.ts b/apps/sim/lib/execution/code-placeholders/shell.ts index efa1660a309..3f8dc574e5c 100644 --- a/apps/sim/lib/execution/code-placeholders/shell.ts +++ b/apps/sim/lib/execution/code-placeholders/shell.ts @@ -28,20 +28,134 @@ interface HeredocDeclaration { type ShellQuote = 'none' | 'single' | 'double' | 'ansi' interface ShellScanFrame { - kind: 'root' | 'command' | 'arithmetic' | 'backtick' + /** + * `parameter` is a `${...}` expansion, which ends on its own `}`, not a command boundary. + * `keysubscript` is an associative array's `[key]`, scanned like a subscript but not arithmetic. + * `arrayliteral` is a compound assignment's `( … )`, where element keys `[k]=` are subscripts. + * `subshell` is a bare `( … )` group — it reads commands but, unlike a substitution, feeds no output. + */ + kind: + | 'root' + | 'command' + | 'arithmetic' + | 'backtick' + | 'parameter' + | 'keysubscript' + | 'arrayliteral' + | 'subshell' quote: ShellQuote parenthesisDepth: number + /** Open `[`/`]` depth of a bracketed frame — a `$[ ]`, an indexed subscript, or an associative key. */ bracketDepth?: number literalRoot: boolean + /** Inside `[[ ]]`, where only `&&` and `||` end a clause. */ + conditional?: boolean + /** On a `parameter` frame opened inside double quotes, where single quotes stay literal. */ + inDoubleQuotes?: boolean + /** On a `parameter` frame, the cursor is still at the operator right after the name. */ + atOperator?: boolean + /** On a `parameter` frame, its substring offset/length is being read as arithmetic. */ + arithmeticTail?: boolean + /** The integer attribute this frame's scope sets (`-i` → true) or clears (`+i` → false) per name. */ + integerAttribute?: Map + /** Names this frame's scope declared associative (`declare -A`), whose subscripts are string keys. */ + associativeArrays?: Set + /** On a `parameter` frame, the expanded name, so a `${name[…]}` subscript can check its array type. */ + parameterName?: string + /** On an `arrayliteral` frame, the array being assigned, so its element keys check the array type. */ + arrayName?: string + /** A substitution opened inside a non-arithmetic assignment prefix, so a keyword must not mark it. */ + prefixExcluded?: boolean + /** Opened inside an integer assignment's value, so everything here is an arithmetic operand. */ + valueOperand?: boolean + command?: ShellCommandScan + /** Earlier commands of a substitution, whose output still reaches the enclosing command. */ + endedCommands?: ShellCommandScan[] +} + +/** + * The command (or `[[ ]]` clause) being scanned in a frame. A later word can make the whole + * command arithmetic — `-eq` after its left operand, `-i` after `declare` — so the contexts + * already recorded in it, and the commands of substitutions it closed, are kept to be marked then. + */ +interface ShellCommandScan { + contexts: ShellOccurrenceContext[] + nested: ShellCommandScan[] + arithmetic: boolean + /** The declaration builtin being read, if any — only `declare`/`typeset` give a global attribute. */ + declarationBuiltin?: 'declare' | 'typeset' | 'local' + /** The integer attribute the last declaration option set (`-i`) or cleared (`+i`). */ + integerOption?: 'set' | 'clear' + /** A `declare -A` was read, so the declared names are associative (string-keyed) arrays. */ + associativeOption?: boolean + /** A `declare -a` was read, so the declared names are indexed arrays (resetting any prior type). */ + indexedOption?: boolean + /** The command is a builtin (`unset`/`read`/…) whose name arguments carry arithmetic subscripts. */ + nameArgumentBuiltin?: boolean + /** True once the command word (past any `name=value` assignment prefix) has been read. */ + sawCommandWord: boolean + /** A non-arithmetic assignment prefix is being read, so a later keyword must not mark its value. */ + inPrefixValue: boolean + /** An integer assignment's value is being read — arithmetic, but only this value, not the command. */ + valueArithmetic: boolean + /** An integer assignment was seen; its value becomes arithmetic once the `=` is passed, not its subscript. */ + pendingValue?: boolean + /** A redirect target (a filename) is being read, never an arithmetic operand of its command. */ + inRedirectTarget: boolean } interface ShellOccurrenceContext { quote: ShellQuote - /** Includes nested command substitutions whose output can become an arithmetic operand. */ + /** Its value reaches an arithmetic evaluation — a frame, or a command marked arithmetic later. */ arithmetic?: boolean unsupported?: 'escaped sequence' } +interface ShellSpan { + start: number + end: number +} + +const ARITHMETIC_CONDITIONAL_OPERATOR = /^-(?:eq|ne|lt|le|gt|ge)$/ +/** A declaration option word (`-i`, `+i`, `-A`, `-iA`, …); its letters set each attribute. */ +const DECLARATION_OPTION = /^([-+])([A-Za-z]+)$/ +/** Invocation prefixes that run the following word as the command, so they are not the command. */ +const SHELL_COMMAND_PREFIXES = new Set(['command', 'builtin', 'exec', 'nohup']) +/** + * Builtins whose every argument names a variable, so an indexed-array subscript there is arithmetic. + * Limited to `unset`, whose arguments are all names; `read`/`mapfile` mix name and option-value + * arguments (`read -p prompt name`), so their subscripts are left to a conservative compile. + */ +const NAME_ARGUMENT_BUILTINS = new Set(['unset']) +/** Reserved words that introduce a command rather than being one, so the next word is the command. */ +const SHELL_RESERVED_WORDS = new Set([ + 'if', + 'then', + 'elif', + 'else', + 'fi', + 'do', + 'done', + 'while', + 'until', + 'for', + 'select', + 'case', + 'esac', + 'time', + '!', +]) +const SHELL_WORD = /[^\s;&|()<>]+/y +const PARAMETER_NAME = /[!#]?(?:[A-Za-z_][A-Za-z0-9_]*|[0-9]+|[@*#?$!-])/y +const SHELL_NAME_SOURCE = '[A-Za-z_][A-Za-z0-9_]*' +const SHELL_NAME = new RegExp(`^${SHELL_NAME_SOURCE}`) +/** A word that is exactly a bare shell name (no subscript or suffix). */ +const SHELL_BARE_NAME = new RegExp(`^${SHELL_NAME_SOURCE}$`) +/** A `name=`, `name+=` or `name[subscript]=` assignment word; group 1 is the bare name. */ +const SHELL_ASSIGNMENT_WORD = new RegExp(`^(${SHELL_NAME_SOURCE})(?:\\[.*\\])?\\+?=`) +/** A command word ending in an unescaped backslash — a line continuation to join with the next line. */ +const TRAILING_LINE_CONTINUATION = /(?:^|[^\\])(?:\\\\)*\\$/ + function lineEndAfterNewline(code: string, start: number): number { const newline = code.indexOf('\n', start) return newline === -1 ? code.length : newline + 1 @@ -63,16 +177,25 @@ function logicalLineEndAfterContinuations(code: string, start: number): number { return end } -function shellCommentStarts(code: string, index: number): boolean { - if (code[index] !== '#') return false +function shellWordStarts(code: string, index: number): boolean { const previous = code[index - 1] return previous === undefined || /\s|[;&|()<>]/.test(previous) } +function shellCommentStarts(code: string, index: number): boolean { + return code[index] === '#' && shellWordStarts(code, index) +} + function shellArithmeticCommandStarts(code: string, index: number): boolean { - if (code[index] !== '(' || code[index + 1] !== '(') return false - const previous = code[index - 1] - return previous === undefined || /\s|[;&|()<>]/.test(previous) + return code[index] === '(' && code[index + 1] === '(' && shellWordStarts(code, index) +} + +/** `;`, `&` and `|` end a command unless they belong to a redirection such as `>&2` or `&>`. */ +function shellCommandSeparator(code: string, index: number): boolean { + const character = code[index] + if (character === '\n') return true + if (character !== ';' && character !== '&' && character !== '|') return false + return !/[<>]/.test(code[index - 1] ?? '') && !(character === '&' && code[index + 1] === '>') } function decodeAnsiCCharacter(code: string, index: number): { value: string; end: number } { @@ -424,7 +547,7 @@ function shellExpansion(name: string, quote: ShellQuote): string { function isLegacyShellPlaceholder(occurrence: CodePlaceholderOccurrence): boolean { const inner = occurrence.raw.slice(2, -2) - return inner.trim() === occurrence.name && /^[A-Za-z_][A-Za-z0-9_]*$/.test(occurrence.name) + return inner.trim() === occurrence.name && SHELL_BARE_NAME.test(occurrence.name) } function blankPreservingLines(value: string): string { @@ -470,30 +593,347 @@ function heredocBodyRanges(heredocs: HeredocDeclaration[]): Array<[number, numbe return heredocs.map((heredoc) => [heredoc.bodyStart, heredoc.removalEnd]) } +/** + * Removes shell quoting from a word so a keyword, option or name is recognized as bash would after + * quote removal — `declare "-i"` and `'let'` match. Values keep their own quoting; this only feeds + * the command-position checks, never a placeholder's emitted expansion. + */ +function unquoteShellWord(word: string): string { + return word.replace(/'([^']*)'|"((?:[^"\\]|\\.)*)"|\\(.)/g, (_match, single, double, escaped) => + single !== undefined ? single : double !== undefined ? double : escaped + ) +} + +/** Frames whose text is a command line, where words are classified and separators end a command. */ +function readsCommands(frame: ShellScanFrame): boolean { + return ( + frame.kind === 'root' || + frame.kind === 'command' || + frame.kind === 'backtick' || + frame.kind === 'subshell' + ) +} + +/** A command substitution — `$( )` or backticks — whose output the enclosing command receives. */ +function isSubstitution(frame: ShellScanFrame): boolean { + return frame.kind === 'command' || frame.kind === 'backtick' +} + +/** + * Whether a frame puts the cursor in arithmetic — an arithmetic expansion, an integer-value operand, + * a `${…}` substring tail, or a command marked arithmetic. `arithmeticDepth` counts these as they + * open and close; this predicate reads the same set for a point-in-time check of the frame stack. + */ +function frameIsArithmetic(frame: ShellScanFrame): boolean { + return ( + frame.kind === 'arithmetic' || + frame.valueOperand === true || + (frame.kind === 'parameter' && frame.arithmeticTail === true) || + frame.command?.arithmetic === true + ) +} + +/** Where a `name=…` word is an assignment: a command prefix, or an argument to a declaration builtin. */ +function inAssignmentPosition(command: ShellCommandScan): boolean { + return !command.sawCommandWord || command.declarationBuiltin !== undefined +} + +function commandScanOf(frame: ShellScanFrame): ShellCommandScan { + frame.command ??= { + contexts: [], + nested: [], + arithmetic: false, + sawCommandWord: false, + inPrefixValue: false, + valueArithmetic: false, + inRedirectTarget: false, + } + return frame.command +} + +/** + * A `[`/`]`-nesting frame: `arithmetic` for a `$[ ]` or an indexed-array subscript (whose index is + * evaluated), `keysubscript` for an associative array's `[key]` (a string, scanned but not arithmetic). + */ +function subscriptFrame(kind: 'arithmetic' | 'keysubscript'): ShellScanFrame { + return { kind, quote: 'none', parenthesisDepth: 0, bracketDepth: 1, literalRoot: false } +} + +/** A `( )`-delimited command frame: a `$( )` command substitution, or a bare `( )` subshell. */ +function parenCommandFrame(kind: 'command' | 'subshell'): ShellScanFrame { + return { kind, quote: 'none', parenthesisDepth: 1, literalRoot: false } +} + +/** + * Whether the `name[...]` opening at `open` is an indexed assignment (`a[i]=`, `a[i]+=`), whose + * subscript bash evaluates as arithmetic. Brackets nest so `a[b[0]]=` pairs correctly; the scan + * stops at the first unbracketed word boundary, since an unquoted assignment word cannot cross it. + */ +function opensIndexedAssignment(code: string, open: number, end: number): boolean { + let depth = 0 + for (let index = open; index < end; index += 1) { + const character = code[index] + if (character === '[') depth += 1 + else if (character === ']') { + depth -= 1 + if (depth === 0) { + const after = code[index + 1] + return after === '=' || (after === '+' && code[index + 2] === '=') + } + } else if (depth === 0 && /[\s;&|()<>]/.test(character)) return false + } + return false +} + /** * Jumps over `skippedRanges` (sorted heredoc bodies, which bash reads as data) so body prose * cannot shift quote context; bodies that need contexts are scanned on their own with `literalRoot`. + * Spans other than placeholders (heredoc operators) get a context too, for where their body lands. */ -function collectShellOccurrenceContexts( +function collectShellOccurrenceContexts( code: string, - occurrences: CodePlaceholderOccurrence[], + occurrences: readonly T[], start: number, end: number, literalRoot: boolean, - skippedRanges: Array<[number, number]> = [] -): Map { + skippedRanges: Array<[number, number]> = [], + operatorStarts: ReadonlySet = new Set() +): Map { const occurrenceByStart = new Map( occurrencesWithin(occurrences, start, end).map( (occurrence) => [occurrence.start, occurrence] as const ) ) - const contexts = new Map() + const contexts = new Map() const frames: ShellScanFrame[] = [ { kind: 'root', quote: 'none', parenthesisDepth: 0, literalRoot }, ] let skippedRangeIndex = 0 + /** + * How many open frames put the cursor in arithmetic: `$(( ))`/`(( ))`/`$[ ]`/subscript frames, + * integer-value operands, `${…}` substring tails, and commands marked arithmetic. Kept as a count + * so each placeholder reads it in O(1) rather than walking the whole frame stack. + */ let arithmeticDepth = 0 + const pushFrame = (frame: ShellScanFrame) => { + const enclosing = frames.at(-1) + if (enclosing?.command?.inPrefixValue) frame.prefixExcluded = true + if (enclosing?.command?.valueArithmetic) frame.valueOperand = true + frames.push(frame) + if (frame.kind === 'arithmetic' || frame.valueOperand) arithmeticDepth += 1 + } + /** + * A closed substitution or `${…}` passes its recorded values to the enclosing command, so a later + * `-eq`/`let`/`-i` still reaches a value read inside it — unless it was part of a non-arithmetic + * assignment prefix, which a keyword must not reach. + */ + const popFrame = () => { + const closed = frames.pop() + if (!closed) return + if (closed.kind === 'arithmetic' || closed.valueOperand) arithmeticDepth -= 1 + if (closed.kind === 'parameter' && closed.arithmeticTail) arithmeticDepth -= 1 + if (closed.command?.arithmetic) arithmeticDepth -= 1 + const enclosing = frames.at(-1) + // Only arithmetic frames have no values to pass up; every other closable frame is a command + // substitution or a `${…}`, whose values the enclosing command may still mark. + if (!enclosing || closed.kind === 'arithmetic' || closed.prefixExcluded) return + const enclosingCommand = commandScanOf(enclosing) + for (const ended of closed.endedCommands ?? []) enclosingCommand.nested.push(ended) + if (closed.command) enclosingCommand.nested.push(closed.command) + } + /** Whether `name` has the integer attribute here: the nearest scope that sets or clears it wins. */ + const declaresInteger = (name: string): boolean => { + for (let depth = frames.length - 1; depth >= 0; depth -= 1) { + const attribute = frames[depth].integerAttribute?.get(name) + if (attribute !== undefined) return attribute + } + return false + } + /** Whether `name` is a `declare -A` associative array, whose subscript is a string key, not arithmetic. */ + const isAssociativeArray = (name: string) => + frames.some((frame) => frame.associativeArrays?.has(name)) + /** A subscript frame for `name`'s array: a string key for an associative array, else an arithmetic index. */ + const subscriptFrameFor = (name: string | undefined) => + subscriptFrame(isAssociativeArray(name ?? '') ? 'keysubscript' : 'arithmetic') + /** Forget a name's tracked type in every scope — on `unset`, or a re-declaration that changes it. */ + const clearNameType = (name: string) => { + for (const frame of frames) { + frame.associativeArrays?.delete(name) + frame.integerAttribute?.delete(name) + } + } + /** Whether a `>`/`<` redirect operator immediately precedes `at`, across any intervening blanks. */ + const precededByRedirect = (at: number): boolean => { + let cursor = at - 1 + while (cursor >= start && (code[cursor] === ' ' || code[cursor] === '\t')) cursor -= 1 + return code[cursor] === '>' || code[cursor] === '<' + } + /** The array name immediately before the subscript `[` at `bracket`, if a valid identifier precedes it. */ + const arrayNameBefore = (bracket: number): { name: string; start: number } | undefined => { + let nameStart = bracket - 1 + while (nameStart >= start && /[A-Za-z0-9_]/.test(code[nameStart])) nameStart -= 1 + nameStart += 1 + if (nameStart >= bracket || !/[A-Za-z_]/.test(code[nameStart])) return undefined + return { name: code.slice(nameStart, bracket), start: nameStart } + } + /** Inside a double-quoted `${…}` single quotes are literal, so the binding keeps its double-quoting. */ + const effectiveQuote = (frame: ShellScanFrame): ShellQuote => + frame.kind === 'parameter' && frame.quote === 'none' + ? frame.inDoubleQuotes + ? 'double' + : 'none' + : frame.quote + const endCommand = (frame: ShellScanFrame) => { + if (!frame.command) return + if (frame.command.arithmetic) arithmeticDepth -= 1 + else if (frame.kind !== 'root') (frame.endedCommands ??= []).push(frame.command) + frame.command = undefined + } + /** True wherever a placeholder's value would be re-read as arithmetic at this point in the scan. */ + const inArithmetic = () => arithmeticDepth > 0 + const record = (occurrence: T, occurrenceContext: ShellOccurrenceContext) => { + contexts.set(occurrence, occurrenceContext) + const innermost = frames.at(-1) + // A non-arithmetic assignment prefix's value, and a redirect target, are left out so a later + // keyword cannot mark them arithmetic. + if (innermost && !innermost.command?.inPrefixValue && !innermost.command?.inRedirectTarget) + commandScanOf(innermost).contexts.push(occurrenceContext) + } + /** Marks the frame's command, and every context already recorded in it, as arithmetic. */ + const markCommandArithmetic = (frame: ShellScanFrame) => { + const command = commandScanOf(frame) + if (command.arithmetic) return + command.arithmetic = true + arithmeticDepth += 1 + const pending = [command] + for (let scan = pending.pop(); scan; scan = pending.pop()) { + for (const commandContext of scan.contexts) commandContext.arithmetic = true + for (const nested of scan.nested) pending.push(nested) + scan.contexts = [] + scan.nested = [] + } + } + /** + * Reads one word at a command boundary and applies the syntax that depends on command position. + * `let`, `[[` and the declaration builtins are keywords only as the command name — past any + * `name=value` prefix, invocation prefix (`command`/`builtin`) or reserved word, and never a + * redirect target — so an argument of the same spelling (`echo let …`) is left alone. The `[[` + * operators and the declaration `-i` option are read wherever they appear; an assignment to an + * integer-declared name counts only in assignment position. + */ + const scanWord = (frame: ShellScanFrame, index: number) => { + SHELL_WORD.lastIndex = index + const raw = SHELL_WORD.exec(code)?.[0] + if (raw === undefined) return + // Join line continuations so a split command name is still recognized (`le\t` → `let`). + // `SHELL_WORD` stops at the newline, so each continued segment is read and appended here. + let joined = raw + let after = index + raw.length + while ( + TRAILING_LINE_CONTINUATION.test(joined) && + (code[after] === '\n' || code[after] === '\r') + ) { + joined = joined.slice(0, -1) + after += code[after] === '\r' && code[after + 1] === '\n' ? 2 : 1 + SHELL_WORD.lastIndex = after + const next = SHELL_WORD.exec(code) + if (!next || next.index !== after) break + joined += next[0] + after += next[0].length + } + const word = unquoteShellWord(joined) + const command = commandScanOf(frame) + command.valueArithmetic = false + command.pendingValue = false + command.inPrefixValue = false + command.inRedirectTarget = false + // `]]` and the `[[` comparison operators are syntax only unquoted; a quoted `"]]"` or `"-eq"` is + // a string operand, so these read the raw word, not the quote-stripped one. + if (raw === ']]') { + frame.conditional = false + return + } + if (frame.conditional) { + if (ARITHMETIC_CONDITIONAL_OPERATOR.test(raw)) markCommandArithmetic(frame) + return + } + // A redirect target (possibly after whitespace, `> file`) or leading file descriptor is not the + // command word, and is a filename — never an arithmetic operand even when the command does + // arithmetic (`let x=1 >file`), so its placeholders are flagged out of the command's marking. + if (precededByRedirect(index)) { + command.inRedirectTarget = true + return + } + if ( + /^\d+$/.test(word) && + (code[index + raw.length] === '>' || code[index + raw.length] === '<') + ) + return + // Declaration options, read together so a combined `-iA` sets both: the integer attribute (`-i` + // sets, `+i` clears — last wins, and `-i` is not marked until a value appears so `+i` can still + // undo it) and `-A` (an associative array, whose subscripts are string keys, not arithmetic). + if (command.declarationBuiltin) { + const option = DECLARATION_OPTION.exec(word) + if (option) { + if (option[2].includes('i')) command.integerOption = option[1] === '-' ? 'set' : 'clear' + if (option[1] === '-' && option[2].includes('A')) command.associativeOption = true + if (option[1] === '-' && option[2].includes('a')) command.indexedOption = true + return + } + } + // A declaration argument names an integer (`-i`), clears one (`+i`), and/or names an associative + // array (`-A`); the name lives in this frame's scope and pops with it. `local` is function-scoped + // and this scanner has no function frame to drop it with, so its attributes are not tracked across + // statements — same-command arithmetic still marks, and an untracked array stays indexed (a + // conservative reject) rather than leaking out as text. + if ( + command.declarationBuiltin && + command.declarationBuiltin !== 'local' && + word[0] !== '-' && + word[0] !== '+' + ) { + const declared = SHELL_NAME.exec(word)?.[0] + if (declared) { + // A re-declaration resets the name's type before this one's attributes apply, so a later + // `declare -a` (indexed) clears an earlier `-A` (associative) and vice versa. + if (command.associativeOption || command.indexedOption) clearNameType(declared) + if (command.associativeOption) (frame.associativeArrays ??= new Set()).add(declared) + if (command.integerOption) { + ;(frame.integerAttribute ??= new Map()).set(declared, command.integerOption === 'set') + } + } + } + // `unset name` removes the variable and its attributes, so a later indexed reuse is arithmetic + // again. A bare name only — `unset name[i]` removes one element, not the array's type. + if (command.nameArgumentBuiltin && command.sawCommandWord && SHELL_BARE_NAME.test(word)) { + clearNameType(word) + } + // An assignment is one only in command-prefix or declaration-argument position; `echo n=1` or + // `printf a[i]=1` passes an ordinary string that bash never evaluates. + const assignment = inAssignmentPosition(command) ? SHELL_ASSIGNMENT_WORD.exec(word) : null + if (assignment) { + // A `declare -i` argument, or any assignment to an already-integer name not cleared here, has + // an arithmetic value. A declaration marks the whole command (every arg shares the attribute); + // a plain prefix marks only this value, so `n=1 printf "{{x}}"` leaves the printed arg as text. + const integerTarget = + (command.declarationBuiltin !== undefined && command.integerOption === 'set') || + (command.integerOption !== 'clear' && declaresInteger(assignment[1])) + if (integerTarget && command.declarationBuiltin) markCommandArithmetic(frame) + else if (integerTarget) command.pendingValue = true + else if (!command.sawCommandWord) command.inPrefixValue = true + return + } + if (command.sawCommandWord) return + if (SHELL_RESERVED_WORDS.has(word) || SHELL_COMMAND_PREFIXES.has(word)) return + command.sawCommandWord = true + if (word === '[[') frame.conditional = true + else if (word === 'let') markCommandArithmetic(frame) + else if (word === 'declare' || word === 'typeset' || word === 'local') { + command.declarationBuiltin = word + } else if (NAME_ARGUMENT_BUILTINS.has(word)) command.nameArgumentBuiltin = true + } + for (let index = start; index < end; ) { const frame = frames.at(-1) if (!frame) break @@ -511,13 +951,112 @@ function collectShellOccurrenceContexts( } const occurrence = occurrenceByStart.get(index) + if (occurrence && operatorStarts.has(occurrence.start)) { + // A heredoc operator: its body is an arithmetic operand when the substitution reading it is in + // an arithmetic context — an enclosing `$(( ))`, or an enclosing command/value that evaluates + // the substitution's output (`let x="$(cat <= 0 && !isSubstitution(frames[sub])) sub -= 1 + const enclosingArithmetic = frames + .slice(0, sub) + .some((enclosing) => frameIsArithmetic(enclosing) || enclosing.command?.valueArithmetic) + const operatorContext: ShellOccurrenceContext = { + quote: 'none', + arithmetic: enclosingArithmetic, + } + contexts.set(occurrence, operatorContext) + if (sub >= 0) commandScanOf(frames[sub - 1]).contexts.push(operatorContext) + index = occurrence.end + continue + } if (occurrence) { - contexts.set(occurrence, { quote: frame.quote, arithmetic: arithmeticDepth > 0 }) + // A placeholder that is itself the redirect target (`> {{x}}`) is a filename, never arithmetic; + // `scanWord` never ran on it (the occurrence is consumed first), so the immediately preceding + // `>`/`<` — across any whitespace — is checked here, but only in a command frame: inside `$(( ))` + // a `<`/`>` is a comparison operator, not a redirect. + const redirectTarget = + frame.command?.inRedirectTarget === true || + (readsCommands(frame) && precededByRedirect(index)) + const arithmetic = + !redirectTarget && (inArithmetic() || frame.command?.valueArithmetic === true) + record(occurrence, { quote: effectiveQuote(frame), arithmetic }) index = occurrence.end continue } const character = code[index] + // The redirect-target flag covers only its one word; unquoted whitespace ends that word, so a + // following argument (`let x=1 >/dev/null {{x}}`) is marked by the command again. + if (frame.command?.inRedirectTarget && frame.quote === 'none' && /\s/.test(character)) { + frame.command.inRedirectTarget = false + } + // Classify each command word before the quote branches consume a quote-initial word such as + // `"-i"` or `'let'`. `scanWord` only sets state, never advances `index`; the characters below + // still process the word's text. + if ( + readsCommands(frame) && + !frame.literalRoot && + frame.quote === 'none' && + shellWordStarts(code, index) && + !shellCommentStarts(code, index) && + !shellCommandSeparator(code, index) + ) { + scanWord(frame, index) + } + // An integer assignment's value turns arithmetic at its `=` — not its subscript, so an index + // placeholder in `m[{{x}}]=1` is judged by the array type, not by the value's attribute. + if (character === '=' && frame.command?.pendingValue) { + frame.command.valueArithmetic = true + frame.command.pendingValue = false + } + // A name-taking builtin's argument subscript (`unset a[i]`, `unset "a[i]"`, `unset 'a[i]'`) is + // arithmetic — even single-quoted, since the subscript is still evaluated. This runs before the + // quote branches, which would otherwise consume the `[` of a quoted argument. + if ( + character === '[' && + !frame.literalRoot && + readsCommands(frame) && + frame.command?.nameArgumentBuiltin && + frame.command.sawCommandWord + ) { + const named = arrayNameBefore(index) + if ( + named && + /[\s;&|()<>"']/.test(code[named.start - 1] ?? ' ') && + !isAssociativeArray(named.name) + ) { + pushFrame(subscriptFrame('arithmetic')) + index += 1 + continue + } + } + // A `${…}` expansion, including one nested in another's value or an associative key. Only an + // `arithmetic` frame reads a nested `${b}` as operand text rather than its own expansion, so the + // gate excludes it; a `keysubscript` (string key) does take nested expansions. + if ( + character === '$' && + code[index + 1] === '{' && + frame.kind !== 'arithmetic' && + (frame.quote === 'none' || frame.quote === 'double') + ) { + PARAMETER_NAME.lastIndex = index + 2 + const name = PARAMETER_NAME.exec(code) + if (name) { + pushFrame({ + kind: 'parameter', + quote: 'none', + inDoubleQuotes: effectiveQuote(frame) === 'double', + parenthesisDepth: 0, + atOperator: true, + parameterName: name[0].replace(/^[!#]/, ''), + literalRoot: false, + }) + index += 2 + name[0].length + continue + } + } if (frame.quote === 'single') { if (character === "'") frame.quote = 'none' index += 1 @@ -527,7 +1066,7 @@ function collectShellOccurrenceContexts( if (character === '\\') { const escaped = occurrenceByStart.get(index + 1) if (escaped) { - contexts.set(escaped, { quote: frame.quote, unsupported: 'escaped sequence' }) + record(escaped, { quote: frame.quote, unsupported: 'escaped sequence' }) index = escaped.end } else { index += 2 @@ -545,14 +1084,11 @@ function collectShellOccurrenceContexts( frame.quote === 'none' && !frame.literalRoot && shellArithmeticCommandStarts(code, index) if (arithmeticExpansion || arithmeticCommand) { const brackets = arithmeticExpansion && code[index + 1] === '[' - frames.push({ - kind: 'arithmetic', - quote: 'none', - parenthesisDepth: brackets ? 0 : 2, - ...(brackets ? { bracketDepth: 1 } : {}), - literalRoot: false, - }) - arithmeticDepth += 1 + pushFrame( + brackets + ? subscriptFrame('arithmetic') + : { kind: 'arithmetic', quote: 'none', parenthesisDepth: 2, literalRoot: false } + ) index += arithmeticExpansion && !brackets ? 3 : 2 continue } @@ -560,7 +1096,7 @@ function collectShellOccurrenceContexts( if (character === '\\') { const escaped = occurrenceByStart.get(index + 1) if (escaped) { - contexts.set(escaped, { quote: frame.quote, unsupported: 'escaped sequence' }) + record(escaped, { quote: frame.quote, unsupported: 'escaped sequence' }) index = escaped.end } else { index += 2 @@ -569,20 +1105,10 @@ function collectShellOccurrenceContexts( frame.quote = 'none' index += 1 } else if (character === '$' && code[index + 1] === '(') { - frames.push({ - kind: 'command', - quote: 'none', - parenthesisDepth: 1, - literalRoot: false, - }) + pushFrame(parenCommandFrame('command')) index += 2 } else if (character === '`') { - frames.push({ - kind: 'backtick', - quote: 'none', - parenthesisDepth: 0, - literalRoot: false, - }) + pushFrame({ kind: 'backtick', quote: 'none', parenthesisDepth: 0, literalRoot: false }) index += 1 } else { index += 1 @@ -591,11 +1117,13 @@ function collectShellOccurrenceContexts( } if (frame.kind === 'backtick' && character === '`') { - frames.pop() + popFrame() index += 1 continue } - if (frame.kind !== 'arithmetic' && !frame.literalRoot && shellCommentStarts(code, index)) { + // `#` starts a comment only where commands are read; inside a `${…}` or a subscript it is literal. + if (readsCommands(frame) && !frame.literalRoot && shellCommentStarts(code, index)) { + if (!frame.conditional) endCommand(frame) const newline = code.indexOf('\n', index) index = newline === -1 || newline >= end ? end : newline + 1 continue @@ -603,19 +1131,26 @@ function collectShellOccurrenceContexts( if (character === '\\') { const escaped = occurrenceByStart.get(index + 1) if (escaped) { - contexts.set(escaped, { quote: frame.quote, unsupported: 'escaped sequence' }) + record(escaped, { quote: frame.quote, unsupported: 'escaped sequence' }) index = escaped.end } else { index += 2 } continue } - if (!frame.literalRoot && character === '$' && code[index + 1] === "'") { + // Inside a double-quoted `${…}`, single quotes are literal text, not a quote boundary. + const singleQuotesLiteral = frame.kind === 'parameter' && frame.inDoubleQuotes === true + if ( + !frame.literalRoot && + !singleQuotesLiteral && + character === '$' && + code[index + 1] === "'" + ) { frame.quote = 'ansi' index += 2 continue } - if (!frame.literalRoot && character === "'") { + if (!frame.literalRoot && !singleQuotesLiteral && character === "'") { frame.quote = 'single' index += 1 continue @@ -626,48 +1161,166 @@ function collectShellOccurrenceContexts( continue } if (character === '$' && code[index + 1] === '(') { - frames.push({ - kind: 'command', - quote: 'none', - parenthesisDepth: 1, - literalRoot: false, - }) + pushFrame(parenCommandFrame('command')) index += 2 continue } if (character === '`') { - frames.push({ - kind: 'backtick', - quote: 'none', - parenthesisDepth: 0, - literalRoot: false, - }) + pushFrame({ kind: 'backtick', quote: 'none', parenthesisDepth: 0, literalRoot: false }) + index += 1 + continue + } + // A `${…}` expansion's arithmetic operands appear only at the operator right after the name: a + // `[` subscript, or a `:` substring offset/length. Any other operator (`:-`, `#`, `/`, …) means + // the rest is pattern or default text, where a later `[` or `:` is literal. + if (frame.kind === 'parameter') { + if (frame.atOperator) { + // A subscript: an indexed array's is arithmetic, an associative array's is a string key. + // Either way it opens a bracket frame so a `:offset` after the `]` is still seen as a tail. + if (character === '[') { + pushFrame(subscriptFrameFor(frame.parameterName)) + index += 1 + continue + } + if (character === ':' && !/[-=?+]/.test(code[index + 1] ?? '')) { + frame.arithmeticTail = true + arithmeticDepth += 1 + frame.atOperator = false + index += 1 + continue + } + frame.atOperator = false + } + // A nested `${…}` opens its own frame above; here a bare `{` is literal text, and the first + // unmatched `}` ends the expansion — so only `}` closes it, never a counted `{`. + if (character === '}') { + popFrame() + index += 1 + continue + } + } + // A compound array assignment `name=( … )`: each element key `[k]=` is a subscript of `name`. + if (frame.kind === 'arrayliteral') { + if (character === '(') { + frame.parenthesisDepth += 1 + index += 1 + continue + } + if (character === ')') { + frame.parenthesisDepth -= 1 + if (frame.parenthesisDepth === 0) popFrame() + index += 1 + continue + } + // A key assignment `[k]=` / `[k]+=`; a bare `[x]` element (no `=`) is ordinary text. + if ( + character === '[' && + /[\s(]/.test(code[index - 1] ?? ' ') && + opensIndexedAssignment(code, index, end) + ) { + pushFrame(subscriptFrameFor(frame.arrayName)) + index += 1 + continue + } + } + // The `(` opening a compound array assignment (`name=(`, `name+=(`) in command-prefix or + // declaration position. Its element keys are then judged against the array's type. + if ( + character === '(' && + code[index - 1] === '=' && + frame.quote === 'none' && + !frame.literalRoot && + readsCommands(frame) && + inAssignmentPosition(commandScanOf(frame)) + ) { + let nameEnd = index - 1 + if (code[nameEnd - 1] === '+') nameEnd -= 1 + const named = arrayNameBefore(nameEnd) + if (named && shellWordStarts(code, named.start)) { + pushFrame({ + kind: 'arrayliteral', + quote: 'none', + parenthesisDepth: 1, + arrayName: named.name, + literalRoot: false, + }) + index += 1 + continue + } + } + // An indexed-assignment subscript (`a[i]=`, `a[i]+=`) is evaluated as arithmetic — but only in + // command-prefix or declaration-argument position; `printf a[i]=1` passes an ordinary string, and + // an associative key (`declare -A m; m[k]=`) is text. + if ( + character === '[' && + frame.quote === 'none' && + !frame.literalRoot && + readsCommands(frame) && + inAssignmentPosition(commandScanOf(frame)) + ) { + const named = arrayNameBefore(index) + if ( + named && + shellWordStarts(code, named.start) && + !isAssociativeArray(named.name) && + opensIndexedAssignment(code, index, end) + ) { + pushFrame(subscriptFrame('arithmetic')) + index += 1 + continue + } + } + if (readsCommands(frame)) { + if (!frame.literalRoot) { + const groupBrace = (character === '{' || character === '}') && shellWordStarts(code, index) + if (shellCommandSeparator(code, index) || groupBrace) { + const clauseEnds = + !frame.conditional || + ((character === '&' || character === '|') && code[index + 1] === character) + if (clauseEnds) endCommand(frame) + } + } + } + // A bare subshell `( … )` at a command position: scope its declarations to a frame so a + // `(declare -A m)` inside does not leak the array type to the enclosing shell. + if ( + character === '(' && + readsCommands(frame) && + !frame.conditional && + frame.quote === 'none' && + !frame.literalRoot && + shellWordStarts(code, index) + ) { + pushFrame(parenCommandFrame('subshell')) index += 1 continue } - if (frame.kind === 'arithmetic' && frame.bracketDepth !== undefined) { + if ( + (frame.kind === 'arithmetic' || frame.kind === 'keysubscript') && + frame.bracketDepth !== undefined + ) { if (character === '[') frame.bracketDepth += 1 - if (character === ']') { + else if (character === ']') { frame.bracketDepth -= 1 - if (frame.bracketDepth === 0) { - frames.pop() - arithmeticDepth -= 1 - } + if (frame.bracketDepth === 0) popFrame() } index += 1 continue } - if ((frame.kind === 'command' || frame.kind === 'arithmetic') && character === '(') { + if ( + (frame.kind === 'command' || frame.kind === 'arithmetic' || frame.kind === 'subshell') && + character === '(' + ) { frame.parenthesisDepth += 1 index += 1 continue } - if ((frame.kind === 'command' || frame.kind === 'arithmetic') && character === ')') { + if ( + (frame.kind === 'command' || frame.kind === 'arithmetic' || frame.kind === 'subshell') && + character === ')' + ) { frame.parenthesisDepth -= 1 - if (frame.parenthesisDepth === 0) { - frames.pop() - if (frame.kind === 'arithmetic') arithmeticDepth -= 1 - } + if (frame.parenthesisDepth === 0) popFrame() index += 1 continue } @@ -788,36 +1441,91 @@ export async function compileShellPlaceholders( validateShellValue(occurrence, context.resolve(occurrence)) const resolveShellValue = (occurrence: CodePlaceholderOccurrence) => validateShellValue(occurrence, context.resolveValue(occurrence)) + /** A placeholder with no value stays as written; analysis still discovers one with a value. */ + const rejectUnsupported = (occurrence: CodePlaceholderOccurrence, position: string) => { + if (!context.hasValue(occurrence.name)) return + if (input.analysisOnly) { + context.resolveValue(occurrence) + return + } + throw new CodePlaceholderCompileError( + `Variable placeholder "${occurrence.name}" is not supported ${position}`, + input.code, + occurrence.start + ) + } + /** Heredoc bodies are data, so only code outside them can name an assignment target. */ + const resolveInContext = ( + occurrence: CodePlaceholderOccurrence, + occurrenceContext: ShellOccurrenceContext | undefined, + inCode: boolean + ): SourceEdit | undefined => { + if (!occurrenceContext) return undefined + if (occurrenceContext.unsupported) { + rejectUnsupported(occurrence, 'in an escaped shell sequence') + return undefined + } + const unsupportedPosition = + getUnsupportedShellPosition(input.code, occurrence, occurrenceContext) ?? + (inCode && occurrenceContext.quote === 'none' && isShellAssignmentName(input.code, occurrence) + ? 'as a shell assignment name' + : undefined) + if (unsupportedPosition) { + rejectUnsupported(occurrence, unsupportedPosition) + return undefined + } + const resolved = resolveShellOccurrence(occurrence) + return { + start: occurrence.start, + end: occurrence.end, + text: resolved ? shellExpansion(resolved.bindingName, occurrenceContext.quote) : '', + } + } const heredocs = collectHeredocs(input.code) const shellOccurrences = context.occurrences.filter(isLegacyShellPlaceholder) + const isExcluded = createOffsetRangeLookup( + heredocs.flatMap( + (heredoc): Array<[number, number]> => [ + [heredoc.operatorStart, heredoc.operatorEnd], + [heredoc.bodyStart, heredoc.removalEnd], + ] + ) + ) + const rootOccurrences = shellOccurrences.filter((occurrence) => !isExcluded(occurrence.start)) + /** A heredoc operator's context is where its body lands, so it is scanned like a placeholder. */ + const heredocOperators = heredocs.map( + (heredoc): ShellSpan => ({ start: heredoc.operatorStart, end: heredoc.operatorEnd }) + ) + const rootContexts = collectShellOccurrenceContexts( + input.code, + [...rootOccurrences, ...heredocOperators].sort((left, right) => left.start - right.start), + 0, + input.code.length, + false, + heredocBodyRanges(heredocs), + new Set(heredocOperators.map((operator) => operator.start)) + ) const edits: SourceEdit[] = [] - const excludedRanges: Array<[number, number]> = [] - - for (const heredoc of heredocs) { - excludedRanges.push([heredoc.operatorStart, heredoc.operatorEnd]) - excludedRanges.push([heredoc.bodyStart, heredoc.removalEnd]) + for (const [heredocIndex, heredoc] of heredocs.entries()) { const delimiterOccurrences = occurrencesWithin( shellOccurrences, heredoc.operatorStart, heredoc.operatorEnd ) for (const occurrence of delimiterOccurrences) { - if (context.hasValue(occurrence.name)) { - if (input.analysisOnly) { - context.resolveValue(occurrence) - continue - } - throw new CodePlaceholderCompileError( - `Variable placeholder "${occurrence.name}" is not supported in a shell heredoc delimiter`, - input.code, - occurrence.start - ) - } + rejectUnsupported(occurrence, 'in a shell heredoc delimiter') } const bodyOccurrences = occurrencesWithin(shellOccurrences, heredoc.bodyStart, heredoc.bodyEnd) + // The operator's context already carries this: `arithmetic` is set only for a heredoc whose + // reading substitution is itself an arithmetic operand (see where operator spans are recorded), + // so a quoted or unquoted body there is rejected, while plain stdin falls to the body scan. + if (rootContexts.get(heredocOperators[heredocIndex])?.arithmetic) { + for (const occurrence of bodyOccurrences) rejectUnsupported(occurrence, 'in shell arithmetic') + continue + } if (heredoc.quoted) { const bodyEdits: SourceEdit[] = [] let hasResolvedPlaceholder = false @@ -868,116 +1576,14 @@ export async function compileShellPlaceholders( true ) for (const occurrence of bodyOccurrences) { - const occurrenceContext = bodyContexts.get(occurrence) - if (!occurrenceContext) continue - if (occurrenceContext.unsupported) { - if (context.hasValue(occurrence.name)) { - if (input.analysisOnly) { - context.resolveValue(occurrence) - continue - } - throw new CodePlaceholderCompileError( - `Variable placeholder "${occurrence.name}" is not supported in an escaped shell sequence`, - input.code, - occurrence.start - ) - } - continue - } - const unsupportedPosition = getUnsupportedShellPosition( - input.code, - occurrence, - occurrenceContext - ) - if (unsupportedPosition) { - if (context.hasValue(occurrence.name)) { - if (input.analysisOnly) { - context.resolveValue(occurrence) - continue - } - throw new CodePlaceholderCompileError( - `Variable placeholder "${occurrence.name}" is not supported ${unsupportedPosition}`, - input.code, - occurrence.start - ) - } - continue - } - const resolved = resolveShellOccurrence(occurrence) - edits.push({ - start: occurrence.start, - end: occurrence.end, - text: resolved ? shellExpansion(resolved.bindingName, occurrenceContext.quote) : '', - }) + const edit = resolveInContext(occurrence, bodyContexts.get(occurrence), false) + if (edit) edits.push(edit) } } - const isExcluded = createOffsetRangeLookup(excludedRanges) - const rootOccurrences = shellOccurrences.filter((occurrence) => !isExcluded(occurrence.start)) - const rootContexts = collectShellOccurrenceContexts( - input.code, - rootOccurrences, - 0, - input.code.length, - false, - heredocBodyRanges(heredocs) - ) for (const occurrence of rootOccurrences) { - const occurrenceContext = rootContexts.get(occurrence) - if (!occurrenceContext) continue - if (occurrenceContext.unsupported) { - if (context.hasValue(occurrence.name)) { - if (input.analysisOnly) { - context.resolveValue(occurrence) - continue - } - throw new CodePlaceholderCompileError( - `Variable placeholder "${occurrence.name}" is not supported in an escaped shell sequence`, - input.code, - occurrence.start - ) - } - continue - } - const unsupportedPosition = getUnsupportedShellPosition( - input.code, - occurrence, - occurrenceContext - ) - if (unsupportedPosition) { - if (context.hasValue(occurrence.name)) { - if (input.analysisOnly) { - context.resolveValue(occurrence) - continue - } - throw new CodePlaceholderCompileError( - `Variable placeholder "${occurrence.name}" is not supported ${unsupportedPosition}`, - input.code, - occurrence.start - ) - } - continue - } - if (occurrenceContext.quote === 'none' && isShellAssignmentName(input.code, occurrence)) { - if (context.hasValue(occurrence.name)) { - if (input.analysisOnly) { - context.resolveValue(occurrence) - continue - } - throw new CodePlaceholderCompileError( - `Variable placeholder "${occurrence.name}" is not supported as a shell assignment name`, - input.code, - occurrence.start - ) - } - continue - } - const resolved = resolveShellOccurrence(occurrence) - edits.push({ - start: occurrence.start, - end: occurrence.end, - text: resolved ? shellExpansion(resolved.bindingName, occurrenceContext.quote) : '', - }) + const edit = resolveInContext(occurrence, rootContexts.get(occurrence), true) + if (edit) edits.push(edit) } return context.finish(applySourceEdits(input.code, edits)) From f6c87bafc3012c1eb68893f577fb2e6bfef74ea9 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 19:34:58 -0700 Subject: [PATCH 2/7] fix(code-placeholders): keep integer attributes across re-declaration and unset -f, guard shell values against subscript expansions --- .../code-placeholders/compiler.test.ts | 65 +++++++++++++++- .../lib/execution/code-placeholders/shell.ts | 77 ++++++++++++++++--- 2 files changed, 128 insertions(+), 14 deletions(-) diff --git a/apps/sim/lib/execution/code-placeholders/compiler.test.ts b/apps/sim/lib/execution/code-placeholders/compiler.test.ts index e1135e5eea6..01547102a81 100644 --- a/apps/sim/lib/execution/code-placeholders/compiler.test.ts +++ b/apps/sim/lib/execution/code-placeholders/compiler.test.ts @@ -1058,6 +1058,8 @@ describe('code placeholder compiler', () => { "echo $(( $(cat <<-'EOF'\n\t{{KEY}}\n\tEOF\n) + 1 ))", '[[ {{KEY}} -eq 0 ]] && echo zero', '[[ "{{KEY}}" -eq 0 ]] && echo zero', + 'declare -i n; declare -a n; n={{KEY}}', + 'declare -i n; unset -f n; n={{KEY}}', 'if [[ 0 -lt {{KEY}} ]]; then echo positive; fi', '[[ -n x && ( "{{KEY}}" -ge 1 ) ]]', '[[ $(printf "%s" "{{KEY}}") -ne 0 ]]', @@ -1136,7 +1138,7 @@ describe('code placeholder compiler', () => { }) it('preserves shell literal arithmetic text and leaves completed arithmetic frames', async () => { - const value = 'values[$(printf injected >&2)]' + const value = 'values[1]' const compiled = await compileCodePlaceholders({ code: [ 'printf "%s\\n" "{{KEY}}"', @@ -1180,6 +1182,7 @@ describe('code placeholder compiler', () => { '$(let x=1 < { compileCodePlaceholders({ code, language: CodeLanguage.Shell, - environmentVariables: { KEY: 'values[$(printf injected >&2)]' }, + environmentVariables: { KEY: 'values[1]' }, }) ).resolves.toBeDefined() }) @@ -1214,7 +1217,7 @@ describe('code placeholder compiler', () => { }) it('compiles shell placeholders beside arithmetic that never evaluates them', async () => { - const value = 'values[$(printf injected >&2)]' + const value = 'values[1]' const compiled = await compileCodePlaceholders({ code: [ '[ "{{KEY}}" -eq 0 ] 2>/dev/null || printf "%s\\n" not-zero', @@ -1249,6 +1252,62 @@ describe('code placeholder compiler', () => { ) }) + it.each([ + ['an expansion in a subscript', 'a[$(cmd)]', 'an array subscript with an expansion'], + [ + 'a quoted bracket ending the walk early', + 'a["]$(cmd)"]', + 'an array subscript with an expansion', + ], + ['a backtick in a subscript', 'a[`cmd`]', 'an array subscript with an expansion'], + ['a process substitution in a subscript', 'a[<(cmd)]', 'an array subscript with an expansion'], + ['an escaped expansion in a subscript', 'a[\\$x]', 'an array subscript with an expansion'], + ['an unclosed subscript', 'a[', 'an unbalanced array subscript'], + ['a stray closing bracket', '$(cmd)]', 'an unbalanced array subscript'], + ['an open quote in a subscript', 'a["]', 'an unbalanced array subscript'], + ])('refuses a shell value with %s wherever it lands', async (_case, value, reason) => { + for (const code of ['printf "%s\\n" "{{KEY}}"', "cat <<'EOF'\n{{KEY}}\nEOF"]) { + await expect( + compileCodePlaceholders({ + code, + language: CodeLanguage.Shell, + environmentVariables: { KEY: value }, + }) + ).rejects.toThrow(reason) + } + }) + + it.each(['values[1]', '["a", "b"]', '{"items": [1, 2]}', '$HOME and $(date)', "it's [done]"])( + 'compiles a shell value whose subscripts hold no expansion: %s', + async (value) => { + const compiled = await compileCodePlaceholders({ + code: 'printf "%s\\n" "{{KEY}}"', + language: CodeLanguage.Shell, + environmentVariables: { KEY: value }, + }) + expect(executeShell(compiled.code, compiled.bindings)).toBe(`${value}\n`) + } + ) + + it('refuses arithmetic positions whose values are safe alone', async () => { + // Code can supply the brackets, or split a subscript across adjacent placeholders, so the value + // guard cannot judge these alone; the position scan refuses them. + await expect( + compileCodePlaceholders({ + code: 'total=$(( a[{{KEY}}] ))', + language: CodeLanguage.Shell, + environmentVariables: { KEY: '$(cmd)' }, + }) + ).rejects.toThrow('is not supported in shell arithmetic') + await expect( + compileCodePlaceholders({ + code: 'total=$(( {{LEFT}}{{RIGHT}} ))', + language: CodeLanguage.Shell, + environmentVariables: { LEFT: 'a', RIGHT: '1' }, + }) + ).rejects.toThrow('is not supported in shell arithmetic') + }) + it('discovers shell arithmetic placeholders without compiling missing values', async () => { const code = 'total=$(( {{MISSING}} + {{KEY}} ))' await expect(analyzeCodePlaceholders(code, CodeLanguage.Shell)).resolves.toEqual([ diff --git a/apps/sim/lib/execution/code-placeholders/shell.ts b/apps/sim/lib/execution/code-placeholders/shell.ts index 3f8dc574e5c..c86d1d72bf1 100644 --- a/apps/sim/lib/execution/code-placeholders/shell.ts +++ b/apps/sim/lib/execution/code-placeholders/shell.ts @@ -92,6 +92,8 @@ interface ShellCommandScan { indexedOption?: boolean /** The command is a builtin (`unset`/`read`/…) whose name arguments carry arithmetic subscripts. */ nameArgumentBuiltin?: boolean + /** An `unset -f` was read, so its arguments name functions and leave variable attributes alone. */ + unsetsFunctions?: boolean /** True once the command word (past any `name=value` assignment prefix) has been read. */ sawCommandWord: boolean /** A non-arithmetic assignment prefix is being read, so a later keyword must not mark its value. */ @@ -545,6 +547,45 @@ function shellExpansion(name: string, quote: ShellQuote): string { return expansion } +/** Whether an expansion (`$…`, a backtick, `<(`/`>(`) starts at `index`. */ +function expansionStarts(value: string, index: number): boolean { + const character = value[index] + if (character === '$' || character === '`') return true + return (character === '<' || character === '>') && value[index + 1] === '(' +} + +/** + * The reason a value could run a command if it reached arithmetic evaluation, or undefined if it + * cannot. Arithmetic expands an array subscript again, so a subscript holding an expansion runs it. + * This backs up the position scan: any `[ … ]` holding an expansion is refused, as is an + * unbalanced bracket or an open quote inside one, since bash would then find the subscript's end in + * surrounding code. Quotes and escapes count inside a subscript, so a quoted `]` does not end it. + */ +function unsafeArithmeticSubscript(value: string): string | undefined { + let depth = 0 + let quote: '"' | "'" | undefined + for (let index = 0; index < value.length; index += 1) { + const character = value[index] + if (depth > 0 && expansionStarts(value, index)) return 'an array subscript with an expansion' + if (quote === "'") { + if (character === "'") quote = undefined + } else if (character === '\\') { + index += 1 + if (depth > 0 && expansionStarts(value, index)) return 'an array subscript with an expansion' + } else if (quote === '"') { + if (character === '"') quote = undefined + } else if (depth > 0 && (character === '"' || character === "'")) { + quote = character + } else if (character === '[') { + depth += 1 + } else if (character === ']') { + if (depth === 0) return 'an unbalanced array subscript' + depth -= 1 + } + } + return depth > 0 || quote ? 'an unbalanced array subscript' : undefined +} + function isLegacyShellPlaceholder(occurrence: CodePlaceholderOccurrence): boolean { const inner = occurrence.raw.slice(2, -2) return inner.trim() === occurrence.name && SHELL_BARE_NAME.test(occurrence.name) @@ -756,12 +797,14 @@ function collectShellOccurrenceContexts( /** A subscript frame for `name`'s array: a string key for an associative array, else an arithmetic index. */ const subscriptFrameFor = (name: string | undefined) => subscriptFrame(isAssociativeArray(name ?? '') ? 'keysubscript' : 'arithmetic') - /** Forget a name's tracked type in every scope — on `unset`, or a re-declaration that changes it. */ + /** Forget a name's tracked array type in every scope — a re-declaration as `-a`/`-A` resets it. */ + const clearArrayType = (name: string) => { + for (const frame of frames) frame.associativeArrays?.delete(name) + } + /** Forget everything tracked about a name in every scope — `unset` removes its attributes too. */ const clearNameType = (name: string) => { - for (const frame of frames) { - frame.associativeArrays?.delete(name) - frame.integerAttribute?.delete(name) - } + clearArrayType(name) + for (const frame of frames) frame.integerAttribute?.delete(name) } /** Whether a `>`/`<` redirect operator immediately precedes `at`, across any intervening blanks. */ const precededByRedirect = (at: number): boolean => { @@ -895,9 +938,10 @@ function collectShellOccurrenceContexts( ) { const declared = SHELL_NAME.exec(word)?.[0] if (declared) { - // A re-declaration resets the name's type before this one's attributes apply, so a later - // `declare -a` (indexed) clears an earlier `-A` (associative) and vice versa. - if (command.associativeOption || command.indexedOption) clearNameType(declared) + // A re-declaration resets the name's array type before this one's attributes apply, so a + // later `declare -a` (indexed) clears an earlier `-A` (associative) and vice versa. The + // integer attribute survives it — only `+i` or `unset` removes that. + if (command.associativeOption || command.indexedOption) clearArrayType(declared) if (command.associativeOption) (frame.associativeArrays ??= new Set()).add(declared) if (command.integerOption) { ;(frame.integerAttribute ??= new Map()).set(declared, command.integerOption === 'set') @@ -905,9 +949,12 @@ function collectShellOccurrenceContexts( } } // `unset name` removes the variable and its attributes, so a later indexed reuse is arithmetic - // again. A bare name only — `unset name[i]` removes one element, not the array's type. - if (command.nameArgumentBuiltin && command.sawCommandWord && SHELL_BARE_NAME.test(word)) { - clearNameType(word) + // again. A bare name only — `unset name[i]` removes one element, not the array's type — and not + // under `-f`, which removes a function of that name and leaves the variable alone. + if (command.nameArgumentBuiltin && command.sawCommandWord) { + const option = DECLARATION_OPTION.exec(word) + if (option?.[1] === '-' && option[2].includes('f')) command.unsetsFunctions = true + else if (!command.unsetsFunctions && SHELL_BARE_NAME.test(word)) clearNameType(word) } // An assignment is one only in command-prefix or declaration-argument position; `echo n=1` or // `printf a[i]=1` passes an ordinary string that bash never evaluates. @@ -1435,6 +1482,14 @@ export async function compileShellPlaceholders( occurrence.start ) } + const unsafeSubscript = resolved && unsafeArithmeticSubscript(resolved.value) + if (unsafeSubscript && !input.analysisOnly) { + throw new CodePlaceholderCompileError( + `Variable placeholder "${occurrence.name}" cannot contain ${unsafeSubscript} in shell code`, + input.code, + occurrence.start + ) + } return resolved } const resolveShellOccurrence = (occurrence: CodePlaceholderOccurrence) => From 268cf41b39111164b9059d66e2bb707f3f12417e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 20:09:32 -0700 Subject: [PATCH 3/7] fix(code-placeholders): end assignment-value scope at the word, carry attributes into heredoc bodies --- .../code-placeholders/compiler.test.ts | 5 ++ .../lib/execution/code-placeholders/shell.ts | 48 ++++++++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/apps/sim/lib/execution/code-placeholders/compiler.test.ts b/apps/sim/lib/execution/code-placeholders/compiler.test.ts index 01547102a81..620daae9537 100644 --- a/apps/sim/lib/execution/code-placeholders/compiler.test.ts +++ b/apps/sim/lib/execution/code-placeholders/compiler.test.ts @@ -1060,6 +1060,8 @@ describe('code placeholder compiler', () => { '[[ "{{KEY}}" -eq 0 ]] && echo zero', 'declare -i n; declare -a n; n={{KEY}}', 'declare -i n; unset -f n; n={{KEY}}', + 'declare -i n; n=1\\ {{KEY}}', + 'declare -i n; cat < { 'let x=1 > {{KEY}}', 'declare -i n=1 >"{{KEY}}"', '[[ "{{KEY}}" == "-eq" ]]', + 'declare -i n; n=1 {{KEY}} hello', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template + 'declare -A m=([hello]=world); cat < { // These forms use the value as pattern, default, or a shielded prefix — never arithmetic — so a // conservative scanner must not reject them. Each would fail compilation if wrongly rejected. diff --git a/apps/sim/lib/execution/code-placeholders/shell.ts b/apps/sim/lib/execution/code-placeholders/shell.ts index c86d1d72bf1..3aa4c75f171 100644 --- a/apps/sim/lib/execution/code-placeholders/shell.ts +++ b/apps/sim/lib/execution/code-placeholders/shell.ts @@ -106,8 +106,16 @@ interface ShellCommandScan { inRedirectTarget: boolean } +/** The variable attributes visible at a point in the scan, which a heredoc body read there inherits. */ +interface ShellAttributeScope { + integerAttribute: Map + associativeArrays: Set +} + interface ShellOccurrenceContext { quote: ShellQuote + /** On a heredoc operator, the attributes in scope there, so its body scan sees the same arrays. */ + scope?: ShellAttributeScope /** Its value reaches an arithmetic evaluation — a frame, or a command marked arithmetic later. */ arithmetic?: boolean unsupported?: 'escaped sequence' @@ -738,7 +746,8 @@ function collectShellOccurrenceContexts( end: number, literalRoot: boolean, skippedRanges: Array<[number, number]> = [], - operatorStarts: ReadonlySet = new Set() + operatorStarts: ReadonlySet = new Set(), + inheritedScope?: ShellAttributeScope ): Map { const occurrenceByStart = new Map( occurrencesWithin(occurrences, start, end).map( @@ -747,7 +756,14 @@ function collectShellOccurrenceContexts( ) const contexts = new Map() const frames: ShellScanFrame[] = [ - { kind: 'root', quote: 'none', parenthesisDepth: 0, literalRoot }, + { + kind: 'root', + quote: 'none', + parenthesisDepth: 0, + literalRoot, + integerAttribute: new Map(inheritedScope?.integerAttribute), + associativeArrays: new Set(inheritedScope?.associativeArrays), + }, ] let skippedRangeIndex = 0 /** @@ -791,6 +807,17 @@ function collectShellOccurrenceContexts( } return false } + /** The attributes visible here, outermost scope first so a nearer one overrides an integer flag. */ + const visibleScope = (): ShellAttributeScope => { + const scope: ShellAttributeScope = { integerAttribute: new Map(), associativeArrays: new Set() } + for (const frame of frames) { + for (const [name, attribute] of frame.integerAttribute ?? []) { + scope.integerAttribute.set(name, attribute) + } + for (const name of frame.associativeArrays ?? []) scope.associativeArrays.add(name) + } + return scope + } /** Whether `name` is a `declare -A` associative array, whose subscript is a string key, not arithmetic. */ const isAssociativeArray = (name: string) => frames.some((frame) => frame.associativeArrays?.has(name)) @@ -1012,6 +1039,7 @@ function collectShellOccurrenceContexts( const operatorContext: ShellOccurrenceContext = { quote: 'none', arithmetic: enclosingArithmetic, + scope: visibleScope(), } contexts.set(occurrence, operatorContext) if (sub >= 0) commandScanOf(frames[sub - 1]).contexts.push(operatorContext) @@ -1034,10 +1062,15 @@ function collectShellOccurrenceContexts( } const character = code[index] - // The redirect-target flag covers only its one word; unquoted whitespace ends that word, so a - // following argument (`let x=1 >/dev/null {{x}}`) is marked by the command again. - if (frame.command?.inRedirectTarget && frame.quote === 'none' && /\s/.test(character)) { + // The redirect-target and assignment-value flags cover only their one word; unquoted whitespace + // ends that word. `scanWord` clears them at the next word too, but a placeholder-only word never + // reaches it, so `let x=1 >/dev/null {{x}}` is marked by the command again while the command name + // in `n=1 {{x}} arg` is not read as `n`'s integer value. + if (frame.command && frame.quote === 'none' && /\s/.test(character)) { frame.command.inRedirectTarget = false + frame.command.valueArithmetic = false + frame.command.pendingValue = false + frame.command.inPrefixValue = false } // Classify each command word before the quote branches consume a quote-initial word such as // `"-i"` or `'let'`. `scanWord` only sets state, never advances `index`; the characters below @@ -1628,7 +1661,10 @@ export async function compileShellPlaceholders( bodyOccurrences, heredoc.bodyStart, heredoc.bodyEnd, - true + true, + [], + undefined, + rootContexts.get(heredocOperators[heredocIndex])?.scope ) for (const occurrence of bodyOccurrences) { const edit = resolveInContext(occurrence, bodyContexts.get(occurrence), false) From dc283dc47355c06c6fb77ec92f8635916079e1f4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 20:20:19 -0700 Subject: [PATCH 4/7] fix(code-placeholders): treat a declaration as pending inside its own command's expansions --- .../code-placeholders/compiler.test.ts | 6 +++ .../lib/execution/code-placeholders/shell.ts | 40 ++++++++++++++++--- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/apps/sim/lib/execution/code-placeholders/compiler.test.ts b/apps/sim/lib/execution/code-placeholders/compiler.test.ts index 620daae9537..e85419d9eef 100644 --- a/apps/sim/lib/execution/code-placeholders/compiler.test.ts +++ b/apps/sim/lib/execution/code-placeholders/compiler.test.ts @@ -1062,6 +1062,11 @@ describe('code placeholder compiler', () => { 'declare -i n; unset -f n; n={{KEY}}', 'declare -i n; n=1\\ {{KEY}}', 'declare -i n; cat < { 'declare -i n=1 >"{{KEY}}"', '[[ "{{KEY}}" == "-eq" ]]', 'declare -i n; n=1 {{KEY}} hello', + 'declare -A m=([{{KEY}}]=x)', // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template 'declare -A m=([hello]=world); cat < { diff --git a/apps/sim/lib/execution/code-placeholders/shell.ts b/apps/sim/lib/execution/code-placeholders/shell.ts index 3aa4c75f171..beb563a7c4d 100644 --- a/apps/sim/lib/execution/code-placeholders/shell.ts +++ b/apps/sim/lib/execution/code-placeholders/shell.ts @@ -94,6 +94,11 @@ interface ShellCommandScan { nameArgumentBuiltin?: boolean /** An `unset -f` was read, so its arguments name functions and leave variable attributes alone. */ unsetsFunctions?: boolean + /** + * Names whose attributes this command changes. Bash expands the command's words and heredocs + * before running it, so an expansion inside the command cannot rely on them yet. + */ + declaredNames?: Set /** True once the command word (past any `name=value` assignment prefix) has been read. */ sawCommandWord: boolean /** A non-arithmetic assignment prefix is being read, so a later keyword must not mark its value. */ @@ -799,8 +804,18 @@ function collectShellOccurrenceContexts( for (const ended of closed.endedCommands ?? []) enclosingCommand.nested.push(ended) if (closed.command) enclosingCommand.nested.push(closed.command) } + /** + * Whether a command still being read, below the innermost `ownFrames` frames, changes `name`'s + * attributes. An expansion inside that command runs before the change, so its attribute is + * unknown there and lookups take the arithmetic side: integer, indexed. + */ + const declaredByEnclosingCommand = (name: string, ownFrames = 1) => + frames + .slice(0, frames.length - ownFrames) + .some((frame) => frame.command?.declaredNames?.has(name)) /** Whether `name` has the integer attribute here: the nearest scope that sets or clears it wins. */ const declaresInteger = (name: string): boolean => { + if (declaredByEnclosingCommand(name)) return true for (let depth = frames.length - 1; depth >= 0; depth -= 1) { const attribute = frames[depth].integerAttribute?.get(name) if (attribute !== undefined) return attribute @@ -816,14 +831,22 @@ function collectShellOccurrenceContexts( } for (const name of frame.associativeArrays ?? []) scope.associativeArrays.add(name) } + // A heredoc body expands before any command still being read runs, its receiver included. + for (const frame of frames) { + for (const name of frame.command?.declaredNames ?? []) { + scope.integerAttribute.set(name, true) + scope.associativeArrays.delete(name) + } + } return scope } /** Whether `name` is a `declare -A` associative array, whose subscript is a string key, not arithmetic. */ - const isAssociativeArray = (name: string) => + const isAssociativeArray = (name: string, ownFrames = 1) => + !declaredByEnclosingCommand(name, ownFrames) && frames.some((frame) => frame.associativeArrays?.has(name)) /** A subscript frame for `name`'s array: a string key for an associative array, else an arithmetic index. */ - const subscriptFrameFor = (name: string | undefined) => - subscriptFrame(isAssociativeArray(name ?? '') ? 'keysubscript' : 'arithmetic') + const subscriptFrameFor = (name: string | undefined, ownFrames = 1) => + subscriptFrame(isAssociativeArray(name ?? '', ownFrames) ? 'keysubscript' : 'arithmetic') /** Forget a name's tracked array type in every scope — a re-declaration as `-a`/`-A` resets it. */ const clearArrayType = (name: string) => { for (const frame of frames) frame.associativeArrays?.delete(name) @@ -968,6 +991,9 @@ function collectShellOccurrenceContexts( // A re-declaration resets the name's array type before this one's attributes apply, so a // later `declare -a` (indexed) clears an earlier `-A` (associative) and vice versa. The // integer attribute survives it — only `+i` or `unset` removes that. + if (command.associativeOption || command.indexedOption || command.integerOption) { + ;(command.declaredNames ??= new Set()).add(declared) + } if (command.associativeOption || command.indexedOption) clearArrayType(declared) if (command.associativeOption) (frame.associativeArrays ??= new Set()).add(declared) if (command.integerOption) { @@ -981,7 +1007,10 @@ function collectShellOccurrenceContexts( if (command.nameArgumentBuiltin && command.sawCommandWord) { const option = DECLARATION_OPTION.exec(word) if (option?.[1] === '-' && option[2].includes('f')) command.unsetsFunctions = true - else if (!command.unsetsFunctions && SHELL_BARE_NAME.test(word)) clearNameType(word) + else if (!command.unsetsFunctions && SHELL_BARE_NAME.test(word)) { + ;(command.declaredNames ??= new Set()).add(word) + clearNameType(word) + } } // An assignment is one only in command-prefix or declaration-argument position; `echo n=1` or // `printf a[i]=1` passes an ordinary string that bash never evaluates. @@ -1298,7 +1327,8 @@ function collectShellOccurrenceContexts( /[\s(]/.test(code[index - 1] ?? ' ') && opensIndexedAssignment(code, index, end) ) { - pushFrame(subscriptFrameFor(frame.arrayName)) + // The literal belongs to the assignment in the frame below, whose own declaration applies. + pushFrame(subscriptFrameFor(frame.arrayName, 2)) index += 1 continue } From d16e6458e1786083f7e21af90b6bb234ea953258 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 22:42:48 -0700 Subject: [PATCH 5/7] fix(code-placeholders): narrow arithmetic guards and preserve literal data --- .github/workflows/checks.yml | 16 + .../code-placeholders/compiler.test.ts | 234 +--- .../lib/execution/code-placeholders/shell.ts | 1022 ++++------------- .../scripts/test-shell-placeholders-e2e.ts | 194 ++++ 4 files changed, 427 insertions(+), 1039 deletions(-) create mode 100644 apps/sim/scripts/test-shell-placeholders-e2e.ts diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index f680a832415..a6d5fbffedc 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -614,6 +614,22 @@ jobs: - name: Install ripgrep run: command -v rg || (sudo apt-get update && sudo apt-get install -y ripgrep) + - name: Verify shell placeholder compilation in Bash + if: matrix.shard == 1 + working-directory: apps/sim + env: + SHELL_PLACEHOLDERS_REPORT_PATH: ${{ runner.temp }}/shell-placeholders.json + run: bun scripts/test-shell-placeholders-e2e.ts + + - name: Upload shell placeholder execution report + if: failure() && matrix.shard == 1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: shell-placeholders + path: ${{ runner.temp }}/shell-placeholders.json + if-no-files-found: warn + retention-days: 7 + - name: Run tests env: NODE_OPTIONS: '--no-warnings --max-old-space-size=8192' diff --git a/apps/sim/lib/execution/code-placeholders/compiler.test.ts b/apps/sim/lib/execution/code-placeholders/compiler.test.ts index e85419d9eef..96cc224341e 100644 --- a/apps/sim/lib/execution/code-placeholders/compiler.test.ts +++ b/apps/sim/lib/execution/code-placeholders/compiler.test.ts @@ -1053,87 +1053,6 @@ describe('code placeholder compiler', () => { 'total=$(( $(( 1 + 1 )) + {{KEY}} ))', 'cat </dev/null let x="{{KEY}}"', - '2>/dev/null let x="{{KEY}}"', - 'builtin let x="{{KEY}}"', - 'command let x="{{KEY}}"', - 'declare "-i" n="{{KEY}}"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - '[[ ${missing:-{{KEY}}} -eq 0 ]]', - 'declare -i n; echo "$(declare +i n)"; n="{{KEY}}"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 'printf "%s" "${missing:-{text}"; let x="{{KEY}}"', - 'declare -i n; n="$(printf "%s" "{{KEY}}")"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 'declare -i n; n="${missing:-{{KEY}}}"', - 'declare -i n; declare n="{{KEY}}"', - '> /dev/null let x="{{KEY}}"', - 'let x="$(cat </dev/null {{KEY}}', - 'a=([{{KEY}}]=text)', - "unset 'a[{{KEY}}]'", - 'le\\\nt "x={{KEY}}"', - 'local -A a; a[{{KEY}}]=1', - 'declare -A m; unset m; m[{{KEY}}]=1', - 'declare -A m; unset m; declare -a m; m[{{KEY}}]=1', - '(declare -A m); m[{{KEY}}]=1', - 'a[{{KEY}}]=1', - 'a[{{KEY}}]+=1', - 'declare -i n; n="{{KEY}}"', - 'declare -i n\nn="{{KEY}}"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 'a=(1 2); echo "${a[{{KEY}}]}"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 'a=(1 2); echo "${a[0]:{{KEY}}}"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 'a=(1 2); echo "${a[0]:0:{{KEY}}}"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 's=abc; echo "${s:{{KEY}}}"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 's=abc; echo "${s:0:{{KEY}}}"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 's=abc; echo "${s: -1:{{KEY}}}"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 'set -- a b; echo "${@:{{KEY}}}"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 's=abc; printf "%s\\n" "${missing:-"${s:{{KEY}}}"}"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 'cat < { await expect( compileCodePlaceholders({ @@ -1145,7 +1064,7 @@ describe('code placeholder compiler', () => { }) it('preserves shell literal arithmetic text and leaves completed arithmetic frames', async () => { - const value = 'values[1]' + const value = 'values[$(printf injected >&2)]' const compiled = await compileCodePlaceholders({ code: [ 'printf "%s\\n" "{{KEY}}"', @@ -1168,157 +1087,6 @@ describe('code placeholder compiler', () => { ) }) - it.each([ - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 'printf "%s\\n" "${missing:-https://{{KEY}}}"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 'printf "%s\\n" "${missing:-items[{{KEY}}]}"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 's=abc; printf "%s\\n" "${s#[{{KEY}}]}"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 's=abc; printf "%s\\n" "${s/[{{KEY}}]/x}"', - 'f() { local -i n; }; n="{{KEY}}"', - 'PREFIX="$(printf "%s" "{{KEY}}")" let total=2', - 'printf "%s" a[{{KEY}}]=1', - 'declare -i n; printf "%s" n={{KEY}}', - 'declare -i n; declare +i n; n="{{KEY}}"', - 'declare -i n; n=1 printf "%s" "{{KEY}}"', - 'declare -i n; n=5 text="{{KEY}}"', - 'declare -i +i n="{{KEY}}"', - 'declare -i n; echo "$(declare +i n; n=\'{{KEY}}\'; printf "%s" "$n")"', - '$(let x=1 <"{{KEY}}"', - 'let x=1 > {{KEY}}', - 'declare -i n=1 >"{{KEY}}"', - '[[ "{{KEY}}" == "-eq" ]]', - 'declare -i n; n=1 {{KEY}} hello', - 'declare -A m=([{{KEY}}]=x)', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 'declare -A m=([hello]=world); cat < { - // These forms use the value as pattern, default, or a shielded prefix — never arithmetic — so a - // conservative scanner must not reject them. Each would fail compilation if wrongly rejected. - await expect( - compileCodePlaceholders({ - code, - language: CodeLanguage.Shell, - environmentVariables: { KEY: 'values[1]' }, - }) - ).resolves.toBeDefined() - }) - - it('keeps literal single quotes inside a double-quoted default expansion', async () => { - const compiled = await compileCodePlaceholders({ - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - code: 'printf "%s\\n" "${missing:-\'{{KEY}}\'}"', - language: CodeLanguage.Shell, - environmentVariables: { KEY: 'hello world' }, - }) - expect(executeShell(compiled.code, compiled.bindings)).toBe("'hello world'\n") - }) - - it('compiles shell placeholders beside arithmetic that never evaluates them', async () => { - const value = 'values[1]' - const compiled = await compileCodePlaceholders({ - code: [ - '[ "{{KEY}}" -eq 0 ] 2>/dev/null || printf "%s\\n" not-zero', - '[[ "{{KEY}}" == values* && 1 -eq 1 ]] && printf "%s\\n" matched', - 'let total=1+1; printf "%s\\n" "{{KEY}}"', - 'f() { local copy="{{KEY}}"; printf "%s\\n" "$copy"; }; f', - 'declare copy="{{KEY}}"; printf "%s\\n" "$copy"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 'printf "%s\\n" "${missing:-{{KEY}}}"', - // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion, not a JS template - 'a=(x y); printf "%s\\n" "${a[1]}{{KEY}}"', - 'printf "%s\\n" "let {{KEY}}"', - 'printf "%s\\n" "items[{{KEY}}]"', - 'PREFIX="{{KEY}}" let total=2', - '# declare -i n', - 'n="{{KEY}}"; printf "%s\\n" "$n"', - 'let total=1 <&1`, compiled.bindings)).toBe( - `not-zero\nmatched\n${value}\n${value}\n${value}\n${value}\ny${value}\n` + - `let ${value}\nitems[${value}]\n${value}\n1\n${value} 2\n` - ) - }) - - it.each([ - ['an expansion in a subscript', 'a[$(cmd)]', 'an array subscript with an expansion'], - [ - 'a quoted bracket ending the walk early', - 'a["]$(cmd)"]', - 'an array subscript with an expansion', - ], - ['a backtick in a subscript', 'a[`cmd`]', 'an array subscript with an expansion'], - ['a process substitution in a subscript', 'a[<(cmd)]', 'an array subscript with an expansion'], - ['an escaped expansion in a subscript', 'a[\\$x]', 'an array subscript with an expansion'], - ['an unclosed subscript', 'a[', 'an unbalanced array subscript'], - ['a stray closing bracket', '$(cmd)]', 'an unbalanced array subscript'], - ['an open quote in a subscript', 'a["]', 'an unbalanced array subscript'], - ])('refuses a shell value with %s wherever it lands', async (_case, value, reason) => { - for (const code of ['printf "%s\\n" "{{KEY}}"', "cat <<'EOF'\n{{KEY}}\nEOF"]) { - await expect( - compileCodePlaceholders({ - code, - language: CodeLanguage.Shell, - environmentVariables: { KEY: value }, - }) - ).rejects.toThrow(reason) - } - }) - - it.each(['values[1]', '["a", "b"]', '{"items": [1, 2]}', '$HOME and $(date)', "it's [done]"])( - 'compiles a shell value whose subscripts hold no expansion: %s', - async (value) => { - const compiled = await compileCodePlaceholders({ - code: 'printf "%s\\n" "{{KEY}}"', - language: CodeLanguage.Shell, - environmentVariables: { KEY: value }, - }) - expect(executeShell(compiled.code, compiled.bindings)).toBe(`${value}\n`) - } - ) - - it('refuses arithmetic positions whose values are safe alone', async () => { - // Code can supply the brackets, or split a subscript across adjacent placeholders, so the value - // guard cannot judge these alone; the position scan refuses them. - await expect( - compileCodePlaceholders({ - code: 'total=$(( a[{{KEY}}] ))', - language: CodeLanguage.Shell, - environmentVariables: { KEY: '$(cmd)' }, - }) - ).rejects.toThrow('is not supported in shell arithmetic') - await expect( - compileCodePlaceholders({ - code: 'total=$(( {{LEFT}}{{RIGHT}} ))', - language: CodeLanguage.Shell, - environmentVariables: { LEFT: 'a', RIGHT: '1' }, - }) - ).rejects.toThrow('is not supported in shell arithmetic') - }) - it('discovers shell arithmetic placeholders without compiling missing values', async () => { const code = 'total=$(( {{MISSING}} + {{KEY}} ))' await expect(analyzeCodePlaceholders(code, CodeLanguage.Shell)).resolves.toEqual([ diff --git a/apps/sim/lib/execution/code-placeholders/shell.ts b/apps/sim/lib/execution/code-placeholders/shell.ts index beb563a7c4d..0728d43d877 100644 --- a/apps/sim/lib/execution/code-placeholders/shell.ts +++ b/apps/sim/lib/execution/code-placeholders/shell.ts @@ -28,100 +28,27 @@ interface HeredocDeclaration { type ShellQuote = 'none' | 'single' | 'double' | 'ansi' interface ShellScanFrame { - /** - * `parameter` is a `${...}` expansion, which ends on its own `}`, not a command boundary. - * `keysubscript` is an associative array's `[key]`, scanned like a subscript but not arithmetic. - * `arrayliteral` is a compound assignment's `( … )`, where element keys `[k]=` are subscripts. - * `subshell` is a bare `( … )` group — it reads commands but, unlike a substitution, feeds no output. - */ - kind: - | 'root' - | 'command' - | 'arithmetic' - | 'backtick' - | 'parameter' - | 'keysubscript' - | 'arrayliteral' - | 'subshell' + kind: 'root' | 'command' | 'arithmetic' | 'backtick' | 'parameter' quote: ShellQuote parenthesisDepth: number - /** Open `[`/`]` depth of a bracketed frame — a `$[ ]`, an indexed subscript, or an associative key. */ bracketDepth?: number literalRoot: boolean - /** Inside `[[ ]]`, where only `&&` and `||` end a clause. */ - conditional?: boolean - /** On a `parameter` frame opened inside double quotes, where single quotes stay literal. */ inDoubleQuotes?: boolean - /** On a `parameter` frame, the cursor is still at the operator right after the name. */ - atOperator?: boolean - /** On a `parameter` frame, its substring offset/length is being read as arithmetic. */ - arithmeticTail?: boolean - /** The integer attribute this frame's scope sets (`-i` → true) or clears (`+i` → false) per name. */ - integerAttribute?: Map - /** Names this frame's scope declared associative (`declare -A`), whose subscripts are string keys. */ - associativeArrays?: Set - /** On a `parameter` frame, the expanded name, so a `${name[…]}` subscript can check its array type. */ - parameterName?: string - /** On an `arrayliteral` frame, the array being assigned, so its element keys check the array type. */ - arrayName?: string - /** A substitution opened inside a non-arithmetic assignment prefix, so a keyword must not mark it. */ - prefixExcluded?: boolean - /** Opened inside an integer assignment's value, so everything here is an arithmetic operand. */ - valueOperand?: boolean - command?: ShellCommandScan - /** Earlier commands of a substitution, whose output still reaches the enclosing command. */ - endedCommands?: ShellCommandScan[] -} - -/** - * The command (or `[[ ]]` clause) being scanned in a frame. A later word can make the whole - * command arithmetic — `-eq` after its left operand, `-i` after `declare` — so the contexts - * already recorded in it, and the commands of substitutions it closed, are kept to be marked then. - */ -interface ShellCommandScan { - contexts: ShellOccurrenceContext[] - nested: ShellCommandScan[] - arithmetic: boolean - /** The declaration builtin being read, if any — only `declare`/`typeset` give a global attribute. */ - declarationBuiltin?: 'declare' | 'typeset' | 'local' - /** The integer attribute the last declaration option set (`-i`) or cleared (`+i`). */ - integerOption?: 'set' | 'clear' - /** A `declare -A` was read, so the declared names are associative (string-keyed) arrays. */ - associativeOption?: boolean - /** A `declare -a` was read, so the declared names are indexed arrays (resetting any prior type). */ - indexedOption?: boolean - /** The command is a builtin (`unset`/`read`/…) whose name arguments carry arithmetic subscripts. */ - nameArgumentBuiltin?: boolean - /** An `unset -f` was read, so its arguments name functions and leave variable attributes alone. */ - unsetsFunctions?: boolean - /** - * Names whose attributes this command changes. Bash expands the command's words and heredocs - * before running it, so an expansion inside the command cannot rely on them yet. - */ - declaredNames?: Set - /** True once the command word (past any `name=value` assignment prefix) has been read. */ - sawCommandWord: boolean - /** A non-arithmetic assignment prefix is being read, so a later keyword must not mark its value. */ - inPrefixValue: boolean - /** An integer assignment's value is being read — arithmetic, but only this value, not the command. */ - valueArithmetic: boolean - /** An integer assignment was seen; its value becomes arithmetic once the `=` is passed, not its subscript. */ - pendingValue?: boolean - /** A redirect target (a filename) is being read, never an arithmetic operand of its command. */ - inRedirectTarget: boolean -} - -/** The variable attributes visible at a point in the scan, which a heredoc body read there inherits. */ -interface ShellAttributeScope { - integerAttribute: Map - associativeArrays: Set + commandStarted?: boolean + commandPrefix?: 'time' | 'coproc' + wordEnd?: number + conditional?: { + start: number + arithmetic: boolean + words: number + wordOpen: boolean + unary: boolean + } } interface ShellOccurrenceContext { quote: ShellQuote - /** On a heredoc operator, the attributes in scope there, so its body scan sees the same arrays. */ - scope?: ShellAttributeScope - /** Its value reaches an arithmetic evaluation — a frame, or a command marked arithmetic later. */ + /** Includes nested command substitutions whose output can become an arithmetic operand. */ arithmetic?: boolean unsupported?: 'escaped sequence' } @@ -131,45 +58,45 @@ interface ShellSpan { end: number } -const ARITHMETIC_CONDITIONAL_OPERATOR = /^-(?:eq|ne|lt|le|gt|ge)$/ -/** A declaration option word (`-i`, `+i`, `-A`, `-iA`, …); its letters set each attribute. */ -const DECLARATION_OPTION = /^([-+])([A-Za-z]+)$/ -/** Invocation prefixes that run the following word as the command, so they are not the command. */ -const SHELL_COMMAND_PREFIXES = new Set(['command', 'builtin', 'exec', 'nohup']) -/** - * Builtins whose every argument names a variable, so an indexed-array subscript there is arithmetic. - * Limited to `unset`, whose arguments are all names; `read`/`mapfile` mix name and option-value - * arguments (`read -p prompt name`), so their subscripts are left to a conservative compile. - */ -const NAME_ARGUMENT_BUILTINS = new Set(['unset']) -/** Reserved words that introduce a command rather than being one, so the next word is the command. */ -const SHELL_RESERVED_WORDS = new Set([ +const ARITHMETIC_COMPARISON = /^-(?:eq|ne|lt|le|gt|ge)$/ +const SHELL_WORD = /(?:\\[\s\S]|[^\s;&|()<>\\])+/y +const COMMAND_INTRODUCERS = new Set([ 'if', 'then', 'elif', 'else', - 'fi', - 'do', - 'done', 'while', 'until', - 'for', - 'select', - 'case', - 'esac', - 'time', + 'do', '!', + 'time', + 'coproc', ]) -const SHELL_WORD = /[^\s;&|()<>]+/y -const PARAMETER_NAME = /[!#]?(?:[A-Za-z_][A-Za-z0-9_]*|[0-9]+|[@*#?$!-])/y -const SHELL_NAME_SOURCE = '[A-Za-z_][A-Za-z0-9_]*' -const SHELL_NAME = new RegExp(`^${SHELL_NAME_SOURCE}`) -/** A word that is exactly a bare shell name (no subscript or suffix). */ -const SHELL_BARE_NAME = new RegExp(`^${SHELL_NAME_SOURCE}$`) -/** A `name=`, `name+=` or `name[subscript]=` assignment word; group 1 is the bare name. */ -const SHELL_ASSIGNMENT_WORD = new RegExp(`^(${SHELL_NAME_SOURCE})(?:\\[.*\\])?\\+?=`) -/** A command word ending in an unescaped backslash — a line continuation to join with the next line. */ -const TRAILING_LINE_CONTINUATION = /(?:^|[^\\])(?:\\\\)*\\$/ + +function shellWordStarts(code: string, index: number): boolean { + return index === 0 || /[\s;&|()<>]/.test(code[index - 1]) +} + +function followsRedirect(code: string, index: number): boolean { + let previous = index - 1 + while (code[previous] === ' ' || code[previous] === '\t') previous -= 1 + return code[previous] === '<' || code[previous] === '>' +} + +function effectiveQuote(frame: ShellScanFrame): ShellQuote { + return frame.kind === 'parameter' && frame.quote === 'none' && frame.inDoubleQuotes + ? 'double' + : frame.quote +} + +function readShellWord(code: string, index: number): { word: string; end: number } { + SHELL_WORD.lastIndex = index + const raw = SHELL_WORD.exec(code)?.[0] ?? '' + return { + word: raw.replace(/\\([\s\S])/g, (escaped, character) => (character === '\n' ? '' : escaped)), + end: index + raw.length, + } +} function lineEndAfterNewline(code: string, start: number): number { const newline = code.indexOf('\n', start) @@ -192,25 +119,16 @@ function logicalLineEndAfterContinuations(code: string, start: number): number { return end } -function shellWordStarts(code: string, index: number): boolean { +function shellCommentStarts(code: string, index: number): boolean { + if (code[index] !== '#') return false const previous = code[index - 1] return previous === undefined || /\s|[;&|()<>]/.test(previous) } -function shellCommentStarts(code: string, index: number): boolean { - return code[index] === '#' && shellWordStarts(code, index) -} - function shellArithmeticCommandStarts(code: string, index: number): boolean { - return code[index] === '(' && code[index + 1] === '(' && shellWordStarts(code, index) -} - -/** `;`, `&` and `|` end a command unless they belong to a redirection such as `>&2` or `&>`. */ -function shellCommandSeparator(code: string, index: number): boolean { - const character = code[index] - if (character === '\n') return true - if (character !== ';' && character !== '&' && character !== '|') return false - return !/[<>]/.test(code[index - 1] ?? '') && !(character === '&' && code[index + 1] === '>') + if (code[index] !== '(' || code[index + 1] !== '(') return false + const previous = code[index - 1] + return previous === undefined || /\s|[;&|()<>]/.test(previous) } function decodeAnsiCCharacter(code: string, index: number): { value: string; end: number } { @@ -560,48 +478,9 @@ function shellExpansion(name: string, quote: ShellQuote): string { return expansion } -/** Whether an expansion (`$…`, a backtick, `<(`/`>(`) starts at `index`. */ -function expansionStarts(value: string, index: number): boolean { - const character = value[index] - if (character === '$' || character === '`') return true - return (character === '<' || character === '>') && value[index + 1] === '(' -} - -/** - * The reason a value could run a command if it reached arithmetic evaluation, or undefined if it - * cannot. Arithmetic expands an array subscript again, so a subscript holding an expansion runs it. - * This backs up the position scan: any `[ … ]` holding an expansion is refused, as is an - * unbalanced bracket or an open quote inside one, since bash would then find the subscript's end in - * surrounding code. Quotes and escapes count inside a subscript, so a quoted `]` does not end it. - */ -function unsafeArithmeticSubscript(value: string): string | undefined { - let depth = 0 - let quote: '"' | "'" | undefined - for (let index = 0; index < value.length; index += 1) { - const character = value[index] - if (depth > 0 && expansionStarts(value, index)) return 'an array subscript with an expansion' - if (quote === "'") { - if (character === "'") quote = undefined - } else if (character === '\\') { - index += 1 - if (depth > 0 && expansionStarts(value, index)) return 'an array subscript with an expansion' - } else if (quote === '"') { - if (character === '"') quote = undefined - } else if (depth > 0 && (character === '"' || character === "'")) { - quote = character - } else if (character === '[') { - depth += 1 - } else if (character === ']') { - if (depth === 0) return 'an unbalanced array subscript' - depth -= 1 - } - } - return depth > 0 || quote ? 'an unbalanced array subscript' : undefined -} - function isLegacyShellPlaceholder(occurrence: CodePlaceholderOccurrence): boolean { const inner = occurrence.raw.slice(2, -2) - return inner.trim() === occurrence.name && SHELL_BARE_NAME.test(occurrence.name) + return inner.trim() === occurrence.name && /^[A-Za-z_][A-Za-z0-9_]*$/.test(occurrence.name) } function blankPreservingLines(value: string): string { @@ -647,102 +526,9 @@ function heredocBodyRanges(heredocs: HeredocDeclaration[]): Array<[number, numbe return heredocs.map((heredoc) => [heredoc.bodyStart, heredoc.removalEnd]) } -/** - * Removes shell quoting from a word so a keyword, option or name is recognized as bash would after - * quote removal — `declare "-i"` and `'let'` match. Values keep their own quoting; this only feeds - * the command-position checks, never a placeholder's emitted expansion. - */ -function unquoteShellWord(word: string): string { - return word.replace(/'([^']*)'|"((?:[^"\\]|\\.)*)"|\\(.)/g, (_match, single, double, escaped) => - single !== undefined ? single : double !== undefined ? double : escaped - ) -} - -/** Frames whose text is a command line, where words are classified and separators end a command. */ -function readsCommands(frame: ShellScanFrame): boolean { - return ( - frame.kind === 'root' || - frame.kind === 'command' || - frame.kind === 'backtick' || - frame.kind === 'subshell' - ) -} - -/** A command substitution — `$( )` or backticks — whose output the enclosing command receives. */ -function isSubstitution(frame: ShellScanFrame): boolean { - return frame.kind === 'command' || frame.kind === 'backtick' -} - -/** - * Whether a frame puts the cursor in arithmetic — an arithmetic expansion, an integer-value operand, - * a `${…}` substring tail, or a command marked arithmetic. `arithmeticDepth` counts these as they - * open and close; this predicate reads the same set for a point-in-time check of the frame stack. - */ -function frameIsArithmetic(frame: ShellScanFrame): boolean { - return ( - frame.kind === 'arithmetic' || - frame.valueOperand === true || - (frame.kind === 'parameter' && frame.arithmeticTail === true) || - frame.command?.arithmetic === true - ) -} - -/** Where a `name=…` word is an assignment: a command prefix, or an argument to a declaration builtin. */ -function inAssignmentPosition(command: ShellCommandScan): boolean { - return !command.sawCommandWord || command.declarationBuiltin !== undefined -} - -function commandScanOf(frame: ShellScanFrame): ShellCommandScan { - frame.command ??= { - contexts: [], - nested: [], - arithmetic: false, - sawCommandWord: false, - inPrefixValue: false, - valueArithmetic: false, - inRedirectTarget: false, - } - return frame.command -} - -/** - * A `[`/`]`-nesting frame: `arithmetic` for a `$[ ]` or an indexed-array subscript (whose index is - * evaluated), `keysubscript` for an associative array's `[key]` (a string, scanned but not arithmetic). - */ -function subscriptFrame(kind: 'arithmetic' | 'keysubscript'): ShellScanFrame { - return { kind, quote: 'none', parenthesisDepth: 0, bracketDepth: 1, literalRoot: false } -} - -/** A `( )`-delimited command frame: a `$( )` command substitution, or a bare `( )` subshell. */ -function parenCommandFrame(kind: 'command' | 'subshell'): ShellScanFrame { - return { kind, quote: 'none', parenthesisDepth: 1, literalRoot: false } -} - -/** - * Whether the `name[...]` opening at `open` is an indexed assignment (`a[i]=`, `a[i]+=`), whose - * subscript bash evaluates as arithmetic. Brackets nest so `a[b[0]]=` pairs correctly; the scan - * stops at the first unbracketed word boundary, since an unquoted assignment word cannot cross it. - */ -function opensIndexedAssignment(code: string, open: number, end: number): boolean { - let depth = 0 - for (let index = open; index < end; index += 1) { - const character = code[index] - if (character === '[') depth += 1 - else if (character === ']') { - depth -= 1 - if (depth === 0) { - const after = code[index + 1] - return after === '=' || (after === '+' && code[index + 2] === '=') - } - } else if (depth === 0 && /[\s;&|()<>]/.test(character)) return false - } - return false -} - /** * Jumps over `skippedRanges` (sorted heredoc bodies, which bash reads as data) so body prose * cannot shift quote context; bodies that need contexts are scanned on their own with `literalRoot`. - * Spans other than placeholders (heredoc operators) get a context too, for where their body lands. */ function collectShellOccurrenceContexts( code: string, @@ -750,9 +536,7 @@ function collectShellOccurrenceContexts( start: number, end: number, literalRoot: boolean, - skippedRanges: Array<[number, number]> = [], - operatorStarts: ReadonlySet = new Set(), - inheritedScope?: ShellAttributeScope + skippedRanges: Array<[number, number]> = [] ): Map { const occurrenceByStart = new Map( occurrencesWithin(occurrences, start, end).map( @@ -761,280 +545,18 @@ function collectShellOccurrenceContexts( ) const contexts = new Map() const frames: ShellScanFrame[] = [ - { - kind: 'root', - quote: 'none', - parenthesisDepth: 0, - literalRoot, - integerAttribute: new Map(inheritedScope?.integerAttribute), - associativeArrays: new Set(inheritedScope?.associativeArrays), - }, + { kind: 'root', quote: 'none', parenthesisDepth: 0, literalRoot }, ] let skippedRangeIndex = 0 - /** - * How many open frames put the cursor in arithmetic: `$(( ))`/`(( ))`/`$[ ]`/subscript frames, - * integer-value operands, `${…}` substring tails, and commands marked arithmetic. Kept as a count - * so each placeholder reads it in O(1) rather than walking the whole frame stack. - */ let arithmeticDepth = 0 - - const pushFrame = (frame: ShellScanFrame) => { - const enclosing = frames.at(-1) - if (enclosing?.command?.inPrefixValue) frame.prefixExcluded = true - if (enclosing?.command?.valueArithmetic) frame.valueOperand = true - frames.push(frame) - if (frame.kind === 'arithmetic' || frame.valueOperand) arithmeticDepth += 1 - } - /** - * A closed substitution or `${…}` passes its recorded values to the enclosing command, so a later - * `-eq`/`let`/`-i` still reaches a value read inside it — unless it was part of a non-arithmetic - * assignment prefix, which a keyword must not reach. - */ - const popFrame = () => { - const closed = frames.pop() - if (!closed) return - if (closed.kind === 'arithmetic' || closed.valueOperand) arithmeticDepth -= 1 - if (closed.kind === 'parameter' && closed.arithmeticTail) arithmeticDepth -= 1 - if (closed.command?.arithmetic) arithmeticDepth -= 1 - const enclosing = frames.at(-1) - // Only arithmetic frames have no values to pass up; every other closable frame is a command - // substitution or a `${…}`, whose values the enclosing command may still mark. - if (!enclosing || closed.kind === 'arithmetic' || closed.prefixExcluded) return - const enclosingCommand = commandScanOf(enclosing) - for (const ended of closed.endedCommands ?? []) enclosingCommand.nested.push(ended) - if (closed.command) enclosingCommand.nested.push(closed.command) - } - /** - * Whether a command still being read, below the innermost `ownFrames` frames, changes `name`'s - * attributes. An expansion inside that command runs before the change, so its attribute is - * unknown there and lookups take the arithmetic side: integer, indexed. - */ - const declaredByEnclosingCommand = (name: string, ownFrames = 1) => - frames - .slice(0, frames.length - ownFrames) - .some((frame) => frame.command?.declaredNames?.has(name)) - /** Whether `name` has the integer attribute here: the nearest scope that sets or clears it wins. */ - const declaresInteger = (name: string): boolean => { - if (declaredByEnclosingCommand(name)) return true - for (let depth = frames.length - 1; depth >= 0; depth -= 1) { - const attribute = frames[depth].integerAttribute?.get(name) - if (attribute !== undefined) return attribute - } - return false - } - /** The attributes visible here, outermost scope first so a nearer one overrides an integer flag. */ - const visibleScope = (): ShellAttributeScope => { - const scope: ShellAttributeScope = { integerAttribute: new Map(), associativeArrays: new Set() } - for (const frame of frames) { - for (const [name, attribute] of frame.integerAttribute ?? []) { - scope.integerAttribute.set(name, attribute) - } - for (const name of frame.associativeArrays ?? []) scope.associativeArrays.add(name) - } - // A heredoc body expands before any command still being read runs, its receiver included. - for (const frame of frames) { - for (const name of frame.command?.declaredNames ?? []) { - scope.integerAttribute.set(name, true) - scope.associativeArrays.delete(name) - } - } - return scope - } - /** Whether `name` is a `declare -A` associative array, whose subscript is a string key, not arithmetic. */ - const isAssociativeArray = (name: string, ownFrames = 1) => - !declaredByEnclosingCommand(name, ownFrames) && - frames.some((frame) => frame.associativeArrays?.has(name)) - /** A subscript frame for `name`'s array: a string key for an associative array, else an arithmetic index. */ - const subscriptFrameFor = (name: string | undefined, ownFrames = 1) => - subscriptFrame(isAssociativeArray(name ?? '', ownFrames) ? 'keysubscript' : 'arithmetic') - /** Forget a name's tracked array type in every scope — a re-declaration as `-a`/`-A` resets it. */ - const clearArrayType = (name: string) => { - for (const frame of frames) frame.associativeArrays?.delete(name) - } - /** Forget everything tracked about a name in every scope — `unset` removes its attributes too. */ - const clearNameType = (name: string) => { - clearArrayType(name) - for (const frame of frames) frame.integerAttribute?.delete(name) - } - /** Whether a `>`/`<` redirect operator immediately precedes `at`, across any intervening blanks. */ - const precededByRedirect = (at: number): boolean => { - let cursor = at - 1 - while (cursor >= start && (code[cursor] === ' ' || code[cursor] === '\t')) cursor -= 1 - return code[cursor] === '>' || code[cursor] === '<' - } - /** The array name immediately before the subscript `[` at `bracket`, if a valid identifier precedes it. */ - const arrayNameBefore = (bracket: number): { name: string; start: number } | undefined => { - let nameStart = bracket - 1 - while (nameStart >= start && /[A-Za-z0-9_]/.test(code[nameStart])) nameStart -= 1 - nameStart += 1 - if (nameStart >= bracket || !/[A-Za-z_]/.test(code[nameStart])) return undefined - return { name: code.slice(nameStart, bracket), start: nameStart } - } - /** Inside a double-quoted `${…}` single quotes are literal, so the binding keeps its double-quoting. */ - const effectiveQuote = (frame: ShellScanFrame): ShellQuote => - frame.kind === 'parameter' && frame.quote === 'none' - ? frame.inDoubleQuotes - ? 'double' - : 'none' - : frame.quote - const endCommand = (frame: ShellScanFrame) => { - if (!frame.command) return - if (frame.command.arithmetic) arithmeticDepth -= 1 - else if (frame.kind !== 'root') (frame.endedCommands ??= []).push(frame.command) - frame.command = undefined - } - /** True wherever a placeholder's value would be re-read as arithmetic at this point in the scan. */ - const inArithmetic = () => arithmeticDepth > 0 - const record = (occurrence: T, occurrenceContext: ShellOccurrenceContext) => { - contexts.set(occurrence, occurrenceContext) - const innermost = frames.at(-1) - // A non-arithmetic assignment prefix's value, and a redirect target, are left out so a later - // keyword cannot mark them arithmetic. - if (innermost && !innermost.command?.inPrefixValue && !innermost.command?.inRedirectTarget) - commandScanOf(innermost).contexts.push(occurrenceContext) - } - /** Marks the frame's command, and every context already recorded in it, as arithmetic. */ - const markCommandArithmetic = (frame: ShellScanFrame) => { - const command = commandScanOf(frame) - if (command.arithmetic) return - command.arithmetic = true - arithmeticDepth += 1 - const pending = [command] - for (let scan = pending.pop(); scan; scan = pending.pop()) { - for (const commandContext of scan.contexts) commandContext.arithmetic = true - for (const nested of scan.nested) pending.push(nested) - scan.contexts = [] - scan.nested = [] - } - } - /** - * Reads one word at a command boundary and applies the syntax that depends on command position. - * `let`, `[[` and the declaration builtins are keywords only as the command name — past any - * `name=value` prefix, invocation prefix (`command`/`builtin`) or reserved word, and never a - * redirect target — so an argument of the same spelling (`echo let …`) is left alone. The `[[` - * operators and the declaration `-i` option are read wherever they appear; an assignment to an - * integer-declared name counts only in assignment position. - */ - const scanWord = (frame: ShellScanFrame, index: number) => { - SHELL_WORD.lastIndex = index - const raw = SHELL_WORD.exec(code)?.[0] - if (raw === undefined) return - // Join line continuations so a split command name is still recognized (`le\t` → `let`). - // `SHELL_WORD` stops at the newline, so each continued segment is read and appended here. - let joined = raw - let after = index + raw.length - while ( - TRAILING_LINE_CONTINUATION.test(joined) && - (code[after] === '\n' || code[after] === '\r') - ) { - joined = joined.slice(0, -1) - after += code[after] === '\r' && code[after + 1] === '\n' ? 2 : 1 - SHELL_WORD.lastIndex = after - const next = SHELL_WORD.exec(code) - if (!next || next.index !== after) break - joined += next[0] - after += next[0].length - } - const word = unquoteShellWord(joined) - const command = commandScanOf(frame) - command.valueArithmetic = false - command.pendingValue = false - command.inPrefixValue = false - command.inRedirectTarget = false - // `]]` and the `[[` comparison operators are syntax only unquoted; a quoted `"]]"` or `"-eq"` is - // a string operand, so these read the raw word, not the quote-stripped one. - if (raw === ']]') { - frame.conditional = false - return + const conditionalArithmeticRanges: Array<[number, number]> = [] + const endConditionalOperand = (frame: ShellScanFrame, end: number) => { + if (frame.conditional?.arithmetic) { + conditionalArithmeticRanges.push([frame.conditional.start, end]) } if (frame.conditional) { - if (ARITHMETIC_CONDITIONAL_OPERATOR.test(raw)) markCommandArithmetic(frame) - return - } - // A redirect target (possibly after whitespace, `> file`) or leading file descriptor is not the - // command word, and is a filename — never an arithmetic operand even when the command does - // arithmetic (`let x=1 >file`), so its placeholders are flagged out of the command's marking. - if (precededByRedirect(index)) { - command.inRedirectTarget = true - return - } - if ( - /^\d+$/.test(word) && - (code[index + raw.length] === '>' || code[index + raw.length] === '<') - ) - return - // Declaration options, read together so a combined `-iA` sets both: the integer attribute (`-i` - // sets, `+i` clears — last wins, and `-i` is not marked until a value appears so `+i` can still - // undo it) and `-A` (an associative array, whose subscripts are string keys, not arithmetic). - if (command.declarationBuiltin) { - const option = DECLARATION_OPTION.exec(word) - if (option) { - if (option[2].includes('i')) command.integerOption = option[1] === '-' ? 'set' : 'clear' - if (option[1] === '-' && option[2].includes('A')) command.associativeOption = true - if (option[1] === '-' && option[2].includes('a')) command.indexedOption = true - return - } - } - // A declaration argument names an integer (`-i`), clears one (`+i`), and/or names an associative - // array (`-A`); the name lives in this frame's scope and pops with it. `local` is function-scoped - // and this scanner has no function frame to drop it with, so its attributes are not tracked across - // statements — same-command arithmetic still marks, and an untracked array stays indexed (a - // conservative reject) rather than leaking out as text. - if ( - command.declarationBuiltin && - command.declarationBuiltin !== 'local' && - word[0] !== '-' && - word[0] !== '+' - ) { - const declared = SHELL_NAME.exec(word)?.[0] - if (declared) { - // A re-declaration resets the name's array type before this one's attributes apply, so a - // later `declare -a` (indexed) clears an earlier `-A` (associative) and vice versa. The - // integer attribute survives it — only `+i` or `unset` removes that. - if (command.associativeOption || command.indexedOption || command.integerOption) { - ;(command.declaredNames ??= new Set()).add(declared) - } - if (command.associativeOption || command.indexedOption) clearArrayType(declared) - if (command.associativeOption) (frame.associativeArrays ??= new Set()).add(declared) - if (command.integerOption) { - ;(frame.integerAttribute ??= new Map()).set(declared, command.integerOption === 'set') - } - } - } - // `unset name` removes the variable and its attributes, so a later indexed reuse is arithmetic - // again. A bare name only — `unset name[i]` removes one element, not the array's type — and not - // under `-f`, which removes a function of that name and leaves the variable alone. - if (command.nameArgumentBuiltin && command.sawCommandWord) { - const option = DECLARATION_OPTION.exec(word) - if (option?.[1] === '-' && option[2].includes('f')) command.unsetsFunctions = true - else if (!command.unsetsFunctions && SHELL_BARE_NAME.test(word)) { - ;(command.declaredNames ??= new Set()).add(word) - clearNameType(word) - } - } - // An assignment is one only in command-prefix or declaration-argument position; `echo n=1` or - // `printf a[i]=1` passes an ordinary string that bash never evaluates. - const assignment = inAssignmentPosition(command) ? SHELL_ASSIGNMENT_WORD.exec(word) : null - if (assignment) { - // A `declare -i` argument, or any assignment to an already-integer name not cleared here, has - // an arithmetic value. A declaration marks the whole command (every arg shares the attribute); - // a plain prefix marks only this value, so `n=1 printf "{{x}}"` leaves the printed arg as text. - const integerTarget = - (command.declarationBuiltin !== undefined && command.integerOption === 'set') || - (command.integerOption !== 'clear' && declaresInteger(assignment[1])) - if (integerTarget && command.declarationBuiltin) markCommandArithmetic(frame) - else if (integerTarget) command.pendingValue = true - else if (!command.sawCommandWord) command.inPrefixValue = true - return + frame.conditional = { start: end, arithmetic: false, words: 0, wordOpen: false, unary: false } } - if (command.sawCommandWord) return - if (SHELL_RESERVED_WORDS.has(word) || SHELL_COMMAND_PREFIXES.has(word)) return - command.sawCommandWord = true - if (word === '[[') frame.conditional = true - else if (word === 'let') markCommandArithmetic(frame) - else if (word === 'declare' || word === 'typeset' || word === 'local') { - command.declarationBuiltin = word - } else if (NAME_ARGUMENT_BUILTINS.has(word)) command.nameArgumentBuiltin = true } for (let index = start; index < end; ) { @@ -1053,118 +575,106 @@ function collectShellOccurrenceContexts( continue } - const occurrence = occurrenceByStart.get(index) - if (occurrence && operatorStarts.has(occurrence.start)) { - // A heredoc operator: its body is an arithmetic operand when the substitution reading it is in - // an arithmetic context — an enclosing `$(( ))`, or an enclosing command/value that evaluates - // the substitution's output (`let x="$(cat <= 0 && !isSubstitution(frames[sub])) sub -= 1 - const enclosingArithmetic = frames - .slice(0, sub) - .some((enclosing) => frameIsArithmetic(enclosing) || enclosing.command?.valueArithmetic) - const operatorContext: ShellOccurrenceContext = { - quote: 'none', - arithmetic: enclosingArithmetic, - scope: visibleScope(), - } - contexts.set(occurrence, operatorContext) - if (sub >= 0) commandScanOf(frames[sub - 1]).contexts.push(operatorContext) - index = occurrence.end - continue - } - if (occurrence) { - // A placeholder that is itself the redirect target (`> {{x}}`) is a filename, never arithmetic; - // `scanWord` never ran on it (the occurrence is consumed first), so the immediately preceding - // `>`/`<` — across any whitespace — is checked here, but only in a command frame: inside `$(( ))` - // a `<`/`>` is a comparison operator, not a redirect. - const redirectTarget = - frame.command?.inRedirectTarget === true || - (readsCommands(frame) && precededByRedirect(index)) - const arithmetic = - !redirectTarget && (inArithmetic() || frame.command?.valueArithmetic === true) - record(occurrence, { quote: effectiveQuote(frame), arithmetic }) - index = occurrence.end - continue - } - const character = code[index] - // The redirect-target and assignment-value flags cover only their one word; unquoted whitespace - // ends that word. `scanWord` clears them at the next word too, but a placeholder-only word never - // reaches it, so `let x=1 >/dev/null {{x}}` is marked by the command again while the command name - // in `n=1 {{x}} arg` is not read as `n`'s integer value. - if (frame.command && frame.quote === 'none' && /\s/.test(character)) { - frame.command.inRedirectTarget = false - frame.command.valueArithmetic = false - frame.command.pendingValue = false - frame.command.inPrefixValue = false - } - // Classify each command word before the quote branches consume a quote-initial word such as - // `"-i"` or `'let'`. `scanWord` only sets state, never advances `index`; the characters below - // still process the word's text. if ( - readsCommands(frame) && - !frame.literalRoot && frame.quote === 'none' && - shellWordStarts(code, index) && - !shellCommentStarts(code, index) && - !shellCommandSeparator(code, index) - ) { - scanWord(frame, index) - } - // An integer assignment's value turns arithmetic at its `=` — not its subscript, so an index - // placeholder in `m[{{x}}]=1` is judged by the array type, not by the value's attribute. - if (character === '=' && frame.command?.pendingValue) { - frame.command.valueArithmetic = true - frame.command.pendingValue = false - } - // A name-taking builtin's argument subscript (`unset a[i]`, `unset "a[i]"`, `unset 'a[i]'`) is - // arithmetic — even single-quoted, since the subscript is still evaluated. This runs before the - // quote branches, which would otherwise consume the `[` of a quoted argument. - if ( - character === '[' && !frame.literalRoot && - readsCommands(frame) && - frame.command?.nameArgumentBuiltin && - frame.command.sawCommandWord + frame.kind !== 'arithmetic' && + frame.kind !== 'parameter' ) { - const named = arrayNameBefore(index) - if ( - named && - /[\s;&|()<>"']/.test(code[named.start - 1] ?? ' ') && - !isAssociativeArray(named.name) + if (frame.conditional) { + if ((character === '&' || character === '|') && code[index + 1] === character) { + endConditionalOperand(frame, index) + } + if (/\s|[()]/.test(character)) { + frame.conditional.wordOpen = false + } else if (!frame.conditional.wordOpen && character !== '&' && character !== '|') { + const { word } = readShellWord(code, index) + if (word === ']]') { + endConditionalOperand(frame, index) + frame.conditional = undefined + } else { + const conditional = frame.conditional + conditional.wordOpen = true + if (word !== '!' || conditional.words > 0) { + if ( + conditional.words === 1 && + !conditional.unary && + word && + ARITHMETIC_COMPARISON.test(word) + ) { + conditional.arithmetic = true + } + if (conditional.words === 0) conditional.unary = /^-[a-zA-Z]$/.test(word ?? '') + conditional.words += 1 + } + } + } + } else if ( + /[\n;()]/.test(character) || + ((character === '&' || character === '|') && + !/[<>]/.test(code[index - 1] ?? '') && + !(character === '&' && code[index + 1] === '>')) || + (character === '{' && + shellWordStarts(code, index) && + readShellWord(code, index).word === '{') ) { - pushFrame(subscriptFrame('arithmetic')) - index += 1 - continue + frame.commandStarted = false + frame.commandPrefix = undefined + } else if ( + !frame.commandStarted && + index >= (frame.wordEnd ?? start) && + shellWordStarts(code, index) + ) { + const { word, end: wordEnd } = readShellWord(code, index) + frame.wordEnd = wordEnd + if ( + word && + !followsRedirect(code, index) && + !/^\d+(?=[<>])/.test(code.slice(index)) && + !/^[A-Za-z_][A-Za-z0-9_]*=/.test(word) + ) { + const prefixArgument = + (frame.commandPrefix === 'time' && word === '-p') || + (frame.commandPrefix === 'coproc' && word !== '[[' && !COMMAND_INTRODUCERS.has(word)) + frame.commandPrefix = word === 'time' || word === 'coproc' ? word : undefined + if (!COMMAND_INTRODUCERS.has(word) && !prefixArgument) { + frame.commandStarted = true + if (word === '[[') { + frame.conditional = { + start: index, + arithmetic: false, + words: 0, + wordOpen: true, + unary: false, + } + } + } + } } } - // A `${…}` expansion, including one nested in another's value or an associative key. Only an - // `arithmetic` frame reads a nested `${b}` as operand text rather than its own expansion, so the - // gate excludes it; a `keysubscript` (string key) does take nested expansions. + const occurrence = occurrenceByStart.get(index) + if (occurrence) { + contexts.set(occurrence, { quote: effectiveQuote(frame), arithmetic: arithmeticDepth > 0 }) + index = occurrence.end + continue + } if ( character === '$' && code[index + 1] === '{' && + !occurrenceByStart.has(index + 1) && frame.kind !== 'arithmetic' && (frame.quote === 'none' || frame.quote === 'double') ) { - PARAMETER_NAME.lastIndex = index + 2 - const name = PARAMETER_NAME.exec(code) - if (name) { - pushFrame({ - kind: 'parameter', - quote: 'none', - inDoubleQuotes: effectiveQuote(frame) === 'double', - parenthesisDepth: 0, - atOperator: true, - parameterName: name[0].replace(/^[!#]/, ''), - literalRoot: false, - }) - index += 2 + name[0].length - continue - } + frames.push({ + kind: 'parameter', + quote: 'none', + parenthesisDepth: 0, + literalRoot: false, + inDoubleQuotes: effectiveQuote(frame) === 'double' || frame.literalRoot, + }) + index += 2 + continue } if (frame.quote === 'single') { if (character === "'") frame.quote = 'none' @@ -1175,7 +685,7 @@ function collectShellOccurrenceContexts( if (character === '\\') { const escaped = occurrenceByStart.get(index + 1) if (escaped) { - record(escaped, { quote: frame.quote, unsupported: 'escaped sequence' }) + contexts.set(escaped, { quote: frame.quote, unsupported: 'escaped sequence' }) index = escaped.end } else { index += 2 @@ -1190,14 +700,20 @@ function collectShellOccurrenceContexts( character === '$' && ((code[index + 1] === '(' && code[index + 2] === '(') || code[index + 1] === '[') const arithmeticCommand = - frame.quote === 'none' && !frame.literalRoot && shellArithmeticCommandStarts(code, index) + frame.quote === 'none' && + !frame.literalRoot && + frame.kind !== 'parameter' && + shellArithmeticCommandStarts(code, index) if (arithmeticExpansion || arithmeticCommand) { const brackets = arithmeticExpansion && code[index + 1] === '[' - pushFrame( - brackets - ? subscriptFrame('arithmetic') - : { kind: 'arithmetic', quote: 'none', parenthesisDepth: 2, literalRoot: false } - ) + frames.push({ + kind: 'arithmetic', + quote: 'none', + parenthesisDepth: brackets ? 0 : 2, + ...(brackets ? { bracketDepth: 1 } : {}), + literalRoot: false, + }) + arithmeticDepth += 1 index += arithmeticExpansion && !brackets ? 3 : 2 continue } @@ -1205,7 +721,7 @@ function collectShellOccurrenceContexts( if (character === '\\') { const escaped = occurrenceByStart.get(index + 1) if (escaped) { - record(escaped, { quote: frame.quote, unsupported: 'escaped sequence' }) + contexts.set(escaped, { quote: frame.quote, unsupported: 'escaped sequence' }) index = escaped.end } else { index += 2 @@ -1214,10 +730,20 @@ function collectShellOccurrenceContexts( frame.quote = 'none' index += 1 } else if (character === '$' && code[index + 1] === '(') { - pushFrame(parenCommandFrame('command')) + frames.push({ + kind: 'command', + quote: 'none', + parenthesisDepth: 1, + literalRoot: false, + }) index += 2 } else if (character === '`') { - pushFrame({ kind: 'backtick', quote: 'none', parenthesisDepth: 0, literalRoot: false }) + frames.push({ + kind: 'backtick', + quote: 'none', + parenthesisDepth: 0, + literalRoot: false, + }) index += 1 } else { index += 1 @@ -1226,13 +752,17 @@ function collectShellOccurrenceContexts( } if (frame.kind === 'backtick' && character === '`') { - popFrame() + frames.pop() index += 1 continue } - // `#` starts a comment only where commands are read; inside a `${…}` or a subscript it is literal. - if (readsCommands(frame) && !frame.literalRoot && shellCommentStarts(code, index)) { - if (!frame.conditional) endCommand(frame) + if ( + frame.kind !== 'arithmetic' && + frame.kind !== 'parameter' && + !frame.literalRoot && + shellCommentStarts(code, index) + ) { + if (!frame.conditional) frame.commandStarted = false const newline = code.indexOf('\n', index) index = newline === -1 || newline >= end ? end : newline + 1 continue @@ -1240,15 +770,14 @@ function collectShellOccurrenceContexts( if (character === '\\') { const escaped = occurrenceByStart.get(index + 1) if (escaped) { - record(escaped, { quote: frame.quote, unsupported: 'escaped sequence' }) + contexts.set(escaped, { quote: frame.quote, unsupported: 'escaped sequence' }) index = escaped.end } else { index += 2 } continue } - // Inside a double-quoted `${…}`, single quotes are literal text, not a quote boundary. - const singleQuotesLiteral = frame.kind === 'parameter' && frame.inDoubleQuotes === true + const singleQuotesLiteral = frame.kind === 'parameter' && frame.inDoubleQuotes if ( !frame.literalRoot && !singleQuotesLiteral && @@ -1270,173 +799,64 @@ function collectShellOccurrenceContexts( continue } if (character === '$' && code[index + 1] === '(') { - pushFrame(parenCommandFrame('command')) + frames.push({ + kind: 'command', + quote: 'none', + parenthesisDepth: 1, + literalRoot: false, + }) index += 2 continue } if (character === '`') { - pushFrame({ kind: 'backtick', quote: 'none', parenthesisDepth: 0, literalRoot: false }) + frames.push({ + kind: 'backtick', + quote: 'none', + parenthesisDepth: 0, + literalRoot: false, + }) index += 1 continue } - // A `${…}` expansion's arithmetic operands appear only at the operator right after the name: a - // `[` subscript, or a `:` substring offset/length. Any other operator (`:-`, `#`, `/`, …) means - // the rest is pattern or default text, where a later `[` or `:` is literal. - if (frame.kind === 'parameter') { - if (frame.atOperator) { - // A subscript: an indexed array's is arithmetic, an associative array's is a string key. - // Either way it opens a bracket frame so a `:offset` after the `]` is still seen as a tail. - if (character === '[') { - pushFrame(subscriptFrameFor(frame.parameterName)) - index += 1 - continue - } - if (character === ':' && !/[-=?+]/.test(code[index + 1] ?? '')) { - frame.arithmeticTail = true - arithmeticDepth += 1 - frame.atOperator = false - index += 1 - continue - } - frame.atOperator = false - } - // A nested `${…}` opens its own frame above; here a bare `{` is literal text, and the first - // unmatched `}` ends the expansion — so only `}` closes it, never a counted `{`. - if (character === '}') { - popFrame() - index += 1 - continue - } - } - // A compound array assignment `name=( … )`: each element key `[k]=` is a subscript of `name`. - if (frame.kind === 'arrayliteral') { - if (character === '(') { - frame.parenthesisDepth += 1 - index += 1 - continue - } - if (character === ')') { - frame.parenthesisDepth -= 1 - if (frame.parenthesisDepth === 0) popFrame() - index += 1 - continue - } - // A key assignment `[k]=` / `[k]+=`; a bare `[x]` element (no `=`) is ordinary text. - if ( - character === '[' && - /[\s(]/.test(code[index - 1] ?? ' ') && - opensIndexedAssignment(code, index, end) - ) { - // The literal belongs to the assignment in the frame below, whose own declaration applies. - pushFrame(subscriptFrameFor(frame.arrayName, 2)) - index += 1 - continue - } - } - // The `(` opening a compound array assignment (`name=(`, `name+=(`) in command-prefix or - // declaration position. Its element keys are then judged against the array's type. - if ( - character === '(' && - code[index - 1] === '=' && - frame.quote === 'none' && - !frame.literalRoot && - readsCommands(frame) && - inAssignmentPosition(commandScanOf(frame)) - ) { - let nameEnd = index - 1 - if (code[nameEnd - 1] === '+') nameEnd -= 1 - const named = arrayNameBefore(nameEnd) - if (named && shellWordStarts(code, named.start)) { - pushFrame({ - kind: 'arrayliteral', - quote: 'none', - parenthesisDepth: 1, - arrayName: named.name, - literalRoot: false, - }) - index += 1 - continue - } - } - // An indexed-assignment subscript (`a[i]=`, `a[i]+=`) is evaluated as arithmetic — but only in - // command-prefix or declaration-argument position; `printf a[i]=1` passes an ordinary string, and - // an associative key (`declare -A m; m[k]=`) is text. - if ( - character === '[' && - frame.quote === 'none' && - !frame.literalRoot && - readsCommands(frame) && - inAssignmentPosition(commandScanOf(frame)) - ) { - const named = arrayNameBefore(index) - if ( - named && - shellWordStarts(code, named.start) && - !isAssociativeArray(named.name) && - opensIndexedAssignment(code, index, end) - ) { - pushFrame(subscriptFrame('arithmetic')) - index += 1 - continue - } - } - if (readsCommands(frame)) { - if (!frame.literalRoot) { - const groupBrace = (character === '{' || character === '}') && shellWordStarts(code, index) - if (shellCommandSeparator(code, index) || groupBrace) { - const clauseEnds = - !frame.conditional || - ((character === '&' || character === '|') && code[index + 1] === character) - if (clauseEnds) endCommand(frame) - } - } - } - // A bare subshell `( … )` at a command position: scope its declarations to a frame so a - // `(declare -A m)` inside does not leak the array type to the enclosing shell. - if ( - character === '(' && - readsCommands(frame) && - !frame.conditional && - frame.quote === 'none' && - !frame.literalRoot && - shellWordStarts(code, index) - ) { - pushFrame(parenCommandFrame('subshell')) + if (frame.kind === 'parameter' && character === '}') { + frames.pop() index += 1 continue } - if ( - (frame.kind === 'arithmetic' || frame.kind === 'keysubscript') && - frame.bracketDepth !== undefined - ) { + if (frame.kind === 'arithmetic' && frame.bracketDepth !== undefined) { if (character === '[') frame.bracketDepth += 1 - else if (character === ']') { + if (character === ']') { frame.bracketDepth -= 1 - if (frame.bracketDepth === 0) popFrame() + if (frame.bracketDepth === 0) { + frames.pop() + arithmeticDepth -= 1 + } } index += 1 continue } - if ( - (frame.kind === 'command' || frame.kind === 'arithmetic' || frame.kind === 'subshell') && - character === '(' - ) { + if ((frame.kind === 'command' || frame.kind === 'arithmetic') && character === '(') { frame.parenthesisDepth += 1 index += 1 continue } - if ( - (frame.kind === 'command' || frame.kind === 'arithmetic' || frame.kind === 'subshell') && - character === ')' - ) { + if ((frame.kind === 'command' || frame.kind === 'arithmetic') && character === ')') { frame.parenthesisDepth -= 1 - if (frame.parenthesisDepth === 0) popFrame() + if (frame.parenthesisDepth === 0) { + frames.pop() + if (frame.kind === 'arithmetic') arithmeticDepth -= 1 + } index += 1 continue } index += 1 } + for (const frame of frames) endConditionalOperand(frame, end) + const inConditionalArithmetic = createOffsetRangeLookup(conditionalArithmeticRanges) + for (const [occurrence, context] of contexts) { + if (inConditionalArithmetic(occurrence.start)) context.arithmetic = true + } return contexts } @@ -1527,6 +947,11 @@ function recordShellDirectEnvironmentReads( } } +/** + * Binds values without inserting them into shell source, rejecting placeholders in explicit + * arithmetic delimiters and arithmetic comparisons. This lexical guard does not follow later + * evaluation through variable attributes, indirect command names, or commands such as `eval`. + */ export async function compileShellPlaceholders( input: InternalCompileCodePlaceholdersInput ): Promise { @@ -1545,14 +970,6 @@ export async function compileShellPlaceholders( occurrence.start ) } - const unsafeSubscript = resolved && unsafeArithmeticSubscript(resolved.value) - if (unsafeSubscript && !input.analysisOnly) { - throw new CodePlaceholderCompileError( - `Variable placeholder "${occurrence.name}" cannot contain ${unsafeSubscript} in shell code`, - input.code, - occurrence.start - ) - } return resolved } const resolveShellOccurrence = (occurrence: CodePlaceholderOccurrence) => @@ -1621,8 +1038,7 @@ export async function compileShellPlaceholders( 0, input.code.length, false, - heredocBodyRanges(heredocs), - new Set(heredocOperators.map((operator) => operator.start)) + heredocBodyRanges(heredocs) ) const edits: SourceEdit[] = [] @@ -1637,9 +1053,6 @@ export async function compileShellPlaceholders( } const bodyOccurrences = occurrencesWithin(shellOccurrences, heredoc.bodyStart, heredoc.bodyEnd) - // The operator's context already carries this: `arithmetic` is set only for a heredoc whose - // reading substitution is itself an arithmetic operand (see where operator spans are recorded), - // so a quoted or unquoted body there is rejected, while plain stdin falls to the body scan. if (rootContexts.get(heredocOperators[heredocIndex])?.arithmetic) { for (const occurrence of bodyOccurrences) rejectUnsupported(occurrence, 'in shell arithmetic') continue @@ -1691,10 +1104,7 @@ export async function compileShellPlaceholders( bodyOccurrences, heredoc.bodyStart, heredoc.bodyEnd, - true, - [], - undefined, - rootContexts.get(heredocOperators[heredocIndex])?.scope + true ) for (const occurrence of bodyOccurrences) { const edit = resolveInContext(occurrence, bodyContexts.get(occurrence), false) diff --git a/apps/sim/scripts/test-shell-placeholders-e2e.ts b/apps/sim/scripts/test-shell-placeholders-e2e.ts new file mode 100644 index 00000000000..e42c28309b8 --- /dev/null +++ b/apps/sim/scripts/test-shell-placeholders-e2e.ts @@ -0,0 +1,194 @@ +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { mkdir, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { + CodePlaceholderCompileError, + type CompiledCodePlaceholders, + compileCodePlaceholders, +} from '@/lib/execution/code-placeholders' +import { CodeLanguage } from '@/lib/execution/languages' + +/** + * Exercises the compiler's environment bindings and private inputs in real Bash. + * Run from apps/sim with SHELL_PLACEHOLDERS_REPORT_PATH and optionally SHELL_PLACEHOLDERS_BASH. + * This covers compilation through process execution, not the hosted sandbox transport. + */ +const logger = createLogger('ShellPlaceholdersE2E') +const reportPath = process.env.SHELL_PLACEHOLDERS_REPORT_PATH +assert(reportPath, 'Set SHELL_PLACEHOLDERS_REPORT_PATH') +const bash = process.env.SHELL_PLACEHOLDERS_BASH ?? '/bin/bash' +const checks: { name: string; status: 'passed' | 'failed'; durationMs: number; error?: string }[] = + [] +const directory = mkdtempSync(join(tmpdir(), 'sim-shell-placeholders-')) +const sentinel = join(directory, 'sentinel') +const payload = `values[$(touch '${sentinel}')]` + +function execute(compiled: CompiledCodePlaceholders): string { + const env: NodeJS.ProcessEnv = { NODE_ENV: 'test', PATH: process.env.PATH ?? '/usr/bin:/bin' } + for (const binding of compiled.bindings) env[binding.name] = binding.value + for (const [index, input] of compiled.privateInputs.entries()) { + const path = join(directory, `input-${index}`) + writeFileSync(path, input.content, { mode: 0o600 }) + env[input.environmentVariable] = path + } + const result = spawnSync(bash, ['--noprofile', '--norc', '-c', compiled.code], { + env, + cwd: directory, + encoding: 'utf8', + timeout: 5_000, + maxBuffer: 1024 * 1024, + }) + if (result.error) throw result.error + assert.equal(result.status, 0, result.stderr) + return result.stdout +} + +async function check(name: string, run: () => Promise): Promise { + const start = performance.now() + rmSync(sentinel, { force: true }) + try { + await run() + checks.push({ name, status: 'passed', durationMs: performance.now() - start }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: performance.now() - start, + error: getErrorMessage(error), + }) + } +} + +async function compile(code: string, value: string): Promise { + return compileCodePlaceholders({ code, language: CodeLanguage.Shell, params: { KEY: value } }) +} + +try { + const arithmetic = [ + 'time -p [[ "{{KEY}}" -eq 0 ]]', + 'coproc [[ "{{KEY}}" -eq 0 ]]; wait', + 'coproc worker [[ "{{KEY}}" -eq 0 ]]; wait', + 'coproc "worker" [[ "{{KEY}}" -eq 0 ]]; wait', + 'coproc "$(printf worker)" [[ "{{KEY}}" -eq 0 ]]; wait', + 'if \\\n [[ "{{KEY}}" -eq 0 ]]; then :; fi', + '[\\\n[ "{{KEY}}" -eq 0 ]]', + '[[ "{{KEY}}" -e\\\nq 0 ]]', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion + '[[ ${missing:- {{KEY}}} -eq 0 ]]', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion + '[[ ${missing:-"${other:- {{KEY}}}"} -eq 0 ]]', + + '[[ "{{KEY}}" -eq 0 ]]', + '[[ 0 -lt "{{KEY}}" ]]', + 'echo ignored >/dev/null # end command\n[[ "{{KEY}}" -eq 0 ]]', + 'case x in x) [[ "{{KEY}}" -eq 0 ]];; esac', + '[[ $(printf %s "{{KEY}}")+0 -eq 0 ]]', + '[[ $(printf "%s" "{{KEY}}") -ge 0 ]]', + '[[ $(printf "%s" "{{KEY}}"; :) -ne 1 ]]', + '[[ $(cat < { + const binding = '__probe_value' + execute({ + code: code.replaceAll('{{KEY}}', `\${${binding}}`), + bindings: [{ name: binding, value: payload }], + privateInputs: [], + runtimeBindings: [], + resolvedSecretNames: [], + internalIdentifiers: [], + }) + assert(existsSync(sentinel), 'The unprotected expression must execute the sentinel') + }) + } + for (const code of arithmetic) { + await check(`rejects arithmetic: ${code}`, async () => { + await assert.rejects( + () => compile(code, payload), + (error: unknown) => + error instanceof CodePlaceholderCompileError && + error.message.includes('in shell arithmetic') + ) + assert(!existsSync(sentinel)) + }) + } + + for (const value of ['["$5"]', '[WIP', "[don't]", payload, 'a[', '$(printf injected)']) { + for (const code of [ + 'printf "%s\\n" "{{KEY}}"', + "cat <<'EOF'\n{{KEY}}\nEOF", + 'cat < { + assert.equal(execute(await compile(code, value)), `${value}\n`) + assert(!existsSync(sentinel), 'Literal data must not execute') + }) + } + } + + for (const code of [ + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion + 'printf "%s\\n" "${missing:-\'{{KEY}}\'}"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion + "cat < { + assert.equal(execute(await compile(code, payload)), `'${payload}'\n`) + assert(!existsSync(sentinel)) + }) + } + + const safePrograms = [ + 'time -p printf "%s\\n" "{{KEY}}" >/dev/null; printf "%s\\n" ok', + 'if (( BASH_VERSINFO[0] >= 4 )); then coproc printf "%s\\n" "{{KEY}}" [[ -eq 0 ]]; wait; fi; printf "%s\\n" ok', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion + 'printf %s ${missing:- } [[ "{{KEY}}" -eq 0 ]] >/dev/null; printf "%s\\n" ok', + 'printf %s &>/dev/null [[ "{{KEY}}" -eq 0 ]]; printf "%s\\n" ok', + '[[ "{{KEY}}" == -eq ]] || printf "%s\\n" ok', + '[[ "{{KEY}}" < -eq ]] || printf "%s\\n" ok', + '[[ "{{KEY}}" == values* && 1 -eq 1 ]] && printf "%s\\n" ok', + '[[ 1 -eq 1 && "{{KEY}}" == values* ]] && printf "%s\\n" ok', + '[[ ( "{{KEY}}" == values* ) && 1 -eq 1 ]] && printf "%s\\n" ok', + '[ "{{KEY}}" -eq 0 ] 2>/dev/null || printf "%s\\n" ok', + 'f() { declare -i n; }; n="{{KEY}}"; [[ "$n" == values* ]] && printf "%s\\n" ok', + 'if [[ 1 -eq 1 ]]; then printf "%s\\n" "{{KEY}}" >/dev/null; fi; printf "%s\\n" ok', + 'printf "%s" [[ "{{KEY}}" -eq 0 ]] >/dev/null; printf "%s\\n" ok', + ] + for (const code of safePrograms) { + await check(`preserves safe command: ${code}`, async () => { + assert.equal(execute(await compile(code, payload)), 'ok\n') + assert(!existsSync(sentinel)) + }) + } + await check('preserves a placeholder command name', async () => { + const compiled = await compileCodePlaceholders({ + code: '{{COMMAND}} [[ "{{KEY}}" -eq 0 ]]', + language: CodeLanguage.Shell, + params: { COMMAND: 'printf', KEY: payload }, + }) + assert.equal(execute(compiled), '[[') + assert(!existsSync(sentinel)) + }) +} finally { + rmSync(directory, { recursive: true, force: true }) + await mkdir(dirname(reportPath), { recursive: true }) + await writeFile(reportPath, JSON.stringify({ bash, checks }, null, 2)) +} +const failed = checks.filter((result) => result.status === 'failed') +logger.info('Shell placeholder execution complete', { + passed: checks.length - failed.length, + failed: failed.length, + reportPath, +}) +if (failed.length) process.exitCode = 1 From f4573b33fa92f3c2735b624e06bc589018d4037e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 22:57:59 -0700 Subject: [PATCH 6/7] fix(code-placeholders): skip continued newlines before comparison operands --- apps/sim/lib/execution/code-placeholders/shell.ts | 4 ++++ apps/sim/scripts/test-shell-placeholders-e2e.ts | 3 +++ 2 files changed, 7 insertions(+) diff --git a/apps/sim/lib/execution/code-placeholders/shell.ts b/apps/sim/lib/execution/code-placeholders/shell.ts index 0728d43d877..78dfc1703fd 100644 --- a/apps/sim/lib/execution/code-placeholders/shell.ts +++ b/apps/sim/lib/execution/code-placeholders/shell.ts @@ -583,6 +583,10 @@ function collectShellOccurrenceContexts( frame.kind !== 'parameter' ) { if (frame.conditional) { + if (character === '\\' && code[index + 1] === '\n') { + index += 2 + continue + } if ((character === '&' || character === '|') && code[index + 1] === character) { endConditionalOperand(frame, index) } diff --git a/apps/sim/scripts/test-shell-placeholders-e2e.ts b/apps/sim/scripts/test-shell-placeholders-e2e.ts index e42c28309b8..26e434f4cd6 100644 --- a/apps/sim/scripts/test-shell-placeholders-e2e.ts +++ b/apps/sim/scripts/test-shell-placeholders-e2e.ts @@ -70,6 +70,8 @@ async function compile(code: string, value: string): Promise/dev/null; printf "%s\\n" ok', 'if (( BASH_VERSINFO[0] >= 4 )); then coproc printf "%s\\n" "{{KEY}}" [[ -eq 0 ]]; wait; fi; printf "%s\\n" ok', // biome-ignore lint/suspicious/noTemplateCurlyInString: shell parameter expansion From 3d993519fbe4266f527486a3a00e69c9c0374ac7 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 23:18:39 -0700 Subject: [PATCH 7/7] fix(code-placeholders): preserve command scope across process substitutions --- apps/sim/lib/execution/code-placeholders/shell.ts | 9 ++++++++- apps/sim/scripts/test-shell-placeholders-e2e.ts | 12 ++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/execution/code-placeholders/shell.ts b/apps/sim/lib/execution/code-placeholders/shell.ts index 78dfc1703fd..9bcf5cec061 100644 --- a/apps/sim/lib/execution/code-placeholders/shell.ts +++ b/apps/sim/lib/execution/code-placeholders/shell.ts @@ -802,7 +802,14 @@ function collectShellOccurrenceContexts( index += 1 continue } - if (character === '$' && code[index + 1] === '(') { + if ( + code[index + 1] === '(' && + (character === '$' || + ((character === '<' || character === '>') && + !frame.literalRoot && + frame.kind !== 'arithmetic' && + effectiveQuote(frame) === 'none')) + ) { frames.push({ kind: 'command', quote: 'none', diff --git a/apps/sim/scripts/test-shell-placeholders-e2e.ts b/apps/sim/scripts/test-shell-placeholders-e2e.ts index 26e434f4cd6..4902879f74c 100644 --- a/apps/sim/scripts/test-shell-placeholders-e2e.ts +++ b/apps/sim/scripts/test-shell-placeholders-e2e.ts @@ -70,6 +70,7 @@ async function compile(code: string, value: string): Promise(cat)', '<(cat <(printf ignored))']) { + await check(`preserves arguments after ${substitution}`, async () => { + const code = `printf "%s\\n" ${substitution} [[ "{{KEY}}" -eq 0 ]] | tail -n +2` + assert.equal(execute(await compile(code, payload)), `[[\n${payload}\n-eq\n0\n]]\n`) + assert(!existsSync(sentinel)) + }) + } + await check('preserves literal values inside process substitutions', async () => { + assert.equal(execute(await compile('cat <(printf "%s\\n" "{{KEY}}")', payload)), `${payload}\n`) + assert(!existsSync(sentinel)) + }) } finally { rmSync(directory, { recursive: true, force: true }) await mkdir(dirname(reportPath), { recursive: true })