From d82d5f9bc1c8dd8c10057fdc13426adeb2c2367e Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 14:18:58 +0200 Subject: [PATCH 01/35] Match CLI downloads to Assembly results --- .changeset/assembly-download-paths.md | 7 + packages/node/README.md | 9 +- packages/node/src/cli/commands/assemblies.ts | 202 +++++++++++----- packages/node/src/cli/resultFiles.ts | 1 + .../test/unit/cli/assemblies-create.test.ts | 36 +-- .../test/unit/cli/download-layout.test.ts | 217 ++++++++++++++++++ packages/transloadit/README.md | 9 +- 7 files changed, 402 insertions(+), 79 deletions(-) create mode 100644 .changeset/assembly-download-paths.md create mode 100644 packages/node/test/unit/cli/download-layout.test.ts diff --git a/.changeset/assembly-download-paths.md b/.changeset/assembly-download-paths.md new file mode 100644 index 00000000..a8aea995 --- /dev/null +++ b/.changeset/assembly-download-paths.md @@ -0,0 +1,7 @@ +--- +'@transloadit/node': patch +'@transloadit/mcp-server': patch +'transloadit': patch +--- + +Keep CLI download filenames and HLS directories aligned with Assembly results, omit input parent folders from output paths, and support HTTP(S) URL inputs for Assembly and prompt commands. diff --git a/packages/node/README.md b/packages/node/README.md index 87cc0eb9..bc7da02b 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -1502,7 +1502,14 @@ The `assemblies create` command additionally supports: Without uploads (for example, `/vimeo/import`, `/http/import`, or prompt-only `/image/generate` Steps), results are saved as `//`, with no input folder, including with `--single-assembly`. -- `--input, -i` - Provide a local upload, or use `-i -` to read stdin explicitly. When `--steps` + With one local input, filenames come from the Assembly results. Multiple result Steps get their + own folders; the input’s parent folders are omitted. Directory inputs retain subfolders relative + to that directory. HLS results retain the directories in `meta.relative_path`, so playlists can + find their renditions and segments. If those paths already exist, the whole result Step is saved + in a new Step folder together, preserving playlist references. +- `--input, -i` - Provide a local upload or an HTTP(S) URL, or use `-i -` to read stdin explicitly. + URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` + requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly without uploads. With `--template`, stdin always requires `-i -`. diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index de4ac96e..eda76ef6 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -32,6 +32,7 @@ import { z } from 'zod' import { formatLintIssue } from '../../alphalib/assembly-linter.lang.en.ts' import { tryCatch } from '../../alphalib/tryCatch.ts' import { ensureUniqueCounterValue } from '../../ensureUniqueCounter.ts' +import { prepareInputFiles } from '../../inputFiles.ts' import { lintAssemblyInstructions } from '../../lintAssemblyInstructions.ts' import { lintingExamples } from '../docs/assemblyLintingExamples.ts' import { @@ -519,7 +520,7 @@ async function createExistingPathOutputPlan(outputPath: string | undefined): Pro } function dirProvider(output: string): OutputPlanProvider { - return async (inpath, indir = process.cwd()) => { + return async (inpath, indir) => { if (inpath == null) { return await createExistingPathOutputPlan(output) } @@ -527,8 +528,7 @@ function dirProvider(output: string): OutputPlanProvider { throw new Error('You must provide an input to output to a directory') } - let relpath = path.relative(indir, inpath) - relpath = relpath.replace(/^(\.\.\/)+/, '') + const relpath = indir == null ? path.basename(inpath) : path.relative(indir, inpath) const outpath = path.join(output, relpath) return await createExistingPathOutputPlan(outpath) } @@ -610,21 +610,18 @@ async function downloadResultToStdout(resultUrl: string, signal: AbortSignal): P function sanitizeResultName(value: string): string { const base = path.basename(value) - return base.replaceAll('\\', '_').replaceAll('/', '_').replaceAll('\u0000', '') + const name = base.replaceAll('\\', '_').replaceAll('/', '_').replaceAll('\u0000', '') + if (name === '' || name === '.' || name === '..') { + throw new Error('Assembly result has an unsafe name') + } + return name } async function ensureUniquePath(targetPath: string, reservedPaths: Set): Promise { const parsed = path.parse(targetPath) return await ensureUniqueCounterValue({ initialValue: targetPath, - isTaken: async (candidate) => { - if (reservedPaths.has(candidate)) { - return true - } - - const [statErr] = await tryCatch(fsp.stat(candidate)) - return statErr == null - }, + isTaken: async (candidate) => await isDownloadPathTaken(candidate, reservedPaths), reserve: (candidate) => { reservedPaths.add(candidate) }, @@ -633,8 +630,30 @@ async function ensureUniquePath(targetPath: string, reservedPaths: Set): }) } -function getResultFileName(file: NormalizedAssemblyResultFile): string { - return sanitizeResultName(file.name) +const resultPathSchema = z.object({ + meta: z.object({ relative_path: z.string().nullish() }).nullish(), +}) + +function getResultRelativePath(file: NormalizedAssemblyResultFile): string | null { + const parsed = resultPathSchema.safeParse(file.file) + if (!parsed.success) throw new Error('Assembly result has invalid relative path metadata') + const relativePath = parsed.data.meta?.relative_path + if (relativePath == null) return null + const name = sanitizeResultName(file.name) + // Adaptive video results use an empty directory for the master playlist. + if (relativePath === '') return name + + const normalizedPath = relativePath.replaceAll('\\', '/') + if ( + normalizedPath.includes('\u0000') || + path.win32.isAbsolute(normalizedPath) || + /^[a-z]:/i.test(normalizedPath) || + normalizedPath.split('/').some((part) => part === '' || part === '.' || part === '..') + ) { + throw new Error('Assembly result has an unsafe relative path') + } + + return path.join(normalizedPath, name) } interface AssemblyDownloadTarget { @@ -642,6 +661,37 @@ interface AssemblyDownloadTarget { targetPath: string | null } +async function assertNoOutputSymlinks(root: string, target: string): Promise { + const relativePath = path.relative(root, target) + if ( + relativePath.startsWith(`..${path.sep}`) || + relativePath === '..' || + path.isAbsolute(relativePath) + ) { + throw new Error('Assembly result path escapes the output directory') + } + + let current = root + for (const part of relativePath.split(path.sep)) { + current = path.join(current, part) + const [error, stats] = await tryCatch(fsp.lstat(current)) + if (error) { + if (isErrnoException(error) && error.code === 'ENOENT') return + throw error + } + if (stats.isSymbolicLink()) { + throw new Error('Assembly result path contains a symbolic link') + } + } +} + +async function isDownloadPathTaken(target: string, reservedPaths: Set): Promise { + if (reservedPaths.has(target)) return true + const [error] = await tryCatch(fsp.lstat(target)) + if (error && (!isErrnoException(error) || error.code !== 'ENOENT')) throw error + return error == null +} + const STALE_OUTPUT_GRACE_MS = 1000 function isMeaningfullyNewer(newer: Date, older: Date): boolean { @@ -652,27 +702,65 @@ async function buildDirectoryDownloadTargets({ allFiles, baseDir, groupByStep, + outputRoot, reservedPaths, }: { allFiles: NormalizedAssemblyResultFile[] baseDir: string groupByStep: boolean + outputRoot: string reservedPaths: Set }): Promise { + await assertNoOutputSymlinks(outputRoot, baseDir) await fsp.mkdir(baseDir, { recursive: true }) const targets: AssemblyDownloadTarget[] = [] + const steps = new Map< + string, + Array<{ file: NormalizedAssemblyResultFile; relativePath: string | null }> + >() for (const resultFile of allFiles) { - const targetDir = groupByStep ? path.join(baseDir, resultFile.stepName) : baseDir - await fsp.mkdir(targetDir, { recursive: true }) - - targets.push({ - resultUrl: resultFile.url, - targetPath: await ensureUniquePath( - path.join(targetDir, getResultFileName(resultFile)), - reservedPaths, - ), - }) + const files = steps.get(resultFile.stepName) ?? [] + files.push({ file: resultFile, relativePath: getResultRelativePath(resultFile) }) + steps.set(resultFile.stepName, files) + } + + for (const [stepName, files] of steps) { + const safeStepName = sanitizeResultName(stepName) + let targetDir = groupByStep ? path.join(baseDir, safeStepName) : baseDir + const linked = files.some((file) => file.relativePath != null) + const names = files.map( + ({ file, relativePath }) => relativePath ?? sanitizeResultName(file.name), + ) + if (linked && new Set(names).size !== names.length) { + throw new Error('Assembly results have duplicate relative paths') + } + + for (const name of names) { + await assertNoOutputSymlinks(outputRoot, path.join(targetDir, name)) + } + + if (linked) { + const collisions = await Promise.all( + names.map((name) => isDownloadPathTaken(path.join(targetDir, name), reservedPaths)), + ) + if (collisions.some(Boolean)) { + // Playlist references must keep their names; move the whole Step together on a rerun. + targetDir = await ensureUniquePath(path.join(baseDir, safeStepName), reservedPaths) + } + } + + for (const [index, { file }] of files.entries()) { + const targetPath = path.join(targetDir, names[index]) + await assertNoOutputSymlinks(outputRoot, targetPath) + if (linked) { + reservedPaths.add(targetPath) + } + targets.push({ + resultUrl: file.url, + targetPath: linked ? targetPath : await ensureUniquePath(targetPath, reservedPaths), + }) + } } return targets @@ -696,7 +784,6 @@ async function resolveResultDownloadTargets({ inPath, inputs, normalizedResults, - outputMode, outputPath, outputRoot, outputRootIsDirectory, @@ -707,7 +794,6 @@ async function resolveResultDownloadTargets({ inPath: string | null inputs: string[] normalizedResults: NormalizedAssemblyResults - outputMode?: 'directory' | 'file' outputPath: string | null outputRoot: string outputRootIsDirectory: boolean @@ -719,15 +805,14 @@ async function resolveResultDownloadTargets({ !singleAssembly && inPath != null && (hasDirectoryInput || inputs.length > 1) const resolveDirectoryBaseDir = (): string => { - if (!shouldGroupByInput || inPath == null) { - return outputRoot + if (hasDirectoryInput && outputPath != null) { + return allFiles.length === 1 && entries.length === 1 + ? path.dirname(outputPath) + : path.join(path.dirname(outputPath), path.parse(outputPath).name) } - if (hasDirectoryInput && outputPath != null) { - const mappedRelative = path.relative(outputRoot, outputPath) - const mappedDir = path.dirname(mappedRelative) - const mappedStem = path.parse(mappedRelative).name - return path.join(outputRoot, mappedDir === '.' ? '' : mappedDir, mappedStem) + if (!shouldGroupByInput || inPath == null) { + return outputRoot } return path.join(outputRoot, path.parse(path.basename(inPath)).name) @@ -752,27 +837,16 @@ async function resolveResultDownloadTargets({ allFiles, baseDir: outputRoot, groupByStep: inputs.length === 0, + outputRoot, reservedPaths, }) } - if (outputMode === 'directory' || outputPath == null || inPath == null) { - return await buildDirectoryDownloadTargets({ - allFiles, - baseDir: resolveDirectoryBaseDir(), - groupByStep: inPath == null || entries.length > 1, - reservedPaths, - }) - } - - if (allFiles.length === 1) { - return getSingleResultDownloadTarget(allFiles, outputPath) - } - return await buildDirectoryDownloadTargets({ allFiles, - baseDir: path.join(path.dirname(outputPath), path.parse(outputPath).name), - groupByStep: true, + baseDir: resolveDirectoryBaseDir(), + groupByStep: inPath == null || entries.length > 1, + outputRoot, reservedPaths, }) } @@ -843,7 +917,6 @@ async function materializeAssemblyResults({ inPath, inputs, normalizedResults, - outputMode, outputPath, outputRoot, outputRootIsDirectory, @@ -856,7 +929,6 @@ async function materializeAssemblyResults({ inPath: string | null inputs: string[] normalizedResults: NormalizedAssemblyResults - outputMode?: 'directory' | 'file' outputPath: string | null outputRoot: string | null outputRootIsDirectory: boolean @@ -873,7 +945,6 @@ async function materializeAssemblyResults({ inPath, inputs, normalizedResults, - outputMode, outputPath, outputRoot, outputRootIsDirectory, @@ -1328,7 +1399,33 @@ export interface AssembliesCreateOptions { const DEFAULT_CONCURRENCY = 5 // --- Main assembly create function --- +/** Create Assemblies after materializing URL inputs as temporary local uploads. */ export async function create( + outputctl: IOutputCtl, + client: Transloadit, + options: AssembliesCreateOptions, +): ReturnType { + const urlInputs = options.inputs.filter((input) => /^https?:\/\//i.test(input)) + if (urlInputs.length === 0) return await createWithLocalInputs(outputctl, client, options) + if (options.watch) throw new Error('--watch requires local inputs') + + const prepared = await prepareInputFiles({ + inputFiles: urlInputs.map((url, index) => ({ kind: 'url', field: String(index), url })), + allowPrivateUrls: false, + urlStrategy: 'download', + }) + const localPaths = new Map(urlInputs.map((url, index) => [url, prepared.files[String(index)]])) + try { + return await createWithLocalInputs(outputctl, client, { + ...options, + inputs: options.inputs.map((input) => localPaths.get(input) ?? input), + }) + } finally { + await Promise.all(prepared.cleanup.map((cleanup) => cleanup())) + } +} + +async function createWithLocalInputs( outputctl: IOutputCtl, client: Transloadit, { @@ -1557,9 +1654,8 @@ export async function create( abortSignal: abortController.signal, hasDirectoryInput: singleAssemblyMode ? false : hasDirectoryInput, inPath, - inputs: inputPaths, + inputs: singleAssemblyMode ? inputPaths : inputs, normalizedResults, - outputMode, outputPath: outputPlan?.path ?? null, outputRoot: resolvedOutput ?? null, outputRootIsDirectory, diff --git a/packages/node/src/cli/resultFiles.ts b/packages/node/src/cli/resultFiles.ts index ee2aa970..0ba559b5 100644 --- a/packages/node/src/cli/resultFiles.ts +++ b/packages/node/src/cli/resultFiles.ts @@ -1,6 +1,7 @@ export interface AssemblyResultEntryLike { basename?: unknown ext?: unknown + meta?: unknown name?: unknown ssl_url?: unknown url?: unknown diff --git a/packages/node/test/unit/cli/assemblies-create.test.ts b/packages/node/test/unit/cli/assemblies-create.test.ts index f304cef9..5affe9e7 100644 --- a/packages/node/test/unit/cli/assemblies-create.test.ts +++ b/packages/node/test/unit/cli/assemblies-create.test.ts @@ -21,10 +21,6 @@ async function createTempDir(prefix: string): Promise { return tempDir } -function getLegacyRelativeInputPath(inputPath: string): string { - return path.relative(process.cwd(), inputPath).replace(/^(\.\.\/)+/, '') -} - async function collectRelativeFiles(rootDir: string, currentDir = rootDir): Promise { const entries = await readdir(currentDir, { withFileTypes: true }) const files: string[] = [] @@ -326,22 +322,20 @@ describe('assemblies create', () => { }, }), ) - const legacyRelative = getLegacyRelativeInputPath(inputPath) - const baseDir = path.join(path.dirname(legacyRelative), path.parse(legacyRelative).name) expect(await collectRelativeFiles(outputDir)).toEqual([ - path.join(baseDir, 'captioned', 'video.mp4'), - path.join(baseDir, 'transcript', 'video.srt'), - path.join(baseDir, 'transcription', 'video.json'), + path.join('captioned', 'video.mp4'), + path.join('transcript', 'video.srt'), + path.join('transcription', 'video.json'), ]) - expect(await readFile(path.join(outputDir, baseDir, 'captioned', 'video.mp4'), 'utf8')).toBe( + expect(await readFile(path.join(outputDir, 'captioned', 'video.mp4'), 'utf8')).toBe( 'captioned video', ) - expect(await readFile(path.join(outputDir, baseDir, 'transcript', 'video.srt'), 'utf8')).toBe( + expect(await readFile(path.join(outputDir, 'transcript', 'video.srt'), 'utf8')).toBe( 'subtitle text', ) - expect( - await readFile(path.join(outputDir, baseDir, 'transcription', 'video.json'), 'utf8'), - ).toBe('{"words":[]}') + expect(await readFile(path.join(outputDir, 'transcription', 'video.json'), 'utf8')).toBe( + '{"words":[]}', + ) expect(downloads.isDone()).toBe(true) }) @@ -1511,7 +1505,7 @@ describe('assemblies create', () => { expect(await readFile(path.join(outputDir, 'thumb__1.jpg'), 'utf8')).toBe('second-thumb') }) - it('preserves legacy step-directory layout for generic directory outputs', async () => { + it('uses result filenames directly for single-step directory outputs', async () => { vi.spyOn(console, 'error').mockImplementation(() => {}) const tempDir = await createTempDir('transloadit-legacy-outdir-') @@ -1554,13 +1548,7 @@ describe('assemblies create', () => { } as never, ) - const legacyRelative = getLegacyRelativeInputPath(inputPath) - const legacyBaseDir = path.join(path.dirname(legacyRelative), path.parse(legacyRelative).name) - - expect(await collectRelativeFiles(outputDir)).toEqual([ - path.join(legacyBaseDir, 'thumbs', 'one.jpg'), - path.join(legacyBaseDir, 'thumbs', 'two.jpg'), - ]) + expect(await collectRelativeFiles(outputDir)).toEqual(['one.jpg', 'two.jpg']) }) it('uses the actual result filename for single-result directory outputs', async () => { @@ -1608,7 +1596,7 @@ describe('assemblies create', () => { expect(await readFile(path.join(outputDir, 'input.txt'), 'utf8')).toBe('hello') }) - it('preserves mapped out paths for legacy single-result directory outputs', async () => { + it('uses result names for directory outputs without an explicit output mode', async () => { vi.spyOn(console, 'error').mockImplementation(() => {}) const tempDir = await createTempDir('transloadit-legacy-single-result-') @@ -1643,7 +1631,7 @@ describe('assemblies create', () => { }, }) - expect(await collectRelativeFiles(outputDir)).toEqual([getLegacyRelativeInputPath(inputPath)]) + expect(await collectRelativeFiles(outputDir)).toEqual(['input.txt']) }) it('does not create an empty output file when assembly creation fails', async () => { diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts new file mode 100644 index 00000000..050c4808 --- /dev/null +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -0,0 +1,217 @@ +import type { AssemblyResultEntryLike } from '../../../src/cli/resultFiles.ts' + +import fs from 'node:fs' +import { mkdir, mkdtemp, readdir, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' + +import { Cli } from 'clipanion' +import nock from 'nock' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' + +import { AssembliesCreateCommand } from '../../../src/cli/commands/assemblies.ts' +import { Transloadit } from '../../../src/Transloadit.ts' + +const tempDirs: string[] = [] + +async function fixture(results: Record): Promise<{ + inputPath: string + outputDir: string + run: (args?: string[]) => Promise + tempDir: string +}> { + const tempDir = await mkdtemp(path.join(tmpdir(), 'transloadit-download-layout-')) + tempDirs.push(tempDir) + const stepsPath = path.join(tempDir, 'steps.json') + const inputPath = path.join(tempDir, 'inputs', 'logo.png') + const outputDir = path.join(tempDir, 'out') + await mkdir(path.dirname(inputPath)) + await writeFile(inputPath, 'input bytes') + await writeFile(stepsPath, JSON.stringify({ output: { robot: '/file/filter' } })) + vi.spyOn(Transloadit.prototype, 'createAssembly').mockResolvedValue({ + assembly_id: 'download-layout', + }) + vi.spyOn(Transloadit.prototype, 'awaitAssemblyCompletion').mockResolvedValue({ + ok: 'ASSEMBLY_COMPLETED', + results, + }) + + return { + inputPath, + outputDir, + tempDir, + run: async (args = []) => + await Cli.from([AssembliesCreateCommand]).run([ + 'assemblies', + 'create', + '--steps', + stepsPath, + '-o', + `${outputDir}/`, + ...args, + ]), + } +} + +beforeEach(async () => { + vi.stubEnv('TRANSLOADIT_KEY', 'key') + vi.stubEnv('TRANSLOADIT_SECRET', 'secret') + vi.spyOn(console, 'error').mockImplementation(() => {}) + vi.spyOn(fs, 'fstatSync').mockReturnValue(await stat('/dev/null')) +}) + +afterEach(async () => { + vi.restoreAllMocks() + vi.unstubAllEnvs() + nock.cleanAll() + nock.abortPendingRequests() + await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))) +}) + +it('keeps every HLS playlist reference reachable from its downloaded playlist', async () => { + const master = '#EXTM3U\nvariant/stream.m3u8\n' + const variant = '#EXTM3U\n#EXTINF:1,\nsegment.ts\n#EXT-X-ENDLIST\n' + const { inputPath, outputDir, run } = await fixture({ + adaptive: [ + { name: 'master.m3u8', url: 'https://downloads.test/master', meta: { relative_path: '' } }, + { + name: 'stream.m3u8', + url: 'https://downloads.test/variant', + meta: { relative_path: 'variant' }, + }, + { + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'variant' }, + }, + ], + }) + nock('https://downloads.test') + .get('/master') + .reply(200, master) + .get('/variant') + .reply(200, variant) + .get('/segment') + .reply(200, 'transport stream') + + expect(await run(['-i', inputPath])).toBe(0) + expect(await readFile(path.join(outputDir, 'master.m3u8'), 'utf8')).toBe(master) + const variantPath = path.join(outputDir, master.split('\n')[1]) + expect(await readFile(variantPath, 'utf8')).toBe(variant) + expect(await readFile(path.join(path.dirname(variantPath), variant.split('\n')[2]), 'utf8')).toBe( + 'transport stream', + ) +}) + +it.each([false, true])('uses the bundle result name (single Assembly: %s)', async (single) => { + const { inputPath, outputDir, run } = await fixture({ + bundled: [{ name: 'archive.zip', url: 'https://downloads.test/archive' }], + }) + nock('https://downloads.test').get('/archive').reply(200, 'archive bytes') + + expect(await run(['-i', inputPath, ...(single ? ['--single-assembly'] : [])])).toBe(0) + expect(await readdir(outputDir)).toEqual(['archive.zip']) + expect(await readFile(path.join(outputDir, 'archive.zip'), 'utf8')).toBe('archive bytes') +}) + +it('leaves the input folders out of a multi-step download', async () => { + const { inputPath, outputDir, run } = await fixture({ + image: [{ name: 'result.png', url: 'https://downloads.test/image' }], + text: [{ name: 'result.txt', url: 'https://downloads.test/text' }], + }) + nock('https://downloads.test') + .get('/image') + .reply(200, 'image bytes') + .get('/text') + .reply(200, 'text bytes') + + expect(await run(['-i', inputPath])).toBe(0) + expect(await readdir(outputDir)).toEqual(['image', 'text']) + expect(await readFile(path.join(outputDir, 'image', 'result.png'), 'utf8')).toBe('image bytes') + expect(await readFile(path.join(outputDir, 'text', 'result.txt'), 'utf8')).toBe('text bytes') +}) + +it.each([ + false, + true, +])('uploads URL inputs and cleans them up (single Assembly: %s)', async (single) => { + const { outputDir, run } = await fixture({ + processed: [{ name: 'remote.png', url: 'https://downloads.test/result' }], + }) + let uploadedPath = '' + vi.mocked(Transloadit.prototype.createAssembly).mockImplementation(async ({ files }) => { + uploadedPath = Object.values(files ?? {})[0] ?? '' + expect(path.basename(uploadedPath)).toBe('remote.png') + expect(await readFile(uploadedPath, 'utf8')).toBe('remote input') + return { assembly_id: 'url-upload' } + }) + nock('https://example.com').get('/remote.png?signature=example').reply(200, 'remote input') + nock('https://downloads.test').get('/result').reply(200, 'processed bytes') + + expect( + await run([ + '-i', + 'https://example.com/remote.png?signature=example', + ...(single ? ['--single-assembly'] : []), + ]), + ).toBe(0) + expect(await readFile(path.join(outputDir, 'remote.png'), 'utf8')).toBe('processed bytes') + await expect(stat(uploadedPath)).rejects.toMatchObject({ code: 'ENOENT' }) +}) + +it('rejects watch mode for URL inputs before making a download request', async () => { + const { run } = await fixture({}) + const download = nock('https://example.com').get('/remote.png').reply(200, 'remote input') + + expect(await run(['-i', 'https://example.com/remote.png', '--watch'])).toBe(1) + expect(download.isDone()).toBe(false) + expect(Transloadit.prototype.createAssembly).not.toHaveBeenCalled() +}) + +it('keeps a repeated linked download together instead of renaming playlist targets', async () => { + const { inputPath, outputDir, run } = await fixture({ + adaptive: [ + { name: 'master.m3u8', url: 'https://downloads.test/master', meta: { relative_path: '' } }, + { + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'variant' }, + }, + ], + }) + const playlist = '#EXTM3U\nvariant/segment.ts\n' + nock('https://downloads.test') + .get('/master') + .twice() + .reply(200, playlist) + .get('/segment') + .twice() + .reply(200, 'stream bytes') + + expect(await run(['-i', inputPath])).toBe(0) + expect(await run(['-i', inputPath])).toBe(0) + expect(await readFile(path.join(outputDir, 'adaptive', 'master.m3u8'), 'utf8')).toBe(playlist) + expect(await readFile(path.join(outputDir, 'adaptive', 'variant', 'segment.ts'), 'utf8')).toBe( + 'stream bytes', + ) +}) + +it('rejects a relative result path through a symlink outside the output directory', async () => { + const { inputPath, outputDir, run, tempDir } = await fixture({ + adaptive: [ + { + name: 'result.txt', + url: 'https://downloads.test/result', + meta: { relative_path: 'nested' }, + }, + ], + }) + const outside = path.join(tempDir, 'outside') + await mkdir(outputDir) + await mkdir(outside) + await symlink(outside, path.join(outputDir, 'nested')) + nock('https://downloads.test').get('/result').reply(200, 'escaped bytes') + + expect(await run(['-i', inputPath])).toBe(1) + expect(await readdir(outside)).toEqual([]) +}) diff --git a/packages/transloadit/README.md b/packages/transloadit/README.md index 87cc0eb9..bc7da02b 100644 --- a/packages/transloadit/README.md +++ b/packages/transloadit/README.md @@ -1502,7 +1502,14 @@ The `assemblies create` command additionally supports: Without uploads (for example, `/vimeo/import`, `/http/import`, or prompt-only `/image/generate` Steps), results are saved as `//`, with no input folder, including with `--single-assembly`. -- `--input, -i` - Provide a local upload, or use `-i -` to read stdin explicitly. When `--steps` + With one local input, filenames come from the Assembly results. Multiple result Steps get their + own folders; the input’s parent folders are omitted. Directory inputs retain subfolders relative + to that directory. HLS results retain the directories in `meta.relative_path`, so playlists can + find their renditions and segments. If those paths already exist, the whole result Step is saved + in a new Step folder together, preserving playlist references. +- `--input, -i` - Provide a local upload or an HTTP(S) URL, or use `-i -` to read stdin explicitly. + URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` + requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly without uploads. With `--template`, stdin always requires `-i -`. From 2fec19af98ad093f47d9a34df73ab2f6841d7a19 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 14:37:42 +0200 Subject: [PATCH 02/35] Preserve download ownership across concurrent and watched inputs --- packages/node/README.md | 6 +- packages/node/src/cli/commands/assemblies.ts | 139 +++++++++++--- .../test/unit/cli/download-layout.test.ts | 171 +++++++++++++++++- packages/transloadit/README.md | 6 +- 4 files changed, 288 insertions(+), 34 deletions(-) diff --git a/packages/node/README.md b/packages/node/README.md index bc7da02b..416c69d7 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -1507,10 +1507,12 @@ The `assemblies create` command additionally supports: to that directory. HLS results retain the directories in `meta.relative_path`, so playlists can find their renditions and segments. If those paths already exist, the whole result Step is saved in a new Step folder together, preserving playlist references. + Watch updates refresh their existing result files. Other directory runs process the inputs again, + since result filenames are unknown before processing; file outputs keep mtime-based skipping. - `--input, -i` - Provide a local upload or an HTTP(S) URL, or use `-i -` to read stdin explicitly. URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` - requires local inputs. When `--steps` - is used without `-i`, stdin is read only when it is a pipe or a redirected file. Other non-TTY + requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe + or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly without uploads. With `--template`, stdin always requires `-i -`. - `--single-assembly` - Pass all input files to a single assembly instead of one assembly per file diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index eda76ef6..4ec9954e 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -520,6 +520,8 @@ async function createExistingPathOutputPlan(outputPath: string | undefined): Pro } function dirProvider(output: string): OutputPlanProvider { + const namesByInput = new Map() + const usedNames = new Set() return async (inpath, indir) => { if (inpath == null) { return await createExistingPathOutputPlan(output) @@ -528,7 +530,18 @@ function dirProvider(output: string): OutputPlanProvider { throw new Error('You must provide an input to output to a directory') } - const relpath = indir == null ? path.basename(inpath) : path.relative(indir, inpath) + let relpath = indir == null ? namesByInput.get(inpath) : path.relative(indir, inpath) + if (relpath == null) { + const parsed = path.parse(inpath) + relpath = await ensureUniqueCounterValue({ + initialValue: parsed.base, + isTaken: (candidate) => usedNames.has(candidate), + reserve: (candidate) => usedNames.add(candidate), + nextValue: (counter) => `${parsed.name}__${counter}${parsed.ext}`, + scope: usedNames, + }) + namesByInput.set(inpath, relpath) + } const outpath = path.join(output, relpath) return await createExistingPathOutputPlan(outpath) } @@ -617,13 +630,19 @@ function sanitizeResultName(value: string): string { return name } -async function ensureUniquePath(targetPath: string, reservedPaths: Set): Promise { +async function ensureUniquePath( + targetPath: string, + reservedPaths: Map, + owner: string, + overwriteExisting: boolean, +): Promise { const parsed = path.parse(targetPath) return await ensureUniqueCounterValue({ initialValue: targetPath, - isTaken: async (candidate) => await isDownloadPathTaken(candidate, reservedPaths), + isTaken: async (candidate) => + await isDownloadPathTaken(candidate, reservedPaths, owner, overwriteExisting), reserve: (candidate) => { - reservedPaths.add(candidate) + reservedPaths.set(candidate, owner) }, nextValue: (counter) => path.join(parsed.dir, `${parsed.name}__${counter}${parsed.ext}`), scope: reservedPaths, @@ -663,6 +682,8 @@ interface AssemblyDownloadTarget { async function assertNoOutputSymlinks(root: string, target: string): Promise { const relativePath = path.relative(root, target) + // The caller explicitly selected the root; only descendants come from result/input metadata. + if (relativePath === '') return if ( relativePath.startsWith(`..${path.sep}`) || relativePath === '..' || @@ -676,7 +697,7 @@ async function assertNoOutputSymlinks(root: string, target: string): Promise): Promise { - if (reservedPaths.has(target)) return true +async function isDownloadPathTaken( + target: string, + reservedPaths: Map, + owner: string, + overwriteExisting = false, +): Promise { + const existingOwner = reservedPaths.get(target) + if (existingOwner != null) return existingOwner !== owner + if (overwriteExisting) return false const [error] = await tryCatch(fsp.lstat(target)) + if (isErrnoException(error) && error.code === 'ENOTDIR') return true if (error && (!isErrnoException(error) || error.code !== 'ENOENT')) throw error return error == null } @@ -702,14 +731,18 @@ async function buildDirectoryDownloadTargets({ allFiles, baseDir, groupByStep, + owner, + overwriteExisting, outputRoot, reservedPaths, }: { allFiles: NormalizedAssemblyResultFile[] baseDir: string groupByStep: boolean + owner: string + overwriteExisting: boolean outputRoot: string - reservedPaths: Set + reservedPaths: Map }): Promise { await assertNoOutputSymlinks(outputRoot, baseDir) await fsp.mkdir(baseDir, { recursive: true }) @@ -741,24 +774,61 @@ async function buildDirectoryDownloadTargets({ } if (linked) { - const collisions = await Promise.all( - names.map((name) => isDownloadPathTaken(path.join(targetDir, name), reservedPaths)), - ) - if (collisions.some(Boolean)) { - // Playlist references must keep their names; move the whole Step together on a rerun. - targetDir = await ensureUniquePath(path.join(baseDir, safeStepName), reservedPaths) - } + const initialDir = targetDir + const groupOwner = JSON.stringify([owner, stepName]) + // Selection and reservation share the same lock as ordinary result paths. A Step must + // move together on collisions, while later watch updates reuse its owned destinations. + targetDir = await ensureUniqueCounterValue({ + initialValue: initialDir, + isTaken: async (candidate) => { + if ( + candidate !== initialDir && + (await isDownloadPathTaken(candidate, reservedPaths, groupOwner)) + ) + return true + const collisions = await Promise.all( + names.map((name) => + isDownloadPathTaken( + path.join(candidate, name), + reservedPaths, + groupOwner, + overwriteExisting, + ), + ), + ) + return collisions.some(Boolean) + }, + reserve: (candidate) => { + if (candidate !== initialDir) reservedPaths.set(candidate, groupOwner) + for (const name of names) reservedPaths.set(path.join(candidate, name), groupOwner) + }, + nextValue: (counter) => + path.join( + baseDir, + groupByStep + ? `${safeStepName}__${counter}` + : counter === 1 + ? safeStepName + : `${safeStepName}__${counter - 1}`, + ), + scope: reservedPaths, + }) } for (const [index, { file }] of files.entries()) { const targetPath = path.join(targetDir, names[index]) - await assertNoOutputSymlinks(outputRoot, targetPath) - if (linked) { - reservedPaths.add(targetPath) - } + const selectedPath = linked + ? targetPath + : await ensureUniquePath( + targetPath, + reservedPaths, + JSON.stringify([owner, stepName, index]), + overwriteExisting, + ) + await assertNoOutputSymlinks(outputRoot, selectedPath) targets.push({ resultUrl: file.url, - targetPath: linked ? targetPath : await ensureUniquePath(targetPath, reservedPaths), + targetPath: selectedPath, }) } } @@ -784,6 +854,7 @@ async function resolveResultDownloadTargets({ inPath, inputs, normalizedResults, + overwriteExisting, outputPath, outputRoot, outputRootIsDirectory, @@ -794,15 +865,19 @@ async function resolveResultDownloadTargets({ inPath: string | null inputs: string[] normalizedResults: NormalizedAssemblyResults + overwriteExisting: boolean outputPath: string | null outputRoot: string outputRootIsDirectory: boolean - reservedPaths: Set + reservedPaths: Map singleAssembly?: boolean }): Promise { const { allFiles, entries } = normalizedResults const shouldGroupByInput = - !singleAssembly && inPath != null && (hasDirectoryInput || inputs.length > 1) + !singleAssembly && + inPath != null && + (hasDirectoryInput || (inputs.length > 1 && allFiles.length > 1)) + const owner = JSON.stringify([inPath, singleAssembly ? inputs : []]) const resolveDirectoryBaseDir = (): string => { if (hasDirectoryInput && outputPath != null) { @@ -815,7 +890,7 @@ async function resolveResultDownloadTargets({ return outputRoot } - return path.join(outputRoot, path.parse(path.basename(inPath)).name) + return path.join(outputRoot, path.parse(path.basename(outputPath ?? inPath)).name) } if (!outputRootIsDirectory) { @@ -837,6 +912,8 @@ async function resolveResultDownloadTargets({ allFiles, baseDir: outputRoot, groupByStep: inputs.length === 0, + owner, + overwriteExisting, outputRoot, reservedPaths, }) @@ -846,6 +923,8 @@ async function resolveResultDownloadTargets({ allFiles, baseDir: resolveDirectoryBaseDir(), groupByStep: inPath == null || entries.length > 1, + owner, + overwriteExisting, outputRoot, reservedPaths, }) @@ -917,6 +996,7 @@ async function materializeAssemblyResults({ inPath, inputs, normalizedResults, + overwriteExisting, outputPath, outputRoot, outputRootIsDirectory, @@ -929,11 +1009,12 @@ async function materializeAssemblyResults({ inPath: string | null inputs: string[] normalizedResults: NormalizedAssemblyResults + overwriteExisting: boolean outputPath: string | null outputRoot: string | null outputRootIsDirectory: boolean outputctl: IOutputCtl - reservedPaths: Set + reservedPaths: Map singleAssembly?: boolean }): Promise { if (outputRoot == null) { @@ -945,6 +1026,7 @@ async function materializeAssemblyResults({ inPath, inputs, normalizedResults, + overwriteExisting, outputPath, outputRoot, outputRootIsDirectory, @@ -1412,6 +1494,7 @@ export async function create( const prepared = await prepareInputFiles({ inputFiles: urlInputs.map((url, index) => ({ kind: 'url', field: String(index), url })), allowPrivateUrls: false, + // Uploading the bytes preserves :original and Template upload semantics, just as Intents do. urlStrategy: 'download', }) const localPaths = new Map(urlInputs.map((url, index) => [url, prepared.files[String(index)]])) @@ -1520,14 +1603,17 @@ async function createWithLocalInputs( recursive, watch: watchOption, singleAssembly, - reprocessStale, + // Directory plans use upload names before the Assembly's result names are known. + // Inferring freshness from those names can skip unrelated results. File outputs retain + // mtime-based skipping; directory caching would require a persistent result manifest. + reprocessStale: reprocessStale || outstat?.isDirectory(), }) // Use p-queue for concurrency management const queue = new PQueue({ concurrency }) const results: unknown[] = [] const resultUrls: ResultUrlRow[] = [] - const reservedResultPaths = new Set() + const reservedResultPaths = new Map() const latestWatchJobTokenByOutputPath = new Map() let hasFailures = false let nextWatchJobToken = 0 @@ -1656,6 +1742,7 @@ async function createWithLocalInputs( inPath, inputs: singleAssemblyMode ? inputPaths : inputs, normalizedResults, + overwriteExisting: Boolean(watchOption), outputPath: outputPlan?.path ?? null, outputRoot: resolvedOutput ?? null, outputRootIsDirectory, diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 050c4808..fcb48ce5 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -1,7 +1,19 @@ import type { AssemblyResultEntryLike } from '../../../src/cli/resultFiles.ts' +import dns from 'node:dns/promises' +import { EventEmitter } from 'node:events' import fs from 'node:fs' -import { mkdir, mkdtemp, readdir, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises' +import { + mkdir, + mkdtemp, + readdir, + readFile, + rm, + stat, + symlink, + utimes, + writeFile, +} from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' @@ -10,6 +22,7 @@ import nock from 'nock' import { afterEach, beforeEach, expect, it, vi } from 'vitest' import { AssembliesCreateCommand } from '../../../src/cli/commands/assemblies.ts' +import OutputCtl from '../../../src/cli/OutputCtl.ts' import { Transloadit } from '../../../src/Transloadit.ts' const tempDirs: string[] = [] @@ -17,7 +30,7 @@ const tempDirs: string[] = [] async function fixture(results: Record): Promise<{ inputPath: string outputDir: string - run: (args?: string[]) => Promise + run: (args?: string[], output?: string) => Promise tempDir: string }> { const tempDir = await mkdtemp(path.join(tmpdir(), 'transloadit-download-layout-')) @@ -40,14 +53,14 @@ async function fixture(results: Record): Prom inputPath, outputDir, tempDir, - run: async (args = []) => + run: async (args = [], output = `${outputDir}/`) => await Cli.from([AssembliesCreateCommand]).run([ 'assemblies', 'create', '--steps', stepsPath, '-o', - `${outputDir}/`, + output, ...args, ]), } @@ -58,11 +71,13 @@ beforeEach(async () => { vi.stubEnv('TRANSLOADIT_SECRET', 'secret') vi.spyOn(console, 'error').mockImplementation(() => {}) vi.spyOn(fs, 'fstatSync').mockReturnValue(await stat('/dev/null')) + vi.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }]) }) afterEach(async () => { vi.restoreAllMocks() vi.unstubAllEnvs() + vi.doUnmock('node-watch') nock.cleanAll() nock.abortPendingRequests() await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))) @@ -215,3 +230,151 @@ it('rejects a relative result path through a symlink outside the output director expect(await run(['-i', inputPath])).toBe(1) expect(await readdir(outside)).toEqual([]) }) + +it('keeps single-result downloads flat for multiple standalone inputs', async () => { + const { inputPath, outputDir, run, tempDir } = await fixture({ + resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], + }) + const secondInput = path.join(tempDir, 'second.png') + await writeFile(secondInput, 'second input') + nock('https://downloads.test').get('/result').twice().reply(200, 'result bytes') + + expect(await run(['-i', inputPath, '-i', secondInput])).toBe(0) + expect(await readdir(outputDir)).toEqual(['result.png', 'result__1.png']) +}) + +it('accepts inputs with the same basename in different folders', async () => { + const { inputPath, outputDir, run, tempDir } = await fixture({ + resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], + }) + const secondInput = path.join(tempDir, 'logo.png') + await writeFile(secondInput, 'second input') + nock('https://downloads.test').get('/result').twice().reply(200, 'result bytes') + + expect(await run(['-i', inputPath, '-i', secondInput])).toBe(0) + expect(await readdir(outputDir)).toEqual(['result.png', 'result__1.png']) +}) + +it('does not mistake an input-named file for a current result with a different filename', async () => { + const { inputPath, outputDir, run } = await fixture({ + bundled: [{ name: 'archive.zip', url: 'https://downloads.test/result' }], + }) + await mkdir(outputDir) + await writeFile(path.join(outputDir, 'logo.png'), 'unrelated file') + await utimes(path.join(outputDir, 'logo.png'), new Date(), new Date(Date.now() + 10_000)) + nock('https://downloads.test').get('/result').reply(200, 'result bytes') + + expect(await run(['-i', inputPath])).toBe(0) + expect(await readFile(path.join(outputDir, 'archive.zip'), 'utf8')).toBe('result bytes') +}) + +it('allows an explicitly selected output root that is a symlink', async () => { + const { inputPath, outputDir, run, tempDir } = await fixture({ + resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], + }) + const destination = path.join(tempDir, 'destination') + await mkdir(destination) + await symlink(destination, outputDir) + nock('https://downloads.test').get('/result').reply(200, 'result bytes') + + expect(await run(['-i', inputPath], outputDir)).toBe(0) + expect(await readFile(path.join(destination, 'result.png'), 'utf8')).toBe('result bytes') +}) + +it('reserves concurrent linked downloads together when input stems match', async () => { + const { inputPath, outputDir, run } = await fixture({ + adaptive: [ + { name: 'master.m3u8', url: 'https://downloads.test/master', meta: { relative_path: '' } }, + { + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'video' }, + }, + ], + }) + await writeFile(path.join(path.dirname(inputPath), 'logo.jpg'), 'second input') + nock('https://downloads.test') + .get('/master') + .twice() + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/segment') + .twice() + .reply(200, 'stream bytes') + + expect(await run(['-i', path.dirname(inputPath), '--concurrency', '2'])).toBe(0) + expect(await readFile(path.join(outputDir, 'logo', 'master.m3u8'), 'utf8')).toBe( + '#EXTM3U\nvideo/segment.ts\n', + ) + expect(await readFile(path.join(outputDir, 'logo', 'adaptive', 'master.m3u8'), 'utf8')).toBe( + '#EXTM3U\nvideo/segment.ts\n', + ) + expect( + await readFile(path.join(outputDir, 'logo', 'adaptive', 'video', 'segment.ts'), 'utf8'), + ).toBe('stream bytes') +}) + +it('refreshes a watched result in place after the input changes', async () => { + const { inputPath, outputDir } = await fixture({ + resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], + }) + class FakeWatcher extends EventEmitter { + close(): void { + this.emit('close') + } + } + const watcher = new FakeWatcher() + vi.doMock('node-watch', () => ({ default: () => watcher })) + vi.resetModules() + const { create } = await import('../../../src/cli/commands/assemblies.ts') + let completions = 0 + const output = new OutputCtl() + vi.spyOn(output, 'debug').mockImplementation((message) => { + if (!String(message).startsWith('COMPLETED')) return + completions += 1 + if (completions === 1) watcher.emit('change', 'update', inputPath) + else watcher.close() + }) + nock('https://downloads.test') + .get('/result') + .reply(200, 'first result') + .get('/result') + .reply(200, 'updated result') + + const result = await create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + inputs: [inputPath], + output: `${outputDir}/`, + watch: true, + stepsData: { resized: { robot: '/image/resize', use: ':original', width: 16 } }, + }) + + expect(result.hasFailures).toBe(false) + expect(completions).toBe(2) + expect(await readdir(outputDir)).toEqual(['result.png']) + expect(await readFile(path.join(outputDir, 'result.png'), 'utf8')).toBe('updated result') +}) + +it('moves a linked Step together when an existing file occupies a rendition directory', async () => { + const { inputPath, outputDir, run } = await fixture({ + adaptive: [ + { name: 'master.m3u8', url: 'https://downloads.test/master', meta: { relative_path: '' } }, + { + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'video' }, + }, + ], + }) + await mkdir(outputDir) + await writeFile(path.join(outputDir, 'video'), 'existing file') + nock('https://downloads.test') + .get('/master') + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/segment') + .reply(200, 'stream bytes') + + expect(await run(['-i', inputPath])).toBe(0) + expect(await readFile(path.join(outputDir, 'adaptive', 'video', 'segment.ts'), 'utf8')).toBe( + 'stream bytes', + ) + expect(await readFile(path.join(outputDir, 'video'), 'utf8')).toBe('existing file') +}) diff --git a/packages/transloadit/README.md b/packages/transloadit/README.md index bc7da02b..416c69d7 100644 --- a/packages/transloadit/README.md +++ b/packages/transloadit/README.md @@ -1507,10 +1507,12 @@ The `assemblies create` command additionally supports: to that directory. HLS results retain the directories in `meta.relative_path`, so playlists can find their renditions and segments. If those paths already exist, the whole result Step is saved in a new Step folder together, preserving playlist references. + Watch updates refresh their existing result files. Other directory runs process the inputs again, + since result filenames are unknown before processing; file outputs keep mtime-based skipping. - `--input, -i` - Provide a local upload or an HTTP(S) URL, or use `-i -` to read stdin explicitly. URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` - requires local inputs. When `--steps` - is used without `-i`, stdin is read only when it is a pipe or a redirected file. Other non-TTY + requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe + or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly without uploads. With `--template`, stdin always requires `-i -`. - `--single-assembly` - Pass all input files to a single assembly instead of one assembly per file From 5d1f9c9e37d584714c927d0795db39555e975cac Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 14:57:17 +0200 Subject: [PATCH 03/35] Guard watched downloads and resolve all input path collisions --- .changeset/assembly-download-paths.md | 2 + packages/node/src/cli/commands/assemblies.ts | 72 ++++---- .../test/e2e/cli/assemblies-create.test.ts | 26 +-- .../test/unit/cli/download-layout.test.ts | 170 +++++++++++++++++- 4 files changed, 205 insertions(+), 65 deletions(-) diff --git a/.changeset/assembly-download-paths.md b/.changeset/assembly-download-paths.md index a8aea995..a0b951d1 100644 --- a/.changeset/assembly-download-paths.md +++ b/.changeset/assembly-download-paths.md @@ -5,3 +5,5 @@ --- Keep CLI download filenames and HLS directories aligned with Assembly results, omit input parent folders from output paths, and support HTTP(S) URL inputs for Assembly and prompt commands. + +Directory runs process inputs again to discover the result filenames instead of guessing freshness from upload names. Watch updates reuse their own result paths, while existing unrelated files and outputs from other inputs remain separate. diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 4ec9954e..7747a20e 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -530,17 +530,19 @@ function dirProvider(output: string): OutputPlanProvider { throw new Error('You must provide an input to output to a directory') } - let relpath = indir == null ? namesByInput.get(inpath) : path.relative(indir, inpath) + const inputKey = path.resolve(inpath) + let relpath = namesByInput.get(inputKey) if (relpath == null) { - const parsed = path.parse(inpath) + const requestedPath = indir == null ? path.basename(inpath) : path.relative(indir, inpath) + const parsed = path.parse(requestedPath) relpath = await ensureUniqueCounterValue({ - initialValue: parsed.base, + initialValue: requestedPath, isTaken: (candidate) => usedNames.has(candidate), reserve: (candidate) => usedNames.add(candidate), - nextValue: (counter) => `${parsed.name}__${counter}${parsed.ext}`, + nextValue: (counter) => path.join(parsed.dir, `${parsed.name}__${counter}${parsed.ext}`), scope: usedNames, }) - namesByInput.set(inpath, relpath) + namesByInput.set(inputKey, relpath) } const outpath = path.join(output, relpath) return await createExistingPathOutputPlan(outpath) @@ -565,6 +567,7 @@ async function downloadResultToFile( resultUrl: string, outPath: string, signal: AbortSignal, + isCurrent: () => boolean, ): Promise { await fsp.mkdir(path.dirname(outPath), { recursive: true }) @@ -581,6 +584,10 @@ async function downloadResultToFile( throw dlErr } + if (!isCurrent()) { + await fsp.rm(tempPath, { force: true }) + return + } await fsp.rename(tempPath, outPath) } @@ -634,13 +641,11 @@ async function ensureUniquePath( targetPath: string, reservedPaths: Map, owner: string, - overwriteExisting: boolean, ): Promise { const parsed = path.parse(targetPath) return await ensureUniqueCounterValue({ initialValue: targetPath, - isTaken: async (candidate) => - await isDownloadPathTaken(candidate, reservedPaths, owner, overwriteExisting), + isTaken: async (candidate) => await isDownloadPathTaken(candidate, reservedPaths, owner), reserve: (candidate) => { reservedPaths.set(candidate, owner) }, @@ -681,6 +686,7 @@ interface AssemblyDownloadTarget { } async function assertNoOutputSymlinks(root: string, target: string): Promise { + // Reject preexisting descendant symlinks; another process can still race filesystem checks. const relativePath = path.relative(root, target) // The caller explicitly selected the root; only descendants come from result/input metadata. if (relativePath === '') return @@ -710,11 +716,9 @@ async function isDownloadPathTaken( target: string, reservedPaths: Map, owner: string, - overwriteExisting = false, ): Promise { const existingOwner = reservedPaths.get(target) if (existingOwner != null) return existingOwner !== owner - if (overwriteExisting) return false const [error] = await tryCatch(fsp.lstat(target)) if (isErrnoException(error) && error.code === 'ENOTDIR') return true if (error && (!isErrnoException(error) || error.code !== 'ENOENT')) throw error @@ -732,7 +736,6 @@ async function buildDirectoryDownloadTargets({ baseDir, groupByStep, owner, - overwriteExisting, outputRoot, reservedPaths, }: { @@ -740,7 +743,6 @@ async function buildDirectoryDownloadTargets({ baseDir: string groupByStep: boolean owner: string - overwriteExisting: boolean outputRoot: string reservedPaths: Map }): Promise { @@ -788,12 +790,7 @@ async function buildDirectoryDownloadTargets({ return true const collisions = await Promise.all( names.map((name) => - isDownloadPathTaken( - path.join(candidate, name), - reservedPaths, - groupOwner, - overwriteExisting, - ), + isDownloadPathTaken(path.join(candidate, name), reservedPaths, groupOwner), ), ) return collisions.some(Boolean) @@ -802,28 +799,27 @@ async function buildDirectoryDownloadTargets({ if (candidate !== initialDir) reservedPaths.set(candidate, groupOwner) for (const name of names) reservedPaths.set(path.join(candidate, name), groupOwner) }, - nextValue: (counter) => - path.join( - baseDir, - groupByStep - ? `${safeStepName}__${counter}` - : counter === 1 - ? safeStepName - : `${safeStepName}__${counter - 1}`, - ), + nextValue: (counter) => { + const suffix = groupByStep ? counter : counter - 1 + return path.join(baseDir, suffix === 0 ? safeStepName : `${safeStepName}__${suffix}`) + }, scope: reservedPaths, }) } + await fsp.mkdir(targetDir, { recursive: true }) + const nameOccurrences = new Map() for (const [index, { file }] of files.entries()) { - const targetPath = path.join(targetDir, names[index]) + const name = names[index] + const occurrence = nameOccurrences.get(name) ?? 0 + nameOccurrences.set(name, occurrence + 1) + const targetPath = path.join(targetDir, name) const selectedPath = linked ? targetPath : await ensureUniquePath( targetPath, reservedPaths, - JSON.stringify([owner, stepName, index]), - overwriteExisting, + JSON.stringify([owner, stepName, name, occurrence]), ) await assertNoOutputSymlinks(outputRoot, selectedPath) targets.push({ @@ -854,7 +850,6 @@ async function resolveResultDownloadTargets({ inPath, inputs, normalizedResults, - overwriteExisting, outputPath, outputRoot, outputRootIsDirectory, @@ -865,7 +860,6 @@ async function resolveResultDownloadTargets({ inPath: string | null inputs: string[] normalizedResults: NormalizedAssemblyResults - overwriteExisting: boolean outputPath: string | null outputRoot: string outputRootIsDirectory: boolean @@ -873,6 +867,8 @@ async function resolveResultDownloadTargets({ singleAssembly?: boolean }): Promise { const { allFiles, entries } = normalizedResults + // Keep single-result jobs flat for multi-file CLI compatibility. Input groups are only + // needed for multi-file results; reservations distinguish colliding result names. const shouldGroupByInput = !singleAssembly && inPath != null && @@ -913,7 +909,6 @@ async function resolveResultDownloadTargets({ baseDir: outputRoot, groupByStep: inputs.length === 0, owner, - overwriteExisting, outputRoot, reservedPaths, }) @@ -924,7 +919,6 @@ async function resolveResultDownloadTargets({ baseDir: resolveDirectoryBaseDir(), groupByStep: inPath == null || entries.length > 1, owner, - overwriteExisting, outputRoot, reservedPaths, }) @@ -994,9 +988,9 @@ async function materializeAssemblyResults({ abortSignal, hasDirectoryInput, inPath, + isCurrent, inputs, normalizedResults, - overwriteExisting, outputPath, outputRoot, outputRootIsDirectory, @@ -1007,9 +1001,9 @@ async function materializeAssemblyResults({ abortSignal: AbortSignal hasDirectoryInput: boolean inPath: string | null + isCurrent: () => boolean inputs: string[] normalizedResults: NormalizedAssemblyResults - overwriteExisting: boolean outputPath: string | null outputRoot: string | null outputRootIsDirectory: boolean @@ -1026,7 +1020,6 @@ async function materializeAssemblyResults({ inPath, inputs, normalizedResults, - overwriteExisting, outputPath, outputRoot, outputRootIsDirectory, @@ -1039,7 +1032,7 @@ async function materializeAssemblyResults({ const [dlErr] = await tryCatch( targetPath == null ? downloadResultToStdout(resultUrl, abortSignal) - : downloadResultToFile(resultUrl, targetPath, abortSignal), + : downloadResultToFile(resultUrl, targetPath, abortSignal, isCurrent), ) if (dlErr) { if (dlErr.name === 'AbortError') { @@ -1740,9 +1733,9 @@ async function createWithLocalInputs( abortSignal: abortController.signal, hasDirectoryInput: singleAssemblyMode ? false : hasDirectoryInput, inPath, + isCurrent: () => !isSupersededWatchJob(outputPlan?.path ?? null, outputToken), inputs: singleAssemblyMode ? inputPaths : inputs, normalizedResults, - overwriteExisting: Boolean(watchOption), outputPath: outputPlan?.path ?? null, outputRoot: resolvedOutput ?? null, outputRootIsDirectory, @@ -1751,6 +1744,7 @@ async function createWithLocalInputs( singleAssembly: singleAssemblyMode, }) + if (isSupersededWatchJob(outputPlan?.path ?? null, outputToken)) return assembly outputctl.debug(`COMPLETED ${inPath ?? 'null'} ${outputPlan?.path ?? 'null'}`) if (del) { diff --git a/packages/node/test/e2e/cli/assemblies-create.test.ts b/packages/node/test/e2e/cli/assemblies-create.test.ts index 9343bc85..d1ec1ba6 100644 --- a/packages/node/test/e2e/cli/assemblies-create.test.ts +++ b/packages/node/test/e2e/cli/assemblies-create.test.ts @@ -234,7 +234,7 @@ describeLive('assemblies', { retry: 1 }, () => { ) it( - 'should detect outdir conflicts', + 'should disambiguate colliding input names in directory outputs', testCase(async (client) => { await fsp.mkdir('in') await Promise.all(['1.jpg', 'in/1.jpg'].map(imgPromise)) @@ -242,24 +242,12 @@ describeLive('assemblies', { retry: 1 }, () => { const steps = await stepsPromise() const output = new OutputCtl() - try { - await assembliesCreate(output, client, { - steps, - inputs: ['1.jpg', 'in'], - output: 'out', - }) - throw new Error('assembliesCreate didnt err; should have') - } catch (_err) { - const result = output.get() as OutputEntry[] - expect(result[result.length - 1]) - .to.have.property('type') - .that.equals('error') - expect(result[result.length - 1]) - .to.have.nested.property('msg.message') - .that.equals("Output collision between 'in/1.jpg' and '1.jpg'") - } - // Small delay to allow abort signals to propagate and streams to close - await new Promise((resolve) => setTimeout(resolve, 50)) + await assembliesCreate(output, client, { + steps, + inputs: ['1.jpg', 'in'], + output: 'out', + }) + expect(await fsp.readdir('out')).to.deep.equal(['1.jpg', '1__1.jpg']) }), ) diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index fcb48ce5..923ac6a2 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -1,6 +1,5 @@ import type { AssemblyResultEntryLike } from '../../../src/cli/resultFiles.ts' -import dns from 'node:dns/promises' import { EventEmitter } from 'node:events' import fs from 'node:fs' import { @@ -25,6 +24,10 @@ import { AssembliesCreateCommand } from '../../../src/cli/commands/assemblies.ts import OutputCtl from '../../../src/cli/OutputCtl.ts' import { Transloadit } from '../../../src/Transloadit.ts' +vi.mock('node:dns/promises', () => ({ + lookup: vi.fn(async () => [{ address: '93.184.216.34', family: 4 }]), +})) + const tempDirs: string[] = [] async function fixture(results: Record): Promise<{ @@ -71,7 +74,6 @@ beforeEach(async () => { vi.stubEnv('TRANSLOADIT_SECRET', 'secret') vi.spyOn(console, 'error').mockImplementation(() => {}) vi.spyOn(fs, 'fstatSync').mockReturnValue(await stat('/dev/null')) - vi.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }]) }) afterEach(async () => { @@ -160,13 +162,13 @@ it.each([ expect(await readFile(uploadedPath, 'utf8')).toBe('remote input') return { assembly_id: 'url-upload' } }) - nock('https://example.com').get('/remote.png?signature=example').reply(200, 'remote input') + nock('https://source.invalid').get('/remote.png?signature=example').reply(200, 'remote input') nock('https://downloads.test').get('/result').reply(200, 'processed bytes') expect( await run([ '-i', - 'https://example.com/remote.png?signature=example', + 'https://source.invalid/remote.png?signature=example', ...(single ? ['--single-assembly'] : []), ]), ).toBe(0) @@ -313,10 +315,21 @@ it('reserves concurrent linked downloads together when input stems match', async ).toBe('stream bytes') }) -it('refreshes a watched result in place after the input changes', async () => { +it.each([ + { existing: false, expectedName: 'result.png', expectedFiles: ['result.png'] }, + { existing: true, expectedName: 'result__1.png', expectedFiles: ['result.png', 'result__1.png'] }, +])('refreshes its owned watch result (preexisting output: $existing)', async ({ + existing, + expectedName, + expectedFiles, +}) => { const { inputPath, outputDir } = await fixture({ resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], }) + if (existing) { + await mkdir(outputDir) + await writeFile(path.join(outputDir, 'result.png'), 'unrelated file') + } class FakeWatcher extends EventEmitter { close(): void { this.emit('close') @@ -349,8 +362,105 @@ it('refreshes a watched result in place after the input changes', async () => { expect(result.hasFailures).toBe(false) expect(completions).toBe(2) - expect(await readdir(outputDir)).toEqual(['result.png']) - expect(await readFile(path.join(outputDir, 'result.png'), 'utf8')).toBe('updated result') + expect(await readdir(outputDir)).toEqual(expectedFiles) + expect(await readFile(path.join(outputDir, expectedName), 'utf8')).toBe('updated result') + if (existing) + expect(await readFile(path.join(outputDir, 'result.png'), 'utf8')).toBe('unrelated file') +}) + +it('accepts a standalone input and a directory containing the same basename', async () => { + const { inputPath, outputDir, run, tempDir } = await fixture({ + resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], + }) + const directoryInput = path.join(tempDir, 'pictures') + await mkdir(directoryInput) + await writeFile(path.join(directoryInput, 'logo.png'), 'second input') + nock('https://downloads.test').get('/result').twice().reply(200, 'result bytes') + + expect(await run(['-i', inputPath, '-i', directoryInput])).toBe(0) + expect(await readdir(outputDir)).toEqual(['result.png', 'result__1.png']) +}) + +it('fails promptly when a regular file blocks an ordinary Step directory', async () => { + const { inputPath, outputDir, run } = await fixture({ + thumbs: [{ name: 'result.png', url: 'https://downloads.test/image' }], + text: [{ name: 'result.txt', url: 'https://downloads.test/text' }], + }) + await mkdir(outputDir) + await writeFile(path.join(outputDir, 'thumbs'), 'existing file') + const lstat = fs.promises.lstat + let probes = 0 + vi.spyOn(fs.promises, 'lstat').mockImplementation(async (...args) => { + probes += 1 + // Bound the broken implementation so a failed regression does not hang the test worker. + if (probes > 30) throw new Error('Filesystem probe budget exceeded') + return await lstat(...args) + }) + + expect(await run(['-i', inputPath])).toBe(1) + expect(probes).toBeLessThan(15) + expect(await readFile(path.join(outputDir, 'thumbs'), 'utf8')).toBe('existing file') +}) + +it('discards a slow watch download when a newer result has already been saved', async () => { + const { inputPath, outputDir } = await fixture({}) + class FakeWatcher extends EventEmitter { + close(): void { + this.emit('close') + } + } + const watcher = new FakeWatcher() + let resolveWatcherReady: (() => void) | undefined + const watcherReady = new Promise((resolve) => { + resolveWatcherReady = resolve + }) + vi.doMock('node-watch', () => ({ + default: () => { + resolveWatcherReady?.() + return watcher + }, + })) + vi.resetModules() + const { create } = await import('../../../src/cli/commands/assemblies.ts') + let count = 0 + vi.mocked(Transloadit.prototype.createAssembly).mockImplementation(async () => ({ + assembly_id: String(++count), + })) + vi.mocked(Transloadit.prototype.awaitAssemblyCompletion).mockImplementation(async (id) => ({ + ok: 'ASSEMBLY_COMPLETED', + results: { resized: [{ name: 'result.png', url: `https://downloads.test/${id}` }] }, + })) + let resolveDownloadStarted: (() => void) | undefined + const downloadStarted = new Promise((resolve) => { + resolveDownloadStarted = resolve + }) + nock('https://downloads.test') + .get('/1') + .delayBody(120) + .reply(200, () => { + resolveDownloadStarted?.() + return 'older result' + }) + .get('/2') + .reply(200, 'newer result') + const output = new OutputCtl() + vi.spyOn(output, 'debug').mockImplementation((message) => { + if (String(message).startsWith('COMPLETED')) watcher.close() + }) + + const processing = create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + inputs: [inputPath], + output: `${outputDir}/`, + watch: true, + stepsData: { resized: { robot: '/image/resize', use: ':original', width: 16 } }, + }) + await watcherReady + await downloadStarted + watcher.emit('change', 'update', inputPath) + + expect((await processing).hasFailures).toBe(false) + expect(count).toBe(2) + expect(await readFile(path.join(outputDir, 'result.png'), 'utf8')).toBe('newer result') }) it('moves a linked Step together when an existing file occupies a rendition directory', async () => { @@ -378,3 +488,49 @@ it('moves a linked Step together when an existing file occupies a rendition dire ) expect(await readFile(path.join(outputDir, 'video'), 'utf8')).toBe('existing file') }) + +it('reuses watched filenames when the Assembly returns its results in a different order', async () => { + const { inputPath, outputDir } = await fixture({}) + class FakeWatcher extends EventEmitter { + close(): void { + this.emit('close') + } + } + const watcher = new FakeWatcher() + vi.doMock('node-watch', () => ({ default: () => watcher })) + vi.resetModules() + const { create } = await import('../../../src/cli/commands/assemblies.ts') + const a = { name: 'a.png', url: 'https://downloads.test/a' } + const b = { name: 'b.png', url: 'https://downloads.test/b' } + vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) + .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', results: { resized: [a, b] } }) + .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', results: { resized: [b, a] } }) + const output = new OutputCtl() + let completions = 0 + vi.spyOn(output, 'debug').mockImplementation((message) => { + if (!String(message).startsWith('COMPLETED')) return + if (++completions === 1) watcher.emit('change', 'update', inputPath) + else watcher.close() + }) + nock('https://downloads.test') + .get('/a') + .reply(200, 'old a') + .get('/b') + .reply(200, 'old b') + .get('/b') + .reply(200, 'new b') + .get('/a') + .reply(200, 'new a') + + const result = await create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + inputs: [inputPath], + output: `${outputDir}/`, + watch: true, + stepsData: { resized: { robot: '/image/resize', use: ':original', width: 16 } }, + }) + + expect(result.hasFailures).toBe(false) + expect(await readdir(outputDir)).toEqual(['a.png', 'b.png']) + expect(await readFile(path.join(outputDir, 'a.png'), 'utf8')).toBe('new a') + expect(await readFile(path.join(outputDir, 'b.png'), 'utf8')).toBe('new b') +}) From a47e9857830c8eb7c3866971aaad8bb594b73d0d Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 15:20:43 +0200 Subject: [PATCH 04/35] Keep watch destinations stable after output collisions --- packages/node/src/cli/commands/assemblies.ts | 46 ++++++--- .../test/e2e/cli/assemblies-create.test.ts | 40 ++++++-- .../test/unit/cli/download-layout.test.ts | 98 +++++++++++++++++-- 3 files changed, 160 insertions(+), 24 deletions(-) diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 7747a20e..03afa748 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -637,17 +637,25 @@ function sanitizeResultName(value: string): string { return name } +interface DownloadPathReservations { + ownersByPath: Map + pathsByOwner: Map +} + async function ensureUniquePath( targetPath: string, - reservedPaths: Map, + reservedPaths: DownloadPathReservations, owner: string, ): Promise { + const selectedPath = reservedPaths.pathsByOwner.get(owner) + if (selectedPath != null) return selectedPath const parsed = path.parse(targetPath) return await ensureUniqueCounterValue({ initialValue: targetPath, isTaken: async (candidate) => await isDownloadPathTaken(candidate, reservedPaths, owner), reserve: (candidate) => { - reservedPaths.set(candidate, owner) + reservedPaths.ownersByPath.set(candidate, owner) + reservedPaths.pathsByOwner.set(owner, candidate) }, nextValue: (counter) => path.join(parsed.dir, `${parsed.name}__${counter}${parsed.ext}`), scope: reservedPaths, @@ -714,10 +722,10 @@ async function assertNoOutputSymlinks(root: string, target: string): Promise, + reservedPaths: DownloadPathReservations, owner: string, ): Promise { - const existingOwner = reservedPaths.get(target) + const existingOwner = reservedPaths.ownersByPath.get(target) if (existingOwner != null) return existingOwner !== owner const [error] = await tryCatch(fsp.lstat(target)) if (isErrnoException(error) && error.code === 'ENOTDIR') return true @@ -744,7 +752,7 @@ async function buildDirectoryDownloadTargets({ groupByStep: boolean owner: string outputRoot: string - reservedPaths: Map + reservedPaths: DownloadPathReservations }): Promise { await assertNoOutputSymlinks(outputRoot, baseDir) await fsp.mkdir(baseDir, { recursive: true }) @@ -764,6 +772,9 @@ async function buildDirectoryDownloadTargets({ const safeStepName = sanitizeResultName(stepName) let targetDir = groupByStep ? path.join(baseDir, safeStepName) : baseDir const linked = files.some((file) => file.relativePath != null) + const groupOwner = JSON.stringify([owner, stepName]) + const selectedGroupDir = linked ? reservedPaths.pathsByOwner.get(groupOwner) : undefined + if (selectedGroupDir != null) targetDir = selectedGroupDir const names = files.map( ({ file, relativePath }) => relativePath ?? sanitizeResultName(file.name), ) @@ -777,7 +788,6 @@ async function buildDirectoryDownloadTargets({ if (linked) { const initialDir = targetDir - const groupOwner = JSON.stringify([owner, stepName]) // Selection and reservation share the same lock as ordinary result paths. A Step must // move together on collisions, while later watch updates reuse its owned destinations. targetDir = await ensureUniqueCounterValue({ @@ -793,11 +803,19 @@ async function buildDirectoryDownloadTargets({ isDownloadPathTaken(path.join(candidate, name), reservedPaths, groupOwner), ), ) - return collisions.some(Boolean) + const collides = collisions.some(Boolean) + // Moving an already downloaded Step would leave its previous playlist stale. + if (collides && selectedGroupDir != null) { + throw new Error('Assembly result path conflicts with an existing download') + } + return collides }, reserve: (candidate) => { - if (candidate !== initialDir) reservedPaths.set(candidate, groupOwner) - for (const name of names) reservedPaths.set(path.join(candidate, name), groupOwner) + if (candidate !== initialDir) reservedPaths.ownersByPath.set(candidate, groupOwner) + for (const name of names) { + reservedPaths.ownersByPath.set(path.join(candidate, name), groupOwner) + } + reservedPaths.pathsByOwner.set(groupOwner, candidate) }, nextValue: (counter) => { const suffix = groupByStep ? counter : counter - 1 @@ -863,7 +881,7 @@ async function resolveResultDownloadTargets({ outputPath: string | null outputRoot: string outputRootIsDirectory: boolean - reservedPaths: Map + reservedPaths: DownloadPathReservations singleAssembly?: boolean }): Promise { const { allFiles, entries } = normalizedResults @@ -1008,7 +1026,7 @@ async function materializeAssemblyResults({ outputRoot: string | null outputRootIsDirectory: boolean outputctl: IOutputCtl - reservedPaths: Map + reservedPaths: DownloadPathReservations singleAssembly?: boolean }): Promise { if (outputRoot == null) { @@ -1606,7 +1624,10 @@ async function createWithLocalInputs( const queue = new PQueue({ concurrency }) const results: unknown[] = [] const resultUrls: ResultUrlRow[] = [] - const reservedResultPaths = new Map() + const reservedResultPaths: DownloadPathReservations = { + ownersByPath: new Map(), + pathsByOwner: new Map(), + } const latestWatchJobTokenByOutputPath = new Map() let hasFailures = false let nextWatchJobToken = 0 @@ -1744,6 +1765,7 @@ async function createWithLocalInputs( singleAssembly: singleAssemblyMode, }) + // Only the current watch job may delete its input; a failed newer job must retain it. if (isSupersededWatchJob(outputPlan?.path ?? null, outputToken)) return assembly outputctl.debug(`COMPLETED ${inPath ?? 'null'} ${outputPlan?.path ?? 'null'}`) diff --git a/packages/node/test/e2e/cli/assemblies-create.test.ts b/packages/node/test/e2e/cli/assemblies-create.test.ts index d1ec1ba6..636a20b3 100644 --- a/packages/node/test/e2e/cli/assemblies-create.test.ts +++ b/packages/node/test/e2e/cli/assemblies-create.test.ts @@ -400,7 +400,7 @@ describeLive('assemblies', { retry: 1 }, () => { ) it( - 'should not reprocess inputs that are older than their output', + 'should reprocess directory outputs to discover result filenames', testCase(async (client) => { const infiles = await Promise.all(['in1.jpg', 'in2.jpg', 'in3.jpg'].map(imgPromise)) const steps = await stepsPromise() @@ -414,16 +414,44 @@ describeLive('assemblies', { retry: 1 }, () => { }) const output2 = new OutputCtl() - await assembliesCreate(output2, client, { + const { results } = await assembliesCreate(output2, client, { steps, inputs: infiles, output: 'out', }) - const result = output2.get(true) as OutputEntry[] + expect(results).to.have.lengthOf(3) + expect(await fsp.readdir('out')).to.deep.equal([ + 'in1.jpg', + 'in1__1.jpg', + 'in2.jpg', + 'in3.jpg', + ]) + }), + 180_000, + ) - expect( - result.map((line) => line.msg).filter((msg) => String(msg).includes('in1.jpg')), - ).to.have.lengthOf(0) + it( + 'should skip an explicit file output newer than its input', + testCase(async (client) => { + const infile = await imgPromise() + const steps = await stepsPromise() + const inputTime = new Date(Date.now() - 60_000) + await fsp.utimes(infile, inputTime, inputTime) + await assembliesCreate(new OutputCtl(), client, { + steps, + inputs: [infile], + output: 'out.jpg', + }) + const originalOutput = await fsp.readFile('out.jpg') + + const { results } = await assembliesCreate(new OutputCtl(), client, { + steps, + inputs: [infile], + output: 'out.jpg', + }) + + expect(results).to.have.lengthOf(0) + expect(await fsp.readFile('out.jpg')).to.deep.equal(originalOutput) }), 180_000, ) diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 923ac6a2..894f64a8 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -10,6 +10,7 @@ import { rm, stat, symlink, + unlink, utimes, writeFile, } from 'node:fs/promises' @@ -316,10 +317,27 @@ it('reserves concurrent linked downloads together when input stems match', async }) it.each([ - { existing: false, expectedName: 'result.png', expectedFiles: ['result.png'] }, - { existing: true, expectedName: 'result__1.png', expectedFiles: ['result.png', 'result__1.png'] }, -])('refreshes its owned watch result (preexisting output: $existing)', async ({ + { + existing: false, + removeConflict: false, + expectedName: 'result.png', + expectedFiles: ['result.png'], + }, + { + existing: true, + removeConflict: false, + expectedName: 'result__1.png', + expectedFiles: ['result.png', 'result__1.png'], + }, + { + existing: true, + removeConflict: true, + expectedName: 'result__1.png', + expectedFiles: ['result__1.png'], + }, +])('refreshes its owned watch result (existing: $existing, removed: $removeConflict)', async ({ existing, + removeConflict, expectedName, expectedFiles, }) => { @@ -340,11 +358,15 @@ it.each([ vi.resetModules() const { create } = await import('../../../src/cli/commands/assemblies.ts') let completions = 0 + let resolveFirstCompletion: (() => void) | undefined + const firstCompletion = new Promise((resolve) => { + resolveFirstCompletion = resolve + }) const output = new OutputCtl() vi.spyOn(output, 'debug').mockImplementation((message) => { if (!String(message).startsWith('COMPLETED')) return completions += 1 - if (completions === 1) watcher.emit('change', 'update', inputPath) + if (completions === 1) resolveFirstCompletion?.() else watcher.close() }) nock('https://downloads.test') @@ -353,21 +375,85 @@ it.each([ .get('/result') .reply(200, 'updated result') - const result = await create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + const processing = create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { inputs: [inputPath], output: `${outputDir}/`, watch: true, stepsData: { resized: { robot: '/image/resize', use: ':original', width: 16 } }, }) + await firstCompletion + if (removeConflict) await unlink(path.join(outputDir, 'result.png')) + watcher.emit('change', 'update', inputPath) + const result = await processing expect(result.hasFailures).toBe(false) expect(completions).toBe(2) expect(await readdir(outputDir)).toEqual(expectedFiles) expect(await readFile(path.join(outputDir, expectedName), 'utf8')).toBe('updated result') - if (existing) + if (existing && !removeConflict) expect(await readFile(path.join(outputDir, 'result.png'), 'utf8')).toBe('unrelated file') }) +it('keeps a watched HLS Step in its selected folder after the original conflict disappears', async () => { + const { inputPath, outputDir } = await fixture({ + adaptive: [ + { name: 'master.m3u8', url: 'https://downloads.test/master', meta: { relative_path: '' } }, + { + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'video' }, + }, + ], + }) + await mkdir(outputDir) + await writeFile(path.join(outputDir, 'master.m3u8'), 'unrelated file') + class FakeWatcher extends EventEmitter { + close(): void { + this.emit('close') + } + } + const watcher = new FakeWatcher() + vi.doMock('node-watch', () => ({ default: () => watcher })) + vi.resetModules() + const { create } = await import('../../../src/cli/commands/assemblies.ts') + let resolveFirstCompletion: (() => void) | undefined + const firstCompletion = new Promise((resolve) => { + resolveFirstCompletion = resolve + }) + let completions = 0 + const output = new OutputCtl() + vi.spyOn(output, 'debug').mockImplementation((message) => { + if (!String(message).startsWith('COMPLETED')) return + if (++completions === 1) resolveFirstCompletion?.() + else watcher.close() + }) + nock('https://downloads.test') + .get('/master') + .twice() + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/segment') + .reply(200, 'old segment') + .get('/segment') + .reply(200, 'new segment') + + const processing = create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + inputs: [inputPath], + output: `${outputDir}/`, + watch: true, + stepsData: { adaptive: { robot: '/video/adaptive', use: ':original' } }, + }) + await firstCompletion + await unlink(path.join(outputDir, 'master.m3u8')) + watcher.emit('change', 'update', inputPath) + + expect((await processing).hasFailures).toBe(false) + expect(completions).toBe(2) + expect(await readdir(outputDir)).toEqual(['adaptive']) + expect(await readFile(path.join(outputDir, 'adaptive', 'video', 'segment.ts'), 'utf8')).toBe( + 'new segment', + ) +}) + it('accepts a standalone input and a directory containing the same basename', async () => { const { inputPath, outputDir, run, tempDir } = await fixture({ resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], From 46d5ef91d77cbb1e686ea5b3120b344eceea53fb Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 15:41:40 +0200 Subject: [PATCH 05/35] Separate independent HLS bundles within a result Step --- .changeset/assembly-download-paths.md | 2 + packages/node/README.md | 9 +- packages/node/src/cli/commands/assemblies.ts | 86 ++++++++++++++++--- .../test/unit/cli/download-layout.test.ts | 71 +++++++++++++++ packages/transloadit/README.md | 9 +- 5 files changed, 155 insertions(+), 22 deletions(-) diff --git a/.changeset/assembly-download-paths.md b/.changeset/assembly-download-paths.md index a0b951d1..e556e8f6 100644 --- a/.changeset/assembly-download-paths.md +++ b/.changeset/assembly-download-paths.md @@ -6,4 +6,6 @@ Keep CLI download filenames and HLS directories aligned with Assembly results, omit input parent folders from output paths, and support HTTP(S) URL inputs for Assembly and prompt commands. +Independent HLS bundles returned by the same Step get separate source folders so their playlist and segment paths can coexist. + Directory runs process inputs again to discover the result filenames instead of guessing freshness from upload names. Watch updates reuse their own result paths, while existing unrelated files and outputs from other inputs remain separate. diff --git a/packages/node/README.md b/packages/node/README.md index 416c69d7..7b2f7ec3 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -1505,15 +1505,16 @@ The `assemblies create` command additionally supports: With one local input, filenames come from the Assembly results. Multiple result Steps get their own folders; the input’s parent folders are omitted. Directory inputs retain subfolders relative to that directory. HLS results retain the directories in `meta.relative_path`, so playlists can - find their renditions and segments. If those paths already exist, the whole result Step is saved - in a new Step folder together, preserving playlist references. + find their renditions and segments. Multiple HLS bundles in one Step get separate folders named + after their sources. On collisions, a linked bundle moves together to a new folder, preserving + playlist references. Watch updates refresh their existing result files. Other directory runs process the inputs again, since result filenames are unknown before processing; file outputs keep mtime-based skipping. - `--input, -i` - Provide a local upload or an HTTP(S) URL, or use `-i -` to read stdin explicitly. URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe - or a redirected file. Other non-TTY - stdin (such as an agent shell or `/dev/null`) runs the Assembly without uploads. With `--template`, + or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly + without uploads. With `--template`, stdin always requires `-i -`. - `--single-assembly` - Pass all input files to a single assembly instead of one assembly per file diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 03afa748..fd5ad32b 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -31,6 +31,7 @@ import { z } from 'zod' import { formatLintIssue } from '../../alphalib/assembly-linter.lang.en.ts' import { tryCatch } from '../../alphalib/tryCatch.ts' +import { assemblyStatusResultSchema } from '../../alphalib/types/assemblyStatus.ts' import { ensureUniqueCounterValue } from '../../ensureUniqueCounter.ts' import { prepareInputFiles } from '../../inputFiles.ts' import { lintAssemblyInstructions } from '../../lintAssemblyInstructions.ts' @@ -739,6 +740,69 @@ function isMeaningfullyNewer(newer: Date, older: Date): boolean { return newer.getTime() - older.getTime() > STALE_OUTPUT_GRACE_MS } +interface DirectoryResultFile { + file: NormalizedAssemblyResultFile + relativePath: string | null +} + +interface DirectoryResultGroup { + directoryName: string | null + files: DirectoryResultFile[] + ownerKey: string | null + stepName: string +} + +const resultSourceSchema = assemblyStatusResultSchema.pick({ + original_basename: true, + original_id: true, + original_name: true, +}) + +function groupDirectoryResults(allFiles: NormalizedAssemblyResultFile[]): DirectoryResultGroup[] { + const steps = new Map() + for (const file of allFiles) { + const files = steps.get(file.stepName) ?? [] + files.push({ file, relativePath: getResultRelativePath(file) }) + steps.set(file.stepName, files) + } + + const groups: DirectoryResultGroup[] = [] + for (const [stepName, stepFiles] of steps) { + if (!stepFiles.some((file) => file.relativePath != null)) { + groups.push({ directoryName: null, files: stepFiles, ownerKey: null, stepName }) + continue + } + + const sources = new Map() + for (const file of stepFiles) { + const parsed = resultSourceSchema.safeParse(file.file.file) + const source = parsed.success ? parsed.data : undefined + const sourceId = JSON.stringify(source?.original_id ?? null) + const group = sources.get(sourceId) ?? { + name: source?.original_name ?? source?.original_basename ?? null, + files: [], + } + group.files.push(file) + sources.set(sourceId, group) + } + + const nameOccurrences = new Map() + for (const { name, files } of sources.values()) { + const occurrence = nameOccurrences.get(name) ?? 0 + nameOccurrences.set(name, occurrence + 1) + groups.push({ + directoryName: + sources.size > 1 ? sanitizeResultName(path.parse(name ?? stepName).name) : null, + files, + // Upload IDs change between watch runs; source names retain logical ownership. + ownerKey: JSON.stringify([name, occurrence]), + stepName, + }) + } + } + return groups +} + async function buildDirectoryDownloadTargets({ allFiles, baseDir, @@ -758,21 +822,12 @@ async function buildDirectoryDownloadTargets({ await fsp.mkdir(baseDir, { recursive: true }) const targets: AssemblyDownloadTarget[] = [] - const steps = new Map< - string, - Array<{ file: NormalizedAssemblyResultFile; relativePath: string | null }> - >() - for (const resultFile of allFiles) { - const files = steps.get(resultFile.stepName) ?? [] - files.push({ file: resultFile, relativePath: getResultRelativePath(resultFile) }) - steps.set(resultFile.stepName, files) - } - - for (const [stepName, files] of steps) { + for (const { directoryName, files, ownerKey, stepName } of groupDirectoryResults(allFiles)) { const safeStepName = sanitizeResultName(stepName) let targetDir = groupByStep ? path.join(baseDir, safeStepName) : baseDir + if (directoryName != null) targetDir = path.join(targetDir, directoryName) const linked = files.some((file) => file.relativePath != null) - const groupOwner = JSON.stringify([owner, stepName]) + const groupOwner = JSON.stringify([owner, stepName, ownerKey]) const selectedGroupDir = linked ? reservedPaths.pathsByOwner.get(groupOwner) : undefined if (selectedGroupDir != null) targetDir = selectedGroupDir const names = files.map( @@ -788,7 +843,7 @@ async function buildDirectoryDownloadTargets({ if (linked) { const initialDir = targetDir - // Selection and reservation share the same lock as ordinary result paths. A Step must + // Selection and reservation share the same lock as ordinary result paths. A bundle must // move together on collisions, while later watch updates reuse its owned destinations. targetDir = await ensureUniqueCounterValue({ initialValue: initialDir, @@ -804,7 +859,7 @@ async function buildDirectoryDownloadTargets({ ), ) const collides = collisions.some(Boolean) - // Moving an already downloaded Step would leave its previous playlist stale. + // Moving an already downloaded bundle would leave its previous playlist stale. if (collides && selectedGroupDir != null) { throw new Error('Assembly result path conflicts with an existing download') } @@ -818,6 +873,9 @@ async function buildDirectoryDownloadTargets({ reservedPaths.pathsByOwner.set(groupOwner, candidate) }, nextValue: (counter) => { + if (directoryName != null) { + return path.join(path.dirname(initialDir), `${directoryName}__${counter}`) + } const suffix = groupByStep ? counter : counter - 1 return path.join(baseDir, suffix === 0 ? safeStepName : `${safeStepName}__${suffix}`) }, diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 894f64a8..777cf394 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -121,6 +121,77 @@ it('keeps every HLS playlist reference reachable from its downloaded playlist', ) }) +it('separates independent HLS bundles with identical paths in one Assembly Step', async () => { + const firstMaster = { + name: 'master.m3u8', + url: 'https://downloads.test/first-master', + original_id: 'first-source', + original_name: 'first.mp4', + meta: { relative_path: '' }, + } + const firstSegment = { + ...firstMaster, + name: 'segment.ts', + url: 'https://downloads.test/first-segment', + meta: { relative_path: 'video' }, + } + const secondMaster = { + ...firstMaster, + url: 'https://downloads.test/second-master', + original_id: 'second-source', + original_name: 'second.mp4', + } + const secondSegment = { + ...secondMaster, + name: 'segment.ts', + url: 'https://downloads.test/second-segment', + meta: { relative_path: 'video' }, + } + const { inputPath, outputDir, run } = await fixture({ + adaptive: [firstMaster, secondMaster, firstSegment, secondSegment], + }) + const secondInput = path.join(path.dirname(inputPath), 'second.mp4') + await writeFile(secondInput, 'second input') + nock('https://downloads.test') + .get('/first-master') + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/second-master') + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/first-segment') + .reply(200, 'first stream') + .get('/second-segment') + .reply(200, 'second stream') + + expect(await run(['--single-assembly', '-i', inputPath, '-i', secondInput])).toBe(0) + expect(await readdir(outputDir)).toEqual(['first', 'second']) + expect(await readFile(path.join(outputDir, 'first', 'master.m3u8'), 'utf8')).toBe( + '#EXTM3U\nvideo/segment.ts\n', + ) + expect(await readFile(path.join(outputDir, 'first', 'video', 'segment.ts'), 'utf8')).toBe( + 'first stream', + ) + expect(await readFile(path.join(outputDir, 'second', 'master.m3u8'), 'utf8')).toBe( + '#EXTM3U\nvideo/segment.ts\n', + ) + expect(await readFile(path.join(outputDir, 'second', 'video', 'segment.ts'), 'utf8')).toBe( + 'second stream', + ) +}) + +it('rejects duplicate paths within a single HLS bundle', async () => { + const segment = { + name: 'segment.ts', + url: 'https://downloads.test/segment', + original_id: 'same-source', + original_name: 'source.mp4', + meta: { relative_path: 'video' }, + } + const { inputPath, outputDir, run } = await fixture({ adaptive: [segment, segment] }) + + expect(await run(['-i', inputPath])).toBe(1) + expect(await readdir(outputDir)).toEqual([]) +}) + it.each([false, true])('uses the bundle result name (single Assembly: %s)', async (single) => { const { inputPath, outputDir, run } = await fixture({ bundled: [{ name: 'archive.zip', url: 'https://downloads.test/archive' }], diff --git a/packages/transloadit/README.md b/packages/transloadit/README.md index 416c69d7..7b2f7ec3 100644 --- a/packages/transloadit/README.md +++ b/packages/transloadit/README.md @@ -1505,15 +1505,16 @@ The `assemblies create` command additionally supports: With one local input, filenames come from the Assembly results. Multiple result Steps get their own folders; the input’s parent folders are omitted. Directory inputs retain subfolders relative to that directory. HLS results retain the directories in `meta.relative_path`, so playlists can - find their renditions and segments. If those paths already exist, the whole result Step is saved - in a new Step folder together, preserving playlist references. + find their renditions and segments. Multiple HLS bundles in one Step get separate folders named + after their sources. On collisions, a linked bundle moves together to a new folder, preserving + playlist references. Watch updates refresh their existing result files. Other directory runs process the inputs again, since result filenames are unknown before processing; file outputs keep mtime-based skipping. - `--input, -i` - Provide a local upload or an HTTP(S) URL, or use `-i -` to read stdin explicitly. URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe - or a redirected file. Other non-TTY - stdin (such as an agent shell or `/dev/null`) runs the Assembly without uploads. With `--template`, + or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly + without uploads. With `--template`, stdin always requires `-i -`. - `--single-assembly` - Pass all input files to a single assembly instead of one assembly per file From b6056613a656b657c82a3a5561985cc92569a882 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 16:11:27 +0200 Subject: [PATCH 06/35] Reserve complete output paths and stage linked downloads --- .changeset/assembly-download-paths.md | 2 + packages/node/README.md | 5 +- packages/node/src/cli/commands/assemblies.ts | 301 ++++++++++++++---- .../test/unit/cli/download-layout.test.ts | 158 +++++++++ packages/transloadit/README.md | 5 +- 5 files changed, 398 insertions(+), 73 deletions(-) diff --git a/.changeset/assembly-download-paths.md b/.changeset/assembly-download-paths.md index e556e8f6..1ee0df58 100644 --- a/.changeset/assembly-download-paths.md +++ b/.changeset/assembly-download-paths.md @@ -8,4 +8,6 @@ Keep CLI download filenames and HLS directories aligned with Assembly results, o Independent HLS bundles returned by the same Step get separate source folders so their playlist and segment paths can coexist. +Reserve result paths with filesystem case sensitivity and file/directory conflicts in mind. Stage linked watch downloads before publishing them together, and validate local steps and output options before fetching URL inputs. + Directory runs process inputs again to discover the result filenames instead of guessing freshness from upload names. Watch updates reuse their own result paths, while existing unrelated files and outputs from other inputs remain separate. diff --git a/packages/node/README.md b/packages/node/README.md index 7b2f7ec3..2656b249 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -1510,12 +1510,11 @@ The `assemblies create` command additionally supports: playlist references. Watch updates refresh their existing result files. Other directory runs process the inputs again, since result filenames are unknown before processing; file outputs keep mtime-based skipping. -- `--input, -i` - Provide a local upload or an HTTP(S) URL, or use `-i -` to read stdin explicitly. +- `--input, -i` - Provide a local upload or a public HTTP(S) URL, or use `-i -` to read stdin explicitly. URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly - without uploads. With `--template`, - stdin always requires `-i -`. + without uploads. With `--template`, stdin always requires `-i -`. - `--single-assembly` - Pass all input files to a single assembly instead of one assembly per file #### Log Levels diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index fd5ad32b..4a009ab6 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -639,10 +639,61 @@ function sanitizeResultName(value: string): string { } interface DownloadPathReservations { + allocations: PQueue + bundleCommits: PQueue + caseSensitive: boolean + directories: Set ownersByPath: Map pathsByOwner: Map } +async function outputNamesAreCaseSensitive(root: string): Promise { + await fsp.mkdir(root, { recursive: true }) + const probe = await fsp.mkdtemp(path.join(root, '.transloadit-case-')) + try { + await fsp.writeFile(path.join(probe, 'case'), '') + const [error] = await tryCatch(fsp.lstat(path.join(probe, 'CASE'))) + if (error && (!isErrnoException(error) || error.code !== 'ENOENT')) throw error + return error != null + } finally { + await fsp.rm(probe, { recursive: true, force: true }) + } +} + +function downloadPathKey(target: string, reservedPaths: DownloadPathReservations): string { + const absolutePath = path.resolve(target) + return reservedPaths.caseSensitive ? absolutePath : absolutePath.normalize('NFC').toLowerCase() +} + +function hasReservedFileAncestor(target: string, reservedPaths: DownloadPathReservations): boolean { + let current = downloadPathKey(target, reservedPaths) + while (true) { + if (reservedPaths.ownersByPath.has(current) && !reservedPaths.directories.has(current)) { + return true + } + const parent = path.dirname(current) + if (parent === current) return false + current = parent + } +} + +function reserveDownloadPath( + target: string, + owner: string, + reservedPaths: DownloadPathReservations, + directory = false, +): void { + const key = downloadPathKey(target, reservedPaths) + reservedPaths.ownersByPath.set(key, owner) + let current = directory ? key : path.dirname(key) + while (true) { + reservedPaths.directories.add(current) + const parent = path.dirname(current) + if (parent === current) return + current = parent + } +} + async function ensureUniquePath( targetPath: string, reservedPaths: DownloadPathReservations, @@ -655,7 +706,7 @@ async function ensureUniquePath( initialValue: targetPath, isTaken: async (candidate) => await isDownloadPathTaken(candidate, reservedPaths, owner), reserve: (candidate) => { - reservedPaths.ownersByPath.set(candidate, owner) + reserveDownloadPath(candidate, owner, reservedPaths) reservedPaths.pathsByOwner.set(owner, candidate) }, nextValue: (counter) => path.join(parsed.dir, `${parsed.name}__${counter}${parsed.ext}`), @@ -690,10 +741,49 @@ function getResultRelativePath(file: NormalizedAssemblyResultFile): string | nul } interface AssemblyDownloadTarget { + bundleRoot?: string resultUrl: string targetPath: string | null } +interface BundleDownloadTarget { + resultUrl: string + targetPath: string +} + +async function downloadResultBundle( + root: string, + targets: BundleDownloadTarget[], + signal: AbortSignal, + isCurrent: () => boolean, + commits: PQueue, + onDownload: () => void, +): Promise { + const stagingDir = await fsp.mkdtemp(path.join(root, '.transloadit-bundle-')) + const staged = targets.map((target, index) => ({ + ...target, + stagingPath: path.join(stagingDir, String(index)), + })) + try { + for (const { resultUrl, stagingPath } of staged) { + if (!isCurrent()) return + onDownload() + await downloadResultToFile(resultUrl, stagingPath, signal, () => true) + } + await commits.add(async () => { + if (!isCurrent()) return + // Finish publication once it starts, even if superseded. A failed newer job must not + // strand this playlist without its segments, and newer commits must wait for this one. + for (const { stagingPath, targetPath } of staged) { + await fsp.mkdir(path.dirname(targetPath), { recursive: true }) + await fsp.rename(stagingPath, targetPath) + } + }) + } finally { + await fsp.rm(stagingDir, { recursive: true, force: true }) + } +} + async function assertNoOutputSymlinks(root: string, target: string): Promise { // Reject preexisting descendant symlinks; another process can still race filesystem checks. const relativePath = path.relative(root, target) @@ -726,8 +816,12 @@ async function isDownloadPathTaken( reservedPaths: DownloadPathReservations, owner: string, ): Promise { - const existingOwner = reservedPaths.ownersByPath.get(target) + const key = downloadPathKey(target, reservedPaths) + const existingOwner = reservedPaths.ownersByPath.get(key) if (existingOwner != null) return existingOwner !== owner + if (reservedPaths.directories.has(key) || hasReservedFileAncestor(target, reservedPaths)) { + return true + } const [error] = await tryCatch(fsp.lstat(target)) if (isErrnoException(error) && error.code === 'ENOTDIR') return true if (error && (!isErrnoException(error) || error.code !== 'ENOENT')) throw error @@ -819,6 +913,9 @@ async function buildDirectoryDownloadTargets({ reservedPaths: DownloadPathReservations }): Promise { await assertNoOutputSymlinks(outputRoot, baseDir) + if (hasReservedFileAncestor(baseDir, reservedPaths)) { + throw new Error('Output directory conflicts with a result file') + } await fsp.mkdir(baseDir, { recursive: true }) const targets: AssemblyDownloadTarget[] = [] @@ -833,7 +930,11 @@ async function buildDirectoryDownloadTargets({ const names = files.map( ({ file, relativePath }) => relativePath ?? sanitizeResultName(file.name), ) - if (linked && new Set(names).size !== names.length) { + if ( + linked && + new Set(names.map((name) => downloadPathKey(path.join(targetDir, name), reservedPaths))) + .size !== names.length + ) { throw new Error('Assembly results have duplicate relative paths') } @@ -866,9 +967,10 @@ async function buildDirectoryDownloadTargets({ return collides }, reserve: (candidate) => { - if (candidate !== initialDir) reservedPaths.ownersByPath.set(candidate, groupOwner) + if (candidate !== initialDir) + reserveDownloadPath(candidate, groupOwner, reservedPaths, true) for (const name of names) { - reservedPaths.ownersByPath.set(path.join(candidate, name), groupOwner) + reserveDownloadPath(path.join(candidate, name), groupOwner, reservedPaths) } reservedPaths.pathsByOwner.set(groupOwner, candidate) }, @@ -883,6 +985,9 @@ async function buildDirectoryDownloadTargets({ }) } + if (hasReservedFileAncestor(targetDir, reservedPaths)) { + throw new Error('Output directory conflicts with a result file') + } await fsp.mkdir(targetDir, { recursive: true }) const nameOccurrences = new Map() for (const [index, { file }] of files.entries()) { @@ -899,6 +1004,7 @@ async function buildDirectoryDownloadTargets({ ) await assertNoOutputSymlinks(outputRoot, selectedPath) targets.push({ + bundleRoot: linked ? targetDir : undefined, resultUrl: file.url, targetPath: selectedPath, }) @@ -1091,25 +1197,53 @@ async function materializeAssemblyResults({ return } - const targets = await resolveResultDownloadTargets({ - hasDirectoryInput, - inPath, - inputs, - normalizedResults, - outputPath, - outputRoot, - outputRootIsDirectory, - reservedPaths, - singleAssembly, - }) + // Directory creation and filename reservation must share a lock across concurrent jobs. + const targets = await reservedPaths.allocations.add( + async () => + await resolveResultDownloadTargets({ + hasDirectoryInput, + inPath, + inputs, + normalizedResults, + outputPath, + outputRoot, + outputRootIsDirectory, + reservedPaths, + singleAssembly, + }), + ) - for (const { resultUrl, targetPath } of targets) { - outputctl.debug('DOWNLOADING') - const [dlErr] = await tryCatch( - targetPath == null - ? downloadResultToStdout(resultUrl, abortSignal) - : downloadResultToFile(resultUrl, targetPath, abortSignal, isCurrent), + const bundles = new Map() + const downloads: Array<() => Promise> = [] + for (const { bundleRoot, resultUrl, targetPath } of targets) { + if (bundleRoot != null && targetPath != null) { + const files = bundles.get(bundleRoot) ?? [] + files.push({ resultUrl, targetPath }) + bundles.set(bundleRoot, files) + continue + } + downloads.push(async () => { + outputctl.debug('DOWNLOADING') + if (targetPath == null) await downloadResultToStdout(resultUrl, abortSignal) + else await downloadResultToFile(resultUrl, targetPath, abortSignal, isCurrent) + }) + } + for (const [root, files] of bundles) { + downloads.push( + async () => + await downloadResultBundle( + root, + files, + abortSignal, + isCurrent, + reservedPaths.bundleCommits, + () => outputctl.debug('DOWNLOADING'), + ), ) + } + + for (const download of downloads) { + const [dlErr] = await tryCatch(download()) if (dlErr) { if (dlErr.name === 'AbortError') { continue @@ -1549,53 +1683,21 @@ export interface AssembliesCreateOptions { const DEFAULT_CONCURRENCY = 5 -// --- Main assembly create function --- -/** Create Assemblies after materializing URL inputs as temporary local uploads. */ -export async function create( - outputctl: IOutputCtl, - client: Transloadit, - options: AssembliesCreateOptions, -): ReturnType { - const urlInputs = options.inputs.filter((input) => /^https?:\/\//i.test(input)) - if (urlInputs.length === 0) return await createWithLocalInputs(outputctl, client, options) - if (options.watch) throw new Error('--watch requires local inputs') +interface PreparedCreateOptions { + effectiveStepsData: StepsInput | undefined + hasDirectoryInput: boolean + outstat: StatLike | undefined + resolvedOutput: string | null | undefined +} - const prepared = await prepareInputFiles({ - inputFiles: urlInputs.map((url, index) => ({ kind: 'url', field: String(index), url })), - allowPrivateUrls: false, - // Uploading the bytes preserves :original and Template upload semantics, just as Intents do. - urlStrategy: 'download', - }) - const localPaths = new Map(urlInputs.map((url, index) => [url, prepared.files[String(index)]])) - try { - return await createWithLocalInputs(outputctl, client, { - ...options, - inputs: options.inputs.map((input) => localPaths.get(input) ?? input), - }) - } finally { - await Promise.all(prepared.cleanup.map((cleanup) => cleanup())) - } +function isUrlInput(input: string): boolean { + return /^https?:\/\//i.test(input) } -async function createWithLocalInputs( +async function prepareCreateOptions( outputctl: IOutputCtl, - client: Transloadit, - { - steps, - stepsData, - template, - fields, - outputMode, - watch: watchOption, - recursive, - inputs, - output, - del, - reprocessStale, - singleAssembly, - concurrency = DEFAULT_CONCURRENCY, - }: AssembliesCreateOptions, -): Promise<{ resultUrls: ResultUrlRow[]; results: unknown[]; hasFailures: boolean }> { + { steps, stepsData, outputMode, output, inputs, singleAssembly }: AssembliesCreateOptions, +): Promise { // Quick fix for https://github.com/transloadit/transloadify/issues/13 // Only default to stdout when output is undefined (not provided), not when explicitly null let resolvedOutput = output @@ -1631,7 +1733,9 @@ async function createWithLocalInputs( if (!outstat.isDirectory() && inputs.length !== 0 && !singleAssembly) { const firstInput = inputs[0] if (firstInput) { - const firstInputStat = await myStat(process.stdin, firstInput) + const firstInputStat = isUrlInput(firstInput) + ? { isDirectory: () => false } + : await myStat(process.stdin, firstInput) if (inputs.length > 1 || firstInputStat.isDirectory()) { const msg = 'Output must be a directory when specifying multiple inputs' outputctl.error(msg) @@ -1643,11 +1747,70 @@ async function createWithLocalInputs( const inputStats = await Promise.all( inputs.map(async (input) => { - if (input === '-') return null + if (input === '-' || isUrlInput(input)) return null return await myStat(process.stdin, input) }), ) const hasDirectoryInput = inputStats.some((stat) => stat?.isDirectory() === true) + return { effectiveStepsData, hasDirectoryInput, outstat, resolvedOutput } +} + +// --- Main assembly create function --- +/** Create Assemblies after materializing URL inputs as temporary local uploads. */ +export async function create( + outputctl: IOutputCtl, + client: Transloadit, + options: AssembliesCreateOptions, +): ReturnType { + const preparedOptions = await prepareCreateOptions(outputctl, options) + const urlInputs = options.inputs.filter(isUrlInput) + if (urlInputs.length === 0) + return await createWithLocalInputs(outputctl, client, options, preparedOptions) + if (options.watch) throw new Error('--watch requires local inputs') + + const prepared = await prepareInputFiles({ + inputFiles: urlInputs.map((url, index) => ({ kind: 'url', field: String(index), url })), + allowPrivateUrls: false, + // Uploading the bytes preserves :original and Template upload semantics, just as Intents do. + urlStrategy: 'download', + }) + const localPaths = new Map(urlInputs.map((url, index) => [url, prepared.files[String(index)]])) + try { + return await createWithLocalInputs( + outputctl, + client, + { + ...options, + inputs: options.inputs.map((input) => localPaths.get(input) ?? input), + }, + preparedOptions, + ) + } finally { + await Promise.all(prepared.cleanup.map((cleanup) => cleanup())) + } +} + +async function createWithLocalInputs( + outputctl: IOutputCtl, + client: Transloadit, + { + template, + fields, + watch: watchOption, + recursive, + inputs, + del, + reprocessStale, + singleAssembly, + concurrency = DEFAULT_CONCURRENCY, + }: AssembliesCreateOptions, + preparedOptions: PreparedCreateOptions, +): Promise<{ resultUrls: ResultUrlRow[]; results: unknown[]; hasFailures: boolean }> { + const { effectiveStepsData, hasDirectoryInput, outstat, resolvedOutput } = preparedOptions + const caseSensitive = + resolvedOutput != null && outstat?.isDirectory() + ? await outputNamesAreCaseSensitive(resolvedOutput) + : true return new Promise((resolve, reject) => { const params: CreateAssemblyParams = ( @@ -1683,6 +1846,10 @@ async function createWithLocalInputs( const results: unknown[] = [] const resultUrls: ResultUrlRow[] = [] const reservedResultPaths: DownloadPathReservations = { + allocations: new PQueue({ concurrency: 1 }), + bundleCommits: new PQueue({ concurrency: 1 }), + caseSensitive, + directories: new Set(), ownersByPath: new Map(), pathsByOwner: new Map(), } diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 777cf394..59f14cfd 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -21,6 +21,7 @@ import { Cli } from 'clipanion' import nock from 'nock' import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { tryCatch } from '../../../src/alphalib/tryCatch.ts' import { AssembliesCreateCommand } from '../../../src/cli/commands/assemblies.ts' import OutputCtl from '../../../src/cli/OutputCtl.ts' import { Transloadit } from '../../../src/Transloadit.ts' @@ -192,6 +193,163 @@ it('rejects duplicate paths within a single HLS bundle', async () => { expect(await readdir(outputDir)).toEqual([]) }) +it('keeps concurrent case-only result names separate on case-insensitive filesystems', async () => { + const { inputPath, outputDir, run } = await fixture({}) + const secondInput = path.join(path.dirname(inputPath), 'second.png') + await writeFile(secondInput, 'second input') + vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) + .mockResolvedValueOnce({ + ok: 'ASSEMBLY_COMPLETED', + results: { resized: [{ name: 'Photo.jpg', url: 'https://downloads.test/first' }] }, + }) + .mockResolvedValueOnce({ + ok: 'ASSEMBLY_COMPLETED', + results: { resized: [{ name: 'photo.jpg', url: 'https://downloads.test/second' }] }, + }) + async function resolveCaseInsensitivePath(value: fs.PathLike): Promise { + if (typeof value !== 'string') return value + const [error, siblings] = await tryCatch(readdir(path.dirname(value))) + if (error) { + if ('code' in error && error.code === 'ENOENT') return value + throw error + } + const match = siblings.find((name) => name.toLowerCase() === path.basename(value).toLowerCase()) + return match == null ? value : path.join(path.dirname(value), match) + } + const { lstat, rename } = fs.promises + vi.spyOn(fs.promises, 'lstat').mockImplementation( + async (target, ...args) => await lstat(await resolveCaseInsensitivePath(target), ...args), + ) + vi.spyOn(fs.promises, 'rename').mockImplementation( + async (source, target) => await rename(source, await resolveCaseInsensitivePath(target)), + ) + nock('https://downloads.test') + .get('/first') + .delayBody(40) + .reply(200, 'first result') + .get('/second') + .delayBody(40) + .reply(200, 'second result') + + expect(await run(['-i', inputPath, '-i', secondInput, '--concurrency', '2'])).toBe(0) + expect(await readdir(outputDir)).toEqual(['Photo.jpg', 'photo__1.jpg']) + expect(await readFile(path.join(outputDir, 'Photo.jpg'), 'utf8')).toBe('first result') + expect(await readFile(path.join(outputDir, 'photo__1.jpg'), 'utf8')).toBe('second result') +}) + +it.each([ + false, + true, +])('resolves file and bundle-directory conflicts (bundle first: %s)', async (bundleFirst) => { + const linked = { + adaptive: [ + { + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'video' }, + }, + ], + } + const ordinary = { output: [{ name: 'video', url: 'https://downloads.test/file' }] } + const { inputPath, outputDir, run } = await fixture( + bundleFirst ? { ...linked, ...ordinary } : { ...ordinary, ...linked }, + ) + nock('https://downloads.test') + .get('/file') + .reply(200, 'ordinary file') + .get('/segment') + .reply(200, 'stream bytes') + + expect(await run(['--single-assembly', '-i', inputPath])).toBe(0) + expect(await readFile(path.join(outputDir, bundleFirst ? 'video__1' : 'video'), 'utf8')).toBe( + 'ordinary file', + ) + expect( + await readFile( + path.join(outputDir, ...(bundleFirst ? [] : ['adaptive']), 'video', 'segment.ts'), + 'utf8', + ), + ).toBe('stream bytes') +}) + +it('finishes publishing a linked watch bundle when a newer Assembly fails', async () => { + const results = { + adaptive: [ + { name: 'master.m3u8', url: 'https://downloads.test/master', meta: { relative_path: '' } }, + { + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'video' }, + }, + ], + } + const { inputPath, outputDir } = await fixture(results) + class FakeWatcher extends EventEmitter { + close(): void { + this.emit('close') + } + } + const watcher = new FakeWatcher() + vi.doMock('node-watch', () => ({ default: () => watcher })) + vi.resetModules() + const { create } = await import('../../../src/cli/commands/assemblies.ts') + vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) + .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', results }) + .mockRejectedValueOnce(new Error('Newer Assembly failed')) + const rename = fs.promises.rename + vi.spyOn(fs.promises, 'rename').mockImplementation(async (source, target) => { + await rename(source, target) + if (target === path.join(outputDir, 'master.m3u8')) watcher.emit('change', 'update', inputPath) + }) + const output = new OutputCtl() + vi.spyOn(output, 'error').mockImplementation(() => watcher.close()) + nock('https://downloads.test') + .get('/master') + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/segment') + .delayBody(40) + .reply(200, 'complete stream') + + const result = await create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + inputs: [inputPath], + output: `${outputDir}/`, + watch: true, + stepsData: { adaptive: { robot: '/video/adaptive', use: ':original' } }, + }) + + expect(result.hasFailures).toBe(true) + expect(Transloadit.prototype.createAssembly).toHaveBeenCalledTimes(2) + expect(await readFile(path.join(outputDir, 'master.m3u8'), 'utf8')).toBe( + '#EXTM3U\nvideo/segment.ts\n', + ) + expect(await readFile(path.join(outputDir, 'video', 'segment.ts'), 'utf8')).toBe( + 'complete stream', + ) +}) + +it.each([ + 'steps', + 'output', +])('validates local %s configuration before downloading a URL', async (invalid) => { + const { outputDir, run } = await fixture({}) + let downloads = 0 + nock('https://source.invalid') + .get('/remote.png') + .reply(200, () => { + downloads += 1 + return 'remote input' + }) + const args = ['-i', 'https://source.invalid/remote.png'] + if (invalid === 'steps') args.push('--steps', path.join(outputDir, 'missing.json')) + else args.push('-i', 'https://source.invalid/second.png') + + expect( + await run(args, invalid === 'output' ? path.join(outputDir, 'result.png') : `${outputDir}/`), + ).toBe(1) + expect(downloads).toBe(0) + expect(Transloadit.prototype.createAssembly).not.toHaveBeenCalled() +}) + it.each([false, true])('uses the bundle result name (single Assembly: %s)', async (single) => { const { inputPath, outputDir, run } = await fixture({ bundled: [{ name: 'archive.zip', url: 'https://downloads.test/archive' }], diff --git a/packages/transloadit/README.md b/packages/transloadit/README.md index 7b2f7ec3..2656b249 100644 --- a/packages/transloadit/README.md +++ b/packages/transloadit/README.md @@ -1510,12 +1510,11 @@ The `assemblies create` command additionally supports: playlist references. Watch updates refresh their existing result files. Other directory runs process the inputs again, since result filenames are unknown before processing; file outputs keep mtime-based skipping. -- `--input, -i` - Provide a local upload or an HTTP(S) URL, or use `-i -` to read stdin explicitly. +- `--input, -i` - Provide a local upload or a public HTTP(S) URL, or use `-i -` to read stdin explicitly. URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly - without uploads. With `--template`, - stdin always requires `-i -`. + without uploads. With `--template`, stdin always requires `-i -`. - `--single-assembly` - Pass all input files to a single assembly instead of one assembly per file #### Log Levels From 8af7b502a43d94f00a757337f7d5910acf45209f Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 16:33:02 +0200 Subject: [PATCH 07/35] Isolate URL uploads and validate HLS source folders --- packages/node/src/cli/commands/assemblies.ts | 68 ++++++++++----- .../test/unit/cli/download-layout.test.ts | 86 +++++++++++++++++-- 2 files changed, 124 insertions(+), 30 deletions(-) diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 4a009ab6..82737775 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -629,10 +629,11 @@ async function downloadResultToStdout(resultUrl: string, signal: AbortSignal): P await pipeline(got.stream(resultUrl, { signal }), stdoutStream) } -function sanitizeResultName(value: string): string { +function sanitizeResultName(value: string, fallback?: string): string { const base = path.basename(value) const name = base.replaceAll('\\', '_').replaceAll('/', '_').replaceAll('\u0000', '') if (name === '' || name === '.' || name === '..') { + if (fallback != null) return sanitizeResultName(fallback) throw new Error('Assembly result has an unsafe name') } return name @@ -828,6 +829,18 @@ async function isDownloadPathTaken( return error == null } +async function prepareDownloadDirectory( + root: string, + target: string, + reservedPaths: DownloadPathReservations, +): Promise { + await assertNoOutputSymlinks(root, target) + if (hasReservedFileAncestor(target, reservedPaths)) { + throw new Error('Output directory conflicts with a result file') + } + await fsp.mkdir(target, { recursive: true }) +} + const STALE_OUTPUT_GRACE_MS = 1000 function isMeaningfullyNewer(newer: Date, older: Date): boolean { @@ -886,7 +899,7 @@ function groupDirectoryResults(allFiles: NormalizedAssemblyResultFile[]): Direct nameOccurrences.set(name, occurrence + 1) groups.push({ directoryName: - sources.size > 1 ? sanitizeResultName(path.parse(name ?? stepName).name) : null, + sources.size > 1 ? sanitizeResultName(path.parse(name ?? stepName).name, stepName) : null, files, // Upload IDs change between watch runs; source names retain logical ownership. ownerKey: JSON.stringify([name, occurrence]), @@ -912,11 +925,7 @@ async function buildDirectoryDownloadTargets({ outputRoot: string reservedPaths: DownloadPathReservations }): Promise { - await assertNoOutputSymlinks(outputRoot, baseDir) - if (hasReservedFileAncestor(baseDir, reservedPaths)) { - throw new Error('Output directory conflicts with a result file') - } - await fsp.mkdir(baseDir, { recursive: true }) + await prepareDownloadDirectory(outputRoot, baseDir, reservedPaths) const targets: AssemblyDownloadTarget[] = [] for (const { directoryName, files, ownerKey, stepName } of groupDirectoryResults(allFiles)) { @@ -943,6 +952,10 @@ async function buildDirectoryDownloadTargets({ } if (linked) { + // Source folders share this parent, so changing their suffix cannot fix a blocked parent. + if (directoryName != null) { + await prepareDownloadDirectory(outputRoot, path.dirname(targetDir), reservedPaths) + } const initialDir = targetDir // Selection and reservation share the same lock as ordinary result paths. A bundle must // move together on collisions, while later watch updates reuse its owned destinations. @@ -985,10 +998,7 @@ async function buildDirectoryDownloadTargets({ }) } - if (hasReservedFileAncestor(targetDir, reservedPaths)) { - throw new Error('Output directory conflicts with a result file') - } - await fsp.mkdir(targetDir, { recursive: true }) + await prepareDownloadDirectory(outputRoot, targetDir, reservedPaths) const nameOccurrences = new Map() for (const [index, { file }] of files.entries()) { const name = names[index] @@ -1763,30 +1773,44 @@ export async function create( options: AssembliesCreateOptions, ): ReturnType { const preparedOptions = await prepareCreateOptions(outputctl, options) - const urlInputs = options.inputs.filter(isUrlInput) - if (urlInputs.length === 0) + if (!options.inputs.some(isUrlInput)) return await createWithLocalInputs(outputctl, client, options, preparedOptions) if (options.watch) throw new Error('--watch requires local inputs') - const prepared = await prepareInputFiles({ - inputFiles: urlInputs.map((url, index) => ({ kind: 'url', field: String(index), url })), - allowPrivateUrls: false, - // Uploading the bytes preserves :original and Template upload semantics, just as Intents do. - urlStrategy: 'download', - }) - const localPaths = new Map(urlInputs.map((url, index) => [url, prepared.files[String(index)]])) + const preparedInputs: Array>> = [] + const localInputs: string[] = [] try { + for (const [index, input] of options.inputs.entries()) { + if (!isUrlInput(input)) { + localInputs.push(input) + continue + } + const field = String(index) + // Separate staging roots protect case-only filenames and repeated URLs from aliasing. + const prepared = await prepareInputFiles({ + inputFiles: [{ kind: 'url', field, url: input }], + allowPrivateUrls: false, + // Uploading preserves :original and Template semantics, just as Intents do. + urlStrategy: 'download', + }) + preparedInputs.push(prepared) + const localPath = prepared.files[field] + if (localPath == null) throw new Error('URL input could not be prepared') + localInputs.push(localPath) + } return await createWithLocalInputs( outputctl, client, { ...options, - inputs: options.inputs.map((input) => localPaths.get(input) ?? input), + inputs: localInputs, }, preparedOptions, ) } finally { - await Promise.all(prepared.cleanup.map((cleanup) => cleanup())) + await Promise.all( + preparedInputs.flatMap((prepared) => prepared.cleanup.map((cleanup) => cleanup())), + ) } } diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 59f14cfd..35924819 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -122,12 +122,21 @@ it('keeps every HLS playlist reference reachable from its downloaded playlist', ) }) -it('separates independent HLS bundles with identical paths in one Assembly Step', async () => { +it.each([ + { firstName: 'first.mp4', secondName: 'second.mp4', firstDir: 'first', secondDir: 'second' }, + { firstName: 'clip.mp4', secondName: 'clip.mp4', firstDir: 'clip', secondDir: 'clip__1' }, + { firstName: '...', secondName: '...', firstDir: 'adaptive', secondDir: 'adaptive__1' }, +])('separates independent HLS bundles (sources: $firstName, $secondName)', async ({ + firstName, + secondName, + firstDir, + secondDir, +}) => { const firstMaster = { name: 'master.m3u8', url: 'https://downloads.test/first-master', original_id: 'first-source', - original_name: 'first.mp4', + original_name: firstName, meta: { relative_path: '' }, } const firstSegment = { @@ -140,7 +149,7 @@ it('separates independent HLS bundles with identical paths in one Assembly Step' ...firstMaster, url: 'https://downloads.test/second-master', original_id: 'second-source', - original_name: 'second.mp4', + original_name: secondName, } const secondSegment = { ...secondMaster, @@ -164,21 +173,82 @@ it('separates independent HLS bundles with identical paths in one Assembly Step' .reply(200, 'second stream') expect(await run(['--single-assembly', '-i', inputPath, '-i', secondInput])).toBe(0) - expect(await readdir(outputDir)).toEqual(['first', 'second']) - expect(await readFile(path.join(outputDir, 'first', 'master.m3u8'), 'utf8')).toBe( + expect(await readdir(outputDir)).toEqual([firstDir, secondDir]) + expect(await readFile(path.join(outputDir, firstDir, 'master.m3u8'), 'utf8')).toBe( '#EXTM3U\nvideo/segment.ts\n', ) - expect(await readFile(path.join(outputDir, 'first', 'video', 'segment.ts'), 'utf8')).toBe( + expect(await readFile(path.join(outputDir, firstDir, 'video', 'segment.ts'), 'utf8')).toBe( 'first stream', ) - expect(await readFile(path.join(outputDir, 'second', 'master.m3u8'), 'utf8')).toBe( + expect(await readFile(path.join(outputDir, secondDir, 'master.m3u8'), 'utf8')).toBe( '#EXTM3U\nvideo/segment.ts\n', ) - expect(await readFile(path.join(outputDir, 'second', 'video', 'segment.ts'), 'utf8')).toBe( + expect(await readFile(path.join(outputDir, secondDir, 'video', 'segment.ts'), 'utf8')).toBe( 'second stream', ) }) +it('fails promptly when a file blocks the parent of grouped HLS source folders', async () => { + const first = { + name: 'master.m3u8', + url: 'https://downloads.test/master', + original_id: 'first', + original_name: 'first.mp4', + meta: { relative_path: '' }, + } + const second = { ...first, original_id: 'second', original_name: 'second.mp4' } + const { outputDir, run } = await fixture({ adaptive: [first, second] }) + await mkdir(outputDir) + await writeFile(path.join(outputDir, 'adaptive'), 'existing file') + const lstat = fs.promises.lstat + let probes = 0 + vi.spyOn(fs.promises, 'lstat').mockImplementation(async (...args) => { + probes += 1 + if (probes > 40) throw new Error('Filesystem probe budget exceeded') + return await lstat(...args) + }) + + expect(await run(['--single-assembly'])).toBe(1) + expect(probes).toBeLessThan(20) + expect(await readFile(path.join(outputDir, 'adaptive'), 'utf8')).toBe('existing file') +}) + +it.each([ + 'photo.png', + 'Photo.png', +])('keeps URL uploads separate (second filename: %s)', async (secondName) => { + const { run } = await fixture({}) + const uploadPaths: string[] = [] + const uploadBytes: string[] = [] + vi.mocked(Transloadit.prototype.createAssembly).mockImplementation(async ({ files }) => { + const uploadedPath = Object.values(files ?? {})[0] + if (uploadedPath == null) throw new Error('Missing upload path') + uploadPaths.push(uploadedPath) + uploadBytes.push(await readFile(uploadedPath, 'utf8')) + return { assembly_id: 'url-upload' } + }) + nock('https://source.invalid') + .get('/Photo.png') + .reply(200, 'first input') + .get(`/${secondName}`) + .reply(200, 'second input') + + expect( + await run([ + '-i', + 'https://source.invalid/Photo.png', + '-i', + `https://source.invalid/${secondName}`, + ]), + ).toBe(0) + expect(uploadPaths).toHaveLength(2) + expect(new Set(uploadPaths.map((file) => file.normalize('NFC').toLowerCase())).size).toBe(2) + expect(uploadBytes.sort()).toEqual(['first input', 'second input']) + for (const uploadedPath of uploadPaths) { + await expect(stat(uploadedPath)).rejects.toHaveProperty('code', 'ENOENT') + } +}) + it('rejects duplicate paths within a single HLS bundle', async () => { const segment = { name: 'segment.ts', From 56c0fca5ab8f59a2006029145230a416c1279948 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 16:54:45 +0200 Subject: [PATCH 08/35] Preserve independent result ownership and local staging --- packages/node/src/cli/commands/assemblies.ts | 57 ++++++--- .../test/unit/cli/download-layout.test.ts | 120 ++++++++++++++++++ 2 files changed, 158 insertions(+), 19 deletions(-) diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 82737775..e9a664ca 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -752,24 +752,34 @@ interface BundleDownloadTarget { targetPath: string } +interface StagedBundleDownloadTarget extends BundleDownloadTarget { + stagingPath: string +} + async function downloadResultBundle( - root: string, targets: BundleDownloadTarget[], signal: AbortSignal, isCurrent: () => boolean, commits: PQueue, onDownload: () => void, ): Promise { - const stagingDir = await fsp.mkdtemp(path.join(root, '.transloadit-bundle-')) - const staged = targets.map((target, index) => ({ - ...target, - stagingPath: path.join(stagingDir, String(index)), - })) + const stagingDirs = new Map() + const staged: StagedBundleDownloadTarget[] = [] try { - for (const { resultUrl, stagingPath } of staged) { + for (const [index, target] of targets.entries()) { if (!isCurrent()) return + const parent = path.dirname(target.targetPath) + await fsp.mkdir(parent, { recursive: true }) + let stagingDir = stagingDirs.get(parent) + if (stagingDir == null) { + // Stage beside each destination so publication also works across nested filesystems. + stagingDir = await fsp.mkdtemp(path.join(parent, '.transloadit-bundle-')) + stagingDirs.set(parent, stagingDir) + } + const stagingPath = path.join(stagingDir, String(index)) + staged.push({ ...target, stagingPath }) onDownload() - await downloadResultToFile(resultUrl, stagingPath, signal, () => true) + await downloadResultToFile(target.resultUrl, stagingPath, signal, () => true) } await commits.add(async () => { if (!isCurrent()) return @@ -781,7 +791,11 @@ async function downloadResultBundle( } }) } finally { - await fsp.rm(stagingDir, { recursive: true, force: true }) + await Promise.all( + [...stagingDirs.values()].map( + async (dir) => await fsp.rm(dir, { recursive: true, force: true }), + ), + ) } } @@ -1010,7 +1024,7 @@ async function buildDirectoryDownloadTargets({ : await ensureUniquePath( targetPath, reservedPaths, - JSON.stringify([owner, stepName, name, occurrence]), + JSON.stringify([groupOwner, name, occurrence]), ) await assertNoOutputSymlinks(outputRoot, selectedPath) targets.push({ @@ -1045,6 +1059,7 @@ async function resolveResultDownloadTargets({ outputPath, outputRoot, outputRootIsDirectory, + owner, reservedPaths, singleAssembly, }: { @@ -1055,6 +1070,7 @@ async function resolveResultDownloadTargets({ outputPath: string | null outputRoot: string outputRootIsDirectory: boolean + owner: string reservedPaths: DownloadPathReservations singleAssembly?: boolean }): Promise { @@ -1065,7 +1081,6 @@ async function resolveResultDownloadTargets({ !singleAssembly && inPath != null && (hasDirectoryInput || (inputs.length > 1 && allFiles.length > 1)) - const owner = JSON.stringify([inPath, singleAssembly ? inputs : []]) const resolveDirectoryBaseDir = (): string => { if (hasDirectoryInput && outputPath != null) { @@ -1187,6 +1202,7 @@ async function materializeAssemblyResults({ outputRoot, outputRootIsDirectory, outputctl, + owner, reservedPaths, singleAssembly, }: { @@ -1200,6 +1216,7 @@ async function materializeAssemblyResults({ outputRoot: string | null outputRootIsDirectory: boolean outputctl: IOutputCtl + owner: string reservedPaths: DownloadPathReservations singleAssembly?: boolean }): Promise { @@ -1218,6 +1235,7 @@ async function materializeAssemblyResults({ outputPath, outputRoot, outputRootIsDirectory, + owner, reservedPaths, singleAssembly, }), @@ -1238,16 +1256,11 @@ async function materializeAssemblyResults({ else await downloadResultToFile(resultUrl, targetPath, abortSignal, isCurrent) }) } - for (const [root, files] of bundles) { + for (const files of bundles.values()) { downloads.push( async () => - await downloadResultBundle( - root, - files, - abortSignal, - isCurrent, - reservedPaths.bundleCommits, - () => outputctl.debug('DOWNLOADING'), + await downloadResultBundle(files, abortSignal, isCurrent, reservedPaths.bundleCommits, () => + outputctl.debug('DOWNLOADING'), ), ) } @@ -2010,6 +2023,12 @@ async function createWithLocalInputs( outputRoot: resolvedOutput ?? null, outputRootIsDirectory, outputctl, + // Only watch updates reuse destinations; independent Assemblies retain every result. + owner: JSON.stringify([ + inPath, + singleAssemblyMode ? inputPaths : [], + watchOption ? null : assemblyId, + ]), reservedPaths: reservedResultPaths, singleAssembly: singleAssemblyMode, }) diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 35924819..8044bb4f 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -263,6 +263,126 @@ it('rejects duplicate paths within a single HLS bundle', async () => { expect(await readdir(outputDir)).toEqual([]) }) +it('retains ordinary results from different source groups in a Step containing HLS', async () => { + const master = { + name: 'master.m3u8', + url: 'https://downloads.test/master', + original_id: 'video', + original_name: 'clip.mp4', + meta: { relative_path: '' }, + } + const segment = { + ...master, + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'video' }, + } + const firstCover = { + name: 'cover.jpg', + url: 'https://downloads.test/first-cover', + original_id: 'first-cover', + original_name: 'same.png', + } + const secondCover = { + ...firstCover, + original_id: 'second-cover', + url: 'https://downloads.test/second-cover', + } + const { inputPath, outputDir, run } = await fixture({ + adaptive: [master, segment, firstCover, secondCover], + }) + nock('https://downloads.test') + .get('/master') + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/segment') + .reply(200, 'stream bytes') + .get('/first-cover') + .reply(200, 'first cover') + .get('/second-cover') + .reply(200, 'second cover') + + expect(await run(['--single-assembly', '-i', inputPath])).toBe(0) + expect(await readdir(path.join(outputDir, 'same'))).toEqual(['cover.jpg', 'cover__1.jpg']) + expect(await readFile(path.join(outputDir, 'same', 'cover.jpg'), 'utf8')).toBe('first cover') + expect(await readFile(path.join(outputDir, 'same', 'cover__1.jpg'), 'utf8')).toBe('second cover') + expect(await readFile(path.join(outputDir, 'clip', 'video', 'segment.ts'), 'utf8')).toBe( + 'stream bytes', + ) +}) + +it('preserves every independent Assembly result when file and directory inputs overlap', async () => { + const { inputPath, outputDir, run } = await fixture({}) + let count = 0 + vi.mocked(Transloadit.prototype.createAssembly).mockImplementation(async () => ({ + assembly_id: String(++count), + })) + vi.mocked(Transloadit.prototype.awaitAssemblyCompletion).mockImplementation(async (id) => ({ + ok: 'ASSEMBLY_COMPLETED', + results: { + images: [ + { name: 'a.png', url: `https://downloads.test/${id}/a` }, + { name: 'b.png', url: `https://downloads.test/${id}/b` }, + ], + }, + })) + nock('https://downloads.test') + .get('/1/a') + .reply(200, 'a1') + .get('/1/b') + .reply(200, 'b1') + .get('/2/a') + .reply(200, 'a2') + .get('/2/b') + .reply(200, 'b2') + + expect(await run(['-i', inputPath, '-i', path.dirname(inputPath)])).toBe(0) + expect(count).toBe(2) + const files = await readdir(path.join(outputDir, 'logo')) + expect(files).toEqual(['a.png', 'a__1.png', 'b.png', 'b__1.png']) + expect( + ( + await Promise.all( + files.map(async (file) => await readFile(path.join(outputDir, 'logo', file), 'utf8')), + ) + ).sort(), + ).toEqual(['a1', 'a2', 'b1', 'b2']) +}) + +it('stages HLS files on their destination filesystem', async () => { + const { inputPath, outputDir, run } = await fixture({ + adaptive: [ + { name: 'master.m3u8', url: 'https://downloads.test/master', meta: { relative_path: '' } }, + { + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'video' }, + }, + ], + }) + const rename = fs.promises.rename + let crossDeviceAttempts = 0 + vi.spyOn(fs.promises, 'rename').mockImplementation(async (source, target) => { + const onRenditionDevice = (file: fs.PathLike): boolean => + String(file).includes(`${path.sep}video${path.sep}`) + if (onRenditionDevice(source) !== onRenditionDevice(target)) { + crossDeviceAttempts += 1 + throw Object.assign(new Error('Cross-device rename'), { code: 'EXDEV' }) + } + await rename(source, target) + }) + nock('https://downloads.test') + .get('/master') + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/segment') + .reply(200, 'stream bytes') + + expect(await run(['-i', inputPath])).toBe(0) + expect(crossDeviceAttempts).toBe(0) + expect(await readdir(outputDir)).toEqual(['master.m3u8', 'video']) + expect(await readdir(path.join(outputDir, 'video'))).toEqual(['segment.ts']) + expect(await readFile(path.join(outputDir, 'video', 'segment.ts'), 'utf8')).toBe('stream bytes') +}) + it('keeps concurrent case-only result names separate on case-insensitive filesystems', async () => { const { inputPath, outputDir, run } = await fixture({}) const secondInput = path.join(path.dirname(inputPath), 'second.png') From f2d1cdb5802dc1beb9621bd69e1cdad284aeda0e Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 17:18:00 +0200 Subject: [PATCH 09/35] Separate mixed HLS results and reserve input directories --- packages/node/src/cli/commands/assemblies.ts | 57 +++++++++++---- .../test/unit/cli/download-layout.test.ts | 71 +++++++++++++++++++ 2 files changed, 114 insertions(+), 14 deletions(-) diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index e9a664ca..7b238b7b 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -699,18 +699,24 @@ async function ensureUniquePath( targetPath: string, reservedPaths: DownloadPathReservations, owner: string, + directory = false, ): Promise { const selectedPath = reservedPaths.pathsByOwner.get(owner) if (selectedPath != null) return selectedPath const parsed = path.parse(targetPath) return await ensureUniqueCounterValue({ initialValue: targetPath, - isTaken: async (candidate) => await isDownloadPathTaken(candidate, reservedPaths, owner), + isTaken: async (candidate) => + await isDownloadPathTaken(candidate, reservedPaths, owner, directory), reserve: (candidate) => { - reserveDownloadPath(candidate, owner, reservedPaths) + reserveDownloadPath(candidate, owner, reservedPaths, directory) reservedPaths.pathsByOwner.set(owner, candidate) }, - nextValue: (counter) => path.join(parsed.dir, `${parsed.name}__${counter}${parsed.ext}`), + nextValue: (counter) => + path.join( + parsed.dir, + directory ? `${parsed.base}__${counter}` : `${parsed.name}__${counter}${parsed.ext}`, + ), scope: reservedPaths, }) } @@ -830,17 +836,19 @@ async function isDownloadPathTaken( target: string, reservedPaths: DownloadPathReservations, owner: string, + directory = false, ): Promise { const key = downloadPathKey(target, reservedPaths) + if (directory && reservedPaths.directories.has(key)) return false const existingOwner = reservedPaths.ownersByPath.get(key) if (existingOwner != null) return existingOwner !== owner if (reservedPaths.directories.has(key) || hasReservedFileAncestor(target, reservedPaths)) { return true } - const [error] = await tryCatch(fsp.lstat(target)) + const [error, stats] = await tryCatch(fsp.lstat(target)) if (isErrnoException(error) && error.code === 'ENOTDIR') return true if (error && (!isErrnoException(error) || error.code !== 'ENOENT')) throw error - return error == null + return error == null && (!directory || !stats?.isDirectory()) } async function prepareDownloadDirectory( @@ -911,14 +919,21 @@ function groupDirectoryResults(allFiles: NormalizedAssemblyResultFile[]): Direct for (const { name, files } of sources.values()) { const occurrence = nameOccurrences.get(name) ?? 0 nameOccurrences.set(name, occurrence + 1) - groups.push({ - directoryName: - sources.size > 1 ? sanitizeResultName(path.parse(name ?? stepName).name, stepName) : null, - files, - // Upload IDs change between watch runs; source names retain logical ownership. - ownerKey: JSON.stringify([name, occurrence]), - stepName, - }) + // Ordinary files can be renamed independently, even when they share an HLS source. + for (const linked of [true, false]) { + const groupFiles = files.filter(({ relativePath }) => (relativePath != null) === linked) + if (groupFiles.length === 0) continue + groups.push({ + directoryName: + sources.size > 1 + ? sanitizeResultName(path.parse(name ?? stepName).name, stepName) + : null, + files: groupFiles, + // Upload IDs change between watch runs; source names retain logical ownership. + ownerKey: JSON.stringify([name, occurrence, linked]), + stepName, + }) + } } } return groups @@ -1121,9 +1136,23 @@ async function resolveResultDownloadTargets({ }) } + let baseDir = resolveDirectoryBaseDir() + if (path.resolve(baseDir) !== path.resolve(outputRoot)) { + // Input grouping directories share the result namespace, including files already reserved + // by another Assembly. Reuse existing directories and suffix a conflicting filename. + await assertNoOutputSymlinks(outputRoot, baseDir) + await prepareDownloadDirectory(outputRoot, path.dirname(baseDir), reservedPaths) + baseDir = await ensureUniquePath( + baseDir, + reservedPaths, + JSON.stringify([owner, 'input-directory']), + true, + ) + } + return await buildDirectoryDownloadTargets({ allFiles, - baseDir: resolveDirectoryBaseDir(), + baseDir, groupByStep: inPath == null || entries.length > 1, owner, outputRoot, diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 8044bb4f..6592dbb5 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -263,6 +263,77 @@ it('rejects duplicate paths within a single HLS bundle', async () => { expect(await readdir(outputDir)).toEqual([]) }) +it('suffixes ordinary collisions within the same source as an HLS bundle', async () => { + const source = { original_id: 'video', original_name: 'clip.mp4' } + const { inputPath, outputDir, run } = await fixture({ + adaptive: [ + { + ...source, + name: 'master.m3u8', + url: 'https://downloads.test/master', + meta: { relative_path: '' }, + }, + { ...source, name: 'cover.jpg', url: 'https://downloads.test/first-cover' }, + { ...source, name: 'cover.jpg', url: 'https://downloads.test/second-cover' }, + ], + }) + nock('https://downloads.test') + .get('/master') + .reply(200, '#EXTM3U\n') + .get('/first-cover') + .reply(200, 'first cover') + .get('/second-cover') + .reply(200, 'second cover') + + expect(await run(['-i', inputPath])).toBe(0) + expect(await readdir(outputDir)).toEqual(['cover.jpg', 'cover__1.jpg', 'master.m3u8']) + expect(await readFile(path.join(outputDir, 'cover.jpg'), 'utf8')).toBe('first cover') + expect(await readFile(path.join(outputDir, 'cover__1.jpg'), 'utf8')).toBe('second cover') +}) + +it.each([ + false, + true, +])('separates input directories from result files (file first: %s)', async (fileFirst) => { + const { inputPath, outputDir, run } = await fixture({}) + const singleInput = path.join(path.dirname(inputPath), 'report.docx') + const multiInput = path.join(path.dirname(inputPath), 'report.pdf.docx') + await writeFile(singleInput, 'single-result input') + await writeFile(multiInput, 'multi-result input') + const single = { output: [{ name: 'report.pdf', url: 'https://downloads.test/pdf' }] } + const multi = { + output: [ + { name: 'page1.jpg', url: 'https://downloads.test/page1' }, + { name: 'page2.jpg', url: 'https://downloads.test/page2' }, + ], + } + vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) + .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', results: fileFirst ? single : multi }) + .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', results: fileFirst ? multi : single }) + nock('https://downloads.test') + .get('/pdf') + .reply(200, 'pdf bytes') + .get('/page1') + .reply(200, 'first page') + .get('/page2') + .reply(200, 'second page') + + expect( + await run([ + '-i', + fileFirst ? singleInput : multiInput, + '-i', + fileFirst ? multiInput : singleInput, + ]), + ).toBe(0) + expect( + await readFile(path.join(outputDir, fileFirst ? 'report.pdf' : 'report__1.pdf'), 'utf8'), + ).toBe('pdf bytes') + const inputDir = path.join(outputDir, fileFirst ? 'report.pdf__1' : 'report.pdf') + expect(await readFile(path.join(inputDir, 'page1.jpg'), 'utf8')).toBe('first page') + expect(await readFile(path.join(inputDir, 'page2.jpg'), 'utf8')).toBe('second page') +}) + it('retains ordinary results from different source groups in a Step containing HLS', async () => { const master = { name: 'master.m3u8', From 8c3cc8a7a9363bac6296e42104ea8eb4a7e06bd6 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 17:50:37 +0200 Subject: [PATCH 10/35] Keep watched HLS ownership stable across source reordering --- packages/node/src/cli/commands/assemblies.ts | 27 +++-- .../test/unit/cli/download-layout.test.ts | 100 ++++++++++++++++++ 2 files changed, 119 insertions(+), 8 deletions(-) diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 7b238b7b..93437258 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -885,6 +885,7 @@ const resultSourceSchema = assemblyStatusResultSchema.pick({ original_basename: true, original_id: true, original_name: true, + original_path: true, }) function groupDirectoryResults(allFiles: NormalizedAssemblyResultFile[]): DirectoryResultGroup[] { @@ -902,23 +903,32 @@ function groupDirectoryResults(allFiles: NormalizedAssemblyResultFile[]): Direct continue } - const sources = new Map() + const sources = new Map< + string, + { identity: string; name: string | null; files: DirectoryResultFile[] } + >() for (const file of stepFiles) { const parsed = resultSourceSchema.safeParse(file.file.file) const source = parsed.success ? parsed.data : undefined const sourceId = JSON.stringify(source?.original_id ?? null) + const name = source?.original_name ?? source?.original_basename ?? null const group = sources.get(sourceId) ?? { - name: source?.original_name ?? source?.original_basename ?? null, + // A path and a fallback name with identical text still describe different sources. + identity: JSON.stringify([ + source?.original_path ?? null, + source?.original_path == null ? name : null, + ]), + name, files: [], } group.files.push(file) sources.set(sourceId, group) } - const nameOccurrences = new Map() - for (const { name, files } of sources.values()) { - const occurrence = nameOccurrences.get(name) ?? 0 - nameOccurrences.set(name, occurrence + 1) + const sourceOccurrences = new Map() + for (const { identity, name, files } of sources.values()) { + const occurrence = sourceOccurrences.get(identity) ?? 0 + sourceOccurrences.set(identity, occurrence + 1) // Ordinary files can be renamed independently, even when they share an HLS source. for (const linked of [true, false]) { const groupFiles = files.filter(({ relativePath }) => (relativePath != null) === linked) @@ -929,8 +939,9 @@ function groupDirectoryResults(allFiles: NormalizedAssemblyResultFile[]): Direct ? sanitizeResultName(path.parse(name ?? stepName).name, stepName) : null, files: groupFiles, - // Upload IDs change between watch runs; source names retain logical ownership. - ownerKey: JSON.stringify([name, occurrence, linked]), + // Upload IDs change between watch runs. Original paths distinguish same-named sources + // even when results arrive in a different order; names are the metadata fallback. + ownerKey: JSON.stringify([identity, occurrence, linked]), stepName, }) } diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 6592dbb5..3e1efe60 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -944,6 +944,106 @@ it('keeps a watched HLS Step in its selected folder after the original conflict ) }) +it.each([ + { firstPath: 'first/clip.mp4', secondPath: 'second/clip.mp4' }, + { firstPath: 'clip.mp4', secondPath: undefined }, +])('keeps watched HLS source paths attached to their folders (paths: $firstPath, $secondPath)', async ({ + firstPath, + secondPath, +}) => { + const firstMaster = { + original_id: 'first-old', + original_name: 'clip.mp4', + original_path: firstPath, + name: 'master.m3u8', + url: 'https://downloads.test/first-master', + meta: { relative_path: '' }, + } + const firstSegment = { + ...firstMaster, + name: 'segment.ts', + url: 'https://downloads.test/first-segment', + meta: { relative_path: 'video' }, + } + const secondMaster = { + ...firstMaster, + original_id: 'second-old', + original_path: secondPath, + url: 'https://downloads.test/second-master', + } + const secondSegment = { + ...firstSegment, + original_id: 'second-old', + original_path: secondPath, + url: 'https://downloads.test/second-segment', + } + const { inputPath, outputDir } = await fixture({ + adaptive: [firstMaster, firstSegment, secondMaster, secondSegment], + }) + vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) + .mockResolvedValueOnce({ + ok: 'ASSEMBLY_COMPLETED', + results: { adaptive: [firstMaster, firstSegment, secondMaster, secondSegment] }, + }) + .mockResolvedValueOnce({ + ok: 'ASSEMBLY_COMPLETED', + results: { + adaptive: [ + { ...secondMaster, original_id: 'second-new' }, + { ...secondSegment, original_id: 'second-new' }, + { ...firstMaster, original_id: 'first-new' }, + { ...firstSegment, original_id: 'first-new' }, + ], + }, + }) + class FakeWatcher extends EventEmitter { + close(): void { + this.emit('close') + } + } + const watcher = new FakeWatcher() + vi.doMock('node-watch', () => ({ default: () => watcher })) + vi.resetModules() + const { create } = await import('../../../src/cli/commands/assemblies.ts') + const output = new OutputCtl() + let completions = 0 + vi.spyOn(output, 'debug').mockImplementation((message) => { + if (!String(message).startsWith('COMPLETED')) return + if (++completions === 1) watcher.emit('change', 'update', inputPath) + else watcher.close() + }) + nock('https://downloads.test') + .get('/first-master') + .twice() + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/second-master') + .twice() + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/first-segment') + .reply(200, 'first original') + .get('/first-segment') + .reply(200, 'first updated') + .get('/second-segment') + .reply(200, 'second original') + .get('/second-segment') + .reply(200, 'second updated') + + const result = await create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + inputs: [inputPath], + output: `${outputDir}/`, + watch: true, + stepsData: { adaptive: { robot: '/video/adaptive', use: ':original' } }, + }) + expect(result.hasFailures).toBe(false) + expect(completions).toBe(2) + expect(await readFile(path.join(outputDir, 'clip', 'video', 'segment.ts'), 'utf8')).toBe( + 'first updated', + ) + expect(await readFile(path.join(outputDir, 'clip__1', 'video', 'segment.ts'), 'utf8')).toBe( + 'second updated', + ) +}) + it('accepts a standalone input and a directory containing the same basename', async () => { const { inputPath, outputDir, run, tempDir } = await fixture({ resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], From ebbba3b0b00f7daf1d078f380053175187190838 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 18:18:17 +0200 Subject: [PATCH 11/35] Stabilize HLS watch ownership and Unicode collision keys --- packages/node/src/cli/commands/assemblies.ts | 47 ++++---- .../test/unit/cli/download-layout.test.ts | 104 +++++++++++++++--- 2 files changed, 116 insertions(+), 35 deletions(-) diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 93437258..d91e324b 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -639,23 +639,34 @@ function sanitizeResultName(value: string, fallback?: string): string { return name } -interface DownloadPathReservations { +interface OutputNameSensitivity { + caseSensitive: boolean + normalizationSensitive: boolean +} + +interface DownloadPathReservations extends OutputNameSensitivity { allocations: PQueue bundleCommits: PQueue - caseSensitive: boolean directories: Set ownersByPath: Map pathsByOwner: Map } -async function outputNamesAreCaseSensitive(root: string): Promise { +async function outputNameSensitivity(root: string): Promise { await fsp.mkdir(root, { recursive: true }) - const probe = await fsp.mkdtemp(path.join(root, '.transloadit-case-')) - try { - await fsp.writeFile(path.join(probe, 'case'), '') - const [error] = await tryCatch(fsp.lstat(path.join(probe, 'CASE'))) + const probe = await fsp.mkdtemp(path.join(root, '.transloadit-names-')) + async function distinguishesNames(first: string, second: string): Promise { + await fsp.writeFile(path.join(probe, first), '') + const [error] = await tryCatch(fsp.lstat(path.join(probe, second))) if (error && (!isErrnoException(error) || error.code !== 'ENOENT')) throw error return error != null + } + try { + // Assume this tree shares the root's naming rules; nested filesystems are tracked in #615. + return { + caseSensitive: await distinguishesNames('case', 'CASE'), + normalizationSensitive: await distinguishesNames('\u00e9', 'e\u0301'), + } } finally { await fsp.rm(probe, { recursive: true, force: true }) } @@ -663,7 +674,8 @@ async function outputNamesAreCaseSensitive(root: string): Promise { function downloadPathKey(target: string, reservedPaths: DownloadPathReservations): string { const absolutePath = path.resolve(target) - return reservedPaths.caseSensitive ? absolutePath : absolutePath.normalize('NFC').toLowerCase() + const folded = reservedPaths.caseSensitive ? absolutePath : absolutePath.toLowerCase() + return reservedPaths.normalizationSensitive ? folded : folded.normalize('NFC') } function hasReservedFileAncestor(target: string, reservedPaths: DownloadPathReservations): boolean { @@ -913,11 +925,8 @@ function groupDirectoryResults(allFiles: NormalizedAssemblyResultFile[]): Direct const sourceId = JSON.stringify(source?.original_id ?? null) const name = source?.original_name ?? source?.original_basename ?? null const group = sources.get(sourceId) ?? { - // A path and a fallback name with identical text still describe different sources. - identity: JSON.stringify([ - source?.original_path ?? null, - source?.original_path == null ? name : null, - ]), + // original_path can be a shared directory, so the source filename is also required. + identity: JSON.stringify([source?.original_path ?? null, name]), name, files: [], } @@ -939,8 +948,8 @@ function groupDirectoryResults(allFiles: NormalizedAssemblyResultFile[]): Direct ? sanitizeResultName(path.parse(name ?? stepName).name, stepName) : null, files: groupFiles, - // Upload IDs change between watch runs. Original paths distinguish same-named sources - // even when results arrive in a different order; names are the metadata fallback. + // Upload IDs change between watch runs. Paths and names keep source ownership stable + // even when results arrive in a different order. ownerKey: JSON.stringify([identity, occurrence, linked]), stepName, }) @@ -1884,10 +1893,10 @@ async function createWithLocalInputs( preparedOptions: PreparedCreateOptions, ): Promise<{ resultUrls: ResultUrlRow[]; results: unknown[]; hasFailures: boolean }> { const { effectiveStepsData, hasDirectoryInput, outstat, resolvedOutput } = preparedOptions - const caseSensitive = + const nameSensitivity = resolvedOutput != null && outstat?.isDirectory() - ? await outputNamesAreCaseSensitive(resolvedOutput) - : true + ? await outputNameSensitivity(resolvedOutput) + : { caseSensitive: true, normalizationSensitive: true } return new Promise((resolve, reject) => { const params: CreateAssemblyParams = ( @@ -1925,7 +1934,7 @@ async function createWithLocalInputs( const reservedResultPaths: DownloadPathReservations = { allocations: new PQueue({ concurrency: 1 }), bundleCommits: new PQueue({ concurrency: 1 }), - caseSensitive, + ...nameSensitivity, directories: new Set(), ownersByPath: new Map(), pathsByOwner: new Map(), diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 3e1efe60..a6424bed 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -308,7 +308,11 @@ it.each([ ], } vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) - .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', results: fileFirst ? single : multi }) + .mockImplementationOnce(async () => { + // Finish the second job first unless the CLI is explicitly asked to run serially. + await new Promise((resolve) => setTimeout(resolve, 60)) + return { ok: 'ASSEMBLY_COMPLETED', results: fileFirst ? single : multi } + }) .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', results: fileFirst ? multi : single }) nock('https://downloads.test') .get('/pdf') @@ -324,6 +328,8 @@ it.each([ fileFirst ? singleInput : multiInput, '-i', fileFirst ? multiInput : singleInput, + '--concurrency', + '1', ]), ).toBe(0) expect( @@ -454,35 +460,69 @@ it('stages HLS files on their destination filesystem', async () => { expect(await readFile(path.join(outputDir, 'video', 'segment.ts'), 'utf8')).toBe('stream bytes') }) -it('keeps concurrent case-only result names separate on case-insensitive filesystems', async () => { +it.each([ + { + firstName: 'Photo.jpg', + secondName: 'photo.jpg', + caseSensitive: false, + normalizationSensitive: true, + }, + { + firstName: '\u0390.jpg', + secondName: '\u03aa\u0301.jpg', + caseSensitive: false, + normalizationSensitive: false, + }, + { + firstName: '\u00e9.jpg', + secondName: 'e\u0301.jpg', + caseSensitive: true, + normalizationSensitive: false, + }, +])('keeps concurrent equivalent result names separate ($firstName, $secondName)', async ({ + firstName, + secondName, + caseSensitive, + normalizationSensitive, +}) => { const { inputPath, outputDir, run } = await fixture({}) const secondInput = path.join(path.dirname(inputPath), 'second.png') await writeFile(secondInput, 'second input') vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', - results: { resized: [{ name: 'Photo.jpg', url: 'https://downloads.test/first' }] }, + results: { resized: [{ name: firstName, url: 'https://downloads.test/first' }] }, }) .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', - results: { resized: [{ name: 'photo.jpg', url: 'https://downloads.test/second' }] }, + results: { resized: [{ name: secondName, url: 'https://downloads.test/second' }] }, }) - async function resolveCaseInsensitivePath(value: fs.PathLike): Promise { + function filenameKey(name: string): string { + const folded = caseSensitive ? name : name.toLowerCase() + return normalizationSensitive ? folded : folded.normalize('NFC') + } + async function resolveEquivalentPath( + value: fs.PathLike, + requireExisting = false, + ): Promise { if (typeof value !== 'string') return value const [error, siblings] = await tryCatch(readdir(path.dirname(value))) if (error) { if ('code' in error && error.code === 'ENOENT') return value throw error } - const match = siblings.find((name) => name.toLowerCase() === path.basename(value).toLowerCase()) + const match = siblings.find((name) => filenameKey(name) === filenameKey(path.basename(value))) + if (match == null && requireExisting) { + throw Object.assign(new Error('No equivalent filename'), { code: 'ENOENT' }) + } return match == null ? value : path.join(path.dirname(value), match) } const { lstat, rename } = fs.promises vi.spyOn(fs.promises, 'lstat').mockImplementation( - async (target, ...args) => await lstat(await resolveCaseInsensitivePath(target), ...args), + async (target, ...args) => await lstat(await resolveEquivalentPath(target, true), ...args), ) vi.spyOn(fs.promises, 'rename').mockImplementation( - async (source, target) => await rename(source, await resolveCaseInsensitivePath(target)), + async (source, target) => await rename(source, await resolveEquivalentPath(target)), ) nock('https://downloads.test') .get('/first') @@ -493,9 +533,13 @@ it('keeps concurrent case-only result names separate on case-insensitive filesys .reply(200, 'second result') expect(await run(['-i', inputPath, '-i', secondInput, '--concurrency', '2'])).toBe(0) - expect(await readdir(outputDir)).toEqual(['Photo.jpg', 'photo__1.jpg']) - expect(await readFile(path.join(outputDir, 'Photo.jpg'), 'utf8')).toBe('first result') - expect(await readFile(path.join(outputDir, 'photo__1.jpg'), 'utf8')).toBe('second result') + const { name, ext } = path.parse(secondName) + const secondOutput = `${name}__1${ext}` + expect((await readdir(outputDir)).map((value) => value.normalize('NFC')).sort()).toEqual( + [firstName, secondOutput].map((value) => value.normalize('NFC')).sort(), + ) + expect(await readFile(path.join(outputDir, firstName), 'utf8')).toBe('first result') + expect(await readFile(path.join(outputDir, secondOutput), 'utf8')).toBe('second result') }) it.each([ @@ -945,15 +989,41 @@ it('keeps a watched HLS Step in its selected folder after the original conflict }) it.each([ - { firstPath: 'first/clip.mp4', secondPath: 'second/clip.mp4' }, - { firstPath: 'clip.mp4', secondPath: undefined }, + { + firstPath: 'first/clip.mp4', + secondPath: 'second/clip.mp4', + firstName: 'clip.mp4', + secondName: 'clip.mp4', + firstDir: 'clip', + secondDir: 'clip__1', + }, + { + firstPath: 'clip.mp4', + secondPath: undefined, + firstName: 'clip.mp4', + secondName: 'clip.mp4', + firstDir: 'clip', + secondDir: 'clip__1', + }, + { + firstPath: '/', + secondPath: '/', + firstName: 'first.mp4', + secondName: 'second.mp4', + firstDir: 'first', + secondDir: 'second', + }, ])('keeps watched HLS source paths attached to their folders (paths: $firstPath, $secondPath)', async ({ firstPath, secondPath, + firstName, + secondName, + firstDir, + secondDir, }) => { const firstMaster = { original_id: 'first-old', - original_name: 'clip.mp4', + original_name: firstName, original_path: firstPath, name: 'master.m3u8', url: 'https://downloads.test/first-master', @@ -968,12 +1038,14 @@ it.each([ const secondMaster = { ...firstMaster, original_id: 'second-old', + original_name: secondName, original_path: secondPath, url: 'https://downloads.test/second-master', } const secondSegment = { ...firstSegment, original_id: 'second-old', + original_name: secondName, original_path: secondPath, url: 'https://downloads.test/second-segment', } @@ -1036,10 +1108,10 @@ it.each([ }) expect(result.hasFailures).toBe(false) expect(completions).toBe(2) - expect(await readFile(path.join(outputDir, 'clip', 'video', 'segment.ts'), 'utf8')).toBe( + expect(await readFile(path.join(outputDir, firstDir, 'video', 'segment.ts'), 'utf8')).toBe( 'first updated', ) - expect(await readFile(path.join(outputDir, 'clip__1', 'video', 'segment.ts'), 'utf8')).toBe( + expect(await readFile(path.join(outputDir, secondDir, 'video', 'segment.ts'), 'utf8')).toBe( 'second updated', ) }) From 53e46bf10510f39887562953341807de7ac370c2 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 18:36:05 +0200 Subject: [PATCH 12/35] Document ambiguous HLS bundle follow-up --- packages/node/src/cli/commands/assemblies.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index d91e324b..ca7e2c34 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -993,6 +993,8 @@ async function buildDirectoryDownloadTargets({ new Set(names.map((name) => downloadPathKey(path.join(targetDir, name), reservedPaths))) .size !== names.length ) { + // Merged Steps can hide distinct bundles sharing one upload (#619). Reject ambiguous + // linked paths rather than guessing bundle boundaries and overwriting their contents. throw new Error('Assembly results have duplicate relative paths') } From bcd8a002785197fe17c56d0a918cfcb5404de5fe Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 18:56:24 +0200 Subject: [PATCH 13/35] Bound URL downloads and classify CLI compatibility changes --- .changeset/assembly-download-paths.md | 8 +++-- packages/node/README.md | 5 ++- packages/node/src/cli/commands/assemblies.ts | 18 ++++++++++ .../test/unit/cli/download-layout.test.ts | 33 +++++++++++++++++++ packages/transloadit/README.md | 5 ++- 5 files changed, 65 insertions(+), 4 deletions(-) diff --git a/.changeset/assembly-download-paths.md b/.changeset/assembly-download-paths.md index 1ee0df58..20f481a4 100644 --- a/.changeset/assembly-download-paths.md +++ b/.changeset/assembly-download-paths.md @@ -1,13 +1,17 @@ --- -'@transloadit/node': patch +'@transloadit/node': major '@transloadit/mcp-server': patch -'transloadit': patch +'transloadit': major --- Keep CLI download filenames and HLS directories aligned with Assembly results, omit input parent folders from output paths, and support HTTP(S) URL inputs for Assembly and prompt commands. +SDK-breaking: CLI directory downloads use Assembly result names and omit input parent folders. Directory reruns process inputs again and preserve existing files with collision suffixes. Update scripts to use the result paths; use an explicit output file when mtime-based skipping is required. + Independent HLS bundles returned by the same Step get separate source folders so their playlist and segment paths can coexist. Reserve result paths with filesystem case sensitivity and file/directory conflicts in mind. Stage linked watch downloads before publishing them together, and validate local steps and output options before fetching URL inputs. Directory runs process inputs again to discover the result filenames instead of guessing freshness from upload names. Watch updates reuse their own result paths, while existing unrelated files and outputs from other inputs remain separate. + +URL input downloads have a five-minute deadline per input, configurable in milliseconds with `--url-download-timeout` on `assemblies create` and `run`. diff --git a/packages/node/README.md b/packages/node/README.md index 2656b249..7e31c862 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -146,7 +146,8 @@ npx -y @transloadit/node run "extract a waveform image from this audio" --input ``` Repeat `--input` for multiple files, or pass a directory. Use `--single-assembly` to process the -inputs together. `run` also supports `--watch`, `--recursive`, and `--concurrency`. +inputs together. `run` also supports `--watch`, `--recursive`, `--concurrency`, and +`--url-download-timeout` (milliseconds per URL input, default: `300000`). ### Compile Assembly Instructions @@ -1516,6 +1517,8 @@ The `assemblies create` command additionally supports: or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly without uploads. With `--template`, stdin always requires `-i -`. - `--single-assembly` - Pass all input files to a single assembly instead of one assembly per file +- `--url-download-timeout` - Limit each URL input download to this many milliseconds (default: + `300000`, or five minutes). DNS checks and redirects share the same deadline. #### Log Levels diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index ca7e2c34..3f6fbb7a 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -1753,9 +1753,11 @@ export interface AssembliesCreateOptions { reprocessStale?: boolean singleAssembly?: boolean concurrency?: number + urlDownloadTimeoutMs?: number } const DEFAULT_CONCURRENCY = 5 +const DEFAULT_URL_DOWNLOAD_TIMEOUT_MS = 5 * 60 * 1000 interface PreparedCreateOptions { effectiveStepsData: StepsInput | undefined @@ -1837,6 +1839,9 @@ export async function create( options: AssembliesCreateOptions, ): ReturnType { const preparedOptions = await prepareCreateOptions(outputctl, options) + const urlDownloadTimeoutMs = + toPositiveInteger(options.urlDownloadTimeoutMs, '--url-download-timeout') ?? + DEFAULT_URL_DOWNLOAD_TIMEOUT_MS if (!options.inputs.some(isUrlInput)) return await createWithLocalInputs(outputctl, client, options, preparedOptions) if (options.watch) throw new Error('--watch requires local inputs') @@ -1856,6 +1861,7 @@ export async function create( allowPrivateUrls: false, // Uploading preserves :original and Template semantics, just as Intents do. urlStrategy: 'download', + urlDownloadTimeoutMs, }) preparedInputs.push(prepared) const localPath = prepared.files[field] @@ -2378,6 +2384,10 @@ export class AssembliesCreateCommand extends AuthenticatedCommand { concurrency = concurrencyOption() + urlDownloadTimeoutMs = Option.String('--url-download-timeout', { + description: 'Maximum time per URL input download in milliseconds (default: 300000)', + }) + printUrls = printUrlsOption() protected async run(): Promise { @@ -2425,6 +2435,8 @@ export class AssembliesCreateCommand extends AuthenticatedCommand { reprocessStale: this.reprocessStale, singleAssembly: this.singleAssembly, concurrency: this.concurrency == null ? undefined : Number(this.concurrency), + urlDownloadTimeoutMs: + this.urlDownloadTimeoutMs == null ? undefined : Number(this.urlDownloadTimeoutMs), }) if (this.printUrls) { printResultUrls(this.output, resultUrls) @@ -2497,6 +2509,10 @@ export class RunCommand extends AuthenticatedCommand { concurrency = concurrencyOption() + urlDownloadTimeoutMs = Option.String('--url-download-timeout', { + description: 'Maximum time per URL input download in milliseconds (default: 300000)', + }) + printUrls = printUrlsOption() protected async run(): Promise { @@ -2555,6 +2571,8 @@ export class RunCommand extends AuthenticatedCommand { reprocessStale: this.reprocessStale, singleAssembly: this.singleAssembly, concurrency: this.concurrency == null ? undefined : Number(this.concurrency), + urlDownloadTimeoutMs: + this.urlDownloadTimeoutMs == null ? undefined : Number(this.urlDownloadTimeoutMs), }) if (this.printUrls) { printResultUrls(this.output, resultUrls) diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index a6424bed..521f418a 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -720,6 +720,39 @@ it('rejects watch mode for URL inputs before making a download request', async ( expect(Transloadit.prototype.createAssembly).not.toHaveBeenCalled() }) +it('times out a stalled URL input before creating an Assembly', async () => { + await fixture({}) + const { create } = await import('../../../src/cli/commands/assemblies.ts') + const download = nock('https://source.invalid') + .get('/slow.png') + .delay(200) + .reply(200, 'delayed input') + + await expect( + create(new OutputCtl(), new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + inputs: ['https://source.invalid/slow.png'], + stepsData: { output: { robot: '/file/filter' } }, + urlDownloadTimeoutMs: 30, + }), + ).rejects.toMatchObject({ name: 'TimeoutError' }) + expect(download.isDone()).toBe(true) + expect(Transloadit.prototype.createAssembly).not.toHaveBeenCalled() +}) + +it('honors the URL download timeout flag before uploading', async () => { + const { run } = await fixture({}) + const download = nock('https://source.invalid') + .get('/slow.png') + .delay(200) + .reply(200, 'delayed input') + + expect(await run(['-i', 'https://source.invalid/slow.png', '--url-download-timeout', '30'])).toBe( + 1, + ) + expect(download.isDone()).toBe(true) + expect(Transloadit.prototype.createAssembly).not.toHaveBeenCalled() +}) + it('keeps a repeated linked download together instead of renaming playlist targets', async () => { const { inputPath, outputDir, run } = await fixture({ adaptive: [ diff --git a/packages/transloadit/README.md b/packages/transloadit/README.md index 2656b249..7e31c862 100644 --- a/packages/transloadit/README.md +++ b/packages/transloadit/README.md @@ -146,7 +146,8 @@ npx -y @transloadit/node run "extract a waveform image from this audio" --input ``` Repeat `--input` for multiple files, or pass a directory. Use `--single-assembly` to process the -inputs together. `run` also supports `--watch`, `--recursive`, and `--concurrency`. +inputs together. `run` also supports `--watch`, `--recursive`, `--concurrency`, and +`--url-download-timeout` (milliseconds per URL input, default: `300000`). ### Compile Assembly Instructions @@ -1516,6 +1517,8 @@ The `assemblies create` command additionally supports: or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly without uploads. With `--template`, stdin always requires `-i -`. - `--single-assembly` - Pass all input files to a single assembly instead of one assembly per file +- `--url-download-timeout` - Limit each URL input download to this many milliseconds (default: + `300000`, or five minutes). DNS checks and redirects share the same deadline. #### Log Levels From 65a9ab819886f5a457a6719a72a9c2260ee1368a Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 19:02:54 +0200 Subject: [PATCH 14/35] Associate collision-test results with their actual inputs --- .../test/unit/cli/download-layout.test.ts | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 521f418a..cf68bbea 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -300,6 +300,14 @@ it.each([ const multiInput = path.join(path.dirname(inputPath), 'report.pdf.docx') await writeFile(singleInput, 'single-result input') await writeFile(multiInput, 'multi-result input') + const { stat: originalStat } = fs.promises + vi.spyOn(fs.promises, 'stat').mockImplementation(async (target, ...args) => { + // Input discovery can queue jobs in a different order from the CLI arguments. + if (target === path.join(outputDir, path.basename(fileFirst ? singleInput : multiInput))) { + await new Promise((resolve) => setTimeout(resolve, 60)) + } + return await originalStat(target, ...args) + }) const single = { output: [{ name: 'report.pdf', url: 'https://downloads.test/pdf' }] } const multi = { output: [ @@ -307,13 +315,15 @@ it.each([ { name: 'page2.jpg', url: 'https://downloads.test/page2' }, ], } - vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) - .mockImplementationOnce(async () => { - // Finish the second job first unless the CLI is explicitly asked to run serially. - await new Promise((resolve) => setTimeout(resolve, 60)) - return { ok: 'ASSEMBLY_COMPLETED', results: fileFirst ? single : multi } - }) - .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', results: fileFirst ? multi : single }) + vi.mocked(Transloadit.prototype.createAssembly).mockImplementation(async ({ files }) => { + const input = Object.values(files ?? {})[0] + expect([singleInput, multiInput]).toContain(input) + return { assembly_id: input === singleInput ? 'single' : 'multi' } + }) + vi.mocked(Transloadit.prototype.awaitAssemblyCompletion).mockImplementation(async (id) => ({ + ok: 'ASSEMBLY_COMPLETED', + results: id === 'single' ? single : multi, + })) nock('https://downloads.test') .get('/pdf') .reply(200, 'pdf bytes') @@ -332,10 +342,12 @@ it.each([ '1', ]), ).toBe(0) + const fileWasFirst = (await stat(path.join(outputDir, 'report.pdf'))).isFile() + expect(fileWasFirst).toBe(!fileFirst) expect( - await readFile(path.join(outputDir, fileFirst ? 'report.pdf' : 'report__1.pdf'), 'utf8'), + await readFile(path.join(outputDir, fileWasFirst ? 'report.pdf' : 'report__1.pdf'), 'utf8'), ).toBe('pdf bytes') - const inputDir = path.join(outputDir, fileFirst ? 'report.pdf__1' : 'report.pdf') + const inputDir = path.join(outputDir, fileWasFirst ? 'report.pdf__1' : 'report.pdf') expect(await readFile(path.join(inputDir, 'page1.jpg'), 'utf8')).toBe('first page') expect(await readFile(path.join(inputDir, 'page2.jpg'), 'utf8')).toBe('second page') }) From fc85a67f0d4624c96684ec07f275511e4c308ae6 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 19:04:05 +0200 Subject: [PATCH 15/35] Use an explicit promise in the collision fixture --- packages/node/test/unit/cli/download-layout.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index cf68bbea..1b74ffd1 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -315,10 +315,10 @@ it.each([ { name: 'page2.jpg', url: 'https://downloads.test/page2' }, ], } - vi.mocked(Transloadit.prototype.createAssembly).mockImplementation(async ({ files }) => { + vi.mocked(Transloadit.prototype.createAssembly).mockImplementation(({ files }) => { const input = Object.values(files ?? {})[0] expect([singleInput, multiInput]).toContain(input) - return { assembly_id: input === singleInput ? 'single' : 'multi' } + return Promise.resolve({ assembly_id: input === singleInput ? 'single' : 'multi' }) }) vi.mocked(Transloadit.prototype.awaitAssemblyCompletion).mockImplementation(async (id) => ({ ok: 'ASSEMBLY_COMPLETED', From 86a708f3fdd4162795a30c0cb34e668dc2aba64f Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 22:28:06 +0200 Subject: [PATCH 16/35] Report Assembly references and concise CLI failures (#610) * Report Assembly references and concise CLI failures * Preserve reporting output modes and sync legacy CLI docs * Skip empty ignored-error warning details * Report command failures once at the boundary * Preserve transport and batch failure diagnostics * Explain missing Template and steps inputs * Keep accepted Assembly references and sanitize parse failures * Retain creation attempt references and sync input errors * Limit ignored-error JSON to diagnostic fields * Validate Assembly diagnostic payloads before reporting * Include structured ignored-error details in CLI warnings --- .changeset/clear-assembly-reports.md | 9 + packages/node/README.md | 13 + packages/node/src/cli/OutputCtl.ts | 3 +- packages/node/src/cli/commands/BaseCommand.ts | 36 +- packages/node/src/cli/commands/assemblies.ts | 123 ++- packages/node/src/cli/commands/templates.ts | 191 +++-- packages/node/src/cli/errors.ts | 34 + packages/node/src/cli/helpers.ts | 21 + packages/node/src/cli/intentRuntime.ts | 5 +- packages/node/src/cli/resultUrls.ts | 14 +- packages/node/src/cli/stepsInput.ts | 23 +- packages/node/src/cli/types.ts | 35 + .../test/e2e/cli/assemblies-create.test.ts | 2 +- .../cli/assemblies-create-command.test.ts | 4 + .../test/unit/cli/assemblies-create.test.ts | 26 +- .../test/unit/cli/assembly-reporting.test.ts | 720 ++++++++++++++++++ .../node/test/unit/cli/run-command.test.ts | 1 + packages/transloadit/README.md | 13 + 18 files changed, 1114 insertions(+), 159 deletions(-) create mode 100644 .changeset/clear-assembly-reports.md create mode 100644 packages/node/src/cli/errors.ts create mode 100644 packages/node/test/unit/cli/assembly-reporting.test.ts diff --git a/.changeset/clear-assembly-reports.md b/.changeset/clear-assembly-reports.md new file mode 100644 index 00000000..5d70a395 --- /dev/null +++ b/.changeset/clear-assembly-reports.md @@ -0,0 +1,9 @@ +--- +'@transloadit/node': patch +'@transloadit/mcp-server': patch +'transloadit': patch +--- + +Report Assembly IDs and URLs in CLI diagnostics and JSON summaries, warn about ignored errors, +and show concise validation and API errors with the reason for failed Assemblies. +Validate diagnostic fields before printing them, excluding nested request metadata. diff --git a/packages/node/README.md b/packages/node/README.md index 87cc0eb9..99809328 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -265,6 +265,19 @@ npx -y transloadit auth token --aud mcp --scope assemblies:write,templates:read ### Processing Media +Assembly runs print their ID and status URL to stderr, including when processing fails. +`--json` writes newline-delimited JSON summaries for terminal Assembly statuses, with their +reference, final status, and `ignored_errors`. Ignored errors also produce a warning with their +count and up to five details. +If polling fails before a terminal status is retrieved, the error and known Assembly reference +are reported on stderr. +With `--print-urls`, JSON remains a single object containing `urls` and `assemblies`. +This document is written after the batch finishes; an input or watch error that aborts the batch +can prevent it from being written. +`--output -` keeps Assembly summaries off stdout; the reference stays on stderr. +When creating Assemblies, invalid Robot parameters and API failures produce a short diagnostic and +a nonzero exit status. + For common one-off tasks, prefer the intent-first commands: The full generated intent reference also lives in [`docs/intent-commands.md`](./docs/intent-commands.md). diff --git a/packages/node/src/cli/OutputCtl.ts b/packages/node/src/cli/OutputCtl.ts index be6fc9a2..9e353138 100644 --- a/packages/node/src/cli/OutputCtl.ts +++ b/packages/node/src/cli/OutputCtl.ts @@ -60,6 +60,7 @@ export interface IOutputCtl { info(msg: unknown): void debug(msg: unknown): void trace(msg: unknown): void + /** An undefined simple value emits JSON only, preserving binary or URL stdout in text mode. */ print(simple: unknown, json: unknown): void } @@ -110,6 +111,6 @@ export default class OutputCtl implements IOutputCtl { print(simple: unknown, json: unknown): void { if (this.json) console.log(JSON.stringify(json)) else if (typeof simple === 'string') console.log(simple) - else console.dir(simple, { depth: null }) + else if (simple !== undefined) console.dir(simple, { depth: null }) } } diff --git a/packages/node/src/cli/commands/BaseCommand.ts b/packages/node/src/cli/commands/BaseCommand.ts index 28ea82cf..10642cac 100644 --- a/packages/node/src/cli/commands/BaseCommand.ts +++ b/packages/node/src/cli/commands/BaseCommand.ts @@ -2,9 +2,14 @@ import type { ResolvedCliConfig } from '../helpers.ts' import type { IOutputCtl } from '../OutputCtl.ts' import { Command, Option } from 'clipanion' +import { RequestError } from 'got' +import { z } from 'zod' +import { ApiError } from '../../ApiError.ts' +import { tryCatch } from '../../alphalib/tryCatch.ts' import { Transloadit as TransloaditClient } from '../../Transloadit.ts' -import { buildMissingAuthMessage, resolveCliConfig } from '../helpers.ts' +import { CliInputError, formatSchemaIssues } from '../errors.ts' +import { buildMissingAuthMessage, formatAPIError, resolveCliConfig } from '../helpers.ts' import OutputCtl, { LOG_LEVEL_DEFAULT, LOG_LEVEL_NAMES, parseLogLevel } from '../OutputCtl.ts' abstract class BaseCommand extends Command { @@ -50,6 +55,27 @@ abstract class BaseCommand extends Command { } abstract override execute(): Promise + + protected abstract run(): Promise + + /** Report schema, API, and transport failures once; other errors retain Clipanion's diagnostics. */ + protected async runWithErrorReporting(): Promise { + const [error, result] = await tryCatch(this.run()) + if (!error) return result + if (error instanceof z.ZodError) { + this.output.error(`Invalid options: ${formatSchemaIssues(error.issues)}`) + return 1 + } + if (error instanceof RequestError) { + this.output.error(formatAPIError(error)) + return 1 + } + if (error instanceof CliInputError || error instanceof ApiError) { + this.output.error(formatAPIError(error)) + return 1 + } + throw error + } } export abstract class AuthenticatedCommand extends BaseCommand { @@ -58,17 +84,13 @@ export abstract class AuthenticatedCommand extends BaseCommand { if (!this.setupClient()) { return 1 } - return await this.run() + return await this.runWithErrorReporting() } - - protected abstract run(): Promise } export abstract class UnauthenticatedCommand extends BaseCommand { override async execute(): Promise { this.setupOutput() - return await this.run() + return await this.runWithErrorReporting() } - - protected abstract run(): Promise } diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index de4ac96e..cd9d90e7 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -1,5 +1,6 @@ import type { Readable } from 'node:stream' +import type { AssemblyStatus } from '../../alphalib/types/assemblyStatus.ts' import type { StepsInput } from '../../alphalib/types/template.ts' import type { CreateAssemblyParams, ReplayAssemblyParams } from '../../apiTypes.ts' import type { LintFatalLevel } from '../../lintAssemblyInstructions.ts' @@ -11,6 +12,7 @@ import type { import type { IOutputCtl } from '../OutputCtl.ts' import type { NormalizedAssemblyResultFile, NormalizedAssemblyResults } from '../resultFiles.ts' import type { ResultUrlRow } from '../resultUrls.ts' +import type { AssemblySummary } from '../types.ts' import { randomUUID } from 'node:crypto' import EventEmitter from 'node:events' @@ -24,7 +26,7 @@ import { setTimeout as delay } from 'node:timers/promises' import { promisify } from 'node:util' import { Command, Option } from 'clipanion' -import got from 'got' +import got, { RequestError } from 'got' import PQueue from 'p-queue' import * as t from 'typanion' import { z } from 'zod' @@ -49,7 +51,7 @@ import { formatAPIError, readCliInput } from '../helpers.ts' import { normalizeAssemblyResults } from '../resultFiles.ts' import { collectNormalizedResultUrlRows, printResultUrls } from '../resultUrls.ts' import { parseStepsInputJson, readStepsInputFile } from '../stepsInput.ts' -import { ensureError, isErrnoException } from '../types.ts' +import { assemblySummarySchema, ensureError, isErrnoException } from '../types.ts' import { AuthenticatedCommand, UnauthenticatedCommand } from './BaseCommand.ts' // --- From assemblies.ts: Schemas and interfaces --- @@ -194,11 +196,7 @@ export async function get( ): Promise { for (const assembly of assemblies) { await delay(1000) - const [err, result] = await tryCatch(client.getAssembly(assembly)) - if (err) { - output.error(formatAPIError(err)) - throw ensureError(err) - } + const result = await client.getAssembly(assembly) output.print(result, result) } } @@ -1323,6 +1321,7 @@ export interface AssembliesCreateOptions { reprocessStale?: boolean singleAssembly?: boolean concurrency?: number + printUrls?: boolean } const DEFAULT_CONCURRENCY = 5 @@ -1345,8 +1344,14 @@ export async function create( reprocessStale, singleAssembly, concurrency = DEFAULT_CONCURRENCY, + printUrls = false, }: AssembliesCreateOptions, -): Promise<{ resultUrls: ResultUrlRow[]; results: unknown[]; hasFailures: boolean }> { +): Promise<{ + assemblies: AssemblySummary[] + resultUrls: ResultUrlRow[] + results: unknown[] + hasFailures: boolean +}> { // Quick fix for https://github.com/transloadit/transloadify/issues/13 // Only default to stdout when output is undefined (not provided), not when explicitly null let resolvedOutput = output @@ -1429,6 +1434,7 @@ export async function create( // Use p-queue for concurrency management const queue = new PQueue({ concurrency }) const results: unknown[] = [] + const assemblies: AssemblySummary[] = [] const resultUrls: ResultUrlRow[] = [] const reservedResultPaths = new Set() const latestWatchJobTokenByOutputPath = new Map() @@ -1480,11 +1486,40 @@ export async function create( assembly: Awaited> assemblyId: string }> { - const result = await client.createAssembly(createOptions) - const assemblyId = result.assembly_id - if (!assemblyId) throw new Error('No assembly_id in result') - const assemblyUrl = result.assembly_ssl_url ?? result.assembly_url ?? null - const assemblyReference = formatAssemblyReference({ assemblyId, assemblyUrl }) + let assemblyReference: string | undefined + function reportCreatedAssembly(rawAssembly: AssemblyStatus): void { + const assembly = assemblySummarySchema.parse(rawAssembly) + if (assemblyReference || !assembly.assembly_id) return + assemblyReference = formatAssemblyReference({ + assemblyId: assembly.assembly_id, + assemblyUrl: assembly.assembly_ssl_url ?? assembly.assembly_url ?? null, + }) + outputctl.notice(assemblyReference) + } + + const creation = client.createAssembly({ + ...createOptions, + onAssemblyCreated: reportCreatedAssembly, + }) + const [creationError, result] = await tryCatch(creation) + if (creationError) { + if (!assemblyReference && creationError instanceof RequestError && creation.assemblyId) { + const requestUrl = creationError.options.url + assemblyReference = formatAssemblyReference({ + assemblyId: creation.assemblyId, + assemblyUrl: + requestUrl instanceof URL ? `${requestUrl.origin}${requestUrl.pathname}` : null, + }) + } + if (!assemblyReference) throw creationError + throw new Error(`${formatAPIError(creationError)} (${assemblyReference})`, { + cause: creationError, + }) + } + reportCreatedAssembly(result) + const createdSummary = assemblySummarySchema.parse(result) + const assemblyId = createdSummary.assembly_id + if (!assemblyId || !assemblyReference) throw new Error('No assembly_id in result') let assembly: Awaited> try { @@ -1492,16 +1527,50 @@ export async function create( signal: abortController.signal, onPoll: () => true, onAssemblyProgress: (status) => { - outputctl.debug(`Assembly status: ${status.ok}`) + const summary = assemblySummarySchema.parse(status) + outputctl.debug(`Assembly status: ${summary.ok ?? 'unknown'}`) }, }) } catch (err) { const error = ensureError(err) - throw new Error(`${error.message} (${assemblyReference})`, { cause: error }) + const message = error instanceof RequestError ? formatAPIError(error) : error.message + throw new Error(`${message} (${assemblyReference})`, { cause: error }) + } + + const summary = assemblySummarySchema.parse(assembly) + summary.assembly_id = assemblyId + summary.assembly_url ??= createdSummary.assembly_url + summary.assembly_ssl_url ??= createdSummary.assembly_ssl_url + const ignoredErrors = summary.ignored_errors + const ignoredErrorCount = Math.max(summary.ignored_error_count, ignoredErrors.length) + summary.ignored_error_count = ignoredErrorCount + // Preserve stdout file bytes. URL mode combines all summaries with its existing JSON object. + if (resolvedOutput !== '-') { + if (printUrls) assemblies.push(summary) + else outputctl.print(undefined, summary) + } + if (ignoredErrorCount > 0) { + const details = ignoredErrors + .map((ignored) => { + const code = typeof ignored.error === 'string' ? ignored.error : ignored.error?.code + const message = + ignored.message || + (typeof ignored.error === 'object' ? ignored.error.message : undefined) + return [ignored.step, code, message].filter(Boolean).join(': ') + }) + .filter(Boolean) + .slice(0, 5) + outputctl.warn( + `${ignoredErrorCount} ignored error${ignoredErrorCount === 1 ? '' : 's'} (${assemblyReference})${details.length > 0 ? `: ${details.join('; ')}` : ''}`, + ) } + // Malformed error fields still signal failure, but only sanitized values enter the message. if (assembly.error || (assembly.ok && assembly.ok !== 'ASSEMBLY_COMPLETED')) { - const msg = `Assembly failed: ${assembly.error || assembly.message} (Status: ${assembly.ok}, ${assemblyReference})` + const details = + [summary.error ?? summary.ok, summary.message].filter(Boolean).join(': ') || + 'Unknown Assembly error' + const msg = `Assembly failed: ${details} (${assemblyReference})` throw new Error(msg) } @@ -1612,7 +1681,7 @@ export async function create( function handleEmitterError(err: Error): void { abortController.abort() queue.clear() - outputctl.error(err) + outputctl.error(formatAPIError(err)) reject(err) } @@ -1643,7 +1712,7 @@ export async function create( }) ) { outputctl.debug(`SKIPPED STALE SINGLE ASSEMBLY ${resolvedOutput ?? 'null'}`) - resolve({ resultUrls, results: [], hasFailures: false }) + resolve({ assemblies, resultUrls, results: [], hasFailures: false }) return } @@ -1691,10 +1760,10 @@ export async function create( results.push(assembly) } catch (err) { hasFailures = true - outputctl.error(err as Error) + outputctl.error(formatAPIError(err)) } - resolve({ resultUrls, results, hasFailures }) + resolve({ assemblies, resultUrls, results, hasFailures }) }) } @@ -1713,13 +1782,13 @@ export async function create( }) .catch((err: unknown) => { hasFailures = true - outputctl.error(err as Error) + outputctl.error(formatAPIError(err)) }) }) emitter.on('end', async () => { await queue.onIdle() - resolve({ resultUrls, results, hasFailures }) + resolve({ assemblies, resultUrls, results, hasFailures }) }) } @@ -1895,7 +1964,7 @@ export class AssembliesCreateCommand extends AuthenticatedCommand { return 1 } - const { hasFailures, resultUrls } = await create(this.output, this.client, { + const { assemblies, hasFailures, resultUrls } = await create(this.output, this.client, { steps: this.steps, template: this.template, fields: fieldsMap ?? {}, @@ -1907,9 +1976,10 @@ export class AssembliesCreateCommand extends AuthenticatedCommand { reprocessStale: this.reprocessStale, singleAssembly: this.singleAssembly, concurrency: this.concurrency == null ? undefined : Number(this.concurrency), + printUrls: this.printUrls, }) if (this.printUrls) { - printResultUrls(this.output, resultUrls) + printResultUrls(this.output, resultUrls, assemblies) } return hasFailures ? 1 : undefined } @@ -2026,7 +2096,7 @@ export class RunCommand extends AuthenticatedCommand { return 1 } - const { hasFailures, resultUrls } = await create(this.output, this.client, { + const { assemblies, hasFailures, resultUrls } = await create(this.output, this.client, { stepsData: parseStepsInputJson(JSON.stringify(compiled.instructions.steps), this.output), fields: fieldsMap ?? {}, watch: this.watch, @@ -2037,9 +2107,10 @@ export class RunCommand extends AuthenticatedCommand { reprocessStale: this.reprocessStale, singleAssembly: this.singleAssembly, concurrency: this.concurrency == null ? undefined : Number(this.concurrency), + printUrls: this.printUrls, }) if (this.printUrls) { - printResultUrls(this.output, resultUrls) + printResultUrls(this.output, resultUrls, assemblies) } return hasFailures ? 1 : undefined } diff --git a/packages/node/src/cli/commands/templates.ts b/packages/node/src/cli/commands/templates.ts index 3a858b47..9afd1933 100644 --- a/packages/node/src/cli/commands/templates.ts +++ b/packages/node/src/cli/commands/templates.ts @@ -11,11 +11,13 @@ import { Command, Option } from 'clipanion' import rreaddir from 'recursive-readdir' import { z } from 'zod' +import { ApiError } from '../../ApiError.ts' import { tryCatch } from '../../alphalib/tryCatch.ts' +import { CliInputError } from '../errors.ts' import { createReadStream, formatAPIError, streamToBuffer } from '../helpers.ts' import { createCliStepsSchema, parseStepsInputJson } from '../stepsInput.ts' import ModifiedLookup from '../template-last-modified.ts' -import { ensureError, isTransloaditAPIError } from '../types.ts' +import { isTransloaditAPIError } from '../types.ts' import { AuthenticatedCommand } from './BaseCommand.ts' const rreaddirAsync = promisify(rreaddir) @@ -54,27 +56,29 @@ export interface TemplateSyncOptions { recursive?: boolean } +async function readTemplateInput(input: Promise): Promise { + const [error, content] = await tryCatch(input) + if (error) { + throw new CliInputError(`Cannot read Template input: ${error.message}`, { cause: error }) + } + return content +} + export async function create( output: IOutputCtl, client: Transloadit, { name, file }: TemplateCreateOptions, ): Promise { - try { - const buf = await streamToBuffer(createReadStream(file)) + const buf = await readTemplateInput(streamToBuffer(createReadStream(file))) - const steps = parseStepsInputJson(buf.toString(), output) + const steps = parseStepsInputJson(buf.toString(), output) - const result = await client.createTemplate({ - name, - template: { steps } as TemplateContent, - }) - output.print(result.id, result) - return result - } catch (err) { - const error = ensureError(err) - output.error(error.message) - throw err - } + const result = await client.createTemplate({ + name, + template: { steps } as TemplateContent, + }) + output.print(result.id, result) + return result } export async function get( @@ -84,11 +88,7 @@ export async function get( ): Promise { const requests = templates.map((template) => client.getTemplate(template)) - const [err, results] = await tryCatch(Promise.all(requests)) - if (err) { - output.error(formatAPIError(err)) - throw err - } + const results = await Promise.all(requests) for (const result of results) { output.print(result, result) @@ -100,53 +100,49 @@ export async function modify( client: Transloadit, { template, name, file }: TemplateModifyOptions, ): Promise { - try { - const buf = await streamToBuffer(createReadStream(file)) + const buf = await readTemplateInput(streamToBuffer(createReadStream(file))) - let steps: TemplateContent['steps'] | null = null - let newName = name + let steps: TemplateContent['steps'] | null = null + let newName = name - if (buf.length > 0) { - steps = parseStepsInputJson(buf.toString(), output) as TemplateContent['steps'] - } - - if (!name || buf.length === 0) { - const tpl = await client.getTemplate(template) - if (!name) newName = tpl.name - if (buf.length === 0 && tpl.content.steps) { - steps = tpl.content.steps as TemplateContent['steps'] - } - } + if (buf.length > 0) { + steps = parseStepsInputJson(buf.toString(), output) as TemplateContent['steps'] + } - if (steps === null) { - throw new Error('No steps to update template with') + if (!name || buf.length === 0) { + const tpl = await client.getTemplate(template) + if (!name) newName = tpl.name + if (buf.length === 0 && tpl.content.steps) { + steps = tpl.content.steps as TemplateContent['steps'] } + } - await client.editTemplate(template, { - name: newName, - // Steps (validated) is assignable to StepsInput at runtime; cast for TS - template: { steps } as TemplateContent, - }) - } catch (err) { - output.error(formatAPIError(err)) - throw err + if (steps === null) { + throw new CliInputError('No steps to update template with') } + + await client.editTemplate(template, { + name: newName, + // Steps (validated) is assignable to StepsInput at runtime; cast for TS + template: { steps } as TemplateContent, + }) } async function deleteTemplates( output: IOutputCtl, client: Transloadit, { templates }: TemplateDeleteOptions, -): Promise { - await Promise.all( - templates.map(async (template) => { - const [err] = await tryCatch(client.deleteTemplate(template)) - if (err) { - output.error(formatAPIError(err)) - throw err - } - }), +): Promise { + const results = await Promise.allSettled( + templates.map((template) => client.deleteTemplate(template)), ) + let exitCode: number | undefined + for (const [index, result] of results.entries()) { + if (result.status === 'fulfilled') continue + output.error(`Template ${templates[index]}: ${formatAPIError(result.reason)}`) + exitCode = 1 + } + return exitCode } // Export with `delete` alias for external consumers @@ -281,54 +277,54 @@ export async function sync( const modified = new ModifiedLookup(client) - const [err] = await tryCatch( - Promise.all( - templates.map(async (template) => { - if (!('steps' in template.data)) { - if (!template.data.transloadit_template_id) { - throw new Error(`Template file has no id and no steps: ${template.file}`) - } - return download(template) + await Promise.all( + templates.map(async (template) => { + if (!('steps' in template.data)) { + if (!template.data.transloadit_template_id) { + throw new CliInputError(`Template file has no id and no steps: ${template.file}`) } + return download(template) + } - if (!template.data.transloadit_template_id) return upload(template) - - const stats = await fsp.stat(template.file) - const fileModified = stats.mtime - - let templateModified: Date - const templateId = template.data.transloadit_template_id - try { - await client.getTemplate(templateId) - templateModified = await new Promise((resolve, reject) => - modified.byId(templateId, (err, res) => { - if (err) { - reject(err) - } else if (res) { - resolve(res) - } else { - reject(new Error('No date returned')) - } - }), - ) - } catch (err) { - if (isTransloaditAPIError(err)) { - if (err.code === 'SERVER_404' || (err.response && err.response.statusCode === 404)) { - throw new Error(`Template file references nonexistent template: ${template.file}`) + if (!template.data.transloadit_template_id) return upload(template) + + const stats = await readTemplateInput(fsp.stat(template.file)) + const fileModified = stats.mtime + + let templateModified: Date + const templateId = template.data.transloadit_template_id + try { + await client.getTemplate(templateId) + templateModified = await new Promise((resolve, reject) => + modified.byId(templateId, (err, res) => { + if (err) { + reject(err) + } else if (res) { + resolve(res) + } else { + reject(new Error('No date returned')) } - } - throw err + }), + ) + } catch (err) { + if ( + (err instanceof ApiError && + (err.code === 'TEMPLATE_NOT_FOUND' || err.cause?.response?.statusCode === 404)) || + (isTransloaditAPIError(err) && + (err.code === 'SERVER_404' || err.response?.statusCode === 404)) + ) { + throw new CliInputError( + `Template file references nonexistent template: ${template.file}`, + { cause: err }, + ) } + throw err + } - if (fileModified > templateModified) return upload(template) - return download(template) - }), - ), + if (fileModified > templateModified) return upload(template) + return download(template) + }), ) - if (err) { - output.error(err) - throw err - } async function upload(template: TemplateFile): Promise { const params = { @@ -479,10 +475,9 @@ export class TemplatesDeleteCommand extends AuthenticatedCommand { templateIds = Option.Rest({ required: 1 }) protected async run(): Promise { - await deleteTemplates(this.output, this.client, { + return await deleteTemplates(this.output, this.client, { templates: this.templateIds, }) - return undefined } } diff --git a/packages/node/src/cli/errors.ts b/packages/node/src/cli/errors.ts new file mode 100644 index 00000000..3514c8ea --- /dev/null +++ b/packages/node/src/cli/errors.ts @@ -0,0 +1,34 @@ +import type { ZodIssue } from 'zod' + +/** An input error the CLI can explain without an internal-error stack trace. */ +export class CliInputError extends Error { + override name = 'CliInputError' +} + +function describeIssues(issues: ZodIssue[]): string[] { + return issues.flatMap((issue) => { + if (issue.code === 'invalid_union') { + const descriptions = issue.unionErrors.flatMap((error) => describeIssues(error.issues)) + // Interpolation is an alternative to literal Robot parameters, not a required input format. + const literals = descriptions.filter( + (description) => !description.endsWith(': Must be a full interpolation string'), + ) + return literals.length > 0 ? literals : descriptions + } + const message = + issue.code === 'invalid_enum_value' + ? `Expected one of: ${issue.options.join(', ')}` + : issue.message + return [`${issue.path.join('.') || '(root)'}: ${message}`] + }) +} + +/** Summarize validation paths and remedies without duplicating Zod's union diagnostics. */ +export function formatSchemaIssues(issues: ZodIssue[]): string { + const descriptions = [...new Set(describeIssues(issues))] + const remaining = descriptions.length - 10 + return [ + ...descriptions.slice(0, 10), + ...(remaining > 0 ? [`${remaining} more validation error${remaining === 1 ? '' : 's'}`] : []), + ].join('; ') +} diff --git a/packages/node/src/cli/helpers.ts b/packages/node/src/cli/helpers.ts index c3f6e75f..e20822d4 100644 --- a/packages/node/src/cli/helpers.ts +++ b/packages/node/src/cli/helpers.ts @@ -8,8 +8,10 @@ import { homedir, userInfo } from 'node:os' import path from 'node:path' import { parse as parseDotenv } from 'dotenv' +import { ParseError, RequestError } from 'got' import { z } from 'zod' +import { ApiError } from '../ApiError.ts' import { isAPIError } from './types.ts' /** API signing algorithms supported by CLI credentials and device authorization. */ @@ -535,6 +537,25 @@ export async function readCliInput(options: ReadCliInputOptions): Promise 0 ? formatResultUrlRows(rows) : undefined, { + urls: rows, + ...(assemblies?.length ? { assemblies } : {}), + }) } diff --git a/packages/node/src/cli/stepsInput.ts b/packages/node/src/cli/stepsInput.ts index 1d4d9349..442159d1 100644 --- a/packages/node/src/cli/stepsInput.ts +++ b/packages/node/src/cli/stepsInput.ts @@ -5,9 +5,11 @@ import fsp from 'node:fs/promises' import { z } from 'zod' +import { tryCatch } from '../alphalib/tryCatch.ts' import { robotsSchema } from '../alphalib/types/robots/_index.ts' import { refineDocumentExtractOcr } from '../alphalib/types/robots/document-extract.ts' import { refineTransloaditImportSelector } from '../alphalib/types/robots/transloadit-import.ts' +import { CliInputError, formatSchemaIssues } from './errors.ts' const stepsWrapperSchema = z.object({ steps: z.record(z.unknown()) }) @@ -35,7 +37,12 @@ export function parseStepsInputJson( content: string, output: Pick = console, ): StepsInput { - const parsed: unknown = JSON.parse(content) + let parsed: unknown + try { + parsed = JSON.parse(content) + } catch (error) { + throw new CliInputError('Invalid steps format: expected valid JSON', { cause: error }) + } const wrapped = stepsWrapperSchema.safeParse(parsed) let stepsInput = parsed // A step named “steps” has a robot, unlike the top-level instructions wrapper. @@ -43,13 +50,18 @@ export function parseStepsInputJson( // --steps only supplies steps; other instructions must not be silently discarded. const validatedWrapper = stepsWrapperSchema.strict().safeParse(parsed) if (!validatedWrapper.success) { - throw new Error(`Invalid steps format: ${validatedWrapper.error.message}`) + throw new CliInputError( + `Invalid steps format: ${formatSchemaIssues(validatedWrapper.error.issues)}`, + { cause: validatedWrapper.error }, + ) } stepsInput = validatedWrapper.data.steps } const validated = createCliStepsSchema(output).safeParse(stepsInput) if (!validated.success) { - throw new Error(`Invalid steps format: ${validated.error.message}`) + throw new CliInputError(`Invalid steps format: ${formatSchemaIssues(validated.error.issues)}`, { + cause: validated.error, + }) } const parsedSteps = stepsInput as Record> @@ -72,6 +84,9 @@ export async function readStepsInputFile( filePath: string, output: Pick = console, ): Promise { - const content = await fsp.readFile(filePath, 'utf8') + const [error, content] = await tryCatch(fsp.readFile(filePath, 'utf8')) + if (error) { + throw new CliInputError(`Cannot read steps input: ${error.message}`, { cause: error }) + } return parseStepsInputJson(content, output) } diff --git a/packages/node/src/cli/types.ts b/packages/node/src/cli/types.ts index 658fe630..8c1a957a 100644 --- a/packages/node/src/cli/types.ts +++ b/packages/node/src/cli/types.ts @@ -2,6 +2,41 @@ import type { Steps } from '../alphalib/types/template.ts' import { z } from 'zod' +const ignoredErrorDetailsSchema = z + .union([ + z.string(), + z.object({ + code: z.string().optional().catch(undefined), + message: z.string().optional().catch(undefined), + }), + ]) + .optional() + .catch(undefined) + +const ignoredErrorSummarySchema = z + .object({ + step: z.string().nullable().optional().catch(undefined), + phase: z.string().optional().catch(undefined), + error: ignoredErrorDetailsSchema, + message: z.string().optional().catch(undefined), + }) + .catch({}) + +/** Project unchecked API responses to CLI diagnostics without nested request metadata. */ +export const assemblySummarySchema = z.object({ + assembly_id: z.string().optional().catch(undefined), + assembly_url: z.string().nullable().optional().catch(undefined), + assembly_ssl_url: z.string().nullable().optional().catch(undefined), + ok: z.string().nullable().optional().catch(undefined), + error: z.string().optional().catch(undefined), + message: z.string().optional().catch(undefined), + ignored_error_count: z.number().finite().int().nonnegative().default(0).catch(0), + ignored_errors: z.array(ignoredErrorSummarySchema).default([]).catch([]), +}) + +/** Public diagnostics from one Assembly, excluding instructions and request internals. */ +export type AssemblySummary = z.infer + // Zod schemas for runtime validation const APIErrorSchema = z.object({ error: z.string(), diff --git a/packages/node/test/e2e/cli/assemblies-create.test.ts b/packages/node/test/e2e/cli/assemblies-create.test.ts index 9343bc85..646a96e3 100644 --- a/packages/node/test/e2e/cli/assemblies-create.test.ts +++ b/packages/node/test/e2e/cli/assemblies-create.test.ts @@ -255,7 +255,7 @@ describeLive('assemblies', { retry: 1 }, () => { .to.have.property('type') .that.equals('error') expect(result[result.length - 1]) - .to.have.nested.property('msg.message') + .to.have.property('msg') .that.equals("Output collision between 'in/1.jpg' and '1.jpg'") } // Small delay to allow abort signals to propagate and streams to close diff --git a/packages/node/test/unit/cli/assemblies-create-command.test.ts b/packages/node/test/unit/cli/assemblies-create-command.test.ts index 0a21a2f2..44afd035 100644 --- a/packages/node/test/unit/cli/assemblies-create-command.test.ts +++ b/packages/node/test/unit/cli/assemblies-create-command.test.ts @@ -99,6 +99,7 @@ describe('assemblies create command', () => { expect(exitCode).toBe(0) expect(createAssembly).toHaveBeenCalledExactlyOnceWith({ params: { steps, fields: {} }, + onAssemblyCreated: expect.any(Function), signal: expect.any(AbortSignal), }) expect(await readdir(outputDir)).toEqual(['imported']) @@ -128,6 +129,7 @@ describe('assemblies create command', () => { ).toBe(0) expect(createAssembly).toHaveBeenCalledExactlyOnceWith({ params: { steps: expect.any(Object), fields: {} }, + onAssemblyCreated: expect.any(Function), signal: expect.any(AbortSignal), }) expect(fs.createReadStream).not.toHaveBeenCalled() @@ -156,6 +158,7 @@ describe('assemblies create command', () => { expect(createAssembly).toHaveBeenCalledExactlyOnceWith({ params: { steps: expect.any(Object), fields: {} }, uploads: { in: process.stdin }, + onAssemblyCreated: expect.any(Function), signal: expect.any(AbortSignal), }) expect(fs.createReadStream).not.toHaveBeenCalled() @@ -182,6 +185,7 @@ describe('assemblies create command', () => { expect(createAssembly).toHaveBeenCalledExactlyOnceWith({ params: { steps: expect.any(Object), fields: {} }, files: { in: inputPath }, + onAssemblyCreated: expect.any(Function), signal: expect.any(AbortSignal), }) }) diff --git a/packages/node/test/unit/cli/assemblies-create.test.ts b/packages/node/test/unit/cli/assemblies-create.test.ts index f304cef9..39960fbb 100644 --- a/packages/node/test/unit/cli/assemblies-create.test.ts +++ b/packages/node/test/unit/cli/assemblies-create.test.ts @@ -267,11 +267,7 @@ describe('assemblies create', () => { ) await expect(stat(outputPath)).rejects.toThrow() - expect(outputError).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('use -o /'), - }), - ) + expect(outputError).toHaveBeenCalledWith(expect.stringContaining('use -o /')) }) it.each([ @@ -549,6 +545,7 @@ describe('assemblies create', () => { expect(client.createAssembly).toHaveBeenCalledExactlyOnceWith({ params: { steps: expect.any(Object) }, + onAssemblyCreated: expect.any(Function), signal: expect.any(AbortSignal), }) expect(await collectRelativeFiles(outputDir)).toEqual([path.join('generated', 'generated.png')]) @@ -1726,14 +1723,11 @@ describe('assemblies create', () => { const loggedError = consoleError.mock.calls.find( ([prefix, value]) => prefix === 'err ' && - value instanceof Error && - value.message.includes( - 'Assembly URL: https://api2.transloadit.com/assemblies/assembly-timeout', - ), + typeof value === 'string' && + value.includes('Assembly URL: https://api2.transloadit.com/assemblies/assembly-timeout'), )?.[1] - expect(loggedError).toBeInstanceOf(Error) - expect((loggedError as Error).message).toContain('Polling timed out') + expect(loggedError).toContain('Polling timed out') }) it('does not report another assembly URL when concurrent polling times out', async () => { @@ -1794,13 +1788,11 @@ describe('assemblies create', () => { const loggedError = consoleError.mock.calls.find( ([prefix, value]) => prefix === 'err ' && - value instanceof Error && - value.message.includes('Assembly ID: assembly-one'), + typeof value === 'string' && + value.includes('Assembly ID: assembly-one'), )?.[1] - expect(loggedError).toBeInstanceOf(Error) - expect((loggedError as Error).message).not.toContain( - 'https://api2.transloadit.com/assemblies/assembly-two', - ) + expect(loggedError).toEqual(expect.any(String)) + expect(loggedError).not.toContain('https://api2.transloadit.com/assemblies/assembly-two') }) }) diff --git a/packages/node/test/unit/cli/assembly-reporting.test.ts b/packages/node/test/unit/cli/assembly-reporting.test.ts new file mode 100644 index 00000000..1e6b7bda --- /dev/null +++ b/packages/node/test/unit/cli/assembly-reporting.test.ts @@ -0,0 +1,720 @@ +import fs from 'node:fs' +import { mkdtemp, rm, stat, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { Writable } from 'node:stream' +import { format } from 'node:util' + +import { Cli } from 'clipanion' +import { Options, RequestError } from 'got' +import nock from 'nock' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { AssembliesCreateCommand } from '../../../src/cli/commands/assemblies.ts' +import { + TemplatesCreateCommand, + TemplatesDeleteCommand, + TemplatesGetCommand, + TemplatesModifyCommand, + TemplatesSyncCommand, +} from '../../../src/cli/commands/templates.ts' +import { Transloadit } from '../../../src/Transloadit.ts' + +const assemblyId = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +const assemblyUrl = `https://api2.transloadit.com/assemblies/${assemblyId}` +let tempDir: string + +beforeEach(async () => { + tempDir = await mkdtemp(path.join(tmpdir(), 'transloadit-reporting-')) + vi.stubEnv('TRANSLOADIT_KEY', 'key') + vi.stubEnv('TRANSLOADIT_SECRET', 'secret') + vi.stubEnv('TRANSLOADIT_AUTH_TOKEN', '') + vi.stubEnv('TRANSLOADIT_ENDPOINT', 'https://api2.transloadit.com') + vi.spyOn(fs, 'fstatSync').mockReturnValue(await stat('/dev/null')) + nock.disableNetConnect() +}) + +afterEach(async () => { + vi.restoreAllMocks() + vi.unstubAllEnvs() + nock.cleanAll() + nock.enableNetConnect() + await rm(tempDir, { recursive: true, force: true }) +}) + +function run(args: string[]): Promise<{ code: number; stdout: string; stderr: string }> { + return runCommand(['assemblies', 'create', ...args]) +} + +async function runCommand( + args: string[], +): Promise<{ code: number; stdout: string; stderr: string }> { + const stdout = vi.spyOn(console, 'log').mockImplementation(() => {}) + const stderr = vi.spyOn(console, 'error').mockImplementation(() => {}) + let parserOutput = '' + const code = await Cli.from([ + AssembliesCreateCommand, + TemplatesCreateCommand, + TemplatesDeleteCommand, + TemplatesGetCommand, + TemplatesModifyCommand, + TemplatesSyncCommand, + ]).run(args, { + stdout: new Writable({ + write(chunk, _encoding, callback) { + parserOutput += chunk.toString() + callback() + }, + }), + }) + return { + code, + stdout: stdout.mock.calls.map((args) => format(...args)).join('\n') + parserOutput, + stderr: stderr.mock.calls.map((args) => format(...args)).join('\n'), + } +} + +function mockAssembly(): void { + vi.spyOn(Transloadit.prototype, 'createAssembly').mockResolvedValue({ + ok: 'ASSEMBLY_UPLOADING', + assembly_id: assemblyId, + assembly_ssl_url: assemblyUrl, + }) + vi.spyOn(Transloadit.prototype, 'awaitAssemblyCompletion').mockResolvedValue({ + ok: 'ASSEMBLY_COMPLETED', + results: {}, + }) +} + +describe('Assembly reporting', () => { + it('reports the Assembly reference on stderr while preserving stdout', async () => { + mockAssembly() + + const result = await run(['--template', 'template']) + + expect(result.code).toBe(0) + expect(result.stdout).toBe('') + expect(result.stderr).toContain(`Assembly ID: ${assemblyId}, Assembly URL: ${assemblyUrl}`) + }) + + it.each([ + { args: [] }, + { args: ['--single-assembly'] }, + ])('includes the reference in JSON: %j', async ({ args }) => { + mockAssembly() + + const result = await run(['--template', 'template', '-j', '-l', 'err', ...args]) + + expect(result.code).toBe(0) + expect(JSON.parse(result.stdout)).toMatchObject({ + assembly_id: assemblyId, + assembly_ssl_url: assemblyUrl, + ok: 'ASSEMBLY_COMPLETED', + }) + expect(result.stderr).toBe('') + }) + + it('summarizes ignored errors and preserves their details in JSON', async () => { + mockAssembly() + const ignoredErrors = [ + { step: 'resized', phase: 'meta', error: 'INVALID_FILE', message: 'Cannot read broken.jpg' }, + ] + vi.spyOn(Transloadit.prototype, 'awaitAssemblyCompletion').mockResolvedValue({ + ok: 'ASSEMBLY_COMPLETED', + ignored_errors: ignoredErrors, + ignored_error_count: 3, + results: {}, + }) + + const result = await run(['--template', 'template', '--single-assembly', '-j']) + + expect(result.code).toBe(0) + expect(result.stderr).toContain('3 ignored errors') + expect(result.stderr).toContain('resized') + expect(result.stderr).toContain('Cannot read broken.jpg') + expect(JSON.parse(result.stdout)).toMatchObject({ + assembly_id: assemblyId, + ignored_errors: ignoredErrors, + ignored_error_count: 3, + }) + }) + + it('limits ignored-error JSON to diagnostic fields and safe structured errors', async () => { + mockAssembly() + vi.spyOn(Transloadit.prototype, 'awaitAssemblyCompletion').mockResolvedValue({ + ok: 'ASSEMBLY_COMPLETED', + ignored_errors: [ + { + step: 'stored', + phase: 'export', + error: 'ACCESS_DENIED', + message: 'Storage refused access.', + request: { headers: { authorization: 'credential' } }, + }, + { + step: 'imported', + error: { + code: 'FETCH_ERROR', + message: 'Could not fetch the input.', + params: { auth: { secret: 'credential' } }, + }, + }, + { phase: 'meta', error: [{ secret: 'credential' }] }, + ], + results: {}, + }) + + const result = await run(['--template', 'template', '-j']) + + expect(result.code).toBe(0) + expect(JSON.parse(result.stdout)).toMatchObject({ ignored_error_count: 3 }) + expect(JSON.parse(result.stdout).ignored_errors).toEqual([ + { + step: 'stored', + phase: 'export', + error: 'ACCESS_DENIED', + message: 'Storage refused access.', + }, + { step: 'imported', error: { code: 'FETCH_ERROR', message: 'Could not fetch the input.' } }, + { phase: 'meta' }, + ]) + }) + + it.each([ + { + diagnostics: { + error: { request: 'credential' }, + message: { request: 'credential' }, + ignored_errors: [ + { + step: { request: 'credential' }, + phase: { request: 'credential' }, + message: { request: 'credential' }, + error: { code: { request: 'credential' }, message: 'Storage refused access.' }, + }, + ], + }, + expected: { + ignored_error_count: 1, + ignored_errors: [{ error: { message: 'Storage refused access.' } }], + }, + failure: 'Unknown Assembly error', + }, + { + diagnostics: { + error: 'S3_STORE_ACCESS_DENIED', + message: 'Storage refused access.', + ignored_errors: { request: 'credential' }, + }, + expected: { + error: 'S3_STORE_ACCESS_DENIED', + message: 'Storage refused access.', + ignored_error_count: 0, + ignored_errors: [], + }, + failure: 'S3_STORE_ACCESS_DENIED: Storage refused access.', + }, + ])('sanitizes malformed API diagnostics before reporting: $failure', async ({ + diagnostics, + expected, + failure, + }) => { + vi.spyOn(Transloadit.prototype, 'createAssembly').mockResolvedValue({ + ok: 'ASSEMBLY_UPLOADING', + assembly_id: assemblyId, + assembly_ssl_url: assemblyUrl, + }) + const api = nock('https://api2.transloadit.com') + .get(`/assemblies/${assemblyId}`) + .query(true) + .reply(200, { + assembly_id: { request: 'credential' }, + assembly_url: { request: 'credential' }, + assembly_ssl_url: { request: 'credential' }, + ok: { request: 'credential' }, + ignored_error_count: { request: 'credential' }, + ...diagnostics, + }) + + const result = await run(['--template', 'template', '-j']) + + expect(api.isDone()).toBe(true) + expect(result.code).toBe(1) + expect(result.stdout).toContain(assemblyId) + expect(JSON.parse(result.stdout)).toEqual({ + assembly_id: assemblyId, + assembly_ssl_url: assemblyUrl, + ...expected, + }) + expect(result.stderr.split('\n').find((line) => line.startsWith('err '))).toBe( + `err Assembly failed: ${failure} (Assembly ID: ${assemblyId}, Assembly URL: ${assemblyUrl})`, + ) + }) + + it('uses the error list length when no ignored error count is supplied', async () => { + mockAssembly() + vi.spyOn(Transloadit.prototype, 'awaitAssemblyCompletion').mockResolvedValue({ + ok: 'ASSEMBLY_COMPLETED', + ignored_errors: [{ step: 'resized', error: 'INVALID_FILE' }], + results: {}, + }) + + const result = await run(['--template', 'template']) + + expect(result.code).toBe(0) + expect(result.stderr).toContain('1 ignored error') + expect(result.stderr).toContain('resized') + }) + + it('includes safe structured ignored-error details in warnings', async () => { + mockAssembly() + vi.spyOn(Transloadit.prototype, 'awaitAssemblyCompletion').mockResolvedValue({ + ok: 'ASSEMBLY_COMPLETED', + ignored_errors: [ + { + step: 'stored', + error: { code: 'S3_STORE_ACCESS_DENIED', message: 'Storage refused access.' }, + }, + ], + results: {}, + }) + + const result = await run(['--template', 'template']) + + expect(result.code).toBe(0) + expect(result.stderr).toContain('stored: S3_STORE_ACCESS_DENIED: Storage refused access.') + }) + + it('omits empty ignored-error details while preserving their count and JSON', async () => { + mockAssembly() + const ignoredErrors = [{ error: {} }, { step: 'resized', error: 'INVALID_FILE' }] + vi.spyOn(Transloadit.prototype, 'awaitAssemblyCompletion').mockResolvedValue({ + ok: 'ASSEMBLY_COMPLETED', + ignored_errors: ignoredErrors, + results: {}, + }) + + const result = await run(['--template', 'template', '-j']) + + expect(result.code).toBe(0) + const warning = result.stderr.split('\n').find((line) => line.startsWith('warn ')) + expect(warning).toBe( + `warn 2 ignored errors (Assembly ID: ${assemblyId}, Assembly URL: ${assemblyUrl}): resized: INVALID_FILE`, + ) + expect(JSON.parse(result.stdout).ignored_errors).toEqual(ignoredErrors) + }) + + it('reports invalid html wait_until once with its path and allowed values', async () => { + const create = vi.spyOn(Transloadit.prototype, 'createAssembly') + const stepsPath = path.join(tempDir, 'steps.json') + await writeFile( + stepsPath, + JSON.stringify({ + shot: { robot: '/html/convert', url: 'https://example.com/', wait_until: 'bogus' }, + }), + ) + + const result = await run(['--steps', stepsPath]) + + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).toBe( + 'err Invalid steps format: shot.wait_until: Expected one of: domcontentloaded, load, networkidle, commit', + ) + expect(create).not.toHaveBeenCalled() + }) + + it('uses singular wording when one validation error is omitted', async () => { + const file = path.join(tempDir, 'steps.json') + await writeFile( + file, + JSON.stringify( + Object.fromEntries( + Array.from({ length: 11 }, (_, index) => [ + `shot${index}`, + { robot: '/html/convert', url: 'https://example.com/', wait_until: 'bogus' }, + ]), + ), + ), + ) + + const result = await run(['--steps', file]) + + expect(result.code).toBe(1) + expect(result.stderr.split('; ')).toHaveLength(11) + expect(result.stderr).toMatch(/; 1 more validation error$/) + }) + + it('reports a template API failure once at the command boundary', async () => { + const api = nock('https://api2.transloadit.com') + .get('/templates/missing-template') + .query(true) + .reply(403, { error: 'TEMPLATE_NOT_FOUND', message: 'There was no Template found.' }) + + const result = await runCommand(['templates', 'get', 'missing-template']) + + expect(api.isDone()).toBe(true) + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).toBe( + 'err API error (HTTP 403) TEMPLATE_NOT_FOUND: There was no Template found.', + ) + }) + + it('reports invalid template JSON once at the command boundary', async () => { + const file = path.join(tempDir, 'invalid.json') + await writeFile(file, '{') + + const result = await runCommand(['templates', 'create', 'name', file]) + + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).toBe('err Invalid steps format: expected valid JSON') + }) + + it('reports template sync API failures without inspecting the request object', async () => { + const file = path.join(tempDir, 'template.json') + await writeFile(file, JSON.stringify({ transloadit_template_id: '', steps: {} })) + const api = nock('https://api2.transloadit.com') + .post('/templates') + .reply(403, { error: 'AUTH_DENIED', message: 'This key cannot create Templates.' }) + + const result = await runCommand(['templates', 'sync', file]) + + expect(api.isDone()).toBe(true) + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).toBe( + 'err API error (HTTP 403) AUTH_DENIED: This key cannot create Templates.', + ) + }) + + it('reports transport failures on stderr without a stack on stdout', async () => { + vi.spyOn(Transloadit.prototype, 'getTemplate').mockRejectedValue( + new RequestError( + 'Connection reset by peer', + { code: 'ECONNRESET' }, + new Options('https://api2.transloadit.com/templates/template'), + ), + ) + + const result = await runCommand(['templates', 'get', 'template']) + + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).toBe('err ECONNRESET: Connection reset by peer') + }) + + it('reports an invalid JSON API response without the signed request URL', async () => { + const api = nock('https://api2.transloadit.com') + .get('/templates/template') + .query(true) + .reply(200, 'invalid JSON') + + const result = await runCommand(['templates', 'get', 'template']) + + expect(api.isDone()).toBe(true) + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).toBe('err ERR_BODY_PARSE_FAILURE: The API returned invalid JSON.') + }) + + it('reports every failed Template deletion once with its ID', async () => { + const api = nock('https://api2.transloadit.com') + .delete('/templates/first-template') + .reply(403, { error: 'AUTH_DENIED', message: 'First Template cannot be deleted.' }) + .delete('/templates/second-template') + .reply(403, { error: 'AUTH_DENIED', message: 'Second Template cannot be deleted.' }) + + const result = await runCommand(['templates', 'delete', 'first-template', 'second-template']) + + expect(api.isDone()).toBe(true) + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr.split('\n')).toEqual([ + 'err Template first-template: API error (HTTP 403) AUTH_DENIED: First Template cannot be deleted.', + 'err Template second-template: API error (HTTP 403) AUTH_DENIED: Second Template cannot be deleted.', + ]) + }) + + it('explains a missing Template input file on stderr', async () => { + const file = path.join(tempDir, 'missing.json') + + const result = await runCommand(['templates', 'create', 'name', file]) + + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).toContain('err Cannot read Template input: ENOENT:') + expect(result.stderr).toContain(file) + expect(result.stderr.split('\n')).toHaveLength(1) + }) + + it('explains missing steps when modifying a Template', async () => { + const file = path.join(tempDir, 'empty.json') + await writeFile(file, '') + const api = nock('https://api2.transloadit.com') + .get('/templates/template') + .query(true) + .reply(200, { name: 'name', content: {} }) + + const result = await runCommand(['templates', 'modify', 'template', file]) + + expect(api.isDone()).toBe(true) + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).toBe('err No steps to update template with') + }) + + it('explains missing steps and ID when syncing a Template', async () => { + const file = path.join(tempDir, 'empty.json') + await writeFile(file, JSON.stringify({ transloadit_template_id: '' })) + + const result = await runCommand(['templates', 'sync', file]) + + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).toBe(`err Template file has no id and no steps: ${file}`) + }) + + it('keeps a sync input filesystem error on stderr after the initial scan', async () => { + const file = path.join(tempDir, 'template.json') + await writeFile(file, JSON.stringify({ transloadit_template_id: 'template', steps: {} })) + vi.spyOn(fs.promises, 'stat') + .mockResolvedValueOnce(await stat(file)) + .mockRejectedValueOnce(new Error('Template input disappeared')) + + const result = await runCommand(['templates', 'sync', file]) + + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).toBe('err Cannot read Template input: Template input disappeared') + }) + + it.each([ + { status: 403, error: 'TEMPLATE_NOT_FOUND' }, + { status: 404, error: 'SERVER_404' }, + ])('explains a nonexistent Template referenced by a sync file: %j', async ({ status, error }) => { + const file = path.join(tempDir, 'missing.json') + await writeFile(file, JSON.stringify({ transloadit_template_id: 'missing', steps: {} })) + const api = nock('https://api2.transloadit.com') + .get('/templates/missing') + .query(true) + .reply(status, { error, message: 'Template not found.' }) + + const result = await runCommand(['templates', 'sync', file]) + + expect(api.isDone()).toBe(true) + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).toBe(`err Template file references nonexistent template: ${file}`) + }) + + it('explains a missing steps input file on stderr', async () => { + const file = path.join(tempDir, 'missing.json') + + const result = await run(['--steps', file]) + + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).toContain('err Cannot read steps input: ENOENT:') + expect(result.stderr).toContain(file) + expect(result.stderr.split('\n')).toHaveLength(1) + }) + + it.each([ + { args: [] }, + { args: ['--single-assembly'] }, + ])('formats real HTTP 400 errors concisely: %j', async ({ args }) => { + const api = nock('https://api2.transloadit.com') + .post(/\/assemblies\/[a-f\d]{32}$/) + .reply(400, { + error: 'TEMPLATE_NOT_FOUND', + message: 'There was no Template found.', + assembly_id: assemblyId, + assembly_ssl_url: assemblyUrl, + }) + + const result = await run(['--template', 'missing-template', ...args]) + + expect(api.isDone()).toBe(true) + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr.split('\n')).toHaveLength(1) + expect(result.stderr).toContain('TEMPLATE_NOT_FOUND: There was no Template found.') + expect(result.stderr).toContain(assemblyUrl) + }) + + it('reports the request Assembly URL when the HTTP error body omits it', async () => { + let requestUrl = '' + const api = nock('https://api2.transloadit.com') + .post(/\/assemblies\/[a-f\d]{32}$/) + .reply((uri) => { + requestUrl = `https://api2.transloadit.com${uri}` + return [400, { error: 'TEMPLATE_NOT_FOUND', message: 'There was no Template found.' }] + }) + + const result = await run(['--template', 'missing-template']) + + expect(api.isDone()).toBe(true) + expect(result.code).toBe(1) + expect(result.stderr.split('\n')).toHaveLength(1) + expect(result.stderr).toContain(`Assembly URL: ${requestUrl}`) + }) + + it('preserves file bytes on stdout with --json --output -', async () => { + mockAssembly() + vi.spyOn(Transloadit.prototype, 'awaitAssemblyCompletion').mockResolvedValue({ + ok: 'ASSEMBLY_COMPLETED', + results: { imported: [{ name: 'file.txt', url: 'http://downloads.test/file.txt' }] }, + }) + nock('http://downloads.test').get('/file.txt').reply(200, 'file contents') + let streamed = '' + vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { + streamed += String(chunk) + return true + }) + + const result = await run(['--template', 'template', '-j', '-o', '-']) + + expect(result.code).toBe(0) + expect(result.stdout).toBe('') + expect(streamed).toBe('file contents') + expect(result.stderr).toContain(assemblyUrl) + }) + + it('keeps --json --print-urls as one document with Assembly metadata', async () => { + mockAssembly() + vi.spyOn(Transloadit.prototype, 'awaitAssemblyCompletion').mockResolvedValue({ + ok: 'ASSEMBLY_COMPLETED', + results: { imported: [{ name: 'file.txt', url: 'https://results.test/file.txt' }] }, + }) + + const result = await run(['--template', 'template', '-j', '--print-urls']) + + expect(result.code).toBe(0) + expect(JSON.parse(result.stdout)).toMatchObject({ + urls: [ + { assemblyId, step: 'imported', name: 'file.txt', url: 'https://results.test/file.txt' }, + ], + assemblies: [{ assembly_id: assemblyId, assembly_ssl_url: assemblyUrl }], + }) + }) + + it('includes the Assembly URL only once in polling error diagnostics', async () => { + vi.spyOn(Transloadit.prototype, 'createAssembly').mockResolvedValue({ + ok: 'ASSEMBLY_UPLOADING', + assembly_id: assemblyId, + assembly_ssl_url: assemblyUrl, + }) + const api = nock('https://api2.transloadit.com') + .get(`/assemblies/${assemblyId}`) + .query(true) + .reply(403, { + error: 'ASSEMBLY_NOT_FOUND', + message: 'The Assembly could not be accessed.', + }) + + const result = await run(['--template', 'template']) + + expect(api.isDone()).toBe(true) + expect(result.code).toBe(1) + const diagnostic = result.stderr.split('\n').find((line) => line.startsWith('err ')) ?? '' + expect(diagnostic).toContain('ASSEMBLY_NOT_FOUND: The Assembly could not be accessed.') + expect(diagnostic.split(assemblyUrl)).toHaveLength(2) + }) + + it('redacts invalid JSON polling responses while preserving the Assembly reference', async () => { + vi.spyOn(Transloadit.prototype, 'createAssembly').mockResolvedValue({ + ok: 'ASSEMBLY_UPLOADING', + assembly_id: assemblyId, + assembly_ssl_url: assemblyUrl, + }) + const api = nock('https://api2.transloadit.com') + .get(`/assemblies/${assemblyId}`) + .query(true) + .reply(200, 'invalid JSON') + + const result = await run(['--template', 'template', '-l', 'err']) + + expect(api.isDone()).toBe(true) + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).toBe( + `err ERR_BODY_PARSE_FAILURE: The API returned invalid JSON. (Assembly ID: ${assemblyId}, Assembly URL: ${assemblyUrl})`, + ) + }) + + it.each([ + { args: [] }, + { args: ['--single-assembly'] }, + ])('preserves the accepted Assembly reference when uploading fails: %j', async ({ args }) => { + const file = path.join(tempDir, 'input.txt') + await writeFile(file, 'input') + const api = nock('https://api2.transloadit.com') + .post(/\/assemblies\/[a-f\d]{32}$/) + .reply(200, { + ok: 'ASSEMBLY_UPLOADING', + assembly_id: assemblyId, + assembly_ssl_url: assemblyUrl, + tus_url: 'https://uploads.test/files', + }) + const upload = nock('https://uploads.test').post('/files').reply(403, 'Access denied.') + + const result = await run(['--template', 'template', '-i', file, '-l', 'err', ...args]) + + expect(api.isDone()).toBe(true) + expect(upload.isDone()).toBe(true) + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).toContain(`Assembly ID: ${assemblyId}, Assembly URL: ${assemblyUrl}`) + expect(result.stderr.split('\n')).toHaveLength(1) + }) + + it.each([ + { args: [] }, + { args: ['--single-assembly'] }, + ])('preserves the attempted Assembly reference on a creation reset: %j', async ({ args }) => { + const error = new RequestError( + 'Connection reset by peer', + { code: 'ECONNRESET' }, + new Options(assemblyUrl), + ) + const create = vi + .spyOn(Transloadit.prototype, 'createAssembly') + .mockImplementation(() => Object.assign(Promise.reject(error), { assemblyId })) + + const result = await run(['--template', 'template', '-l', 'err', ...args]) + + expect(create).toHaveBeenCalledOnce() + expect(result.code).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).toBe( + `err ECONNRESET: Connection reset by peer (Assembly ID: ${assemblyId}, Assembly URL: ${assemblyUrl})`, + ) + }) + + it.each([ + 'S3_STORE_ACCESS_DENIED', + 'TIGRIS_STORE_ACCESS_DENIED', + ])('reports a terminal %s with its message and reference', async (error) => { + mockAssembly() + vi.spyOn(Transloadit.prototype, 'awaitAssemblyCompletion').mockResolvedValue({ + error, + message: 'The storage credentials do not allow access to this bucket.', + }) + + const result = await run(['--template', 'template', '-j']) + + expect(result.code).toBe(1) + expect(result.stderr.split('\n')).toHaveLength(2) + expect(result.stderr).toContain( + `Assembly failed: ${error}: The storage credentials do not allow access to this bucket. (Assembly ID: ${assemblyId}, Assembly URL: ${assemblyUrl})`, + ) + expect(JSON.parse(result.stdout)).toMatchObject({ + assembly_id: assemblyId, + assembly_ssl_url: assemblyUrl, + error, + message: 'The storage credentials do not allow access to this bucket.', + }) + }) +}) diff --git a/packages/node/test/unit/cli/run-command.test.ts b/packages/node/test/unit/cli/run-command.test.ts index ced85177..08e87b7b 100644 --- a/packages/node/test/unit/cli/run-command.test.ts +++ b/packages/node/test/unit/cli/run-command.test.ts @@ -68,6 +68,7 @@ describe('run command', () => { expect(process.exitCode).toBeUndefined() expect(createAssembly).toHaveBeenCalledExactlyOnceWith({ params: { steps: instructions.steps, fields: {} }, + onAssemblyCreated: expect.any(Function), signal: expect.any(AbortSignal), }) expect(fs.createReadStream).not.toHaveBeenCalled() diff --git a/packages/transloadit/README.md b/packages/transloadit/README.md index 87cc0eb9..99809328 100644 --- a/packages/transloadit/README.md +++ b/packages/transloadit/README.md @@ -265,6 +265,19 @@ npx -y transloadit auth token --aud mcp --scope assemblies:write,templates:read ### Processing Media +Assembly runs print their ID and status URL to stderr, including when processing fails. +`--json` writes newline-delimited JSON summaries for terminal Assembly statuses, with their +reference, final status, and `ignored_errors`. Ignored errors also produce a warning with their +count and up to five details. +If polling fails before a terminal status is retrieved, the error and known Assembly reference +are reported on stderr. +With `--print-urls`, JSON remains a single object containing `urls` and `assemblies`. +This document is written after the batch finishes; an input or watch error that aborts the batch +can prevent it from being written. +`--output -` keeps Assembly summaries off stdout; the reference stays on stderr. +When creating Assemblies, invalid Robot parameters and API failures produce a short diagnostic and +a nonzero exit status. + For common one-off tasks, prefer the intent-first commands: The full generated intent reference also lives in [`docs/intent-commands.md`](./docs/intent-commands.md). From 4fbd847be134ddffcc66bf135913400fa76b38b2 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 22:28:11 +0200 Subject: [PATCH 17/35] Close upload file streams when Assembly creation fails (#612) * Close upload file streams when Assembly creation fails * Track and close tus chunk streams on upload failure * Track tus chunks from caller-supplied file streams * Wait for pending file slices before rejecting Assembly creation * Stop sends after abort and clean up caller cancellation * Guard queued cancellation at the HTTP send boundary * Stop tus retry timers after startup cancellation * Allow upload listeners on the private cancellation signal * Synchronize retry cancellation tests with send attempts --- .changeset/close-upload-streams.md | 7 + packages/mcp-server/src/server.ts | 4 +- packages/node/src/Transloadit.ts | 69 ++- packages/node/src/tus.ts | 77 +++- .../test/unit/create-assembly-streams.test.ts | 421 ++++++++++++++++++ 5 files changed, 566 insertions(+), 12 deletions(-) create mode 100644 .changeset/close-upload-streams.md create mode 100644 packages/node/test/unit/create-assembly-streams.test.ts diff --git a/.changeset/close-upload-streams.md b/.changeset/close-upload-streams.md new file mode 100644 index 00000000..65921b7d --- /dev/null +++ b/.changeset/close-upload-streams.md @@ -0,0 +1,7 @@ +--- +'@transloadit/node': patch +'@transloadit/mcp-server': patch +'transloadit': patch +--- + +Close SDK-opened upload file streams before a failed or aborted `createAssembly()` rejects, so callers can immediately remove temporary files without leaking file handles. diff --git a/packages/mcp-server/src/server.ts b/packages/mcp-server/src/server.ts index 789c80be..1d7762c2 100644 --- a/packages/mcp-server/src/server.ts +++ b/packages/mcp-server/src/server.ts @@ -1717,9 +1717,7 @@ export const createTransloaditMcpServer = ( const known = await settleWithin(Promise.race([created, work]), statusReadTimeoutMs) if (known === deadlineReached) { // Without the request, nothing can create this Assembly later, so a status check - // that finds no Assembly makes creating it again safe. The SDK keeps the read - // streams it opened for the files until they are collected, as on any failed - // creation; closing them needs an SDK change. + // that finds no Assembly makes creating it again safe. creationAbort.abort() return creationAssemblyId ? buildCreationPending( diff --git a/packages/node/src/Transloadit.ts b/packages/node/src/Transloadit.ts index e2eddb60..f1d97749 100644 --- a/packages/node/src/Transloadit.ts +++ b/packages/node/src/Transloadit.ts @@ -1,3 +1,4 @@ +import type { ReadStream } from 'node:fs' import type { Readable } from 'node:stream' import type { @@ -72,9 +73,11 @@ import type { Stream, UploadBehavior } from './tus.ts' import * as assert from 'node:assert' import { randomUUID } from 'node:crypto' +import { addAbortListener, setMaxListeners } from 'node:events' import { constants, createReadStream } from 'node:fs' import { access, stat } from 'node:fs/promises' import { basename } from 'node:path' +import { finished } from 'node:stream/promises' import { setTimeout as delay } from 'node:timers/promises' import { compileAssemblyInstructionsFromPrompt } from '@transloadit/utils' @@ -241,11 +244,15 @@ const toReadableUpload = (label: string, value: Readable | IntoStreamInput): Rea return readable ? value : intoStream(value) } -const buildStreamsMap = (descriptors: UploadDescriptor[]): Record => +const buildStreamsMap = ( + descriptors: UploadDescriptor[], + onFileStreamOpened?: (stream: ReadStream) => void, +): Record => Object.fromEntries( descriptors.map((descriptor) => { if (descriptor.path) { const stream = createReadStream(descriptor.path) + onFileStreamOpened?.(stream) return [descriptor.label, { stream, path: descriptor.path }] } @@ -705,7 +712,7 @@ export class Transloadit { assemblyId, expectedUploads, onAssemblyCreated, - signal, + signal: callerSignal, uploadBehavior = 'await', } = opts @@ -722,6 +729,34 @@ export class Transloadit { effectiveAssemblyId = randomUUID().replace(/-/g, '') } const urlSuffix = `/assemblies/${effectiveAssemblyId}` + const uploadAbort = new AbortController() + const signal = uploadAbort.signal + // This private signal coordinates every concurrent upload and its queue. + setMaxListeners(0, signal) + if (callerSignal?.aborted) uploadAbort.abort(callerSignal.reason) + // Older supported Node releases retain AbortSignal.any() dependencies indefinitely. + const callerAbort = + callerSignal && !callerSignal.aborted + ? addAbortListener(callerSignal, () => uploadAbort.abort(callerSignal.reason)) + : undefined + const detachCallerAbort = (): void => { + callerAbort?.[Symbol.dispose]() + } + let backgroundUploadsComplete: Promise | undefined + const openedFileStreams = new Set() + const pendingFileSlices = new Set>() + const onFileStreamOpened = (stream: ReadStream): void => { + // Register before I/O can emit close, so cleanup cannot miss that event. + openedFileStreams.add(stream) + stream.once('close', () => openedFileStreams.delete(stream)) + } + const onFileSliceStarted = (slice: Promise): void => { + pendingFileSlices.add(slice) + const remove = (): void => { + pendingFileSlices.delete(slice) + } + slice.then(remove, remove) + } // We want to be able to return the promise immediately with custom data const promise = (async () => { @@ -746,7 +781,7 @@ export class Transloadit { })), ] - const allStreamsMap = buildStreamsMap(descriptors) + const allStreamsMap = buildStreamsMap(descriptors, onFileStreamOpened) const allStreams = Object.values(allStreamsMap) @@ -779,7 +814,7 @@ export class Transloadit { onAssemblyCreated?.(result) if (Object.keys(allStreamsMap).length > 0) { - const { uploadUrls } = await sendTusRequest({ + const { uploadUrls, uploadsComplete } = await sendTusRequest({ streamsMap: allStreamsMap, assembly: result, onProgress: onUploadProgress, @@ -787,7 +822,10 @@ export class Transloadit { uploadConcurrency, signal, uploadBehavior, + onFileStreamOpened, + onFileSliceStarted, }) + if (uploadBehavior === 'background') backgroundUploadsComplete = uploadsComplete if (uploadBehavior !== 'await' && Object.keys(uploadUrls).length > 0) { result.upload_urls = uploadUrls } @@ -813,6 +851,29 @@ export class Transloadit { return Promise.race([createAssemblyAndUpload(), streamErrorPromise]) })() + .catch(async (error: unknown) => { + uploadAbort.abort() + // File descriptors must be closed before callers can remove their temporary files. + // Destroyed, unread streams reject finished(); cleanup must preserve the creation error. + while (openedFileStreams.size > 0 || pendingFileSlices.size > 0) { + await Promise.allSettled(pendingFileSlices) + await Promise.allSettled( + [...openedFileStreams].map((stream) => { + const closed = finished(stream, { cleanup: true }) + stream.destroy() + return closed + }), + ) + } + throw error + }) + .finally(() => { + if (backgroundUploadsComplete && !signal.aborted) { + backgroundUploadsComplete.then(detachCallerAbort, detachCallerAbort) + return + } + detachCallerAbort() + }) // This allows the user to use or log the assemblyId even before it has been created for easier debugging return Object.assign(promise, { assemblyId: effectiveAssemblyId }) diff --git a/packages/node/src/tus.ts b/packages/node/src/tus.ts index f7b48b1b..09d952c7 100644 --- a/packages/node/src/tus.ts +++ b/packages/node/src/tus.ts @@ -1,16 +1,24 @@ import type { Readable } from 'node:stream' -import type { OnSuccessPayload, UploadOptions } from 'tus-js-client' +import type { + FileSource, + HttpRequest, + HttpResponse, + OnSuccessPayload, + SliceResult, + UploadOptions, +} from 'tus-js-client' import type { AssemblyStatus } from './alphalib/types/assemblyStatus.ts' import type { UploadProgress } from './Transloadit.ts' +import { ReadStream } from 'node:fs' import { stat } from 'node:fs/promises' import { basename } from 'node:path' import debug from 'debug' import pMap from 'p-map' -import { Upload } from 'tus-js-client' +import { defaultOptions, Upload } from 'tus-js-client' const log = debug('transloadit') const logWarn = debug('transloadit:warn') @@ -31,6 +39,13 @@ interface SendTusRequestOptions { signal?: AbortSignal uploadUrls?: Record uploadBehavior?: UploadBehavior + onFileStreamOpened?: (stream: ReadStream) => void + onFileSliceStarted?: (slice: Promise) => void +} + +interface SendTusRequestResult { + uploadUrls: Record + uploadsComplete?: Promise } export async function sendTusRequest({ @@ -42,7 +57,9 @@ export async function sendTusRequest({ signal, uploadUrls, uploadBehavior = 'await', -}: SendTusRequestOptions) { + onFileStreamOpened, + onFileSliceStarted, +}: SendTusRequestOptions): Promise { const streamLabels = Object.keys(streamsMap) let totalBytes = 0 @@ -203,6 +220,25 @@ export async function sendTusRequest({ fieldname: label, filename, }, + httpStack: { + createRequest(method: string, url: string): HttpRequest { + const request = defaultOptions.httpStack.createRequest(method, url) + const send = request.send.bind(request) + request.send = (body: unknown): Promise => { + // tus awaits pre-request hooks, so queued aborts must be checked at send itself. + if (signal?.aborted) { + // File setup can finish after abort() and reset tus's aborted state. + tusUpload.abort() + return Promise.reject(new Error('Upload aborted')) + } + return send(body) + } + return request + }, + getName(): string { + return defaultOptions.httpStack.getName() + }, + }, onError: reject, onProgress: onTusProgress, onSuccess: resolve, @@ -218,6 +254,36 @@ export async function sendTusRequest({ } }, } + if (onFileStreamOpened) { + // tus opens a separate ReadStream for each chunk; abort() leaves those streams open. + // Track those chunks even when the original file stream belongs to the caller. + tusOptions.fileReader = { + async openFile(input: unknown, chunkSize: number): Promise { + const source = await defaultOptions.fileReader.openFile(input, chunkSize) + return { + size: source.size, + async slice(start: number, end: number): Promise { + if (signal?.aborted) throw new Error('Upload aborted') + const reading = source.slice(start, end).then((chunk) => { + // tus-js-client declares platform-specific chunk values as any. + const value: unknown = chunk.value + if (value instanceof ReadStream) onFileStreamOpened(value) + if (signal?.aborted) throw new Error('Upload aborted') + return chunk + }) + // A file slice can open its stream before the promise resolves. + if (input instanceof ReadStream && input.path != null) { + onFileSliceStarted?.(reading) + } + return await reading + }, + close() { + source.close() + }, + } + }, + } + } // tus-js-client doesn't like undefined/null if (size != null) tusOptions.uploadSize = size if (chunkSize) tusOptions.chunkSize = chunkSize @@ -255,13 +321,14 @@ export async function sendTusRequest({ await Promise.all(uploadUrlPromises) + const uploadsComplete = Promise.allSettled(completionPromises) if (uploadBehavior === 'await') { await Promise.all(completionPromises) } else { - Promise.allSettled(completionPromises).catch((err) => { + uploadsComplete.catch((err) => { logWarn('Background upload failed', err) }) } - return { uploadUrls: uploadUrlsResult } + return { uploadUrls: uploadUrlsResult, uploadsComplete } } diff --git a/packages/node/test/unit/create-assembly-streams.test.ts b/packages/node/test/unit/create-assembly-streams.test.ts new file mode 100644 index 00000000..5bbff99a --- /dev/null +++ b/packages/node/test/unit/create-assembly-streams.test.ts @@ -0,0 +1,421 @@ +import type { ReadStream } from 'node:fs' + +import type { HttpRequest } from 'tus-js-client' + +import { getEventListeners, setMaxListeners } from 'node:events' +import fs from 'node:fs' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { ClientRequest } from 'node:http' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { PassThrough } from 'node:stream' +import { setImmediate } from 'node:timers/promises' + +import nock from 'nock' +import { defaultOptions, Upload } from 'tus-js-client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { Transloadit } from '../../src/Transloadit.ts' +import * as tus from '../../src/tus.ts' + +vi.mock('node:fs', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + createReadStream: vi.fn((...args: Parameters) => { + const stream = actual.createReadStream(...args) + openedStreams.push(stream) + return stream + }), + } +}) + +const openedStreams = vi.hoisted((): ReadStream[] => []) +const endpoint = 'http://assembly.test' + +async function mockTusUpload(directory: string, retryOffset?: number): Promise { + // The Node build of tus-js-client uses require('fs'), outside Vitest's module mock. + const createReadStream = fs.createReadStream + vi.spyOn(fs, 'createReadStream').mockImplementation((...args) => { + const stream = createReadStream(...args) + openedStreams.push(stream) + return stream + }) + await writeFile(join(directory, 'first.txt'), Buffer.alloc(4 * 1024 * 1024)) + nock(endpoint) + .post(/\/assemblies\/[a-f\d]{32}$/) + .reply(200, { + ok: 'ASSEMBLY_UPLOADING', + assembly_url: `${endpoint}/assemblies/test`, + assembly_ssl_url: `${endpoint}/assemblies/test`, + tus_url: `${endpoint}/tus/`, + }) + const tus = nock(endpoint) + .post('/tus/') + .reply(201, '', { Location: `${endpoint}/tus/file` }) + if (retryOffset !== undefined) { + tus + .patch('/tus/file') + .reply(500) + .head('/tus/file') + .reply(200, '', { + 'Upload-Offset': String(retryOffset), + 'Upload-Length': String(4 * 1024 * 1024), + }) + } + tus.patch('/tus/file').reply(204, '', { 'Upload-Offset': String(4 * 1024 * 1024) }) +} + +describe('createAssembly file stream cleanup', () => { + let directory: string + let files: Record + let client: Transloadit + + beforeEach(async () => { + openedStreams.length = 0 + directory = await mkdtemp(join(tmpdir(), 'assembly-streams-')) + files = { first: join(directory, 'first.txt'), second: join(directory, 'second.txt') } + await Promise.all(Object.values(files).map((path) => writeFile(path, 'upload contents'))) + client = new Transloadit({ authKey: 'key', authSecret: 'secret', endpoint, maxRetries: 0 }) + }) + + afterEach(async () => { + nock.abortPendingRequests() + nock.cleanAll() + await Promise.all( + openedStreams.map((stream) => new Promise((resolve) => stream.close(() => resolve()))), + ) + await rm(directory, { recursive: true, force: true }) + vi.restoreAllMocks() + }) + + it('closes every local file stream before a failed creation settles', async () => { + const request = nock(endpoint) + .post(/\/assemblies\/[a-f\d]{32}$/) + .reply(403, { error: 'INVALID_SIGNATURE', message: 'Invalid signature' }) + + await expect(client.createAssembly({ files })).rejects.toThrow('INVALID_SIGNATURE') + + request.done() + expect(openedStreams).toHaveLength(2) + expect(openedStreams.map((stream) => stream.destroyed)).toEqual([true, true]) + expect(openedStreams.map((stream) => stream.closed)).toEqual([true, true]) + }) + + it('closes every local file stream before an aborted creation settles', async () => { + const controller = new AbortController() + const request = nock(endpoint) + .post(/\/assemblies\/[a-f\d]{32}$/) + .delay(1000) + .reply(200, {}) + .on('request', () => controller.abort()) + + await expect(client.createAssembly({ files, signal: controller.signal })).rejects.toMatchObject( + { + name: 'AbortError', + }, + ) + + request.done() + expect(openedStreams).toHaveLength(2) + expect(openedStreams.map((stream) => stream.destroyed)).toEqual([true, true]) + expect(openedStreams.map((stream) => stream.closed)).toEqual([true, true]) + }) + + it('closes local files when another upload fails stream setup', async () => { + const invalidUpload = new PassThrough({ readable: false }) + + await expect( + client.createAssembly({ files, uploads: { invalid: invalidUpload } }), + ).rejects.toThrow('Upload named "invalid" is not a Readable stream') + + expect(openedStreams).toHaveLength(2) + expect(openedStreams.map((stream) => stream.closed)).toEqual([true, true]) + invalidUpload.destroy() + }) + + it('closes tus chunk streams before an upload abort settles', async () => { + await mockTusUpload(directory) + const controller = new AbortController() + + await expect( + client.createAssembly({ + files, + signal: controller.signal, + uploadConcurrency: 1, + onUploadProgress: () => controller.abort(), + }), + ).rejects.toThrow(/aborted/i) + + expect(openedStreams).toHaveLength(3) + expect(openedStreams.map((stream) => stream.closed)).toEqual([true, true, true]) + }) + + it('closes tus chunks for caller-owned file streams without closing the original', async () => { + await mockTusUpload(directory) + const controller = new AbortController() + const upload = fs.createReadStream(join(directory, 'first.txt')) + + await expect( + client.createAssembly({ + uploads: { file: upload }, + signal: controller.signal, + onUploadProgress: () => controller.abort(), + }), + ).rejects.toThrow(/aborted/i) + + expect(upload.destroyed).toBe(false) + const chunks = openedStreams.filter((stream) => stream !== upload) + expect(chunks).toHaveLength(1) + expect(chunks.map((stream) => stream.closed)).toEqual([true]) + }) + + it('waits for a file slice pending when creation fails', async () => { + await mockTusUpload(directory) + const openFile = defaultOptions.fileReader.openFile.bind(defaultOptions.fileReader) + let releaseSlice: () => void = () => {} + let markSliceCreated: () => void = () => {} + const sliceGate = new Promise((resolve) => { + releaseSlice = resolve + }) + const sliceCreated = new Promise((resolve) => { + markSliceCreated = resolve + }) + vi.spyOn(defaultOptions.fileReader, 'openFile').mockImplementation(async (input, chunkSize) => { + const source = await openFile(input, chunkSize) + return { + size: source.size, + async slice(start: number, end: number) { + const chunk = await source.slice(start, end) + markSliceCreated() + await sliceGate + return chunk + }, + close() { + source.close() + }, + } + }) + const upload = fs.createReadStream(join(directory, 'first.txt')) + const failure = new Error('Upload source failed') + const result = client.createAssembly({ uploads: { file: upload } }).catch((error: unknown) => ({ + error, + chunksClosed: openedStreams + .filter((stream) => stream !== upload) + .map((stream) => stream.closed), + })) + + await sliceCreated + upload.emit('error', failure) + await setImmediate() + releaseSlice() + + expect(await result).toEqual({ error: failure, chunksClosed: [true] }) + expect(upload.destroyed).toBe(false) + }) + + it.each(['synchronous', 'queued'])('stops HTTP sends for %s retry cancellation', async (mode) => { + await mockTusUpload(directory, 128 * 1024) + const controller = new AbortController() + const abortUpload = + mode === 'queued' ? () => queueMicrotask(() => controller.abort()) : () => controller.abort() + let retrying = false + let markRetrySendAttempted: () => void = () => {} + const retrySendAttempted = new Promise((resolve) => { + markRetrySendAttempted = resolve + }) + const sentAfterCancellation: string[] = [] + const requests: HttpRequest[] = [] + const startedRequests: ClientRequest[] = [] + const requestClosures: Promise[] = [] + const createRequest = defaultOptions.httpStack.createRequest.bind(defaultOptions.httpStack) + vi.spyOn(defaultOptions.httpStack, 'createRequest').mockImplementation((method, url) => { + const request = createRequest(method, url) + requests.push(request) + const send = request.send.bind(request) + request.send = (body: unknown) => { + if (controller.signal.aborted) sentAfterCancellation.push(method) + const response = send(body) + const underlyingRequest: unknown = request.getUnderlyingObject() + if (underlyingRequest instanceof ClientRequest) { + startedRequests.push(underlyingRequest) + requestClosures.push( + new Promise((resolve) => underlyingRequest.once('close', resolve)), + ) + } + return response.then((result) => { + if (result.getStatus() === 500) retrying = true + return result + }) + } + return request + }) + const start = Upload.prototype.start + vi.spyOn(Upload.prototype, 'start').mockImplementation(function (this: Upload) { + this.options.onBeforeRequest = (request) => { + if (!retrying || request.getMethod() !== 'PATCH') return + // Cancel at a retry boundary, independent of throttled progress and HEAD ordering. + const send = request.send.bind(request) + request.send = (body: unknown) => { + const response = send(body) + markRetrySendAttempted() + return response + } + abortUpload() + } + start.call(this) + }) + + try { + await expect( + client.createAssembly({ + files, + signal: controller.signal, + uploadConcurrency: 1, + }), + ).rejects.toThrow(/aborted/i) + await retrySendAttempted + + expect(sentAfterCancellation).toEqual([]) + expect(openedStreams.length).toBeGreaterThan(2) + expect(openedStreams.every((stream) => stream.destroyed && stream.closed)).toBe(true) + await Promise.all(requestClosures) + expect(startedRequests.every((request) => request.destroyed)).toBe(true) + } finally { + await Promise.all(requests.map((request) => request.abort())) + } + }) + + it('does not retry after cancellation while tus opens its file source', async () => { + await mockTusUpload(directory) + const controller = new AbortController() + const openFile = defaultOptions.fileReader.openFile.bind(defaultOptions.fileReader) + let releaseSource: () => void = () => {} + let markSourceOpened: () => void = () => {} + const sourceGate = new Promise((resolve) => { + releaseSource = resolve + }) + const sourceOpened = new Promise((resolve) => { + markSourceOpened = resolve + }) + vi.spyOn(defaultOptions.fileReader, 'openFile').mockImplementation(async (input, chunkSize) => { + const source = await openFile(input, chunkSize) + markSourceOpened() + await sourceGate + return source + }) + const start = Upload.prototype.start + const starts = vi.spyOn(Upload.prototype, 'start').mockImplementation(function (this: Upload) { + // Count unexpected retries without letting a failing test start more work. + if (!controller.signal.aborted) start.call(this) + }) + const rejected = expect( + client.createAssembly({ files, signal: controller.signal, uploadConcurrency: 1 }), + ).rejects.toThrow(/aborted/i) + + await sourceOpened + controller.abort() + await rejected + expect(openedStreams.map((stream) => stream.closed)).toEqual([true, true]) + + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + try { + releaseSource() + await vi.advanceTimersByTimeAsync(0) + expect(starts).toHaveBeenCalledTimes(1) + } finally { + vi.clearAllTimers() + vi.useRealTimers() + } + }) + + it('does not warn at default upload concurrency on legacy abort signals', async () => { + nock(endpoint) + .post(/\/assemblies\/[a-f\d]{32}$/) + .reply(200, { + ok: 'ASSEMBLY_UPLOADING', + assembly_ssl_url: `${endpoint}/assemblies/test`, + tus_url: `${endpoint}/tus/`, + }) + nock(endpoint) + .post('/tus/') + .times(10) + .reply(201, '', { Location: `${endpoint}/tus/file` }) + .patch('/tus/file') + .times(10) + .reply(204, '', { 'Upload-Offset': String('upload contents'.length) }) + const warnings: Error[] = [] + const onWarning = (warning: Error): void => { + if (warning.name === 'MaxListenersExceededWarning') warnings.push(warning) + } + class LegacyAbortController extends AbortController { + constructor() { + super() + // Node 20.10 applies this limit; newer releases default AbortSignals to unlimited. + setMaxListeners(10, this.signal) + } + } + vi.stubGlobal('AbortController', LegacyAbortController) + process.on('warning', onWarning) + try { + await client.createAssembly({ + files: Object.fromEntries( + Array.from({ length: 10 }, (_, index) => [`file${index}`, join(directory, 'first.txt')]), + ), + }) + await setImmediate() + + expect(warnings).toEqual([]) + } finally { + process.off('warning', onWarning) + vi.unstubAllGlobals() + } + }) + + it.each([ + { name: 'during upload', abortWhileUploading: true, abortCalls: 1 }, + { name: 'after completion', abortWhileUploading: false, abortCalls: 0 }, + ])('forwards background cancellation only $name', async ({ abortWhileUploading, abortCalls }) => { + const controller = new AbortController() + const onAbort = vi.fn() + let completeUploads: () => void = () => {} + const uploadsComplete = new Promise((resolve) => { + completeUploads = resolve + }) + vi.spyOn(tus, 'sendTusRequest').mockImplementation(({ signal }) => { + signal?.addEventListener('abort', onAbort, { once: true }) + return Promise.resolve({ uploadUrls: {}, uploadsComplete }) + }) + nock(endpoint) + .post(/\/assemblies\/[a-f\d]{32}$/) + .reply(200, { ok: 'ASSEMBLY_UPLOADING' }) + + await client.createAssembly({ files, signal: controller.signal, uploadBehavior: 'background' }) + if (abortWhileUploading) controller.abort() + completeUploads() + await setImmediate() + controller.abort() + + expect(onAbort).toHaveBeenCalledTimes(abortCalls) + expect(getEventListeners(controller.signal, 'abort')).toEqual([]) + }) + + it('closes files overwritten by an upload label while preserving caller-owned streams', async () => { + nock(endpoint) + .post(/\/assemblies\/[a-f\d]{32}$/) + .reply(403, { error: 'INVALID_SIGNATURE', message: 'Invalid signature' }) + const upload = new PassThrough() + + try { + await expect(client.createAssembly({ files, uploads: { first: upload } })).rejects.toThrow( + 'INVALID_SIGNATURE', + ) + + expect(openedStreams).toHaveLength(2) + expect(openedStreams.map((stream) => stream.closed)).toEqual([true, true]) + expect(upload.destroyed).toBe(false) + } finally { + upload.destroy() + } + }) +}) From 36eeeda2d8c3932fc95a91846d771ae156e8abc1 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 14:18:58 +0200 Subject: [PATCH 18/35] Match CLI downloads to Assembly results --- .changeset/assembly-download-paths.md | 7 + packages/node/README.md | 9 +- packages/node/src/cli/commands/assemblies.ts | 202 +++++++++++----- packages/node/src/cli/resultFiles.ts | 1 + .../test/unit/cli/assemblies-create.test.ts | 36 +-- .../test/unit/cli/download-layout.test.ts | 217 ++++++++++++++++++ packages/transloadit/README.md | 9 +- 7 files changed, 402 insertions(+), 79 deletions(-) create mode 100644 .changeset/assembly-download-paths.md create mode 100644 packages/node/test/unit/cli/download-layout.test.ts diff --git a/.changeset/assembly-download-paths.md b/.changeset/assembly-download-paths.md new file mode 100644 index 00000000..a8aea995 --- /dev/null +++ b/.changeset/assembly-download-paths.md @@ -0,0 +1,7 @@ +--- +'@transloadit/node': patch +'@transloadit/mcp-server': patch +'transloadit': patch +--- + +Keep CLI download filenames and HLS directories aligned with Assembly results, omit input parent folders from output paths, and support HTTP(S) URL inputs for Assembly and prompt commands. diff --git a/packages/node/README.md b/packages/node/README.md index 99809328..029388db 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -1515,7 +1515,14 @@ The `assemblies create` command additionally supports: Without uploads (for example, `/vimeo/import`, `/http/import`, or prompt-only `/image/generate` Steps), results are saved as `//`, with no input folder, including with `--single-assembly`. -- `--input, -i` - Provide a local upload, or use `-i -` to read stdin explicitly. When `--steps` + With one local input, filenames come from the Assembly results. Multiple result Steps get their + own folders; the input’s parent folders are omitted. Directory inputs retain subfolders relative + to that directory. HLS results retain the directories in `meta.relative_path`, so playlists can + find their renditions and segments. If those paths already exist, the whole result Step is saved + in a new Step folder together, preserving playlist references. +- `--input, -i` - Provide a local upload or an HTTP(S) URL, or use `-i -` to read stdin explicitly. + URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` + requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly without uploads. With `--template`, stdin always requires `-i -`. diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index cd9d90e7..40ec51f5 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -34,6 +34,7 @@ import { z } from 'zod' import { formatLintIssue } from '../../alphalib/assembly-linter.lang.en.ts' import { tryCatch } from '../../alphalib/tryCatch.ts' import { ensureUniqueCounterValue } from '../../ensureUniqueCounter.ts' +import { prepareInputFiles } from '../../inputFiles.ts' import { lintAssemblyInstructions } from '../../lintAssemblyInstructions.ts' import { lintingExamples } from '../docs/assemblyLintingExamples.ts' import { @@ -517,7 +518,7 @@ async function createExistingPathOutputPlan(outputPath: string | undefined): Pro } function dirProvider(output: string): OutputPlanProvider { - return async (inpath, indir = process.cwd()) => { + return async (inpath, indir) => { if (inpath == null) { return await createExistingPathOutputPlan(output) } @@ -525,8 +526,7 @@ function dirProvider(output: string): OutputPlanProvider { throw new Error('You must provide an input to output to a directory') } - let relpath = path.relative(indir, inpath) - relpath = relpath.replace(/^(\.\.\/)+/, '') + const relpath = indir == null ? path.basename(inpath) : path.relative(indir, inpath) const outpath = path.join(output, relpath) return await createExistingPathOutputPlan(outpath) } @@ -608,21 +608,18 @@ async function downloadResultToStdout(resultUrl: string, signal: AbortSignal): P function sanitizeResultName(value: string): string { const base = path.basename(value) - return base.replaceAll('\\', '_').replaceAll('/', '_').replaceAll('\u0000', '') + const name = base.replaceAll('\\', '_').replaceAll('/', '_').replaceAll('\u0000', '') + if (name === '' || name === '.' || name === '..') { + throw new Error('Assembly result has an unsafe name') + } + return name } async function ensureUniquePath(targetPath: string, reservedPaths: Set): Promise { const parsed = path.parse(targetPath) return await ensureUniqueCounterValue({ initialValue: targetPath, - isTaken: async (candidate) => { - if (reservedPaths.has(candidate)) { - return true - } - - const [statErr] = await tryCatch(fsp.stat(candidate)) - return statErr == null - }, + isTaken: async (candidate) => await isDownloadPathTaken(candidate, reservedPaths), reserve: (candidate) => { reservedPaths.add(candidate) }, @@ -631,8 +628,30 @@ async function ensureUniquePath(targetPath: string, reservedPaths: Set): }) } -function getResultFileName(file: NormalizedAssemblyResultFile): string { - return sanitizeResultName(file.name) +const resultPathSchema = z.object({ + meta: z.object({ relative_path: z.string().nullish() }).nullish(), +}) + +function getResultRelativePath(file: NormalizedAssemblyResultFile): string | null { + const parsed = resultPathSchema.safeParse(file.file) + if (!parsed.success) throw new Error('Assembly result has invalid relative path metadata') + const relativePath = parsed.data.meta?.relative_path + if (relativePath == null) return null + const name = sanitizeResultName(file.name) + // Adaptive video results use an empty directory for the master playlist. + if (relativePath === '') return name + + const normalizedPath = relativePath.replaceAll('\\', '/') + if ( + normalizedPath.includes('\u0000') || + path.win32.isAbsolute(normalizedPath) || + /^[a-z]:/i.test(normalizedPath) || + normalizedPath.split('/').some((part) => part === '' || part === '.' || part === '..') + ) { + throw new Error('Assembly result has an unsafe relative path') + } + + return path.join(normalizedPath, name) } interface AssemblyDownloadTarget { @@ -640,6 +659,37 @@ interface AssemblyDownloadTarget { targetPath: string | null } +async function assertNoOutputSymlinks(root: string, target: string): Promise { + const relativePath = path.relative(root, target) + if ( + relativePath.startsWith(`..${path.sep}`) || + relativePath === '..' || + path.isAbsolute(relativePath) + ) { + throw new Error('Assembly result path escapes the output directory') + } + + let current = root + for (const part of relativePath.split(path.sep)) { + current = path.join(current, part) + const [error, stats] = await tryCatch(fsp.lstat(current)) + if (error) { + if (isErrnoException(error) && error.code === 'ENOENT') return + throw error + } + if (stats.isSymbolicLink()) { + throw new Error('Assembly result path contains a symbolic link') + } + } +} + +async function isDownloadPathTaken(target: string, reservedPaths: Set): Promise { + if (reservedPaths.has(target)) return true + const [error] = await tryCatch(fsp.lstat(target)) + if (error && (!isErrnoException(error) || error.code !== 'ENOENT')) throw error + return error == null +} + const STALE_OUTPUT_GRACE_MS = 1000 function isMeaningfullyNewer(newer: Date, older: Date): boolean { @@ -650,27 +700,65 @@ async function buildDirectoryDownloadTargets({ allFiles, baseDir, groupByStep, + outputRoot, reservedPaths, }: { allFiles: NormalizedAssemblyResultFile[] baseDir: string groupByStep: boolean + outputRoot: string reservedPaths: Set }): Promise { + await assertNoOutputSymlinks(outputRoot, baseDir) await fsp.mkdir(baseDir, { recursive: true }) const targets: AssemblyDownloadTarget[] = [] + const steps = new Map< + string, + Array<{ file: NormalizedAssemblyResultFile; relativePath: string | null }> + >() for (const resultFile of allFiles) { - const targetDir = groupByStep ? path.join(baseDir, resultFile.stepName) : baseDir - await fsp.mkdir(targetDir, { recursive: true }) - - targets.push({ - resultUrl: resultFile.url, - targetPath: await ensureUniquePath( - path.join(targetDir, getResultFileName(resultFile)), - reservedPaths, - ), - }) + const files = steps.get(resultFile.stepName) ?? [] + files.push({ file: resultFile, relativePath: getResultRelativePath(resultFile) }) + steps.set(resultFile.stepName, files) + } + + for (const [stepName, files] of steps) { + const safeStepName = sanitizeResultName(stepName) + let targetDir = groupByStep ? path.join(baseDir, safeStepName) : baseDir + const linked = files.some((file) => file.relativePath != null) + const names = files.map( + ({ file, relativePath }) => relativePath ?? sanitizeResultName(file.name), + ) + if (linked && new Set(names).size !== names.length) { + throw new Error('Assembly results have duplicate relative paths') + } + + for (const name of names) { + await assertNoOutputSymlinks(outputRoot, path.join(targetDir, name)) + } + + if (linked) { + const collisions = await Promise.all( + names.map((name) => isDownloadPathTaken(path.join(targetDir, name), reservedPaths)), + ) + if (collisions.some(Boolean)) { + // Playlist references must keep their names; move the whole Step together on a rerun. + targetDir = await ensureUniquePath(path.join(baseDir, safeStepName), reservedPaths) + } + } + + for (const [index, { file }] of files.entries()) { + const targetPath = path.join(targetDir, names[index]) + await assertNoOutputSymlinks(outputRoot, targetPath) + if (linked) { + reservedPaths.add(targetPath) + } + targets.push({ + resultUrl: file.url, + targetPath: linked ? targetPath : await ensureUniquePath(targetPath, reservedPaths), + }) + } } return targets @@ -694,7 +782,6 @@ async function resolveResultDownloadTargets({ inPath, inputs, normalizedResults, - outputMode, outputPath, outputRoot, outputRootIsDirectory, @@ -705,7 +792,6 @@ async function resolveResultDownloadTargets({ inPath: string | null inputs: string[] normalizedResults: NormalizedAssemblyResults - outputMode?: 'directory' | 'file' outputPath: string | null outputRoot: string outputRootIsDirectory: boolean @@ -717,15 +803,14 @@ async function resolveResultDownloadTargets({ !singleAssembly && inPath != null && (hasDirectoryInput || inputs.length > 1) const resolveDirectoryBaseDir = (): string => { - if (!shouldGroupByInput || inPath == null) { - return outputRoot + if (hasDirectoryInput && outputPath != null) { + return allFiles.length === 1 && entries.length === 1 + ? path.dirname(outputPath) + : path.join(path.dirname(outputPath), path.parse(outputPath).name) } - if (hasDirectoryInput && outputPath != null) { - const mappedRelative = path.relative(outputRoot, outputPath) - const mappedDir = path.dirname(mappedRelative) - const mappedStem = path.parse(mappedRelative).name - return path.join(outputRoot, mappedDir === '.' ? '' : mappedDir, mappedStem) + if (!shouldGroupByInput || inPath == null) { + return outputRoot } return path.join(outputRoot, path.parse(path.basename(inPath)).name) @@ -750,27 +835,16 @@ async function resolveResultDownloadTargets({ allFiles, baseDir: outputRoot, groupByStep: inputs.length === 0, + outputRoot, reservedPaths, }) } - if (outputMode === 'directory' || outputPath == null || inPath == null) { - return await buildDirectoryDownloadTargets({ - allFiles, - baseDir: resolveDirectoryBaseDir(), - groupByStep: inPath == null || entries.length > 1, - reservedPaths, - }) - } - - if (allFiles.length === 1) { - return getSingleResultDownloadTarget(allFiles, outputPath) - } - return await buildDirectoryDownloadTargets({ allFiles, - baseDir: path.join(path.dirname(outputPath), path.parse(outputPath).name), - groupByStep: true, + baseDir: resolveDirectoryBaseDir(), + groupByStep: inPath == null || entries.length > 1, + outputRoot, reservedPaths, }) } @@ -841,7 +915,6 @@ async function materializeAssemblyResults({ inPath, inputs, normalizedResults, - outputMode, outputPath, outputRoot, outputRootIsDirectory, @@ -854,7 +927,6 @@ async function materializeAssemblyResults({ inPath: string | null inputs: string[] normalizedResults: NormalizedAssemblyResults - outputMode?: 'directory' | 'file' outputPath: string | null outputRoot: string | null outputRootIsDirectory: boolean @@ -871,7 +943,6 @@ async function materializeAssemblyResults({ inPath, inputs, normalizedResults, - outputMode, outputPath, outputRoot, outputRootIsDirectory, @@ -1327,7 +1398,33 @@ export interface AssembliesCreateOptions { const DEFAULT_CONCURRENCY = 5 // --- Main assembly create function --- +/** Create Assemblies after materializing URL inputs as temporary local uploads. */ export async function create( + outputctl: IOutputCtl, + client: Transloadit, + options: AssembliesCreateOptions, +): ReturnType { + const urlInputs = options.inputs.filter((input) => /^https?:\/\//i.test(input)) + if (urlInputs.length === 0) return await createWithLocalInputs(outputctl, client, options) + if (options.watch) throw new Error('--watch requires local inputs') + + const prepared = await prepareInputFiles({ + inputFiles: urlInputs.map((url, index) => ({ kind: 'url', field: String(index), url })), + allowPrivateUrls: false, + urlStrategy: 'download', + }) + const localPaths = new Map(urlInputs.map((url, index) => [url, prepared.files[String(index)]])) + try { + return await createWithLocalInputs(outputctl, client, { + ...options, + inputs: options.inputs.map((input) => localPaths.get(input) ?? input), + }) + } finally { + await Promise.all(prepared.cleanup.map((cleanup) => cleanup())) + } +} + +async function createWithLocalInputs( outputctl: IOutputCtl, client: Transloadit, { @@ -1626,9 +1723,8 @@ export async function create( abortSignal: abortController.signal, hasDirectoryInput: singleAssemblyMode ? false : hasDirectoryInput, inPath, - inputs: inputPaths, + inputs: singleAssemblyMode ? inputPaths : inputs, normalizedResults, - outputMode, outputPath: outputPlan?.path ?? null, outputRoot: resolvedOutput ?? null, outputRootIsDirectory, diff --git a/packages/node/src/cli/resultFiles.ts b/packages/node/src/cli/resultFiles.ts index ee2aa970..0ba559b5 100644 --- a/packages/node/src/cli/resultFiles.ts +++ b/packages/node/src/cli/resultFiles.ts @@ -1,6 +1,7 @@ export interface AssemblyResultEntryLike { basename?: unknown ext?: unknown + meta?: unknown name?: unknown ssl_url?: unknown url?: unknown diff --git a/packages/node/test/unit/cli/assemblies-create.test.ts b/packages/node/test/unit/cli/assemblies-create.test.ts index 39960fbb..5a0afb5b 100644 --- a/packages/node/test/unit/cli/assemblies-create.test.ts +++ b/packages/node/test/unit/cli/assemblies-create.test.ts @@ -21,10 +21,6 @@ async function createTempDir(prefix: string): Promise { return tempDir } -function getLegacyRelativeInputPath(inputPath: string): string { - return path.relative(process.cwd(), inputPath).replace(/^(\.\.\/)+/, '') -} - async function collectRelativeFiles(rootDir: string, currentDir = rootDir): Promise { const entries = await readdir(currentDir, { withFileTypes: true }) const files: string[] = [] @@ -322,22 +318,20 @@ describe('assemblies create', () => { }, }), ) - const legacyRelative = getLegacyRelativeInputPath(inputPath) - const baseDir = path.join(path.dirname(legacyRelative), path.parse(legacyRelative).name) expect(await collectRelativeFiles(outputDir)).toEqual([ - path.join(baseDir, 'captioned', 'video.mp4'), - path.join(baseDir, 'transcript', 'video.srt'), - path.join(baseDir, 'transcription', 'video.json'), + path.join('captioned', 'video.mp4'), + path.join('transcript', 'video.srt'), + path.join('transcription', 'video.json'), ]) - expect(await readFile(path.join(outputDir, baseDir, 'captioned', 'video.mp4'), 'utf8')).toBe( + expect(await readFile(path.join(outputDir, 'captioned', 'video.mp4'), 'utf8')).toBe( 'captioned video', ) - expect(await readFile(path.join(outputDir, baseDir, 'transcript', 'video.srt'), 'utf8')).toBe( + expect(await readFile(path.join(outputDir, 'transcript', 'video.srt'), 'utf8')).toBe( 'subtitle text', ) - expect( - await readFile(path.join(outputDir, baseDir, 'transcription', 'video.json'), 'utf8'), - ).toBe('{"words":[]}') + expect(await readFile(path.join(outputDir, 'transcription', 'video.json'), 'utf8')).toBe( + '{"words":[]}', + ) expect(downloads.isDone()).toBe(true) }) @@ -1508,7 +1502,7 @@ describe('assemblies create', () => { expect(await readFile(path.join(outputDir, 'thumb__1.jpg'), 'utf8')).toBe('second-thumb') }) - it('preserves legacy step-directory layout for generic directory outputs', async () => { + it('uses result filenames directly for single-step directory outputs', async () => { vi.spyOn(console, 'error').mockImplementation(() => {}) const tempDir = await createTempDir('transloadit-legacy-outdir-') @@ -1551,13 +1545,7 @@ describe('assemblies create', () => { } as never, ) - const legacyRelative = getLegacyRelativeInputPath(inputPath) - const legacyBaseDir = path.join(path.dirname(legacyRelative), path.parse(legacyRelative).name) - - expect(await collectRelativeFiles(outputDir)).toEqual([ - path.join(legacyBaseDir, 'thumbs', 'one.jpg'), - path.join(legacyBaseDir, 'thumbs', 'two.jpg'), - ]) + expect(await collectRelativeFiles(outputDir)).toEqual(['one.jpg', 'two.jpg']) }) it('uses the actual result filename for single-result directory outputs', async () => { @@ -1605,7 +1593,7 @@ describe('assemblies create', () => { expect(await readFile(path.join(outputDir, 'input.txt'), 'utf8')).toBe('hello') }) - it('preserves mapped out paths for legacy single-result directory outputs', async () => { + it('uses result names for directory outputs without an explicit output mode', async () => { vi.spyOn(console, 'error').mockImplementation(() => {}) const tempDir = await createTempDir('transloadit-legacy-single-result-') @@ -1640,7 +1628,7 @@ describe('assemblies create', () => { }, }) - expect(await collectRelativeFiles(outputDir)).toEqual([getLegacyRelativeInputPath(inputPath)]) + expect(await collectRelativeFiles(outputDir)).toEqual(['input.txt']) }) it('does not create an empty output file when assembly creation fails', async () => { diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts new file mode 100644 index 00000000..050c4808 --- /dev/null +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -0,0 +1,217 @@ +import type { AssemblyResultEntryLike } from '../../../src/cli/resultFiles.ts' + +import fs from 'node:fs' +import { mkdir, mkdtemp, readdir, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' + +import { Cli } from 'clipanion' +import nock from 'nock' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' + +import { AssembliesCreateCommand } from '../../../src/cli/commands/assemblies.ts' +import { Transloadit } from '../../../src/Transloadit.ts' + +const tempDirs: string[] = [] + +async function fixture(results: Record): Promise<{ + inputPath: string + outputDir: string + run: (args?: string[]) => Promise + tempDir: string +}> { + const tempDir = await mkdtemp(path.join(tmpdir(), 'transloadit-download-layout-')) + tempDirs.push(tempDir) + const stepsPath = path.join(tempDir, 'steps.json') + const inputPath = path.join(tempDir, 'inputs', 'logo.png') + const outputDir = path.join(tempDir, 'out') + await mkdir(path.dirname(inputPath)) + await writeFile(inputPath, 'input bytes') + await writeFile(stepsPath, JSON.stringify({ output: { robot: '/file/filter' } })) + vi.spyOn(Transloadit.prototype, 'createAssembly').mockResolvedValue({ + assembly_id: 'download-layout', + }) + vi.spyOn(Transloadit.prototype, 'awaitAssemblyCompletion').mockResolvedValue({ + ok: 'ASSEMBLY_COMPLETED', + results, + }) + + return { + inputPath, + outputDir, + tempDir, + run: async (args = []) => + await Cli.from([AssembliesCreateCommand]).run([ + 'assemblies', + 'create', + '--steps', + stepsPath, + '-o', + `${outputDir}/`, + ...args, + ]), + } +} + +beforeEach(async () => { + vi.stubEnv('TRANSLOADIT_KEY', 'key') + vi.stubEnv('TRANSLOADIT_SECRET', 'secret') + vi.spyOn(console, 'error').mockImplementation(() => {}) + vi.spyOn(fs, 'fstatSync').mockReturnValue(await stat('/dev/null')) +}) + +afterEach(async () => { + vi.restoreAllMocks() + vi.unstubAllEnvs() + nock.cleanAll() + nock.abortPendingRequests() + await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))) +}) + +it('keeps every HLS playlist reference reachable from its downloaded playlist', async () => { + const master = '#EXTM3U\nvariant/stream.m3u8\n' + const variant = '#EXTM3U\n#EXTINF:1,\nsegment.ts\n#EXT-X-ENDLIST\n' + const { inputPath, outputDir, run } = await fixture({ + adaptive: [ + { name: 'master.m3u8', url: 'https://downloads.test/master', meta: { relative_path: '' } }, + { + name: 'stream.m3u8', + url: 'https://downloads.test/variant', + meta: { relative_path: 'variant' }, + }, + { + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'variant' }, + }, + ], + }) + nock('https://downloads.test') + .get('/master') + .reply(200, master) + .get('/variant') + .reply(200, variant) + .get('/segment') + .reply(200, 'transport stream') + + expect(await run(['-i', inputPath])).toBe(0) + expect(await readFile(path.join(outputDir, 'master.m3u8'), 'utf8')).toBe(master) + const variantPath = path.join(outputDir, master.split('\n')[1]) + expect(await readFile(variantPath, 'utf8')).toBe(variant) + expect(await readFile(path.join(path.dirname(variantPath), variant.split('\n')[2]), 'utf8')).toBe( + 'transport stream', + ) +}) + +it.each([false, true])('uses the bundle result name (single Assembly: %s)', async (single) => { + const { inputPath, outputDir, run } = await fixture({ + bundled: [{ name: 'archive.zip', url: 'https://downloads.test/archive' }], + }) + nock('https://downloads.test').get('/archive').reply(200, 'archive bytes') + + expect(await run(['-i', inputPath, ...(single ? ['--single-assembly'] : [])])).toBe(0) + expect(await readdir(outputDir)).toEqual(['archive.zip']) + expect(await readFile(path.join(outputDir, 'archive.zip'), 'utf8')).toBe('archive bytes') +}) + +it('leaves the input folders out of a multi-step download', async () => { + const { inputPath, outputDir, run } = await fixture({ + image: [{ name: 'result.png', url: 'https://downloads.test/image' }], + text: [{ name: 'result.txt', url: 'https://downloads.test/text' }], + }) + nock('https://downloads.test') + .get('/image') + .reply(200, 'image bytes') + .get('/text') + .reply(200, 'text bytes') + + expect(await run(['-i', inputPath])).toBe(0) + expect(await readdir(outputDir)).toEqual(['image', 'text']) + expect(await readFile(path.join(outputDir, 'image', 'result.png'), 'utf8')).toBe('image bytes') + expect(await readFile(path.join(outputDir, 'text', 'result.txt'), 'utf8')).toBe('text bytes') +}) + +it.each([ + false, + true, +])('uploads URL inputs and cleans them up (single Assembly: %s)', async (single) => { + const { outputDir, run } = await fixture({ + processed: [{ name: 'remote.png', url: 'https://downloads.test/result' }], + }) + let uploadedPath = '' + vi.mocked(Transloadit.prototype.createAssembly).mockImplementation(async ({ files }) => { + uploadedPath = Object.values(files ?? {})[0] ?? '' + expect(path.basename(uploadedPath)).toBe('remote.png') + expect(await readFile(uploadedPath, 'utf8')).toBe('remote input') + return { assembly_id: 'url-upload' } + }) + nock('https://example.com').get('/remote.png?signature=example').reply(200, 'remote input') + nock('https://downloads.test').get('/result').reply(200, 'processed bytes') + + expect( + await run([ + '-i', + 'https://example.com/remote.png?signature=example', + ...(single ? ['--single-assembly'] : []), + ]), + ).toBe(0) + expect(await readFile(path.join(outputDir, 'remote.png'), 'utf8')).toBe('processed bytes') + await expect(stat(uploadedPath)).rejects.toMatchObject({ code: 'ENOENT' }) +}) + +it('rejects watch mode for URL inputs before making a download request', async () => { + const { run } = await fixture({}) + const download = nock('https://example.com').get('/remote.png').reply(200, 'remote input') + + expect(await run(['-i', 'https://example.com/remote.png', '--watch'])).toBe(1) + expect(download.isDone()).toBe(false) + expect(Transloadit.prototype.createAssembly).not.toHaveBeenCalled() +}) + +it('keeps a repeated linked download together instead of renaming playlist targets', async () => { + const { inputPath, outputDir, run } = await fixture({ + adaptive: [ + { name: 'master.m3u8', url: 'https://downloads.test/master', meta: { relative_path: '' } }, + { + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'variant' }, + }, + ], + }) + const playlist = '#EXTM3U\nvariant/segment.ts\n' + nock('https://downloads.test') + .get('/master') + .twice() + .reply(200, playlist) + .get('/segment') + .twice() + .reply(200, 'stream bytes') + + expect(await run(['-i', inputPath])).toBe(0) + expect(await run(['-i', inputPath])).toBe(0) + expect(await readFile(path.join(outputDir, 'adaptive', 'master.m3u8'), 'utf8')).toBe(playlist) + expect(await readFile(path.join(outputDir, 'adaptive', 'variant', 'segment.ts'), 'utf8')).toBe( + 'stream bytes', + ) +}) + +it('rejects a relative result path through a symlink outside the output directory', async () => { + const { inputPath, outputDir, run, tempDir } = await fixture({ + adaptive: [ + { + name: 'result.txt', + url: 'https://downloads.test/result', + meta: { relative_path: 'nested' }, + }, + ], + }) + const outside = path.join(tempDir, 'outside') + await mkdir(outputDir) + await mkdir(outside) + await symlink(outside, path.join(outputDir, 'nested')) + nock('https://downloads.test').get('/result').reply(200, 'escaped bytes') + + expect(await run(['-i', inputPath])).toBe(1) + expect(await readdir(outside)).toEqual([]) +}) diff --git a/packages/transloadit/README.md b/packages/transloadit/README.md index 99809328..029388db 100644 --- a/packages/transloadit/README.md +++ b/packages/transloadit/README.md @@ -1515,7 +1515,14 @@ The `assemblies create` command additionally supports: Without uploads (for example, `/vimeo/import`, `/http/import`, or prompt-only `/image/generate` Steps), results are saved as `//`, with no input folder, including with `--single-assembly`. -- `--input, -i` - Provide a local upload, or use `-i -` to read stdin explicitly. When `--steps` + With one local input, filenames come from the Assembly results. Multiple result Steps get their + own folders; the input’s parent folders are omitted. Directory inputs retain subfolders relative + to that directory. HLS results retain the directories in `meta.relative_path`, so playlists can + find their renditions and segments. If those paths already exist, the whole result Step is saved + in a new Step folder together, preserving playlist references. +- `--input, -i` - Provide a local upload or an HTTP(S) URL, or use `-i -` to read stdin explicitly. + URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` + requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly without uploads. With `--template`, stdin always requires `-i -`. From 12d2c55387c79607f8ed23ba64a15ff2fbdd6644 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 14:37:42 +0200 Subject: [PATCH 19/35] Preserve download ownership across concurrent and watched inputs --- packages/node/README.md | 6 +- packages/node/src/cli/commands/assemblies.ts | 139 +++++++++++--- .../test/unit/cli/download-layout.test.ts | 171 +++++++++++++++++- packages/transloadit/README.md | 6 +- 4 files changed, 288 insertions(+), 34 deletions(-) diff --git a/packages/node/README.md b/packages/node/README.md index 029388db..aec379f9 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -1520,10 +1520,12 @@ The `assemblies create` command additionally supports: to that directory. HLS results retain the directories in `meta.relative_path`, so playlists can find their renditions and segments. If those paths already exist, the whole result Step is saved in a new Step folder together, preserving playlist references. + Watch updates refresh their existing result files. Other directory runs process the inputs again, + since result filenames are unknown before processing; file outputs keep mtime-based skipping. - `--input, -i` - Provide a local upload or an HTTP(S) URL, or use `-i -` to read stdin explicitly. URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` - requires local inputs. When `--steps` - is used without `-i`, stdin is read only when it is a pipe or a redirected file. Other non-TTY + requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe + or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly without uploads. With `--template`, stdin always requires `-i -`. - `--single-assembly` - Pass all input files to a single assembly instead of one assembly per file diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 40ec51f5..62c0c46c 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -518,6 +518,8 @@ async function createExistingPathOutputPlan(outputPath: string | undefined): Pro } function dirProvider(output: string): OutputPlanProvider { + const namesByInput = new Map() + const usedNames = new Set() return async (inpath, indir) => { if (inpath == null) { return await createExistingPathOutputPlan(output) @@ -526,7 +528,18 @@ function dirProvider(output: string): OutputPlanProvider { throw new Error('You must provide an input to output to a directory') } - const relpath = indir == null ? path.basename(inpath) : path.relative(indir, inpath) + let relpath = indir == null ? namesByInput.get(inpath) : path.relative(indir, inpath) + if (relpath == null) { + const parsed = path.parse(inpath) + relpath = await ensureUniqueCounterValue({ + initialValue: parsed.base, + isTaken: (candidate) => usedNames.has(candidate), + reserve: (candidate) => usedNames.add(candidate), + nextValue: (counter) => `${parsed.name}__${counter}${parsed.ext}`, + scope: usedNames, + }) + namesByInput.set(inpath, relpath) + } const outpath = path.join(output, relpath) return await createExistingPathOutputPlan(outpath) } @@ -615,13 +628,19 @@ function sanitizeResultName(value: string): string { return name } -async function ensureUniquePath(targetPath: string, reservedPaths: Set): Promise { +async function ensureUniquePath( + targetPath: string, + reservedPaths: Map, + owner: string, + overwriteExisting: boolean, +): Promise { const parsed = path.parse(targetPath) return await ensureUniqueCounterValue({ initialValue: targetPath, - isTaken: async (candidate) => await isDownloadPathTaken(candidate, reservedPaths), + isTaken: async (candidate) => + await isDownloadPathTaken(candidate, reservedPaths, owner, overwriteExisting), reserve: (candidate) => { - reservedPaths.add(candidate) + reservedPaths.set(candidate, owner) }, nextValue: (counter) => path.join(parsed.dir, `${parsed.name}__${counter}${parsed.ext}`), scope: reservedPaths, @@ -661,6 +680,8 @@ interface AssemblyDownloadTarget { async function assertNoOutputSymlinks(root: string, target: string): Promise { const relativePath = path.relative(root, target) + // The caller explicitly selected the root; only descendants come from result/input metadata. + if (relativePath === '') return if ( relativePath.startsWith(`..${path.sep}`) || relativePath === '..' || @@ -674,7 +695,7 @@ async function assertNoOutputSymlinks(root: string, target: string): Promise): Promise { - if (reservedPaths.has(target)) return true +async function isDownloadPathTaken( + target: string, + reservedPaths: Map, + owner: string, + overwriteExisting = false, +): Promise { + const existingOwner = reservedPaths.get(target) + if (existingOwner != null) return existingOwner !== owner + if (overwriteExisting) return false const [error] = await tryCatch(fsp.lstat(target)) + if (isErrnoException(error) && error.code === 'ENOTDIR') return true if (error && (!isErrnoException(error) || error.code !== 'ENOENT')) throw error return error == null } @@ -700,14 +729,18 @@ async function buildDirectoryDownloadTargets({ allFiles, baseDir, groupByStep, + owner, + overwriteExisting, outputRoot, reservedPaths, }: { allFiles: NormalizedAssemblyResultFile[] baseDir: string groupByStep: boolean + owner: string + overwriteExisting: boolean outputRoot: string - reservedPaths: Set + reservedPaths: Map }): Promise { await assertNoOutputSymlinks(outputRoot, baseDir) await fsp.mkdir(baseDir, { recursive: true }) @@ -739,24 +772,61 @@ async function buildDirectoryDownloadTargets({ } if (linked) { - const collisions = await Promise.all( - names.map((name) => isDownloadPathTaken(path.join(targetDir, name), reservedPaths)), - ) - if (collisions.some(Boolean)) { - // Playlist references must keep their names; move the whole Step together on a rerun. - targetDir = await ensureUniquePath(path.join(baseDir, safeStepName), reservedPaths) - } + const initialDir = targetDir + const groupOwner = JSON.stringify([owner, stepName]) + // Selection and reservation share the same lock as ordinary result paths. A Step must + // move together on collisions, while later watch updates reuse its owned destinations. + targetDir = await ensureUniqueCounterValue({ + initialValue: initialDir, + isTaken: async (candidate) => { + if ( + candidate !== initialDir && + (await isDownloadPathTaken(candidate, reservedPaths, groupOwner)) + ) + return true + const collisions = await Promise.all( + names.map((name) => + isDownloadPathTaken( + path.join(candidate, name), + reservedPaths, + groupOwner, + overwriteExisting, + ), + ), + ) + return collisions.some(Boolean) + }, + reserve: (candidate) => { + if (candidate !== initialDir) reservedPaths.set(candidate, groupOwner) + for (const name of names) reservedPaths.set(path.join(candidate, name), groupOwner) + }, + nextValue: (counter) => + path.join( + baseDir, + groupByStep + ? `${safeStepName}__${counter}` + : counter === 1 + ? safeStepName + : `${safeStepName}__${counter - 1}`, + ), + scope: reservedPaths, + }) } for (const [index, { file }] of files.entries()) { const targetPath = path.join(targetDir, names[index]) - await assertNoOutputSymlinks(outputRoot, targetPath) - if (linked) { - reservedPaths.add(targetPath) - } + const selectedPath = linked + ? targetPath + : await ensureUniquePath( + targetPath, + reservedPaths, + JSON.stringify([owner, stepName, index]), + overwriteExisting, + ) + await assertNoOutputSymlinks(outputRoot, selectedPath) targets.push({ resultUrl: file.url, - targetPath: linked ? targetPath : await ensureUniquePath(targetPath, reservedPaths), + targetPath: selectedPath, }) } } @@ -782,6 +852,7 @@ async function resolveResultDownloadTargets({ inPath, inputs, normalizedResults, + overwriteExisting, outputPath, outputRoot, outputRootIsDirectory, @@ -792,15 +863,19 @@ async function resolveResultDownloadTargets({ inPath: string | null inputs: string[] normalizedResults: NormalizedAssemblyResults + overwriteExisting: boolean outputPath: string | null outputRoot: string outputRootIsDirectory: boolean - reservedPaths: Set + reservedPaths: Map singleAssembly?: boolean }): Promise { const { allFiles, entries } = normalizedResults const shouldGroupByInput = - !singleAssembly && inPath != null && (hasDirectoryInput || inputs.length > 1) + !singleAssembly && + inPath != null && + (hasDirectoryInput || (inputs.length > 1 && allFiles.length > 1)) + const owner = JSON.stringify([inPath, singleAssembly ? inputs : []]) const resolveDirectoryBaseDir = (): string => { if (hasDirectoryInput && outputPath != null) { @@ -813,7 +888,7 @@ async function resolveResultDownloadTargets({ return outputRoot } - return path.join(outputRoot, path.parse(path.basename(inPath)).name) + return path.join(outputRoot, path.parse(path.basename(outputPath ?? inPath)).name) } if (!outputRootIsDirectory) { @@ -835,6 +910,8 @@ async function resolveResultDownloadTargets({ allFiles, baseDir: outputRoot, groupByStep: inputs.length === 0, + owner, + overwriteExisting, outputRoot, reservedPaths, }) @@ -844,6 +921,8 @@ async function resolveResultDownloadTargets({ allFiles, baseDir: resolveDirectoryBaseDir(), groupByStep: inPath == null || entries.length > 1, + owner, + overwriteExisting, outputRoot, reservedPaths, }) @@ -915,6 +994,7 @@ async function materializeAssemblyResults({ inPath, inputs, normalizedResults, + overwriteExisting, outputPath, outputRoot, outputRootIsDirectory, @@ -927,11 +1007,12 @@ async function materializeAssemblyResults({ inPath: string | null inputs: string[] normalizedResults: NormalizedAssemblyResults + overwriteExisting: boolean outputPath: string | null outputRoot: string | null outputRootIsDirectory: boolean outputctl: IOutputCtl - reservedPaths: Set + reservedPaths: Map singleAssembly?: boolean }): Promise { if (outputRoot == null) { @@ -943,6 +1024,7 @@ async function materializeAssemblyResults({ inPath, inputs, normalizedResults, + overwriteExisting, outputPath, outputRoot, outputRootIsDirectory, @@ -1411,6 +1493,7 @@ export async function create( const prepared = await prepareInputFiles({ inputFiles: urlInputs.map((url, index) => ({ kind: 'url', field: String(index), url })), allowPrivateUrls: false, + // Uploading the bytes preserves :original and Template upload semantics, just as Intents do. urlStrategy: 'download', }) const localPaths = new Map(urlInputs.map((url, index) => [url, prepared.files[String(index)]])) @@ -1525,7 +1608,10 @@ async function createWithLocalInputs( recursive, watch: watchOption, singleAssembly, - reprocessStale, + // Directory plans use upload names before the Assembly's result names are known. + // Inferring freshness from those names can skip unrelated results. File outputs retain + // mtime-based skipping; directory caching would require a persistent result manifest. + reprocessStale: reprocessStale || outstat?.isDirectory(), }) // Use p-queue for concurrency management @@ -1533,7 +1619,7 @@ async function createWithLocalInputs( const results: unknown[] = [] const assemblies: AssemblySummary[] = [] const resultUrls: ResultUrlRow[] = [] - const reservedResultPaths = new Set() + const reservedResultPaths = new Map() const latestWatchJobTokenByOutputPath = new Map() let hasFailures = false let nextWatchJobToken = 0 @@ -1725,6 +1811,7 @@ async function createWithLocalInputs( inPath, inputs: singleAssemblyMode ? inputPaths : inputs, normalizedResults, + overwriteExisting: Boolean(watchOption), outputPath: outputPlan?.path ?? null, outputRoot: resolvedOutput ?? null, outputRootIsDirectory, diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 050c4808..fcb48ce5 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -1,7 +1,19 @@ import type { AssemblyResultEntryLike } from '../../../src/cli/resultFiles.ts' +import dns from 'node:dns/promises' +import { EventEmitter } from 'node:events' import fs from 'node:fs' -import { mkdir, mkdtemp, readdir, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises' +import { + mkdir, + mkdtemp, + readdir, + readFile, + rm, + stat, + symlink, + utimes, + writeFile, +} from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' @@ -10,6 +22,7 @@ import nock from 'nock' import { afterEach, beforeEach, expect, it, vi } from 'vitest' import { AssembliesCreateCommand } from '../../../src/cli/commands/assemblies.ts' +import OutputCtl from '../../../src/cli/OutputCtl.ts' import { Transloadit } from '../../../src/Transloadit.ts' const tempDirs: string[] = [] @@ -17,7 +30,7 @@ const tempDirs: string[] = [] async function fixture(results: Record): Promise<{ inputPath: string outputDir: string - run: (args?: string[]) => Promise + run: (args?: string[], output?: string) => Promise tempDir: string }> { const tempDir = await mkdtemp(path.join(tmpdir(), 'transloadit-download-layout-')) @@ -40,14 +53,14 @@ async function fixture(results: Record): Prom inputPath, outputDir, tempDir, - run: async (args = []) => + run: async (args = [], output = `${outputDir}/`) => await Cli.from([AssembliesCreateCommand]).run([ 'assemblies', 'create', '--steps', stepsPath, '-o', - `${outputDir}/`, + output, ...args, ]), } @@ -58,11 +71,13 @@ beforeEach(async () => { vi.stubEnv('TRANSLOADIT_SECRET', 'secret') vi.spyOn(console, 'error').mockImplementation(() => {}) vi.spyOn(fs, 'fstatSync').mockReturnValue(await stat('/dev/null')) + vi.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }]) }) afterEach(async () => { vi.restoreAllMocks() vi.unstubAllEnvs() + vi.doUnmock('node-watch') nock.cleanAll() nock.abortPendingRequests() await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))) @@ -215,3 +230,151 @@ it('rejects a relative result path through a symlink outside the output director expect(await run(['-i', inputPath])).toBe(1) expect(await readdir(outside)).toEqual([]) }) + +it('keeps single-result downloads flat for multiple standalone inputs', async () => { + const { inputPath, outputDir, run, tempDir } = await fixture({ + resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], + }) + const secondInput = path.join(tempDir, 'second.png') + await writeFile(secondInput, 'second input') + nock('https://downloads.test').get('/result').twice().reply(200, 'result bytes') + + expect(await run(['-i', inputPath, '-i', secondInput])).toBe(0) + expect(await readdir(outputDir)).toEqual(['result.png', 'result__1.png']) +}) + +it('accepts inputs with the same basename in different folders', async () => { + const { inputPath, outputDir, run, tempDir } = await fixture({ + resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], + }) + const secondInput = path.join(tempDir, 'logo.png') + await writeFile(secondInput, 'second input') + nock('https://downloads.test').get('/result').twice().reply(200, 'result bytes') + + expect(await run(['-i', inputPath, '-i', secondInput])).toBe(0) + expect(await readdir(outputDir)).toEqual(['result.png', 'result__1.png']) +}) + +it('does not mistake an input-named file for a current result with a different filename', async () => { + const { inputPath, outputDir, run } = await fixture({ + bundled: [{ name: 'archive.zip', url: 'https://downloads.test/result' }], + }) + await mkdir(outputDir) + await writeFile(path.join(outputDir, 'logo.png'), 'unrelated file') + await utimes(path.join(outputDir, 'logo.png'), new Date(), new Date(Date.now() + 10_000)) + nock('https://downloads.test').get('/result').reply(200, 'result bytes') + + expect(await run(['-i', inputPath])).toBe(0) + expect(await readFile(path.join(outputDir, 'archive.zip'), 'utf8')).toBe('result bytes') +}) + +it('allows an explicitly selected output root that is a symlink', async () => { + const { inputPath, outputDir, run, tempDir } = await fixture({ + resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], + }) + const destination = path.join(tempDir, 'destination') + await mkdir(destination) + await symlink(destination, outputDir) + nock('https://downloads.test').get('/result').reply(200, 'result bytes') + + expect(await run(['-i', inputPath], outputDir)).toBe(0) + expect(await readFile(path.join(destination, 'result.png'), 'utf8')).toBe('result bytes') +}) + +it('reserves concurrent linked downloads together when input stems match', async () => { + const { inputPath, outputDir, run } = await fixture({ + adaptive: [ + { name: 'master.m3u8', url: 'https://downloads.test/master', meta: { relative_path: '' } }, + { + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'video' }, + }, + ], + }) + await writeFile(path.join(path.dirname(inputPath), 'logo.jpg'), 'second input') + nock('https://downloads.test') + .get('/master') + .twice() + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/segment') + .twice() + .reply(200, 'stream bytes') + + expect(await run(['-i', path.dirname(inputPath), '--concurrency', '2'])).toBe(0) + expect(await readFile(path.join(outputDir, 'logo', 'master.m3u8'), 'utf8')).toBe( + '#EXTM3U\nvideo/segment.ts\n', + ) + expect(await readFile(path.join(outputDir, 'logo', 'adaptive', 'master.m3u8'), 'utf8')).toBe( + '#EXTM3U\nvideo/segment.ts\n', + ) + expect( + await readFile(path.join(outputDir, 'logo', 'adaptive', 'video', 'segment.ts'), 'utf8'), + ).toBe('stream bytes') +}) + +it('refreshes a watched result in place after the input changes', async () => { + const { inputPath, outputDir } = await fixture({ + resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], + }) + class FakeWatcher extends EventEmitter { + close(): void { + this.emit('close') + } + } + const watcher = new FakeWatcher() + vi.doMock('node-watch', () => ({ default: () => watcher })) + vi.resetModules() + const { create } = await import('../../../src/cli/commands/assemblies.ts') + let completions = 0 + const output = new OutputCtl() + vi.spyOn(output, 'debug').mockImplementation((message) => { + if (!String(message).startsWith('COMPLETED')) return + completions += 1 + if (completions === 1) watcher.emit('change', 'update', inputPath) + else watcher.close() + }) + nock('https://downloads.test') + .get('/result') + .reply(200, 'first result') + .get('/result') + .reply(200, 'updated result') + + const result = await create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + inputs: [inputPath], + output: `${outputDir}/`, + watch: true, + stepsData: { resized: { robot: '/image/resize', use: ':original', width: 16 } }, + }) + + expect(result.hasFailures).toBe(false) + expect(completions).toBe(2) + expect(await readdir(outputDir)).toEqual(['result.png']) + expect(await readFile(path.join(outputDir, 'result.png'), 'utf8')).toBe('updated result') +}) + +it('moves a linked Step together when an existing file occupies a rendition directory', async () => { + const { inputPath, outputDir, run } = await fixture({ + adaptive: [ + { name: 'master.m3u8', url: 'https://downloads.test/master', meta: { relative_path: '' } }, + { + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'video' }, + }, + ], + }) + await mkdir(outputDir) + await writeFile(path.join(outputDir, 'video'), 'existing file') + nock('https://downloads.test') + .get('/master') + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/segment') + .reply(200, 'stream bytes') + + expect(await run(['-i', inputPath])).toBe(0) + expect(await readFile(path.join(outputDir, 'adaptive', 'video', 'segment.ts'), 'utf8')).toBe( + 'stream bytes', + ) + expect(await readFile(path.join(outputDir, 'video'), 'utf8')).toBe('existing file') +}) diff --git a/packages/transloadit/README.md b/packages/transloadit/README.md index 029388db..aec379f9 100644 --- a/packages/transloadit/README.md +++ b/packages/transloadit/README.md @@ -1520,10 +1520,12 @@ The `assemblies create` command additionally supports: to that directory. HLS results retain the directories in `meta.relative_path`, so playlists can find their renditions and segments. If those paths already exist, the whole result Step is saved in a new Step folder together, preserving playlist references. + Watch updates refresh their existing result files. Other directory runs process the inputs again, + since result filenames are unknown before processing; file outputs keep mtime-based skipping. - `--input, -i` - Provide a local upload or an HTTP(S) URL, or use `-i -` to read stdin explicitly. URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` - requires local inputs. When `--steps` - is used without `-i`, stdin is read only when it is a pipe or a redirected file. Other non-TTY + requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe + or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly without uploads. With `--template`, stdin always requires `-i -`. - `--single-assembly` - Pass all input files to a single assembly instead of one assembly per file From e9c04358d726b30b3048f8bf9dd0d6e99e98d722 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 14:57:17 +0200 Subject: [PATCH 20/35] Guard watched downloads and resolve all input path collisions --- .changeset/assembly-download-paths.md | 2 + packages/node/src/cli/commands/assemblies.ts | 72 ++++---- .../test/e2e/cli/assemblies-create.test.ts | 26 +-- .../test/unit/cli/download-layout.test.ts | 170 +++++++++++++++++- 4 files changed, 205 insertions(+), 65 deletions(-) diff --git a/.changeset/assembly-download-paths.md b/.changeset/assembly-download-paths.md index a8aea995..a0b951d1 100644 --- a/.changeset/assembly-download-paths.md +++ b/.changeset/assembly-download-paths.md @@ -5,3 +5,5 @@ --- Keep CLI download filenames and HLS directories aligned with Assembly results, omit input parent folders from output paths, and support HTTP(S) URL inputs for Assembly and prompt commands. + +Directory runs process inputs again to discover the result filenames instead of guessing freshness from upload names. Watch updates reuse their own result paths, while existing unrelated files and outputs from other inputs remain separate. diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 62c0c46c..81dd8ed3 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -528,17 +528,19 @@ function dirProvider(output: string): OutputPlanProvider { throw new Error('You must provide an input to output to a directory') } - let relpath = indir == null ? namesByInput.get(inpath) : path.relative(indir, inpath) + const inputKey = path.resolve(inpath) + let relpath = namesByInput.get(inputKey) if (relpath == null) { - const parsed = path.parse(inpath) + const requestedPath = indir == null ? path.basename(inpath) : path.relative(indir, inpath) + const parsed = path.parse(requestedPath) relpath = await ensureUniqueCounterValue({ - initialValue: parsed.base, + initialValue: requestedPath, isTaken: (candidate) => usedNames.has(candidate), reserve: (candidate) => usedNames.add(candidate), - nextValue: (counter) => `${parsed.name}__${counter}${parsed.ext}`, + nextValue: (counter) => path.join(parsed.dir, `${parsed.name}__${counter}${parsed.ext}`), scope: usedNames, }) - namesByInput.set(inpath, relpath) + namesByInput.set(inputKey, relpath) } const outpath = path.join(output, relpath) return await createExistingPathOutputPlan(outpath) @@ -563,6 +565,7 @@ async function downloadResultToFile( resultUrl: string, outPath: string, signal: AbortSignal, + isCurrent: () => boolean, ): Promise { await fsp.mkdir(path.dirname(outPath), { recursive: true }) @@ -579,6 +582,10 @@ async function downloadResultToFile( throw dlErr } + if (!isCurrent()) { + await fsp.rm(tempPath, { force: true }) + return + } await fsp.rename(tempPath, outPath) } @@ -632,13 +639,11 @@ async function ensureUniquePath( targetPath: string, reservedPaths: Map, owner: string, - overwriteExisting: boolean, ): Promise { const parsed = path.parse(targetPath) return await ensureUniqueCounterValue({ initialValue: targetPath, - isTaken: async (candidate) => - await isDownloadPathTaken(candidate, reservedPaths, owner, overwriteExisting), + isTaken: async (candidate) => await isDownloadPathTaken(candidate, reservedPaths, owner), reserve: (candidate) => { reservedPaths.set(candidate, owner) }, @@ -679,6 +684,7 @@ interface AssemblyDownloadTarget { } async function assertNoOutputSymlinks(root: string, target: string): Promise { + // Reject preexisting descendant symlinks; another process can still race filesystem checks. const relativePath = path.relative(root, target) // The caller explicitly selected the root; only descendants come from result/input metadata. if (relativePath === '') return @@ -708,11 +714,9 @@ async function isDownloadPathTaken( target: string, reservedPaths: Map, owner: string, - overwriteExisting = false, ): Promise { const existingOwner = reservedPaths.get(target) if (existingOwner != null) return existingOwner !== owner - if (overwriteExisting) return false const [error] = await tryCatch(fsp.lstat(target)) if (isErrnoException(error) && error.code === 'ENOTDIR') return true if (error && (!isErrnoException(error) || error.code !== 'ENOENT')) throw error @@ -730,7 +734,6 @@ async function buildDirectoryDownloadTargets({ baseDir, groupByStep, owner, - overwriteExisting, outputRoot, reservedPaths, }: { @@ -738,7 +741,6 @@ async function buildDirectoryDownloadTargets({ baseDir: string groupByStep: boolean owner: string - overwriteExisting: boolean outputRoot: string reservedPaths: Map }): Promise { @@ -786,12 +788,7 @@ async function buildDirectoryDownloadTargets({ return true const collisions = await Promise.all( names.map((name) => - isDownloadPathTaken( - path.join(candidate, name), - reservedPaths, - groupOwner, - overwriteExisting, - ), + isDownloadPathTaken(path.join(candidate, name), reservedPaths, groupOwner), ), ) return collisions.some(Boolean) @@ -800,28 +797,27 @@ async function buildDirectoryDownloadTargets({ if (candidate !== initialDir) reservedPaths.set(candidate, groupOwner) for (const name of names) reservedPaths.set(path.join(candidate, name), groupOwner) }, - nextValue: (counter) => - path.join( - baseDir, - groupByStep - ? `${safeStepName}__${counter}` - : counter === 1 - ? safeStepName - : `${safeStepName}__${counter - 1}`, - ), + nextValue: (counter) => { + const suffix = groupByStep ? counter : counter - 1 + return path.join(baseDir, suffix === 0 ? safeStepName : `${safeStepName}__${suffix}`) + }, scope: reservedPaths, }) } + await fsp.mkdir(targetDir, { recursive: true }) + const nameOccurrences = new Map() for (const [index, { file }] of files.entries()) { - const targetPath = path.join(targetDir, names[index]) + const name = names[index] + const occurrence = nameOccurrences.get(name) ?? 0 + nameOccurrences.set(name, occurrence + 1) + const targetPath = path.join(targetDir, name) const selectedPath = linked ? targetPath : await ensureUniquePath( targetPath, reservedPaths, - JSON.stringify([owner, stepName, index]), - overwriteExisting, + JSON.stringify([owner, stepName, name, occurrence]), ) await assertNoOutputSymlinks(outputRoot, selectedPath) targets.push({ @@ -852,7 +848,6 @@ async function resolveResultDownloadTargets({ inPath, inputs, normalizedResults, - overwriteExisting, outputPath, outputRoot, outputRootIsDirectory, @@ -863,7 +858,6 @@ async function resolveResultDownloadTargets({ inPath: string | null inputs: string[] normalizedResults: NormalizedAssemblyResults - overwriteExisting: boolean outputPath: string | null outputRoot: string outputRootIsDirectory: boolean @@ -871,6 +865,8 @@ async function resolveResultDownloadTargets({ singleAssembly?: boolean }): Promise { const { allFiles, entries } = normalizedResults + // Keep single-result jobs flat for multi-file CLI compatibility. Input groups are only + // needed for multi-file results; reservations distinguish colliding result names. const shouldGroupByInput = !singleAssembly && inPath != null && @@ -911,7 +907,6 @@ async function resolveResultDownloadTargets({ baseDir: outputRoot, groupByStep: inputs.length === 0, owner, - overwriteExisting, outputRoot, reservedPaths, }) @@ -922,7 +917,6 @@ async function resolveResultDownloadTargets({ baseDir: resolveDirectoryBaseDir(), groupByStep: inPath == null || entries.length > 1, owner, - overwriteExisting, outputRoot, reservedPaths, }) @@ -992,9 +986,9 @@ async function materializeAssemblyResults({ abortSignal, hasDirectoryInput, inPath, + isCurrent, inputs, normalizedResults, - overwriteExisting, outputPath, outputRoot, outputRootIsDirectory, @@ -1005,9 +999,9 @@ async function materializeAssemblyResults({ abortSignal: AbortSignal hasDirectoryInput: boolean inPath: string | null + isCurrent: () => boolean inputs: string[] normalizedResults: NormalizedAssemblyResults - overwriteExisting: boolean outputPath: string | null outputRoot: string | null outputRootIsDirectory: boolean @@ -1024,7 +1018,6 @@ async function materializeAssemblyResults({ inPath, inputs, normalizedResults, - overwriteExisting, outputPath, outputRoot, outputRootIsDirectory, @@ -1037,7 +1030,7 @@ async function materializeAssemblyResults({ const [dlErr] = await tryCatch( targetPath == null ? downloadResultToStdout(resultUrl, abortSignal) - : downloadResultToFile(resultUrl, targetPath, abortSignal), + : downloadResultToFile(resultUrl, targetPath, abortSignal, isCurrent), ) if (dlErr) { if (dlErr.name === 'AbortError') { @@ -1809,9 +1802,9 @@ async function createWithLocalInputs( abortSignal: abortController.signal, hasDirectoryInput: singleAssemblyMode ? false : hasDirectoryInput, inPath, + isCurrent: () => !isSupersededWatchJob(outputPlan?.path ?? null, outputToken), inputs: singleAssemblyMode ? inputPaths : inputs, normalizedResults, - overwriteExisting: Boolean(watchOption), outputPath: outputPlan?.path ?? null, outputRoot: resolvedOutput ?? null, outputRootIsDirectory, @@ -1820,6 +1813,7 @@ async function createWithLocalInputs( singleAssembly: singleAssemblyMode, }) + if (isSupersededWatchJob(outputPlan?.path ?? null, outputToken)) return assembly outputctl.debug(`COMPLETED ${inPath ?? 'null'} ${outputPlan?.path ?? 'null'}`) if (del) { diff --git a/packages/node/test/e2e/cli/assemblies-create.test.ts b/packages/node/test/e2e/cli/assemblies-create.test.ts index 646a96e3..d1ec1ba6 100644 --- a/packages/node/test/e2e/cli/assemblies-create.test.ts +++ b/packages/node/test/e2e/cli/assemblies-create.test.ts @@ -234,7 +234,7 @@ describeLive('assemblies', { retry: 1 }, () => { ) it( - 'should detect outdir conflicts', + 'should disambiguate colliding input names in directory outputs', testCase(async (client) => { await fsp.mkdir('in') await Promise.all(['1.jpg', 'in/1.jpg'].map(imgPromise)) @@ -242,24 +242,12 @@ describeLive('assemblies', { retry: 1 }, () => { const steps = await stepsPromise() const output = new OutputCtl() - try { - await assembliesCreate(output, client, { - steps, - inputs: ['1.jpg', 'in'], - output: 'out', - }) - throw new Error('assembliesCreate didnt err; should have') - } catch (_err) { - const result = output.get() as OutputEntry[] - expect(result[result.length - 1]) - .to.have.property('type') - .that.equals('error') - expect(result[result.length - 1]) - .to.have.property('msg') - .that.equals("Output collision between 'in/1.jpg' and '1.jpg'") - } - // Small delay to allow abort signals to propagate and streams to close - await new Promise((resolve) => setTimeout(resolve, 50)) + await assembliesCreate(output, client, { + steps, + inputs: ['1.jpg', 'in'], + output: 'out', + }) + expect(await fsp.readdir('out')).to.deep.equal(['1.jpg', '1__1.jpg']) }), ) diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index fcb48ce5..923ac6a2 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -1,6 +1,5 @@ import type { AssemblyResultEntryLike } from '../../../src/cli/resultFiles.ts' -import dns from 'node:dns/promises' import { EventEmitter } from 'node:events' import fs from 'node:fs' import { @@ -25,6 +24,10 @@ import { AssembliesCreateCommand } from '../../../src/cli/commands/assemblies.ts import OutputCtl from '../../../src/cli/OutputCtl.ts' import { Transloadit } from '../../../src/Transloadit.ts' +vi.mock('node:dns/promises', () => ({ + lookup: vi.fn(async () => [{ address: '93.184.216.34', family: 4 }]), +})) + const tempDirs: string[] = [] async function fixture(results: Record): Promise<{ @@ -71,7 +74,6 @@ beforeEach(async () => { vi.stubEnv('TRANSLOADIT_SECRET', 'secret') vi.spyOn(console, 'error').mockImplementation(() => {}) vi.spyOn(fs, 'fstatSync').mockReturnValue(await stat('/dev/null')) - vi.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }]) }) afterEach(async () => { @@ -160,13 +162,13 @@ it.each([ expect(await readFile(uploadedPath, 'utf8')).toBe('remote input') return { assembly_id: 'url-upload' } }) - nock('https://example.com').get('/remote.png?signature=example').reply(200, 'remote input') + nock('https://source.invalid').get('/remote.png?signature=example').reply(200, 'remote input') nock('https://downloads.test').get('/result').reply(200, 'processed bytes') expect( await run([ '-i', - 'https://example.com/remote.png?signature=example', + 'https://source.invalid/remote.png?signature=example', ...(single ? ['--single-assembly'] : []), ]), ).toBe(0) @@ -313,10 +315,21 @@ it('reserves concurrent linked downloads together when input stems match', async ).toBe('stream bytes') }) -it('refreshes a watched result in place after the input changes', async () => { +it.each([ + { existing: false, expectedName: 'result.png', expectedFiles: ['result.png'] }, + { existing: true, expectedName: 'result__1.png', expectedFiles: ['result.png', 'result__1.png'] }, +])('refreshes its owned watch result (preexisting output: $existing)', async ({ + existing, + expectedName, + expectedFiles, +}) => { const { inputPath, outputDir } = await fixture({ resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], }) + if (existing) { + await mkdir(outputDir) + await writeFile(path.join(outputDir, 'result.png'), 'unrelated file') + } class FakeWatcher extends EventEmitter { close(): void { this.emit('close') @@ -349,8 +362,105 @@ it('refreshes a watched result in place after the input changes', async () => { expect(result.hasFailures).toBe(false) expect(completions).toBe(2) - expect(await readdir(outputDir)).toEqual(['result.png']) - expect(await readFile(path.join(outputDir, 'result.png'), 'utf8')).toBe('updated result') + expect(await readdir(outputDir)).toEqual(expectedFiles) + expect(await readFile(path.join(outputDir, expectedName), 'utf8')).toBe('updated result') + if (existing) + expect(await readFile(path.join(outputDir, 'result.png'), 'utf8')).toBe('unrelated file') +}) + +it('accepts a standalone input and a directory containing the same basename', async () => { + const { inputPath, outputDir, run, tempDir } = await fixture({ + resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], + }) + const directoryInput = path.join(tempDir, 'pictures') + await mkdir(directoryInput) + await writeFile(path.join(directoryInput, 'logo.png'), 'second input') + nock('https://downloads.test').get('/result').twice().reply(200, 'result bytes') + + expect(await run(['-i', inputPath, '-i', directoryInput])).toBe(0) + expect(await readdir(outputDir)).toEqual(['result.png', 'result__1.png']) +}) + +it('fails promptly when a regular file blocks an ordinary Step directory', async () => { + const { inputPath, outputDir, run } = await fixture({ + thumbs: [{ name: 'result.png', url: 'https://downloads.test/image' }], + text: [{ name: 'result.txt', url: 'https://downloads.test/text' }], + }) + await mkdir(outputDir) + await writeFile(path.join(outputDir, 'thumbs'), 'existing file') + const lstat = fs.promises.lstat + let probes = 0 + vi.spyOn(fs.promises, 'lstat').mockImplementation(async (...args) => { + probes += 1 + // Bound the broken implementation so a failed regression does not hang the test worker. + if (probes > 30) throw new Error('Filesystem probe budget exceeded') + return await lstat(...args) + }) + + expect(await run(['-i', inputPath])).toBe(1) + expect(probes).toBeLessThan(15) + expect(await readFile(path.join(outputDir, 'thumbs'), 'utf8')).toBe('existing file') +}) + +it('discards a slow watch download when a newer result has already been saved', async () => { + const { inputPath, outputDir } = await fixture({}) + class FakeWatcher extends EventEmitter { + close(): void { + this.emit('close') + } + } + const watcher = new FakeWatcher() + let resolveWatcherReady: (() => void) | undefined + const watcherReady = new Promise((resolve) => { + resolveWatcherReady = resolve + }) + vi.doMock('node-watch', () => ({ + default: () => { + resolveWatcherReady?.() + return watcher + }, + })) + vi.resetModules() + const { create } = await import('../../../src/cli/commands/assemblies.ts') + let count = 0 + vi.mocked(Transloadit.prototype.createAssembly).mockImplementation(async () => ({ + assembly_id: String(++count), + })) + vi.mocked(Transloadit.prototype.awaitAssemblyCompletion).mockImplementation(async (id) => ({ + ok: 'ASSEMBLY_COMPLETED', + results: { resized: [{ name: 'result.png', url: `https://downloads.test/${id}` }] }, + })) + let resolveDownloadStarted: (() => void) | undefined + const downloadStarted = new Promise((resolve) => { + resolveDownloadStarted = resolve + }) + nock('https://downloads.test') + .get('/1') + .delayBody(120) + .reply(200, () => { + resolveDownloadStarted?.() + return 'older result' + }) + .get('/2') + .reply(200, 'newer result') + const output = new OutputCtl() + vi.spyOn(output, 'debug').mockImplementation((message) => { + if (String(message).startsWith('COMPLETED')) watcher.close() + }) + + const processing = create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + inputs: [inputPath], + output: `${outputDir}/`, + watch: true, + stepsData: { resized: { robot: '/image/resize', use: ':original', width: 16 } }, + }) + await watcherReady + await downloadStarted + watcher.emit('change', 'update', inputPath) + + expect((await processing).hasFailures).toBe(false) + expect(count).toBe(2) + expect(await readFile(path.join(outputDir, 'result.png'), 'utf8')).toBe('newer result') }) it('moves a linked Step together when an existing file occupies a rendition directory', async () => { @@ -378,3 +488,49 @@ it('moves a linked Step together when an existing file occupies a rendition dire ) expect(await readFile(path.join(outputDir, 'video'), 'utf8')).toBe('existing file') }) + +it('reuses watched filenames when the Assembly returns its results in a different order', async () => { + const { inputPath, outputDir } = await fixture({}) + class FakeWatcher extends EventEmitter { + close(): void { + this.emit('close') + } + } + const watcher = new FakeWatcher() + vi.doMock('node-watch', () => ({ default: () => watcher })) + vi.resetModules() + const { create } = await import('../../../src/cli/commands/assemblies.ts') + const a = { name: 'a.png', url: 'https://downloads.test/a' } + const b = { name: 'b.png', url: 'https://downloads.test/b' } + vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) + .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', results: { resized: [a, b] } }) + .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', results: { resized: [b, a] } }) + const output = new OutputCtl() + let completions = 0 + vi.spyOn(output, 'debug').mockImplementation((message) => { + if (!String(message).startsWith('COMPLETED')) return + if (++completions === 1) watcher.emit('change', 'update', inputPath) + else watcher.close() + }) + nock('https://downloads.test') + .get('/a') + .reply(200, 'old a') + .get('/b') + .reply(200, 'old b') + .get('/b') + .reply(200, 'new b') + .get('/a') + .reply(200, 'new a') + + const result = await create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + inputs: [inputPath], + output: `${outputDir}/`, + watch: true, + stepsData: { resized: { robot: '/image/resize', use: ':original', width: 16 } }, + }) + + expect(result.hasFailures).toBe(false) + expect(await readdir(outputDir)).toEqual(['a.png', 'b.png']) + expect(await readFile(path.join(outputDir, 'a.png'), 'utf8')).toBe('new a') + expect(await readFile(path.join(outputDir, 'b.png'), 'utf8')).toBe('new b') +}) From 0769ad1b06f9a5612737d252d131cdec8bc442c6 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 15:20:43 +0200 Subject: [PATCH 21/35] Keep watch destinations stable after output collisions --- packages/node/src/cli/commands/assemblies.ts | 46 ++++++--- .../test/e2e/cli/assemblies-create.test.ts | 40 ++++++-- .../test/unit/cli/download-layout.test.ts | 98 +++++++++++++++++-- 3 files changed, 160 insertions(+), 24 deletions(-) diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 81dd8ed3..dc52633a 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -635,17 +635,25 @@ function sanitizeResultName(value: string): string { return name } +interface DownloadPathReservations { + ownersByPath: Map + pathsByOwner: Map +} + async function ensureUniquePath( targetPath: string, - reservedPaths: Map, + reservedPaths: DownloadPathReservations, owner: string, ): Promise { + const selectedPath = reservedPaths.pathsByOwner.get(owner) + if (selectedPath != null) return selectedPath const parsed = path.parse(targetPath) return await ensureUniqueCounterValue({ initialValue: targetPath, isTaken: async (candidate) => await isDownloadPathTaken(candidate, reservedPaths, owner), reserve: (candidate) => { - reservedPaths.set(candidate, owner) + reservedPaths.ownersByPath.set(candidate, owner) + reservedPaths.pathsByOwner.set(owner, candidate) }, nextValue: (counter) => path.join(parsed.dir, `${parsed.name}__${counter}${parsed.ext}`), scope: reservedPaths, @@ -712,10 +720,10 @@ async function assertNoOutputSymlinks(root: string, target: string): Promise, + reservedPaths: DownloadPathReservations, owner: string, ): Promise { - const existingOwner = reservedPaths.get(target) + const existingOwner = reservedPaths.ownersByPath.get(target) if (existingOwner != null) return existingOwner !== owner const [error] = await tryCatch(fsp.lstat(target)) if (isErrnoException(error) && error.code === 'ENOTDIR') return true @@ -742,7 +750,7 @@ async function buildDirectoryDownloadTargets({ groupByStep: boolean owner: string outputRoot: string - reservedPaths: Map + reservedPaths: DownloadPathReservations }): Promise { await assertNoOutputSymlinks(outputRoot, baseDir) await fsp.mkdir(baseDir, { recursive: true }) @@ -762,6 +770,9 @@ async function buildDirectoryDownloadTargets({ const safeStepName = sanitizeResultName(stepName) let targetDir = groupByStep ? path.join(baseDir, safeStepName) : baseDir const linked = files.some((file) => file.relativePath != null) + const groupOwner = JSON.stringify([owner, stepName]) + const selectedGroupDir = linked ? reservedPaths.pathsByOwner.get(groupOwner) : undefined + if (selectedGroupDir != null) targetDir = selectedGroupDir const names = files.map( ({ file, relativePath }) => relativePath ?? sanitizeResultName(file.name), ) @@ -775,7 +786,6 @@ async function buildDirectoryDownloadTargets({ if (linked) { const initialDir = targetDir - const groupOwner = JSON.stringify([owner, stepName]) // Selection and reservation share the same lock as ordinary result paths. A Step must // move together on collisions, while later watch updates reuse its owned destinations. targetDir = await ensureUniqueCounterValue({ @@ -791,11 +801,19 @@ async function buildDirectoryDownloadTargets({ isDownloadPathTaken(path.join(candidate, name), reservedPaths, groupOwner), ), ) - return collisions.some(Boolean) + const collides = collisions.some(Boolean) + // Moving an already downloaded Step would leave its previous playlist stale. + if (collides && selectedGroupDir != null) { + throw new Error('Assembly result path conflicts with an existing download') + } + return collides }, reserve: (candidate) => { - if (candidate !== initialDir) reservedPaths.set(candidate, groupOwner) - for (const name of names) reservedPaths.set(path.join(candidate, name), groupOwner) + if (candidate !== initialDir) reservedPaths.ownersByPath.set(candidate, groupOwner) + for (const name of names) { + reservedPaths.ownersByPath.set(path.join(candidate, name), groupOwner) + } + reservedPaths.pathsByOwner.set(groupOwner, candidate) }, nextValue: (counter) => { const suffix = groupByStep ? counter : counter - 1 @@ -861,7 +879,7 @@ async function resolveResultDownloadTargets({ outputPath: string | null outputRoot: string outputRootIsDirectory: boolean - reservedPaths: Map + reservedPaths: DownloadPathReservations singleAssembly?: boolean }): Promise { const { allFiles, entries } = normalizedResults @@ -1006,7 +1024,7 @@ async function materializeAssemblyResults({ outputRoot: string | null outputRootIsDirectory: boolean outputctl: IOutputCtl - reservedPaths: Map + reservedPaths: DownloadPathReservations singleAssembly?: boolean }): Promise { if (outputRoot == null) { @@ -1612,7 +1630,10 @@ async function createWithLocalInputs( const results: unknown[] = [] const assemblies: AssemblySummary[] = [] const resultUrls: ResultUrlRow[] = [] - const reservedResultPaths = new Map() + const reservedResultPaths: DownloadPathReservations = { + ownersByPath: new Map(), + pathsByOwner: new Map(), + } const latestWatchJobTokenByOutputPath = new Map() let hasFailures = false let nextWatchJobToken = 0 @@ -1813,6 +1834,7 @@ async function createWithLocalInputs( singleAssembly: singleAssemblyMode, }) + // Only the current watch job may delete its input; a failed newer job must retain it. if (isSupersededWatchJob(outputPlan?.path ?? null, outputToken)) return assembly outputctl.debug(`COMPLETED ${inPath ?? 'null'} ${outputPlan?.path ?? 'null'}`) diff --git a/packages/node/test/e2e/cli/assemblies-create.test.ts b/packages/node/test/e2e/cli/assemblies-create.test.ts index d1ec1ba6..636a20b3 100644 --- a/packages/node/test/e2e/cli/assemblies-create.test.ts +++ b/packages/node/test/e2e/cli/assemblies-create.test.ts @@ -400,7 +400,7 @@ describeLive('assemblies', { retry: 1 }, () => { ) it( - 'should not reprocess inputs that are older than their output', + 'should reprocess directory outputs to discover result filenames', testCase(async (client) => { const infiles = await Promise.all(['in1.jpg', 'in2.jpg', 'in3.jpg'].map(imgPromise)) const steps = await stepsPromise() @@ -414,16 +414,44 @@ describeLive('assemblies', { retry: 1 }, () => { }) const output2 = new OutputCtl() - await assembliesCreate(output2, client, { + const { results } = await assembliesCreate(output2, client, { steps, inputs: infiles, output: 'out', }) - const result = output2.get(true) as OutputEntry[] + expect(results).to.have.lengthOf(3) + expect(await fsp.readdir('out')).to.deep.equal([ + 'in1.jpg', + 'in1__1.jpg', + 'in2.jpg', + 'in3.jpg', + ]) + }), + 180_000, + ) - expect( - result.map((line) => line.msg).filter((msg) => String(msg).includes('in1.jpg')), - ).to.have.lengthOf(0) + it( + 'should skip an explicit file output newer than its input', + testCase(async (client) => { + const infile = await imgPromise() + const steps = await stepsPromise() + const inputTime = new Date(Date.now() - 60_000) + await fsp.utimes(infile, inputTime, inputTime) + await assembliesCreate(new OutputCtl(), client, { + steps, + inputs: [infile], + output: 'out.jpg', + }) + const originalOutput = await fsp.readFile('out.jpg') + + const { results } = await assembliesCreate(new OutputCtl(), client, { + steps, + inputs: [infile], + output: 'out.jpg', + }) + + expect(results).to.have.lengthOf(0) + expect(await fsp.readFile('out.jpg')).to.deep.equal(originalOutput) }), 180_000, ) diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 923ac6a2..894f64a8 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -10,6 +10,7 @@ import { rm, stat, symlink, + unlink, utimes, writeFile, } from 'node:fs/promises' @@ -316,10 +317,27 @@ it('reserves concurrent linked downloads together when input stems match', async }) it.each([ - { existing: false, expectedName: 'result.png', expectedFiles: ['result.png'] }, - { existing: true, expectedName: 'result__1.png', expectedFiles: ['result.png', 'result__1.png'] }, -])('refreshes its owned watch result (preexisting output: $existing)', async ({ + { + existing: false, + removeConflict: false, + expectedName: 'result.png', + expectedFiles: ['result.png'], + }, + { + existing: true, + removeConflict: false, + expectedName: 'result__1.png', + expectedFiles: ['result.png', 'result__1.png'], + }, + { + existing: true, + removeConflict: true, + expectedName: 'result__1.png', + expectedFiles: ['result__1.png'], + }, +])('refreshes its owned watch result (existing: $existing, removed: $removeConflict)', async ({ existing, + removeConflict, expectedName, expectedFiles, }) => { @@ -340,11 +358,15 @@ it.each([ vi.resetModules() const { create } = await import('../../../src/cli/commands/assemblies.ts') let completions = 0 + let resolveFirstCompletion: (() => void) | undefined + const firstCompletion = new Promise((resolve) => { + resolveFirstCompletion = resolve + }) const output = new OutputCtl() vi.spyOn(output, 'debug').mockImplementation((message) => { if (!String(message).startsWith('COMPLETED')) return completions += 1 - if (completions === 1) watcher.emit('change', 'update', inputPath) + if (completions === 1) resolveFirstCompletion?.() else watcher.close() }) nock('https://downloads.test') @@ -353,21 +375,85 @@ it.each([ .get('/result') .reply(200, 'updated result') - const result = await create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + const processing = create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { inputs: [inputPath], output: `${outputDir}/`, watch: true, stepsData: { resized: { robot: '/image/resize', use: ':original', width: 16 } }, }) + await firstCompletion + if (removeConflict) await unlink(path.join(outputDir, 'result.png')) + watcher.emit('change', 'update', inputPath) + const result = await processing expect(result.hasFailures).toBe(false) expect(completions).toBe(2) expect(await readdir(outputDir)).toEqual(expectedFiles) expect(await readFile(path.join(outputDir, expectedName), 'utf8')).toBe('updated result') - if (existing) + if (existing && !removeConflict) expect(await readFile(path.join(outputDir, 'result.png'), 'utf8')).toBe('unrelated file') }) +it('keeps a watched HLS Step in its selected folder after the original conflict disappears', async () => { + const { inputPath, outputDir } = await fixture({ + adaptive: [ + { name: 'master.m3u8', url: 'https://downloads.test/master', meta: { relative_path: '' } }, + { + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'video' }, + }, + ], + }) + await mkdir(outputDir) + await writeFile(path.join(outputDir, 'master.m3u8'), 'unrelated file') + class FakeWatcher extends EventEmitter { + close(): void { + this.emit('close') + } + } + const watcher = new FakeWatcher() + vi.doMock('node-watch', () => ({ default: () => watcher })) + vi.resetModules() + const { create } = await import('../../../src/cli/commands/assemblies.ts') + let resolveFirstCompletion: (() => void) | undefined + const firstCompletion = new Promise((resolve) => { + resolveFirstCompletion = resolve + }) + let completions = 0 + const output = new OutputCtl() + vi.spyOn(output, 'debug').mockImplementation((message) => { + if (!String(message).startsWith('COMPLETED')) return + if (++completions === 1) resolveFirstCompletion?.() + else watcher.close() + }) + nock('https://downloads.test') + .get('/master') + .twice() + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/segment') + .reply(200, 'old segment') + .get('/segment') + .reply(200, 'new segment') + + const processing = create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + inputs: [inputPath], + output: `${outputDir}/`, + watch: true, + stepsData: { adaptive: { robot: '/video/adaptive', use: ':original' } }, + }) + await firstCompletion + await unlink(path.join(outputDir, 'master.m3u8')) + watcher.emit('change', 'update', inputPath) + + expect((await processing).hasFailures).toBe(false) + expect(completions).toBe(2) + expect(await readdir(outputDir)).toEqual(['adaptive']) + expect(await readFile(path.join(outputDir, 'adaptive', 'video', 'segment.ts'), 'utf8')).toBe( + 'new segment', + ) +}) + it('accepts a standalone input and a directory containing the same basename', async () => { const { inputPath, outputDir, run, tempDir } = await fixture({ resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], From d92138cf6bc8950882f9c532f9c2be43ed185a29 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 15:41:40 +0200 Subject: [PATCH 22/35] Separate independent HLS bundles within a result Step --- .changeset/assembly-download-paths.md | 2 + packages/node/README.md | 9 +- packages/node/src/cli/commands/assemblies.ts | 86 ++++++++++++++++--- .../test/unit/cli/download-layout.test.ts | 71 +++++++++++++++ packages/transloadit/README.md | 9 +- 5 files changed, 155 insertions(+), 22 deletions(-) diff --git a/.changeset/assembly-download-paths.md b/.changeset/assembly-download-paths.md index a0b951d1..e556e8f6 100644 --- a/.changeset/assembly-download-paths.md +++ b/.changeset/assembly-download-paths.md @@ -6,4 +6,6 @@ Keep CLI download filenames and HLS directories aligned with Assembly results, omit input parent folders from output paths, and support HTTP(S) URL inputs for Assembly and prompt commands. +Independent HLS bundles returned by the same Step get separate source folders so their playlist and segment paths can coexist. + Directory runs process inputs again to discover the result filenames instead of guessing freshness from upload names. Watch updates reuse their own result paths, while existing unrelated files and outputs from other inputs remain separate. diff --git a/packages/node/README.md b/packages/node/README.md index aec379f9..93ae1cf0 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -1518,15 +1518,16 @@ The `assemblies create` command additionally supports: With one local input, filenames come from the Assembly results. Multiple result Steps get their own folders; the input’s parent folders are omitted. Directory inputs retain subfolders relative to that directory. HLS results retain the directories in `meta.relative_path`, so playlists can - find their renditions and segments. If those paths already exist, the whole result Step is saved - in a new Step folder together, preserving playlist references. + find their renditions and segments. Multiple HLS bundles in one Step get separate folders named + after their sources. On collisions, a linked bundle moves together to a new folder, preserving + playlist references. Watch updates refresh their existing result files. Other directory runs process the inputs again, since result filenames are unknown before processing; file outputs keep mtime-based skipping. - `--input, -i` - Provide a local upload or an HTTP(S) URL, or use `-i -` to read stdin explicitly. URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe - or a redirected file. Other non-TTY - stdin (such as an agent shell or `/dev/null`) runs the Assembly without uploads. With `--template`, + or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly + without uploads. With `--template`, stdin always requires `-i -`. - `--single-assembly` - Pass all input files to a single assembly instead of one assembly per file diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index dc52633a..305c931a 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -33,6 +33,7 @@ import { z } from 'zod' import { formatLintIssue } from '../../alphalib/assembly-linter.lang.en.ts' import { tryCatch } from '../../alphalib/tryCatch.ts' +import { assemblyStatusResultSchema } from '../../alphalib/types/assemblyStatus.ts' import { ensureUniqueCounterValue } from '../../ensureUniqueCounter.ts' import { prepareInputFiles } from '../../inputFiles.ts' import { lintAssemblyInstructions } from '../../lintAssemblyInstructions.ts' @@ -737,6 +738,69 @@ function isMeaningfullyNewer(newer: Date, older: Date): boolean { return newer.getTime() - older.getTime() > STALE_OUTPUT_GRACE_MS } +interface DirectoryResultFile { + file: NormalizedAssemblyResultFile + relativePath: string | null +} + +interface DirectoryResultGroup { + directoryName: string | null + files: DirectoryResultFile[] + ownerKey: string | null + stepName: string +} + +const resultSourceSchema = assemblyStatusResultSchema.pick({ + original_basename: true, + original_id: true, + original_name: true, +}) + +function groupDirectoryResults(allFiles: NormalizedAssemblyResultFile[]): DirectoryResultGroup[] { + const steps = new Map() + for (const file of allFiles) { + const files = steps.get(file.stepName) ?? [] + files.push({ file, relativePath: getResultRelativePath(file) }) + steps.set(file.stepName, files) + } + + const groups: DirectoryResultGroup[] = [] + for (const [stepName, stepFiles] of steps) { + if (!stepFiles.some((file) => file.relativePath != null)) { + groups.push({ directoryName: null, files: stepFiles, ownerKey: null, stepName }) + continue + } + + const sources = new Map() + for (const file of stepFiles) { + const parsed = resultSourceSchema.safeParse(file.file.file) + const source = parsed.success ? parsed.data : undefined + const sourceId = JSON.stringify(source?.original_id ?? null) + const group = sources.get(sourceId) ?? { + name: source?.original_name ?? source?.original_basename ?? null, + files: [], + } + group.files.push(file) + sources.set(sourceId, group) + } + + const nameOccurrences = new Map() + for (const { name, files } of sources.values()) { + const occurrence = nameOccurrences.get(name) ?? 0 + nameOccurrences.set(name, occurrence + 1) + groups.push({ + directoryName: + sources.size > 1 ? sanitizeResultName(path.parse(name ?? stepName).name) : null, + files, + // Upload IDs change between watch runs; source names retain logical ownership. + ownerKey: JSON.stringify([name, occurrence]), + stepName, + }) + } + } + return groups +} + async function buildDirectoryDownloadTargets({ allFiles, baseDir, @@ -756,21 +820,12 @@ async function buildDirectoryDownloadTargets({ await fsp.mkdir(baseDir, { recursive: true }) const targets: AssemblyDownloadTarget[] = [] - const steps = new Map< - string, - Array<{ file: NormalizedAssemblyResultFile; relativePath: string | null }> - >() - for (const resultFile of allFiles) { - const files = steps.get(resultFile.stepName) ?? [] - files.push({ file: resultFile, relativePath: getResultRelativePath(resultFile) }) - steps.set(resultFile.stepName, files) - } - - for (const [stepName, files] of steps) { + for (const { directoryName, files, ownerKey, stepName } of groupDirectoryResults(allFiles)) { const safeStepName = sanitizeResultName(stepName) let targetDir = groupByStep ? path.join(baseDir, safeStepName) : baseDir + if (directoryName != null) targetDir = path.join(targetDir, directoryName) const linked = files.some((file) => file.relativePath != null) - const groupOwner = JSON.stringify([owner, stepName]) + const groupOwner = JSON.stringify([owner, stepName, ownerKey]) const selectedGroupDir = linked ? reservedPaths.pathsByOwner.get(groupOwner) : undefined if (selectedGroupDir != null) targetDir = selectedGroupDir const names = files.map( @@ -786,7 +841,7 @@ async function buildDirectoryDownloadTargets({ if (linked) { const initialDir = targetDir - // Selection and reservation share the same lock as ordinary result paths. A Step must + // Selection and reservation share the same lock as ordinary result paths. A bundle must // move together on collisions, while later watch updates reuse its owned destinations. targetDir = await ensureUniqueCounterValue({ initialValue: initialDir, @@ -802,7 +857,7 @@ async function buildDirectoryDownloadTargets({ ), ) const collides = collisions.some(Boolean) - // Moving an already downloaded Step would leave its previous playlist stale. + // Moving an already downloaded bundle would leave its previous playlist stale. if (collides && selectedGroupDir != null) { throw new Error('Assembly result path conflicts with an existing download') } @@ -816,6 +871,9 @@ async function buildDirectoryDownloadTargets({ reservedPaths.pathsByOwner.set(groupOwner, candidate) }, nextValue: (counter) => { + if (directoryName != null) { + return path.join(path.dirname(initialDir), `${directoryName}__${counter}`) + } const suffix = groupByStep ? counter : counter - 1 return path.join(baseDir, suffix === 0 ? safeStepName : `${safeStepName}__${suffix}`) }, diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 894f64a8..777cf394 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -121,6 +121,77 @@ it('keeps every HLS playlist reference reachable from its downloaded playlist', ) }) +it('separates independent HLS bundles with identical paths in one Assembly Step', async () => { + const firstMaster = { + name: 'master.m3u8', + url: 'https://downloads.test/first-master', + original_id: 'first-source', + original_name: 'first.mp4', + meta: { relative_path: '' }, + } + const firstSegment = { + ...firstMaster, + name: 'segment.ts', + url: 'https://downloads.test/first-segment', + meta: { relative_path: 'video' }, + } + const secondMaster = { + ...firstMaster, + url: 'https://downloads.test/second-master', + original_id: 'second-source', + original_name: 'second.mp4', + } + const secondSegment = { + ...secondMaster, + name: 'segment.ts', + url: 'https://downloads.test/second-segment', + meta: { relative_path: 'video' }, + } + const { inputPath, outputDir, run } = await fixture({ + adaptive: [firstMaster, secondMaster, firstSegment, secondSegment], + }) + const secondInput = path.join(path.dirname(inputPath), 'second.mp4') + await writeFile(secondInput, 'second input') + nock('https://downloads.test') + .get('/first-master') + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/second-master') + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/first-segment') + .reply(200, 'first stream') + .get('/second-segment') + .reply(200, 'second stream') + + expect(await run(['--single-assembly', '-i', inputPath, '-i', secondInput])).toBe(0) + expect(await readdir(outputDir)).toEqual(['first', 'second']) + expect(await readFile(path.join(outputDir, 'first', 'master.m3u8'), 'utf8')).toBe( + '#EXTM3U\nvideo/segment.ts\n', + ) + expect(await readFile(path.join(outputDir, 'first', 'video', 'segment.ts'), 'utf8')).toBe( + 'first stream', + ) + expect(await readFile(path.join(outputDir, 'second', 'master.m3u8'), 'utf8')).toBe( + '#EXTM3U\nvideo/segment.ts\n', + ) + expect(await readFile(path.join(outputDir, 'second', 'video', 'segment.ts'), 'utf8')).toBe( + 'second stream', + ) +}) + +it('rejects duplicate paths within a single HLS bundle', async () => { + const segment = { + name: 'segment.ts', + url: 'https://downloads.test/segment', + original_id: 'same-source', + original_name: 'source.mp4', + meta: { relative_path: 'video' }, + } + const { inputPath, outputDir, run } = await fixture({ adaptive: [segment, segment] }) + + expect(await run(['-i', inputPath])).toBe(1) + expect(await readdir(outputDir)).toEqual([]) +}) + it.each([false, true])('uses the bundle result name (single Assembly: %s)', async (single) => { const { inputPath, outputDir, run } = await fixture({ bundled: [{ name: 'archive.zip', url: 'https://downloads.test/archive' }], diff --git a/packages/transloadit/README.md b/packages/transloadit/README.md index aec379f9..93ae1cf0 100644 --- a/packages/transloadit/README.md +++ b/packages/transloadit/README.md @@ -1518,15 +1518,16 @@ The `assemblies create` command additionally supports: With one local input, filenames come from the Assembly results. Multiple result Steps get their own folders; the input’s parent folders are omitted. Directory inputs retain subfolders relative to that directory. HLS results retain the directories in `meta.relative_path`, so playlists can - find their renditions and segments. If those paths already exist, the whole result Step is saved - in a new Step folder together, preserving playlist references. + find their renditions and segments. Multiple HLS bundles in one Step get separate folders named + after their sources. On collisions, a linked bundle moves together to a new folder, preserving + playlist references. Watch updates refresh their existing result files. Other directory runs process the inputs again, since result filenames are unknown before processing; file outputs keep mtime-based skipping. - `--input, -i` - Provide a local upload or an HTTP(S) URL, or use `-i -` to read stdin explicitly. URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe - or a redirected file. Other non-TTY - stdin (such as an agent shell or `/dev/null`) runs the Assembly without uploads. With `--template`, + or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly + without uploads. With `--template`, stdin always requires `-i -`. - `--single-assembly` - Pass all input files to a single assembly instead of one assembly per file From 603049bdb9aed3f9f49a80a518acddf1b3b9a90b Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 16:11:27 +0200 Subject: [PATCH 23/35] Reserve complete output paths and stage linked downloads --- .changeset/assembly-download-paths.md | 2 + packages/node/README.md | 5 +- packages/node/src/cli/commands/assemblies.ts | 313 ++++++++++++++---- .../test/unit/cli/download-layout.test.ts | 158 +++++++++ packages/transloadit/README.md | 5 +- 5 files changed, 404 insertions(+), 79 deletions(-) diff --git a/.changeset/assembly-download-paths.md b/.changeset/assembly-download-paths.md index e556e8f6..1ee0df58 100644 --- a/.changeset/assembly-download-paths.md +++ b/.changeset/assembly-download-paths.md @@ -8,4 +8,6 @@ Keep CLI download filenames and HLS directories aligned with Assembly results, o Independent HLS bundles returned by the same Step get separate source folders so their playlist and segment paths can coexist. +Reserve result paths with filesystem case sensitivity and file/directory conflicts in mind. Stage linked watch downloads before publishing them together, and validate local steps and output options before fetching URL inputs. + Directory runs process inputs again to discover the result filenames instead of guessing freshness from upload names. Watch updates reuse their own result paths, while existing unrelated files and outputs from other inputs remain separate. diff --git a/packages/node/README.md b/packages/node/README.md index 93ae1cf0..7e3dc1ed 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -1523,12 +1523,11 @@ The `assemblies create` command additionally supports: playlist references. Watch updates refresh their existing result files. Other directory runs process the inputs again, since result filenames are unknown before processing; file outputs keep mtime-based skipping. -- `--input, -i` - Provide a local upload or an HTTP(S) URL, or use `-i -` to read stdin explicitly. +- `--input, -i` - Provide a local upload or a public HTTP(S) URL, or use `-i -` to read stdin explicitly. URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly - without uploads. With `--template`, - stdin always requires `-i -`. + without uploads. With `--template`, stdin always requires `-i -`. - `--single-assembly` - Pass all input files to a single assembly instead of one assembly per file #### Log Levels diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 305c931a..095e86d3 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -637,10 +637,61 @@ function sanitizeResultName(value: string): string { } interface DownloadPathReservations { + allocations: PQueue + bundleCommits: PQueue + caseSensitive: boolean + directories: Set ownersByPath: Map pathsByOwner: Map } +async function outputNamesAreCaseSensitive(root: string): Promise { + await fsp.mkdir(root, { recursive: true }) + const probe = await fsp.mkdtemp(path.join(root, '.transloadit-case-')) + try { + await fsp.writeFile(path.join(probe, 'case'), '') + const [error] = await tryCatch(fsp.lstat(path.join(probe, 'CASE'))) + if (error && (!isErrnoException(error) || error.code !== 'ENOENT')) throw error + return error != null + } finally { + await fsp.rm(probe, { recursive: true, force: true }) + } +} + +function downloadPathKey(target: string, reservedPaths: DownloadPathReservations): string { + const absolutePath = path.resolve(target) + return reservedPaths.caseSensitive ? absolutePath : absolutePath.normalize('NFC').toLowerCase() +} + +function hasReservedFileAncestor(target: string, reservedPaths: DownloadPathReservations): boolean { + let current = downloadPathKey(target, reservedPaths) + while (true) { + if (reservedPaths.ownersByPath.has(current) && !reservedPaths.directories.has(current)) { + return true + } + const parent = path.dirname(current) + if (parent === current) return false + current = parent + } +} + +function reserveDownloadPath( + target: string, + owner: string, + reservedPaths: DownloadPathReservations, + directory = false, +): void { + const key = downloadPathKey(target, reservedPaths) + reservedPaths.ownersByPath.set(key, owner) + let current = directory ? key : path.dirname(key) + while (true) { + reservedPaths.directories.add(current) + const parent = path.dirname(current) + if (parent === current) return + current = parent + } +} + async function ensureUniquePath( targetPath: string, reservedPaths: DownloadPathReservations, @@ -653,7 +704,7 @@ async function ensureUniquePath( initialValue: targetPath, isTaken: async (candidate) => await isDownloadPathTaken(candidate, reservedPaths, owner), reserve: (candidate) => { - reservedPaths.ownersByPath.set(candidate, owner) + reserveDownloadPath(candidate, owner, reservedPaths) reservedPaths.pathsByOwner.set(owner, candidate) }, nextValue: (counter) => path.join(parsed.dir, `${parsed.name}__${counter}${parsed.ext}`), @@ -688,10 +739,49 @@ function getResultRelativePath(file: NormalizedAssemblyResultFile): string | nul } interface AssemblyDownloadTarget { + bundleRoot?: string resultUrl: string targetPath: string | null } +interface BundleDownloadTarget { + resultUrl: string + targetPath: string +} + +async function downloadResultBundle( + root: string, + targets: BundleDownloadTarget[], + signal: AbortSignal, + isCurrent: () => boolean, + commits: PQueue, + onDownload: () => void, +): Promise { + const stagingDir = await fsp.mkdtemp(path.join(root, '.transloadit-bundle-')) + const staged = targets.map((target, index) => ({ + ...target, + stagingPath: path.join(stagingDir, String(index)), + })) + try { + for (const { resultUrl, stagingPath } of staged) { + if (!isCurrent()) return + onDownload() + await downloadResultToFile(resultUrl, stagingPath, signal, () => true) + } + await commits.add(async () => { + if (!isCurrent()) return + // Finish publication once it starts, even if superseded. A failed newer job must not + // strand this playlist without its segments, and newer commits must wait for this one. + for (const { stagingPath, targetPath } of staged) { + await fsp.mkdir(path.dirname(targetPath), { recursive: true }) + await fsp.rename(stagingPath, targetPath) + } + }) + } finally { + await fsp.rm(stagingDir, { recursive: true, force: true }) + } +} + async function assertNoOutputSymlinks(root: string, target: string): Promise { // Reject preexisting descendant symlinks; another process can still race filesystem checks. const relativePath = path.relative(root, target) @@ -724,8 +814,12 @@ async function isDownloadPathTaken( reservedPaths: DownloadPathReservations, owner: string, ): Promise { - const existingOwner = reservedPaths.ownersByPath.get(target) + const key = downloadPathKey(target, reservedPaths) + const existingOwner = reservedPaths.ownersByPath.get(key) if (existingOwner != null) return existingOwner !== owner + if (reservedPaths.directories.has(key) || hasReservedFileAncestor(target, reservedPaths)) { + return true + } const [error] = await tryCatch(fsp.lstat(target)) if (isErrnoException(error) && error.code === 'ENOTDIR') return true if (error && (!isErrnoException(error) || error.code !== 'ENOENT')) throw error @@ -817,6 +911,9 @@ async function buildDirectoryDownloadTargets({ reservedPaths: DownloadPathReservations }): Promise { await assertNoOutputSymlinks(outputRoot, baseDir) + if (hasReservedFileAncestor(baseDir, reservedPaths)) { + throw new Error('Output directory conflicts with a result file') + } await fsp.mkdir(baseDir, { recursive: true }) const targets: AssemblyDownloadTarget[] = [] @@ -831,7 +928,11 @@ async function buildDirectoryDownloadTargets({ const names = files.map( ({ file, relativePath }) => relativePath ?? sanitizeResultName(file.name), ) - if (linked && new Set(names).size !== names.length) { + if ( + linked && + new Set(names.map((name) => downloadPathKey(path.join(targetDir, name), reservedPaths))) + .size !== names.length + ) { throw new Error('Assembly results have duplicate relative paths') } @@ -864,9 +965,10 @@ async function buildDirectoryDownloadTargets({ return collides }, reserve: (candidate) => { - if (candidate !== initialDir) reservedPaths.ownersByPath.set(candidate, groupOwner) + if (candidate !== initialDir) + reserveDownloadPath(candidate, groupOwner, reservedPaths, true) for (const name of names) { - reservedPaths.ownersByPath.set(path.join(candidate, name), groupOwner) + reserveDownloadPath(path.join(candidate, name), groupOwner, reservedPaths) } reservedPaths.pathsByOwner.set(groupOwner, candidate) }, @@ -881,6 +983,9 @@ async function buildDirectoryDownloadTargets({ }) } + if (hasReservedFileAncestor(targetDir, reservedPaths)) { + throw new Error('Output directory conflicts with a result file') + } await fsp.mkdir(targetDir, { recursive: true }) const nameOccurrences = new Map() for (const [index, { file }] of files.entries()) { @@ -897,6 +1002,7 @@ async function buildDirectoryDownloadTargets({ ) await assertNoOutputSymlinks(outputRoot, selectedPath) targets.push({ + bundleRoot: linked ? targetDir : undefined, resultUrl: file.url, targetPath: selectedPath, }) @@ -1089,25 +1195,53 @@ async function materializeAssemblyResults({ return } - const targets = await resolveResultDownloadTargets({ - hasDirectoryInput, - inPath, - inputs, - normalizedResults, - outputPath, - outputRoot, - outputRootIsDirectory, - reservedPaths, - singleAssembly, - }) + // Directory creation and filename reservation must share a lock across concurrent jobs. + const targets = await reservedPaths.allocations.add( + async () => + await resolveResultDownloadTargets({ + hasDirectoryInput, + inPath, + inputs, + normalizedResults, + outputPath, + outputRoot, + outputRootIsDirectory, + reservedPaths, + singleAssembly, + }), + ) - for (const { resultUrl, targetPath } of targets) { - outputctl.debug('DOWNLOADING') - const [dlErr] = await tryCatch( - targetPath == null - ? downloadResultToStdout(resultUrl, abortSignal) - : downloadResultToFile(resultUrl, targetPath, abortSignal, isCurrent), + const bundles = new Map() + const downloads: Array<() => Promise> = [] + for (const { bundleRoot, resultUrl, targetPath } of targets) { + if (bundleRoot != null && targetPath != null) { + const files = bundles.get(bundleRoot) ?? [] + files.push({ resultUrl, targetPath }) + bundles.set(bundleRoot, files) + continue + } + downloads.push(async () => { + outputctl.debug('DOWNLOADING') + if (targetPath == null) await downloadResultToStdout(resultUrl, abortSignal) + else await downloadResultToFile(resultUrl, targetPath, abortSignal, isCurrent) + }) + } + for (const [root, files] of bundles) { + downloads.push( + async () => + await downloadResultBundle( + root, + files, + abortSignal, + isCurrent, + reservedPaths.bundleCommits, + () => outputctl.debug('DOWNLOADING'), + ), ) + } + + for (const download of downloads) { + const [dlErr] = await tryCatch(download()) if (dlErr) { if (dlErr.name === 'AbortError') { continue @@ -1548,59 +1682,21 @@ export interface AssembliesCreateOptions { const DEFAULT_CONCURRENCY = 5 -// --- Main assembly create function --- -/** Create Assemblies after materializing URL inputs as temporary local uploads. */ -export async function create( - outputctl: IOutputCtl, - client: Transloadit, - options: AssembliesCreateOptions, -): ReturnType { - const urlInputs = options.inputs.filter((input) => /^https?:\/\//i.test(input)) - if (urlInputs.length === 0) return await createWithLocalInputs(outputctl, client, options) - if (options.watch) throw new Error('--watch requires local inputs') +interface PreparedCreateOptions { + effectiveStepsData: StepsInput | undefined + hasDirectoryInput: boolean + outstat: StatLike | undefined + resolvedOutput: string | null | undefined +} - const prepared = await prepareInputFiles({ - inputFiles: urlInputs.map((url, index) => ({ kind: 'url', field: String(index), url })), - allowPrivateUrls: false, - // Uploading the bytes preserves :original and Template upload semantics, just as Intents do. - urlStrategy: 'download', - }) - const localPaths = new Map(urlInputs.map((url, index) => [url, prepared.files[String(index)]])) - try { - return await createWithLocalInputs(outputctl, client, { - ...options, - inputs: options.inputs.map((input) => localPaths.get(input) ?? input), - }) - } finally { - await Promise.all(prepared.cleanup.map((cleanup) => cleanup())) - } +function isUrlInput(input: string): boolean { + return /^https?:\/\//i.test(input) } -async function createWithLocalInputs( +async function prepareCreateOptions( outputctl: IOutputCtl, - client: Transloadit, - { - steps, - stepsData, - template, - fields, - outputMode, - watch: watchOption, - recursive, - inputs, - output, - del, - reprocessStale, - singleAssembly, - concurrency = DEFAULT_CONCURRENCY, - printUrls = false, - }: AssembliesCreateOptions, -): Promise<{ - assemblies: AssemblySummary[] - resultUrls: ResultUrlRow[] - results: unknown[] - hasFailures: boolean -}> { + { steps, stepsData, outputMode, output, inputs, singleAssembly }: AssembliesCreateOptions, +): Promise { // Quick fix for https://github.com/transloadit/transloadify/issues/13 // Only default to stdout when output is undefined (not provided), not when explicitly null let resolvedOutput = output @@ -1636,7 +1732,9 @@ async function createWithLocalInputs( if (!outstat.isDirectory() && inputs.length !== 0 && !singleAssembly) { const firstInput = inputs[0] if (firstInput) { - const firstInputStat = await myStat(process.stdin, firstInput) + const firstInputStat = isUrlInput(firstInput) + ? { isDirectory: () => false } + : await myStat(process.stdin, firstInput) if (inputs.length > 1 || firstInputStat.isDirectory()) { const msg = 'Output must be a directory when specifying multiple inputs' outputctl.error(msg) @@ -1648,11 +1746,76 @@ async function createWithLocalInputs( const inputStats = await Promise.all( inputs.map(async (input) => { - if (input === '-') return null + if (input === '-' || isUrlInput(input)) return null return await myStat(process.stdin, input) }), ) const hasDirectoryInput = inputStats.some((stat) => stat?.isDirectory() === true) + return { effectiveStepsData, hasDirectoryInput, outstat, resolvedOutput } +} + +// --- Main assembly create function --- +/** Create Assemblies after materializing URL inputs as temporary local uploads. */ +export async function create( + outputctl: IOutputCtl, + client: Transloadit, + options: AssembliesCreateOptions, +): ReturnType { + const preparedOptions = await prepareCreateOptions(outputctl, options) + const urlInputs = options.inputs.filter(isUrlInput) + if (urlInputs.length === 0) + return await createWithLocalInputs(outputctl, client, options, preparedOptions) + if (options.watch) throw new Error('--watch requires local inputs') + + const prepared = await prepareInputFiles({ + inputFiles: urlInputs.map((url, index) => ({ kind: 'url', field: String(index), url })), + allowPrivateUrls: false, + // Uploading the bytes preserves :original and Template upload semantics, just as Intents do. + urlStrategy: 'download', + }) + const localPaths = new Map(urlInputs.map((url, index) => [url, prepared.files[String(index)]])) + try { + return await createWithLocalInputs( + outputctl, + client, + { + ...options, + inputs: options.inputs.map((input) => localPaths.get(input) ?? input), + }, + preparedOptions, + ) + } finally { + await Promise.all(prepared.cleanup.map((cleanup) => cleanup())) + } +} + +async function createWithLocalInputs( + outputctl: IOutputCtl, + client: Transloadit, + { + template, + fields, + watch: watchOption, + recursive, + inputs, + del, + reprocessStale, + singleAssembly, + concurrency = DEFAULT_CONCURRENCY, + printUrls = false, + }: AssembliesCreateOptions, + preparedOptions: PreparedCreateOptions, +): Promise<{ + assemblies: AssemblySummary[] + resultUrls: ResultUrlRow[] + results: unknown[] + hasFailures: boolean +}> { + const { effectiveStepsData, hasDirectoryInput, outstat, resolvedOutput } = preparedOptions + const caseSensitive = + resolvedOutput != null && outstat?.isDirectory() + ? await outputNamesAreCaseSensitive(resolvedOutput) + : true return new Promise((resolve, reject) => { const params: CreateAssemblyParams = ( @@ -1689,6 +1852,10 @@ async function createWithLocalInputs( const assemblies: AssemblySummary[] = [] const resultUrls: ResultUrlRow[] = [] const reservedResultPaths: DownloadPathReservations = { + allocations: new PQueue({ concurrency: 1 }), + bundleCommits: new PQueue({ concurrency: 1 }), + caseSensitive, + directories: new Set(), ownersByPath: new Map(), pathsByOwner: new Map(), } diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 777cf394..59f14cfd 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -21,6 +21,7 @@ import { Cli } from 'clipanion' import nock from 'nock' import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { tryCatch } from '../../../src/alphalib/tryCatch.ts' import { AssembliesCreateCommand } from '../../../src/cli/commands/assemblies.ts' import OutputCtl from '../../../src/cli/OutputCtl.ts' import { Transloadit } from '../../../src/Transloadit.ts' @@ -192,6 +193,163 @@ it('rejects duplicate paths within a single HLS bundle', async () => { expect(await readdir(outputDir)).toEqual([]) }) +it('keeps concurrent case-only result names separate on case-insensitive filesystems', async () => { + const { inputPath, outputDir, run } = await fixture({}) + const secondInput = path.join(path.dirname(inputPath), 'second.png') + await writeFile(secondInput, 'second input') + vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) + .mockResolvedValueOnce({ + ok: 'ASSEMBLY_COMPLETED', + results: { resized: [{ name: 'Photo.jpg', url: 'https://downloads.test/first' }] }, + }) + .mockResolvedValueOnce({ + ok: 'ASSEMBLY_COMPLETED', + results: { resized: [{ name: 'photo.jpg', url: 'https://downloads.test/second' }] }, + }) + async function resolveCaseInsensitivePath(value: fs.PathLike): Promise { + if (typeof value !== 'string') return value + const [error, siblings] = await tryCatch(readdir(path.dirname(value))) + if (error) { + if ('code' in error && error.code === 'ENOENT') return value + throw error + } + const match = siblings.find((name) => name.toLowerCase() === path.basename(value).toLowerCase()) + return match == null ? value : path.join(path.dirname(value), match) + } + const { lstat, rename } = fs.promises + vi.spyOn(fs.promises, 'lstat').mockImplementation( + async (target, ...args) => await lstat(await resolveCaseInsensitivePath(target), ...args), + ) + vi.spyOn(fs.promises, 'rename').mockImplementation( + async (source, target) => await rename(source, await resolveCaseInsensitivePath(target)), + ) + nock('https://downloads.test') + .get('/first') + .delayBody(40) + .reply(200, 'first result') + .get('/second') + .delayBody(40) + .reply(200, 'second result') + + expect(await run(['-i', inputPath, '-i', secondInput, '--concurrency', '2'])).toBe(0) + expect(await readdir(outputDir)).toEqual(['Photo.jpg', 'photo__1.jpg']) + expect(await readFile(path.join(outputDir, 'Photo.jpg'), 'utf8')).toBe('first result') + expect(await readFile(path.join(outputDir, 'photo__1.jpg'), 'utf8')).toBe('second result') +}) + +it.each([ + false, + true, +])('resolves file and bundle-directory conflicts (bundle first: %s)', async (bundleFirst) => { + const linked = { + adaptive: [ + { + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'video' }, + }, + ], + } + const ordinary = { output: [{ name: 'video', url: 'https://downloads.test/file' }] } + const { inputPath, outputDir, run } = await fixture( + bundleFirst ? { ...linked, ...ordinary } : { ...ordinary, ...linked }, + ) + nock('https://downloads.test') + .get('/file') + .reply(200, 'ordinary file') + .get('/segment') + .reply(200, 'stream bytes') + + expect(await run(['--single-assembly', '-i', inputPath])).toBe(0) + expect(await readFile(path.join(outputDir, bundleFirst ? 'video__1' : 'video'), 'utf8')).toBe( + 'ordinary file', + ) + expect( + await readFile( + path.join(outputDir, ...(bundleFirst ? [] : ['adaptive']), 'video', 'segment.ts'), + 'utf8', + ), + ).toBe('stream bytes') +}) + +it('finishes publishing a linked watch bundle when a newer Assembly fails', async () => { + const results = { + adaptive: [ + { name: 'master.m3u8', url: 'https://downloads.test/master', meta: { relative_path: '' } }, + { + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'video' }, + }, + ], + } + const { inputPath, outputDir } = await fixture(results) + class FakeWatcher extends EventEmitter { + close(): void { + this.emit('close') + } + } + const watcher = new FakeWatcher() + vi.doMock('node-watch', () => ({ default: () => watcher })) + vi.resetModules() + const { create } = await import('../../../src/cli/commands/assemblies.ts') + vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) + .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', results }) + .mockRejectedValueOnce(new Error('Newer Assembly failed')) + const rename = fs.promises.rename + vi.spyOn(fs.promises, 'rename').mockImplementation(async (source, target) => { + await rename(source, target) + if (target === path.join(outputDir, 'master.m3u8')) watcher.emit('change', 'update', inputPath) + }) + const output = new OutputCtl() + vi.spyOn(output, 'error').mockImplementation(() => watcher.close()) + nock('https://downloads.test') + .get('/master') + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/segment') + .delayBody(40) + .reply(200, 'complete stream') + + const result = await create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + inputs: [inputPath], + output: `${outputDir}/`, + watch: true, + stepsData: { adaptive: { robot: '/video/adaptive', use: ':original' } }, + }) + + expect(result.hasFailures).toBe(true) + expect(Transloadit.prototype.createAssembly).toHaveBeenCalledTimes(2) + expect(await readFile(path.join(outputDir, 'master.m3u8'), 'utf8')).toBe( + '#EXTM3U\nvideo/segment.ts\n', + ) + expect(await readFile(path.join(outputDir, 'video', 'segment.ts'), 'utf8')).toBe( + 'complete stream', + ) +}) + +it.each([ + 'steps', + 'output', +])('validates local %s configuration before downloading a URL', async (invalid) => { + const { outputDir, run } = await fixture({}) + let downloads = 0 + nock('https://source.invalid') + .get('/remote.png') + .reply(200, () => { + downloads += 1 + return 'remote input' + }) + const args = ['-i', 'https://source.invalid/remote.png'] + if (invalid === 'steps') args.push('--steps', path.join(outputDir, 'missing.json')) + else args.push('-i', 'https://source.invalid/second.png') + + expect( + await run(args, invalid === 'output' ? path.join(outputDir, 'result.png') : `${outputDir}/`), + ).toBe(1) + expect(downloads).toBe(0) + expect(Transloadit.prototype.createAssembly).not.toHaveBeenCalled() +}) + it.each([false, true])('uses the bundle result name (single Assembly: %s)', async (single) => { const { inputPath, outputDir, run } = await fixture({ bundled: [{ name: 'archive.zip', url: 'https://downloads.test/archive' }], diff --git a/packages/transloadit/README.md b/packages/transloadit/README.md index 93ae1cf0..7e3dc1ed 100644 --- a/packages/transloadit/README.md +++ b/packages/transloadit/README.md @@ -1523,12 +1523,11 @@ The `assemblies create` command additionally supports: playlist references. Watch updates refresh their existing result files. Other directory runs process the inputs again, since result filenames are unknown before processing; file outputs keep mtime-based skipping. -- `--input, -i` - Provide a local upload or an HTTP(S) URL, or use `-i -` to read stdin explicitly. +- `--input, -i` - Provide a local upload or a public HTTP(S) URL, or use `-i -` to read stdin explicitly. URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly - without uploads. With `--template`, - stdin always requires `-i -`. + without uploads. With `--template`, stdin always requires `-i -`. - `--single-assembly` - Pass all input files to a single assembly instead of one assembly per file #### Log Levels From c1186447992c8f5bfedb4845257ace25b391b7d4 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 16:33:02 +0200 Subject: [PATCH 24/35] Isolate URL uploads and validate HLS source folders --- packages/node/src/cli/commands/assemblies.ts | 68 ++++++++++----- .../test/unit/cli/download-layout.test.ts | 86 +++++++++++++++++-- 2 files changed, 124 insertions(+), 30 deletions(-) diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 095e86d3..07d154e6 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -627,10 +627,11 @@ async function downloadResultToStdout(resultUrl: string, signal: AbortSignal): P await pipeline(got.stream(resultUrl, { signal }), stdoutStream) } -function sanitizeResultName(value: string): string { +function sanitizeResultName(value: string, fallback?: string): string { const base = path.basename(value) const name = base.replaceAll('\\', '_').replaceAll('/', '_').replaceAll('\u0000', '') if (name === '' || name === '.' || name === '..') { + if (fallback != null) return sanitizeResultName(fallback) throw new Error('Assembly result has an unsafe name') } return name @@ -826,6 +827,18 @@ async function isDownloadPathTaken( return error == null } +async function prepareDownloadDirectory( + root: string, + target: string, + reservedPaths: DownloadPathReservations, +): Promise { + await assertNoOutputSymlinks(root, target) + if (hasReservedFileAncestor(target, reservedPaths)) { + throw new Error('Output directory conflicts with a result file') + } + await fsp.mkdir(target, { recursive: true }) +} + const STALE_OUTPUT_GRACE_MS = 1000 function isMeaningfullyNewer(newer: Date, older: Date): boolean { @@ -884,7 +897,7 @@ function groupDirectoryResults(allFiles: NormalizedAssemblyResultFile[]): Direct nameOccurrences.set(name, occurrence + 1) groups.push({ directoryName: - sources.size > 1 ? sanitizeResultName(path.parse(name ?? stepName).name) : null, + sources.size > 1 ? sanitizeResultName(path.parse(name ?? stepName).name, stepName) : null, files, // Upload IDs change between watch runs; source names retain logical ownership. ownerKey: JSON.stringify([name, occurrence]), @@ -910,11 +923,7 @@ async function buildDirectoryDownloadTargets({ outputRoot: string reservedPaths: DownloadPathReservations }): Promise { - await assertNoOutputSymlinks(outputRoot, baseDir) - if (hasReservedFileAncestor(baseDir, reservedPaths)) { - throw new Error('Output directory conflicts with a result file') - } - await fsp.mkdir(baseDir, { recursive: true }) + await prepareDownloadDirectory(outputRoot, baseDir, reservedPaths) const targets: AssemblyDownloadTarget[] = [] for (const { directoryName, files, ownerKey, stepName } of groupDirectoryResults(allFiles)) { @@ -941,6 +950,10 @@ async function buildDirectoryDownloadTargets({ } if (linked) { + // Source folders share this parent, so changing their suffix cannot fix a blocked parent. + if (directoryName != null) { + await prepareDownloadDirectory(outputRoot, path.dirname(targetDir), reservedPaths) + } const initialDir = targetDir // Selection and reservation share the same lock as ordinary result paths. A bundle must // move together on collisions, while later watch updates reuse its owned destinations. @@ -983,10 +996,7 @@ async function buildDirectoryDownloadTargets({ }) } - if (hasReservedFileAncestor(targetDir, reservedPaths)) { - throw new Error('Output directory conflicts with a result file') - } - await fsp.mkdir(targetDir, { recursive: true }) + await prepareDownloadDirectory(outputRoot, targetDir, reservedPaths) const nameOccurrences = new Map() for (const [index, { file }] of files.entries()) { const name = names[index] @@ -1762,30 +1772,44 @@ export async function create( options: AssembliesCreateOptions, ): ReturnType { const preparedOptions = await prepareCreateOptions(outputctl, options) - const urlInputs = options.inputs.filter(isUrlInput) - if (urlInputs.length === 0) + if (!options.inputs.some(isUrlInput)) return await createWithLocalInputs(outputctl, client, options, preparedOptions) if (options.watch) throw new Error('--watch requires local inputs') - const prepared = await prepareInputFiles({ - inputFiles: urlInputs.map((url, index) => ({ kind: 'url', field: String(index), url })), - allowPrivateUrls: false, - // Uploading the bytes preserves :original and Template upload semantics, just as Intents do. - urlStrategy: 'download', - }) - const localPaths = new Map(urlInputs.map((url, index) => [url, prepared.files[String(index)]])) + const preparedInputs: Array>> = [] + const localInputs: string[] = [] try { + for (const [index, input] of options.inputs.entries()) { + if (!isUrlInput(input)) { + localInputs.push(input) + continue + } + const field = String(index) + // Separate staging roots protect case-only filenames and repeated URLs from aliasing. + const prepared = await prepareInputFiles({ + inputFiles: [{ kind: 'url', field, url: input }], + allowPrivateUrls: false, + // Uploading preserves :original and Template semantics, just as Intents do. + urlStrategy: 'download', + }) + preparedInputs.push(prepared) + const localPath = prepared.files[field] + if (localPath == null) throw new Error('URL input could not be prepared') + localInputs.push(localPath) + } return await createWithLocalInputs( outputctl, client, { ...options, - inputs: options.inputs.map((input) => localPaths.get(input) ?? input), + inputs: localInputs, }, preparedOptions, ) } finally { - await Promise.all(prepared.cleanup.map((cleanup) => cleanup())) + await Promise.all( + preparedInputs.flatMap((prepared) => prepared.cleanup.map((cleanup) => cleanup())), + ) } } diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 59f14cfd..35924819 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -122,12 +122,21 @@ it('keeps every HLS playlist reference reachable from its downloaded playlist', ) }) -it('separates independent HLS bundles with identical paths in one Assembly Step', async () => { +it.each([ + { firstName: 'first.mp4', secondName: 'second.mp4', firstDir: 'first', secondDir: 'second' }, + { firstName: 'clip.mp4', secondName: 'clip.mp4', firstDir: 'clip', secondDir: 'clip__1' }, + { firstName: '...', secondName: '...', firstDir: 'adaptive', secondDir: 'adaptive__1' }, +])('separates independent HLS bundles (sources: $firstName, $secondName)', async ({ + firstName, + secondName, + firstDir, + secondDir, +}) => { const firstMaster = { name: 'master.m3u8', url: 'https://downloads.test/first-master', original_id: 'first-source', - original_name: 'first.mp4', + original_name: firstName, meta: { relative_path: '' }, } const firstSegment = { @@ -140,7 +149,7 @@ it('separates independent HLS bundles with identical paths in one Assembly Step' ...firstMaster, url: 'https://downloads.test/second-master', original_id: 'second-source', - original_name: 'second.mp4', + original_name: secondName, } const secondSegment = { ...secondMaster, @@ -164,21 +173,82 @@ it('separates independent HLS bundles with identical paths in one Assembly Step' .reply(200, 'second stream') expect(await run(['--single-assembly', '-i', inputPath, '-i', secondInput])).toBe(0) - expect(await readdir(outputDir)).toEqual(['first', 'second']) - expect(await readFile(path.join(outputDir, 'first', 'master.m3u8'), 'utf8')).toBe( + expect(await readdir(outputDir)).toEqual([firstDir, secondDir]) + expect(await readFile(path.join(outputDir, firstDir, 'master.m3u8'), 'utf8')).toBe( '#EXTM3U\nvideo/segment.ts\n', ) - expect(await readFile(path.join(outputDir, 'first', 'video', 'segment.ts'), 'utf8')).toBe( + expect(await readFile(path.join(outputDir, firstDir, 'video', 'segment.ts'), 'utf8')).toBe( 'first stream', ) - expect(await readFile(path.join(outputDir, 'second', 'master.m3u8'), 'utf8')).toBe( + expect(await readFile(path.join(outputDir, secondDir, 'master.m3u8'), 'utf8')).toBe( '#EXTM3U\nvideo/segment.ts\n', ) - expect(await readFile(path.join(outputDir, 'second', 'video', 'segment.ts'), 'utf8')).toBe( + expect(await readFile(path.join(outputDir, secondDir, 'video', 'segment.ts'), 'utf8')).toBe( 'second stream', ) }) +it('fails promptly when a file blocks the parent of grouped HLS source folders', async () => { + const first = { + name: 'master.m3u8', + url: 'https://downloads.test/master', + original_id: 'first', + original_name: 'first.mp4', + meta: { relative_path: '' }, + } + const second = { ...first, original_id: 'second', original_name: 'second.mp4' } + const { outputDir, run } = await fixture({ adaptive: [first, second] }) + await mkdir(outputDir) + await writeFile(path.join(outputDir, 'adaptive'), 'existing file') + const lstat = fs.promises.lstat + let probes = 0 + vi.spyOn(fs.promises, 'lstat').mockImplementation(async (...args) => { + probes += 1 + if (probes > 40) throw new Error('Filesystem probe budget exceeded') + return await lstat(...args) + }) + + expect(await run(['--single-assembly'])).toBe(1) + expect(probes).toBeLessThan(20) + expect(await readFile(path.join(outputDir, 'adaptive'), 'utf8')).toBe('existing file') +}) + +it.each([ + 'photo.png', + 'Photo.png', +])('keeps URL uploads separate (second filename: %s)', async (secondName) => { + const { run } = await fixture({}) + const uploadPaths: string[] = [] + const uploadBytes: string[] = [] + vi.mocked(Transloadit.prototype.createAssembly).mockImplementation(async ({ files }) => { + const uploadedPath = Object.values(files ?? {})[0] + if (uploadedPath == null) throw new Error('Missing upload path') + uploadPaths.push(uploadedPath) + uploadBytes.push(await readFile(uploadedPath, 'utf8')) + return { assembly_id: 'url-upload' } + }) + nock('https://source.invalid') + .get('/Photo.png') + .reply(200, 'first input') + .get(`/${secondName}`) + .reply(200, 'second input') + + expect( + await run([ + '-i', + 'https://source.invalid/Photo.png', + '-i', + `https://source.invalid/${secondName}`, + ]), + ).toBe(0) + expect(uploadPaths).toHaveLength(2) + expect(new Set(uploadPaths.map((file) => file.normalize('NFC').toLowerCase())).size).toBe(2) + expect(uploadBytes.sort()).toEqual(['first input', 'second input']) + for (const uploadedPath of uploadPaths) { + await expect(stat(uploadedPath)).rejects.toHaveProperty('code', 'ENOENT') + } +}) + it('rejects duplicate paths within a single HLS bundle', async () => { const segment = { name: 'segment.ts', From c1882256ccefb1ebb8db758473daa6130f9d28b8 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 16:54:45 +0200 Subject: [PATCH 25/35] Preserve independent result ownership and local staging --- packages/node/src/cli/commands/assemblies.ts | 57 ++++++--- .../test/unit/cli/download-layout.test.ts | 120 ++++++++++++++++++ 2 files changed, 158 insertions(+), 19 deletions(-) diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 07d154e6..da103e01 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -750,24 +750,34 @@ interface BundleDownloadTarget { targetPath: string } +interface StagedBundleDownloadTarget extends BundleDownloadTarget { + stagingPath: string +} + async function downloadResultBundle( - root: string, targets: BundleDownloadTarget[], signal: AbortSignal, isCurrent: () => boolean, commits: PQueue, onDownload: () => void, ): Promise { - const stagingDir = await fsp.mkdtemp(path.join(root, '.transloadit-bundle-')) - const staged = targets.map((target, index) => ({ - ...target, - stagingPath: path.join(stagingDir, String(index)), - })) + const stagingDirs = new Map() + const staged: StagedBundleDownloadTarget[] = [] try { - for (const { resultUrl, stagingPath } of staged) { + for (const [index, target] of targets.entries()) { if (!isCurrent()) return + const parent = path.dirname(target.targetPath) + await fsp.mkdir(parent, { recursive: true }) + let stagingDir = stagingDirs.get(parent) + if (stagingDir == null) { + // Stage beside each destination so publication also works across nested filesystems. + stagingDir = await fsp.mkdtemp(path.join(parent, '.transloadit-bundle-')) + stagingDirs.set(parent, stagingDir) + } + const stagingPath = path.join(stagingDir, String(index)) + staged.push({ ...target, stagingPath }) onDownload() - await downloadResultToFile(resultUrl, stagingPath, signal, () => true) + await downloadResultToFile(target.resultUrl, stagingPath, signal, () => true) } await commits.add(async () => { if (!isCurrent()) return @@ -779,7 +789,11 @@ async function downloadResultBundle( } }) } finally { - await fsp.rm(stagingDir, { recursive: true, force: true }) + await Promise.all( + [...stagingDirs.values()].map( + async (dir) => await fsp.rm(dir, { recursive: true, force: true }), + ), + ) } } @@ -1008,7 +1022,7 @@ async function buildDirectoryDownloadTargets({ : await ensureUniquePath( targetPath, reservedPaths, - JSON.stringify([owner, stepName, name, occurrence]), + JSON.stringify([groupOwner, name, occurrence]), ) await assertNoOutputSymlinks(outputRoot, selectedPath) targets.push({ @@ -1043,6 +1057,7 @@ async function resolveResultDownloadTargets({ outputPath, outputRoot, outputRootIsDirectory, + owner, reservedPaths, singleAssembly, }: { @@ -1053,6 +1068,7 @@ async function resolveResultDownloadTargets({ outputPath: string | null outputRoot: string outputRootIsDirectory: boolean + owner: string reservedPaths: DownloadPathReservations singleAssembly?: boolean }): Promise { @@ -1063,7 +1079,6 @@ async function resolveResultDownloadTargets({ !singleAssembly && inPath != null && (hasDirectoryInput || (inputs.length > 1 && allFiles.length > 1)) - const owner = JSON.stringify([inPath, singleAssembly ? inputs : []]) const resolveDirectoryBaseDir = (): string => { if (hasDirectoryInput && outputPath != null) { @@ -1185,6 +1200,7 @@ async function materializeAssemblyResults({ outputRoot, outputRootIsDirectory, outputctl, + owner, reservedPaths, singleAssembly, }: { @@ -1198,6 +1214,7 @@ async function materializeAssemblyResults({ outputRoot: string | null outputRootIsDirectory: boolean outputctl: IOutputCtl + owner: string reservedPaths: DownloadPathReservations singleAssembly?: boolean }): Promise { @@ -1216,6 +1233,7 @@ async function materializeAssemblyResults({ outputPath, outputRoot, outputRootIsDirectory, + owner, reservedPaths, singleAssembly, }), @@ -1236,16 +1254,11 @@ async function materializeAssemblyResults({ else await downloadResultToFile(resultUrl, targetPath, abortSignal, isCurrent) }) } - for (const [root, files] of bundles) { + for (const files of bundles.values()) { downloads.push( async () => - await downloadResultBundle( - root, - files, - abortSignal, - isCurrent, - reservedPaths.bundleCommits, - () => outputctl.debug('DOWNLOADING'), + await downloadResultBundle(files, abortSignal, isCurrent, reservedPaths.bundleCommits, () => + outputctl.debug('DOWNLOADING'), ), ) } @@ -2079,6 +2092,12 @@ async function createWithLocalInputs( outputRoot: resolvedOutput ?? null, outputRootIsDirectory, outputctl, + // Only watch updates reuse destinations; independent Assemblies retain every result. + owner: JSON.stringify([ + inPath, + singleAssemblyMode ? inputPaths : [], + watchOption ? null : assemblyId, + ]), reservedPaths: reservedResultPaths, singleAssembly: singleAssemblyMode, }) diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 35924819..8044bb4f 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -263,6 +263,126 @@ it('rejects duplicate paths within a single HLS bundle', async () => { expect(await readdir(outputDir)).toEqual([]) }) +it('retains ordinary results from different source groups in a Step containing HLS', async () => { + const master = { + name: 'master.m3u8', + url: 'https://downloads.test/master', + original_id: 'video', + original_name: 'clip.mp4', + meta: { relative_path: '' }, + } + const segment = { + ...master, + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'video' }, + } + const firstCover = { + name: 'cover.jpg', + url: 'https://downloads.test/first-cover', + original_id: 'first-cover', + original_name: 'same.png', + } + const secondCover = { + ...firstCover, + original_id: 'second-cover', + url: 'https://downloads.test/second-cover', + } + const { inputPath, outputDir, run } = await fixture({ + adaptive: [master, segment, firstCover, secondCover], + }) + nock('https://downloads.test') + .get('/master') + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/segment') + .reply(200, 'stream bytes') + .get('/first-cover') + .reply(200, 'first cover') + .get('/second-cover') + .reply(200, 'second cover') + + expect(await run(['--single-assembly', '-i', inputPath])).toBe(0) + expect(await readdir(path.join(outputDir, 'same'))).toEqual(['cover.jpg', 'cover__1.jpg']) + expect(await readFile(path.join(outputDir, 'same', 'cover.jpg'), 'utf8')).toBe('first cover') + expect(await readFile(path.join(outputDir, 'same', 'cover__1.jpg'), 'utf8')).toBe('second cover') + expect(await readFile(path.join(outputDir, 'clip', 'video', 'segment.ts'), 'utf8')).toBe( + 'stream bytes', + ) +}) + +it('preserves every independent Assembly result when file and directory inputs overlap', async () => { + const { inputPath, outputDir, run } = await fixture({}) + let count = 0 + vi.mocked(Transloadit.prototype.createAssembly).mockImplementation(async () => ({ + assembly_id: String(++count), + })) + vi.mocked(Transloadit.prototype.awaitAssemblyCompletion).mockImplementation(async (id) => ({ + ok: 'ASSEMBLY_COMPLETED', + results: { + images: [ + { name: 'a.png', url: `https://downloads.test/${id}/a` }, + { name: 'b.png', url: `https://downloads.test/${id}/b` }, + ], + }, + })) + nock('https://downloads.test') + .get('/1/a') + .reply(200, 'a1') + .get('/1/b') + .reply(200, 'b1') + .get('/2/a') + .reply(200, 'a2') + .get('/2/b') + .reply(200, 'b2') + + expect(await run(['-i', inputPath, '-i', path.dirname(inputPath)])).toBe(0) + expect(count).toBe(2) + const files = await readdir(path.join(outputDir, 'logo')) + expect(files).toEqual(['a.png', 'a__1.png', 'b.png', 'b__1.png']) + expect( + ( + await Promise.all( + files.map(async (file) => await readFile(path.join(outputDir, 'logo', file), 'utf8')), + ) + ).sort(), + ).toEqual(['a1', 'a2', 'b1', 'b2']) +}) + +it('stages HLS files on their destination filesystem', async () => { + const { inputPath, outputDir, run } = await fixture({ + adaptive: [ + { name: 'master.m3u8', url: 'https://downloads.test/master', meta: { relative_path: '' } }, + { + name: 'segment.ts', + url: 'https://downloads.test/segment', + meta: { relative_path: 'video' }, + }, + ], + }) + const rename = fs.promises.rename + let crossDeviceAttempts = 0 + vi.spyOn(fs.promises, 'rename').mockImplementation(async (source, target) => { + const onRenditionDevice = (file: fs.PathLike): boolean => + String(file).includes(`${path.sep}video${path.sep}`) + if (onRenditionDevice(source) !== onRenditionDevice(target)) { + crossDeviceAttempts += 1 + throw Object.assign(new Error('Cross-device rename'), { code: 'EXDEV' }) + } + await rename(source, target) + }) + nock('https://downloads.test') + .get('/master') + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/segment') + .reply(200, 'stream bytes') + + expect(await run(['-i', inputPath])).toBe(0) + expect(crossDeviceAttempts).toBe(0) + expect(await readdir(outputDir)).toEqual(['master.m3u8', 'video']) + expect(await readdir(path.join(outputDir, 'video'))).toEqual(['segment.ts']) + expect(await readFile(path.join(outputDir, 'video', 'segment.ts'), 'utf8')).toBe('stream bytes') +}) + it('keeps concurrent case-only result names separate on case-insensitive filesystems', async () => { const { inputPath, outputDir, run } = await fixture({}) const secondInput = path.join(path.dirname(inputPath), 'second.png') From 759a04af6e022db1e9b8ce344fc42ef815a772aa Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 17:18:00 +0200 Subject: [PATCH 26/35] Separate mixed HLS results and reserve input directories --- packages/node/src/cli/commands/assemblies.ts | 57 +++++++++++---- .../test/unit/cli/download-layout.test.ts | 71 +++++++++++++++++++ 2 files changed, 114 insertions(+), 14 deletions(-) diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index da103e01..0af4124b 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -697,18 +697,24 @@ async function ensureUniquePath( targetPath: string, reservedPaths: DownloadPathReservations, owner: string, + directory = false, ): Promise { const selectedPath = reservedPaths.pathsByOwner.get(owner) if (selectedPath != null) return selectedPath const parsed = path.parse(targetPath) return await ensureUniqueCounterValue({ initialValue: targetPath, - isTaken: async (candidate) => await isDownloadPathTaken(candidate, reservedPaths, owner), + isTaken: async (candidate) => + await isDownloadPathTaken(candidate, reservedPaths, owner, directory), reserve: (candidate) => { - reserveDownloadPath(candidate, owner, reservedPaths) + reserveDownloadPath(candidate, owner, reservedPaths, directory) reservedPaths.pathsByOwner.set(owner, candidate) }, - nextValue: (counter) => path.join(parsed.dir, `${parsed.name}__${counter}${parsed.ext}`), + nextValue: (counter) => + path.join( + parsed.dir, + directory ? `${parsed.base}__${counter}` : `${parsed.name}__${counter}${parsed.ext}`, + ), scope: reservedPaths, }) } @@ -828,17 +834,19 @@ async function isDownloadPathTaken( target: string, reservedPaths: DownloadPathReservations, owner: string, + directory = false, ): Promise { const key = downloadPathKey(target, reservedPaths) + if (directory && reservedPaths.directories.has(key)) return false const existingOwner = reservedPaths.ownersByPath.get(key) if (existingOwner != null) return existingOwner !== owner if (reservedPaths.directories.has(key) || hasReservedFileAncestor(target, reservedPaths)) { return true } - const [error] = await tryCatch(fsp.lstat(target)) + const [error, stats] = await tryCatch(fsp.lstat(target)) if (isErrnoException(error) && error.code === 'ENOTDIR') return true if (error && (!isErrnoException(error) || error.code !== 'ENOENT')) throw error - return error == null + return error == null && (!directory || !stats?.isDirectory()) } async function prepareDownloadDirectory( @@ -909,14 +917,21 @@ function groupDirectoryResults(allFiles: NormalizedAssemblyResultFile[]): Direct for (const { name, files } of sources.values()) { const occurrence = nameOccurrences.get(name) ?? 0 nameOccurrences.set(name, occurrence + 1) - groups.push({ - directoryName: - sources.size > 1 ? sanitizeResultName(path.parse(name ?? stepName).name, stepName) : null, - files, - // Upload IDs change between watch runs; source names retain logical ownership. - ownerKey: JSON.stringify([name, occurrence]), - stepName, - }) + // Ordinary files can be renamed independently, even when they share an HLS source. + for (const linked of [true, false]) { + const groupFiles = files.filter(({ relativePath }) => (relativePath != null) === linked) + if (groupFiles.length === 0) continue + groups.push({ + directoryName: + sources.size > 1 + ? sanitizeResultName(path.parse(name ?? stepName).name, stepName) + : null, + files: groupFiles, + // Upload IDs change between watch runs; source names retain logical ownership. + ownerKey: JSON.stringify([name, occurrence, linked]), + stepName, + }) + } } } return groups @@ -1119,9 +1134,23 @@ async function resolveResultDownloadTargets({ }) } + let baseDir = resolveDirectoryBaseDir() + if (path.resolve(baseDir) !== path.resolve(outputRoot)) { + // Input grouping directories share the result namespace, including files already reserved + // by another Assembly. Reuse existing directories and suffix a conflicting filename. + await assertNoOutputSymlinks(outputRoot, baseDir) + await prepareDownloadDirectory(outputRoot, path.dirname(baseDir), reservedPaths) + baseDir = await ensureUniquePath( + baseDir, + reservedPaths, + JSON.stringify([owner, 'input-directory']), + true, + ) + } + return await buildDirectoryDownloadTargets({ allFiles, - baseDir: resolveDirectoryBaseDir(), + baseDir, groupByStep: inPath == null || entries.length > 1, owner, outputRoot, diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 8044bb4f..6592dbb5 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -263,6 +263,77 @@ it('rejects duplicate paths within a single HLS bundle', async () => { expect(await readdir(outputDir)).toEqual([]) }) +it('suffixes ordinary collisions within the same source as an HLS bundle', async () => { + const source = { original_id: 'video', original_name: 'clip.mp4' } + const { inputPath, outputDir, run } = await fixture({ + adaptive: [ + { + ...source, + name: 'master.m3u8', + url: 'https://downloads.test/master', + meta: { relative_path: '' }, + }, + { ...source, name: 'cover.jpg', url: 'https://downloads.test/first-cover' }, + { ...source, name: 'cover.jpg', url: 'https://downloads.test/second-cover' }, + ], + }) + nock('https://downloads.test') + .get('/master') + .reply(200, '#EXTM3U\n') + .get('/first-cover') + .reply(200, 'first cover') + .get('/second-cover') + .reply(200, 'second cover') + + expect(await run(['-i', inputPath])).toBe(0) + expect(await readdir(outputDir)).toEqual(['cover.jpg', 'cover__1.jpg', 'master.m3u8']) + expect(await readFile(path.join(outputDir, 'cover.jpg'), 'utf8')).toBe('first cover') + expect(await readFile(path.join(outputDir, 'cover__1.jpg'), 'utf8')).toBe('second cover') +}) + +it.each([ + false, + true, +])('separates input directories from result files (file first: %s)', async (fileFirst) => { + const { inputPath, outputDir, run } = await fixture({}) + const singleInput = path.join(path.dirname(inputPath), 'report.docx') + const multiInput = path.join(path.dirname(inputPath), 'report.pdf.docx') + await writeFile(singleInput, 'single-result input') + await writeFile(multiInput, 'multi-result input') + const single = { output: [{ name: 'report.pdf', url: 'https://downloads.test/pdf' }] } + const multi = { + output: [ + { name: 'page1.jpg', url: 'https://downloads.test/page1' }, + { name: 'page2.jpg', url: 'https://downloads.test/page2' }, + ], + } + vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) + .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', results: fileFirst ? single : multi }) + .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', results: fileFirst ? multi : single }) + nock('https://downloads.test') + .get('/pdf') + .reply(200, 'pdf bytes') + .get('/page1') + .reply(200, 'first page') + .get('/page2') + .reply(200, 'second page') + + expect( + await run([ + '-i', + fileFirst ? singleInput : multiInput, + '-i', + fileFirst ? multiInput : singleInput, + ]), + ).toBe(0) + expect( + await readFile(path.join(outputDir, fileFirst ? 'report.pdf' : 'report__1.pdf'), 'utf8'), + ).toBe('pdf bytes') + const inputDir = path.join(outputDir, fileFirst ? 'report.pdf__1' : 'report.pdf') + expect(await readFile(path.join(inputDir, 'page1.jpg'), 'utf8')).toBe('first page') + expect(await readFile(path.join(inputDir, 'page2.jpg'), 'utf8')).toBe('second page') +}) + it('retains ordinary results from different source groups in a Step containing HLS', async () => { const master = { name: 'master.m3u8', From 25907ecddd0c9f9b6c6960c88be76a9352d788b9 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 17:50:37 +0200 Subject: [PATCH 27/35] Keep watched HLS ownership stable across source reordering --- packages/node/src/cli/commands/assemblies.ts | 27 +++-- .../test/unit/cli/download-layout.test.ts | 100 ++++++++++++++++++ 2 files changed, 119 insertions(+), 8 deletions(-) diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 0af4124b..1b7b1282 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -883,6 +883,7 @@ const resultSourceSchema = assemblyStatusResultSchema.pick({ original_basename: true, original_id: true, original_name: true, + original_path: true, }) function groupDirectoryResults(allFiles: NormalizedAssemblyResultFile[]): DirectoryResultGroup[] { @@ -900,23 +901,32 @@ function groupDirectoryResults(allFiles: NormalizedAssemblyResultFile[]): Direct continue } - const sources = new Map() + const sources = new Map< + string, + { identity: string; name: string | null; files: DirectoryResultFile[] } + >() for (const file of stepFiles) { const parsed = resultSourceSchema.safeParse(file.file.file) const source = parsed.success ? parsed.data : undefined const sourceId = JSON.stringify(source?.original_id ?? null) + const name = source?.original_name ?? source?.original_basename ?? null const group = sources.get(sourceId) ?? { - name: source?.original_name ?? source?.original_basename ?? null, + // A path and a fallback name with identical text still describe different sources. + identity: JSON.stringify([ + source?.original_path ?? null, + source?.original_path == null ? name : null, + ]), + name, files: [], } group.files.push(file) sources.set(sourceId, group) } - const nameOccurrences = new Map() - for (const { name, files } of sources.values()) { - const occurrence = nameOccurrences.get(name) ?? 0 - nameOccurrences.set(name, occurrence + 1) + const sourceOccurrences = new Map() + for (const { identity, name, files } of sources.values()) { + const occurrence = sourceOccurrences.get(identity) ?? 0 + sourceOccurrences.set(identity, occurrence + 1) // Ordinary files can be renamed independently, even when they share an HLS source. for (const linked of [true, false]) { const groupFiles = files.filter(({ relativePath }) => (relativePath != null) === linked) @@ -927,8 +937,9 @@ function groupDirectoryResults(allFiles: NormalizedAssemblyResultFile[]): Direct ? sanitizeResultName(path.parse(name ?? stepName).name, stepName) : null, files: groupFiles, - // Upload IDs change between watch runs; source names retain logical ownership. - ownerKey: JSON.stringify([name, occurrence, linked]), + // Upload IDs change between watch runs. Original paths distinguish same-named sources + // even when results arrive in a different order; names are the metadata fallback. + ownerKey: JSON.stringify([identity, occurrence, linked]), stepName, }) } diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 6592dbb5..3e1efe60 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -944,6 +944,106 @@ it('keeps a watched HLS Step in its selected folder after the original conflict ) }) +it.each([ + { firstPath: 'first/clip.mp4', secondPath: 'second/clip.mp4' }, + { firstPath: 'clip.mp4', secondPath: undefined }, +])('keeps watched HLS source paths attached to their folders (paths: $firstPath, $secondPath)', async ({ + firstPath, + secondPath, +}) => { + const firstMaster = { + original_id: 'first-old', + original_name: 'clip.mp4', + original_path: firstPath, + name: 'master.m3u8', + url: 'https://downloads.test/first-master', + meta: { relative_path: '' }, + } + const firstSegment = { + ...firstMaster, + name: 'segment.ts', + url: 'https://downloads.test/first-segment', + meta: { relative_path: 'video' }, + } + const secondMaster = { + ...firstMaster, + original_id: 'second-old', + original_path: secondPath, + url: 'https://downloads.test/second-master', + } + const secondSegment = { + ...firstSegment, + original_id: 'second-old', + original_path: secondPath, + url: 'https://downloads.test/second-segment', + } + const { inputPath, outputDir } = await fixture({ + adaptive: [firstMaster, firstSegment, secondMaster, secondSegment], + }) + vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) + .mockResolvedValueOnce({ + ok: 'ASSEMBLY_COMPLETED', + results: { adaptive: [firstMaster, firstSegment, secondMaster, secondSegment] }, + }) + .mockResolvedValueOnce({ + ok: 'ASSEMBLY_COMPLETED', + results: { + adaptive: [ + { ...secondMaster, original_id: 'second-new' }, + { ...secondSegment, original_id: 'second-new' }, + { ...firstMaster, original_id: 'first-new' }, + { ...firstSegment, original_id: 'first-new' }, + ], + }, + }) + class FakeWatcher extends EventEmitter { + close(): void { + this.emit('close') + } + } + const watcher = new FakeWatcher() + vi.doMock('node-watch', () => ({ default: () => watcher })) + vi.resetModules() + const { create } = await import('../../../src/cli/commands/assemblies.ts') + const output = new OutputCtl() + let completions = 0 + vi.spyOn(output, 'debug').mockImplementation((message) => { + if (!String(message).startsWith('COMPLETED')) return + if (++completions === 1) watcher.emit('change', 'update', inputPath) + else watcher.close() + }) + nock('https://downloads.test') + .get('/first-master') + .twice() + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/second-master') + .twice() + .reply(200, '#EXTM3U\nvideo/segment.ts\n') + .get('/first-segment') + .reply(200, 'first original') + .get('/first-segment') + .reply(200, 'first updated') + .get('/second-segment') + .reply(200, 'second original') + .get('/second-segment') + .reply(200, 'second updated') + + const result = await create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + inputs: [inputPath], + output: `${outputDir}/`, + watch: true, + stepsData: { adaptive: { robot: '/video/adaptive', use: ':original' } }, + }) + expect(result.hasFailures).toBe(false) + expect(completions).toBe(2) + expect(await readFile(path.join(outputDir, 'clip', 'video', 'segment.ts'), 'utf8')).toBe( + 'first updated', + ) + expect(await readFile(path.join(outputDir, 'clip__1', 'video', 'segment.ts'), 'utf8')).toBe( + 'second updated', + ) +}) + it('accepts a standalone input and a directory containing the same basename', async () => { const { inputPath, outputDir, run, tempDir } = await fixture({ resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], From ac3e40829716930887b8c6b1a3881013e1e92fbb Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 18:18:17 +0200 Subject: [PATCH 28/35] Stabilize HLS watch ownership and Unicode collision keys --- packages/node/src/cli/commands/assemblies.ts | 47 ++++---- .../test/unit/cli/download-layout.test.ts | 104 +++++++++++++++--- 2 files changed, 116 insertions(+), 35 deletions(-) diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 1b7b1282..313eccb9 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -637,23 +637,34 @@ function sanitizeResultName(value: string, fallback?: string): string { return name } -interface DownloadPathReservations { +interface OutputNameSensitivity { + caseSensitive: boolean + normalizationSensitive: boolean +} + +interface DownloadPathReservations extends OutputNameSensitivity { allocations: PQueue bundleCommits: PQueue - caseSensitive: boolean directories: Set ownersByPath: Map pathsByOwner: Map } -async function outputNamesAreCaseSensitive(root: string): Promise { +async function outputNameSensitivity(root: string): Promise { await fsp.mkdir(root, { recursive: true }) - const probe = await fsp.mkdtemp(path.join(root, '.transloadit-case-')) - try { - await fsp.writeFile(path.join(probe, 'case'), '') - const [error] = await tryCatch(fsp.lstat(path.join(probe, 'CASE'))) + const probe = await fsp.mkdtemp(path.join(root, '.transloadit-names-')) + async function distinguishesNames(first: string, second: string): Promise { + await fsp.writeFile(path.join(probe, first), '') + const [error] = await tryCatch(fsp.lstat(path.join(probe, second))) if (error && (!isErrnoException(error) || error.code !== 'ENOENT')) throw error return error != null + } + try { + // Assume this tree shares the root's naming rules; nested filesystems are tracked in #615. + return { + caseSensitive: await distinguishesNames('case', 'CASE'), + normalizationSensitive: await distinguishesNames('\u00e9', 'e\u0301'), + } } finally { await fsp.rm(probe, { recursive: true, force: true }) } @@ -661,7 +672,8 @@ async function outputNamesAreCaseSensitive(root: string): Promise { function downloadPathKey(target: string, reservedPaths: DownloadPathReservations): string { const absolutePath = path.resolve(target) - return reservedPaths.caseSensitive ? absolutePath : absolutePath.normalize('NFC').toLowerCase() + const folded = reservedPaths.caseSensitive ? absolutePath : absolutePath.toLowerCase() + return reservedPaths.normalizationSensitive ? folded : folded.normalize('NFC') } function hasReservedFileAncestor(target: string, reservedPaths: DownloadPathReservations): boolean { @@ -911,11 +923,8 @@ function groupDirectoryResults(allFiles: NormalizedAssemblyResultFile[]): Direct const sourceId = JSON.stringify(source?.original_id ?? null) const name = source?.original_name ?? source?.original_basename ?? null const group = sources.get(sourceId) ?? { - // A path and a fallback name with identical text still describe different sources. - identity: JSON.stringify([ - source?.original_path ?? null, - source?.original_path == null ? name : null, - ]), + // original_path can be a shared directory, so the source filename is also required. + identity: JSON.stringify([source?.original_path ?? null, name]), name, files: [], } @@ -937,8 +946,8 @@ function groupDirectoryResults(allFiles: NormalizedAssemblyResultFile[]): Direct ? sanitizeResultName(path.parse(name ?? stepName).name, stepName) : null, files: groupFiles, - // Upload IDs change between watch runs. Original paths distinguish same-named sources - // even when results arrive in a different order; names are the metadata fallback. + // Upload IDs change between watch runs. Paths and names keep source ownership stable + // even when results arrive in a different order. ownerKey: JSON.stringify([identity, occurrence, linked]), stepName, }) @@ -1889,10 +1898,10 @@ async function createWithLocalInputs( hasFailures: boolean }> { const { effectiveStepsData, hasDirectoryInput, outstat, resolvedOutput } = preparedOptions - const caseSensitive = + const nameSensitivity = resolvedOutput != null && outstat?.isDirectory() - ? await outputNamesAreCaseSensitive(resolvedOutput) - : true + ? await outputNameSensitivity(resolvedOutput) + : { caseSensitive: true, normalizationSensitive: true } return new Promise((resolve, reject) => { const params: CreateAssemblyParams = ( @@ -1931,7 +1940,7 @@ async function createWithLocalInputs( const reservedResultPaths: DownloadPathReservations = { allocations: new PQueue({ concurrency: 1 }), bundleCommits: new PQueue({ concurrency: 1 }), - caseSensitive, + ...nameSensitivity, directories: new Set(), ownersByPath: new Map(), pathsByOwner: new Map(), diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 3e1efe60..a6424bed 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -308,7 +308,11 @@ it.each([ ], } vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) - .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', results: fileFirst ? single : multi }) + .mockImplementationOnce(async () => { + // Finish the second job first unless the CLI is explicitly asked to run serially. + await new Promise((resolve) => setTimeout(resolve, 60)) + return { ok: 'ASSEMBLY_COMPLETED', results: fileFirst ? single : multi } + }) .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', results: fileFirst ? multi : single }) nock('https://downloads.test') .get('/pdf') @@ -324,6 +328,8 @@ it.each([ fileFirst ? singleInput : multiInput, '-i', fileFirst ? multiInput : singleInput, + '--concurrency', + '1', ]), ).toBe(0) expect( @@ -454,35 +460,69 @@ it('stages HLS files on their destination filesystem', async () => { expect(await readFile(path.join(outputDir, 'video', 'segment.ts'), 'utf8')).toBe('stream bytes') }) -it('keeps concurrent case-only result names separate on case-insensitive filesystems', async () => { +it.each([ + { + firstName: 'Photo.jpg', + secondName: 'photo.jpg', + caseSensitive: false, + normalizationSensitive: true, + }, + { + firstName: '\u0390.jpg', + secondName: '\u03aa\u0301.jpg', + caseSensitive: false, + normalizationSensitive: false, + }, + { + firstName: '\u00e9.jpg', + secondName: 'e\u0301.jpg', + caseSensitive: true, + normalizationSensitive: false, + }, +])('keeps concurrent equivalent result names separate ($firstName, $secondName)', async ({ + firstName, + secondName, + caseSensitive, + normalizationSensitive, +}) => { const { inputPath, outputDir, run } = await fixture({}) const secondInput = path.join(path.dirname(inputPath), 'second.png') await writeFile(secondInput, 'second input') vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', - results: { resized: [{ name: 'Photo.jpg', url: 'https://downloads.test/first' }] }, + results: { resized: [{ name: firstName, url: 'https://downloads.test/first' }] }, }) .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', - results: { resized: [{ name: 'photo.jpg', url: 'https://downloads.test/second' }] }, + results: { resized: [{ name: secondName, url: 'https://downloads.test/second' }] }, }) - async function resolveCaseInsensitivePath(value: fs.PathLike): Promise { + function filenameKey(name: string): string { + const folded = caseSensitive ? name : name.toLowerCase() + return normalizationSensitive ? folded : folded.normalize('NFC') + } + async function resolveEquivalentPath( + value: fs.PathLike, + requireExisting = false, + ): Promise { if (typeof value !== 'string') return value const [error, siblings] = await tryCatch(readdir(path.dirname(value))) if (error) { if ('code' in error && error.code === 'ENOENT') return value throw error } - const match = siblings.find((name) => name.toLowerCase() === path.basename(value).toLowerCase()) + const match = siblings.find((name) => filenameKey(name) === filenameKey(path.basename(value))) + if (match == null && requireExisting) { + throw Object.assign(new Error('No equivalent filename'), { code: 'ENOENT' }) + } return match == null ? value : path.join(path.dirname(value), match) } const { lstat, rename } = fs.promises vi.spyOn(fs.promises, 'lstat').mockImplementation( - async (target, ...args) => await lstat(await resolveCaseInsensitivePath(target), ...args), + async (target, ...args) => await lstat(await resolveEquivalentPath(target, true), ...args), ) vi.spyOn(fs.promises, 'rename').mockImplementation( - async (source, target) => await rename(source, await resolveCaseInsensitivePath(target)), + async (source, target) => await rename(source, await resolveEquivalentPath(target)), ) nock('https://downloads.test') .get('/first') @@ -493,9 +533,13 @@ it('keeps concurrent case-only result names separate on case-insensitive filesys .reply(200, 'second result') expect(await run(['-i', inputPath, '-i', secondInput, '--concurrency', '2'])).toBe(0) - expect(await readdir(outputDir)).toEqual(['Photo.jpg', 'photo__1.jpg']) - expect(await readFile(path.join(outputDir, 'Photo.jpg'), 'utf8')).toBe('first result') - expect(await readFile(path.join(outputDir, 'photo__1.jpg'), 'utf8')).toBe('second result') + const { name, ext } = path.parse(secondName) + const secondOutput = `${name}__1${ext}` + expect((await readdir(outputDir)).map((value) => value.normalize('NFC')).sort()).toEqual( + [firstName, secondOutput].map((value) => value.normalize('NFC')).sort(), + ) + expect(await readFile(path.join(outputDir, firstName), 'utf8')).toBe('first result') + expect(await readFile(path.join(outputDir, secondOutput), 'utf8')).toBe('second result') }) it.each([ @@ -945,15 +989,41 @@ it('keeps a watched HLS Step in its selected folder after the original conflict }) it.each([ - { firstPath: 'first/clip.mp4', secondPath: 'second/clip.mp4' }, - { firstPath: 'clip.mp4', secondPath: undefined }, + { + firstPath: 'first/clip.mp4', + secondPath: 'second/clip.mp4', + firstName: 'clip.mp4', + secondName: 'clip.mp4', + firstDir: 'clip', + secondDir: 'clip__1', + }, + { + firstPath: 'clip.mp4', + secondPath: undefined, + firstName: 'clip.mp4', + secondName: 'clip.mp4', + firstDir: 'clip', + secondDir: 'clip__1', + }, + { + firstPath: '/', + secondPath: '/', + firstName: 'first.mp4', + secondName: 'second.mp4', + firstDir: 'first', + secondDir: 'second', + }, ])('keeps watched HLS source paths attached to their folders (paths: $firstPath, $secondPath)', async ({ firstPath, secondPath, + firstName, + secondName, + firstDir, + secondDir, }) => { const firstMaster = { original_id: 'first-old', - original_name: 'clip.mp4', + original_name: firstName, original_path: firstPath, name: 'master.m3u8', url: 'https://downloads.test/first-master', @@ -968,12 +1038,14 @@ it.each([ const secondMaster = { ...firstMaster, original_id: 'second-old', + original_name: secondName, original_path: secondPath, url: 'https://downloads.test/second-master', } const secondSegment = { ...firstSegment, original_id: 'second-old', + original_name: secondName, original_path: secondPath, url: 'https://downloads.test/second-segment', } @@ -1036,10 +1108,10 @@ it.each([ }) expect(result.hasFailures).toBe(false) expect(completions).toBe(2) - expect(await readFile(path.join(outputDir, 'clip', 'video', 'segment.ts'), 'utf8')).toBe( + expect(await readFile(path.join(outputDir, firstDir, 'video', 'segment.ts'), 'utf8')).toBe( 'first updated', ) - expect(await readFile(path.join(outputDir, 'clip__1', 'video', 'segment.ts'), 'utf8')).toBe( + expect(await readFile(path.join(outputDir, secondDir, 'video', 'segment.ts'), 'utf8')).toBe( 'second updated', ) }) From 22591e6126c2b6b8b803bc4c0225ecbb8f0dc23e Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 18:36:05 +0200 Subject: [PATCH 29/35] Document ambiguous HLS bundle follow-up --- packages/node/src/cli/commands/assemblies.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 313eccb9..ccf45d99 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -991,6 +991,8 @@ async function buildDirectoryDownloadTargets({ new Set(names.map((name) => downloadPathKey(path.join(targetDir, name), reservedPaths))) .size !== names.length ) { + // Merged Steps can hide distinct bundles sharing one upload (#619). Reject ambiguous + // linked paths rather than guessing bundle boundaries and overwriting their contents. throw new Error('Assembly results have duplicate relative paths') } From ffc8eda16f23d5b3be7aa2be952bbc4365b41224 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 18:56:24 +0200 Subject: [PATCH 30/35] Bound URL downloads and classify CLI compatibility changes --- .changeset/assembly-download-paths.md | 8 +++-- packages/node/README.md | 5 ++- packages/node/src/cli/commands/assemblies.ts | 18 ++++++++++ .../test/unit/cli/download-layout.test.ts | 33 +++++++++++++++++++ packages/transloadit/README.md | 5 ++- 5 files changed, 65 insertions(+), 4 deletions(-) diff --git a/.changeset/assembly-download-paths.md b/.changeset/assembly-download-paths.md index 1ee0df58..20f481a4 100644 --- a/.changeset/assembly-download-paths.md +++ b/.changeset/assembly-download-paths.md @@ -1,13 +1,17 @@ --- -'@transloadit/node': patch +'@transloadit/node': major '@transloadit/mcp-server': patch -'transloadit': patch +'transloadit': major --- Keep CLI download filenames and HLS directories aligned with Assembly results, omit input parent folders from output paths, and support HTTP(S) URL inputs for Assembly and prompt commands. +SDK-breaking: CLI directory downloads use Assembly result names and omit input parent folders. Directory reruns process inputs again and preserve existing files with collision suffixes. Update scripts to use the result paths; use an explicit output file when mtime-based skipping is required. + Independent HLS bundles returned by the same Step get separate source folders so their playlist and segment paths can coexist. Reserve result paths with filesystem case sensitivity and file/directory conflicts in mind. Stage linked watch downloads before publishing them together, and validate local steps and output options before fetching URL inputs. Directory runs process inputs again to discover the result filenames instead of guessing freshness from upload names. Watch updates reuse their own result paths, while existing unrelated files and outputs from other inputs remain separate. + +URL input downloads have a five-minute deadline per input, configurable in milliseconds with `--url-download-timeout` on `assemblies create` and `run`. diff --git a/packages/node/README.md b/packages/node/README.md index 7e3dc1ed..9d175c60 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -146,7 +146,8 @@ npx -y @transloadit/node run "extract a waveform image from this audio" --input ``` Repeat `--input` for multiple files, or pass a directory. Use `--single-assembly` to process the -inputs together. `run` also supports `--watch`, `--recursive`, and `--concurrency`. +inputs together. `run` also supports `--watch`, `--recursive`, `--concurrency`, and +`--url-download-timeout` (milliseconds per URL input, default: `300000`). ### Compile Assembly Instructions @@ -1529,6 +1530,8 @@ The `assemblies create` command additionally supports: or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly without uploads. With `--template`, stdin always requires `-i -`. - `--single-assembly` - Pass all input files to a single assembly instead of one assembly per file +- `--url-download-timeout` - Limit each URL input download to this many milliseconds (default: + `300000`, or five minutes). DNS checks and redirects share the same deadline. #### Log Levels diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index ccf45d99..00c81770 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -1752,9 +1752,11 @@ export interface AssembliesCreateOptions { singleAssembly?: boolean concurrency?: number printUrls?: boolean + urlDownloadTimeoutMs?: number } const DEFAULT_CONCURRENCY = 5 +const DEFAULT_URL_DOWNLOAD_TIMEOUT_MS = 5 * 60 * 1000 interface PreparedCreateOptions { effectiveStepsData: StepsInput | undefined @@ -1836,6 +1838,9 @@ export async function create( options: AssembliesCreateOptions, ): ReturnType { const preparedOptions = await prepareCreateOptions(outputctl, options) + const urlDownloadTimeoutMs = + toPositiveInteger(options.urlDownloadTimeoutMs, '--url-download-timeout') ?? + DEFAULT_URL_DOWNLOAD_TIMEOUT_MS if (!options.inputs.some(isUrlInput)) return await createWithLocalInputs(outputctl, client, options, preparedOptions) if (options.watch) throw new Error('--watch requires local inputs') @@ -1855,6 +1860,7 @@ export async function create( allowPrivateUrls: false, // Uploading preserves :original and Template semantics, just as Intents do. urlStrategy: 'download', + urlDownloadTimeoutMs, }) preparedInputs.push(prepared) const localPath = prepared.files[field] @@ -2447,6 +2453,10 @@ export class AssembliesCreateCommand extends AuthenticatedCommand { concurrency = concurrencyOption() + urlDownloadTimeoutMs = Option.String('--url-download-timeout', { + description: 'Maximum time per URL input download in milliseconds (default: 300000)', + }) + printUrls = printUrlsOption() protected async run(): Promise { @@ -2495,6 +2505,8 @@ export class AssembliesCreateCommand extends AuthenticatedCommand { singleAssembly: this.singleAssembly, concurrency: this.concurrency == null ? undefined : Number(this.concurrency), printUrls: this.printUrls, + urlDownloadTimeoutMs: + this.urlDownloadTimeoutMs == null ? undefined : Number(this.urlDownloadTimeoutMs), }) if (this.printUrls) { printResultUrls(this.output, resultUrls, assemblies) @@ -2567,6 +2579,10 @@ export class RunCommand extends AuthenticatedCommand { concurrency = concurrencyOption() + urlDownloadTimeoutMs = Option.String('--url-download-timeout', { + description: 'Maximum time per URL input download in milliseconds (default: 300000)', + }) + printUrls = printUrlsOption() protected async run(): Promise { @@ -2626,6 +2642,8 @@ export class RunCommand extends AuthenticatedCommand { singleAssembly: this.singleAssembly, concurrency: this.concurrency == null ? undefined : Number(this.concurrency), printUrls: this.printUrls, + urlDownloadTimeoutMs: + this.urlDownloadTimeoutMs == null ? undefined : Number(this.urlDownloadTimeoutMs), }) if (this.printUrls) { printResultUrls(this.output, resultUrls, assemblies) diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index a6424bed..521f418a 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -720,6 +720,39 @@ it('rejects watch mode for URL inputs before making a download request', async ( expect(Transloadit.prototype.createAssembly).not.toHaveBeenCalled() }) +it('times out a stalled URL input before creating an Assembly', async () => { + await fixture({}) + const { create } = await import('../../../src/cli/commands/assemblies.ts') + const download = nock('https://source.invalid') + .get('/slow.png') + .delay(200) + .reply(200, 'delayed input') + + await expect( + create(new OutputCtl(), new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + inputs: ['https://source.invalid/slow.png'], + stepsData: { output: { robot: '/file/filter' } }, + urlDownloadTimeoutMs: 30, + }), + ).rejects.toMatchObject({ name: 'TimeoutError' }) + expect(download.isDone()).toBe(true) + expect(Transloadit.prototype.createAssembly).not.toHaveBeenCalled() +}) + +it('honors the URL download timeout flag before uploading', async () => { + const { run } = await fixture({}) + const download = nock('https://source.invalid') + .get('/slow.png') + .delay(200) + .reply(200, 'delayed input') + + expect(await run(['-i', 'https://source.invalid/slow.png', '--url-download-timeout', '30'])).toBe( + 1, + ) + expect(download.isDone()).toBe(true) + expect(Transloadit.prototype.createAssembly).not.toHaveBeenCalled() +}) + it('keeps a repeated linked download together instead of renaming playlist targets', async () => { const { inputPath, outputDir, run } = await fixture({ adaptive: [ diff --git a/packages/transloadit/README.md b/packages/transloadit/README.md index 7e3dc1ed..9d175c60 100644 --- a/packages/transloadit/README.md +++ b/packages/transloadit/README.md @@ -146,7 +146,8 @@ npx -y @transloadit/node run "extract a waveform image from this audio" --input ``` Repeat `--input` for multiple files, or pass a directory. Use `--single-assembly` to process the -inputs together. `run` also supports `--watch`, `--recursive`, and `--concurrency`. +inputs together. `run` also supports `--watch`, `--recursive`, `--concurrency`, and +`--url-download-timeout` (milliseconds per URL input, default: `300000`). ### Compile Assembly Instructions @@ -1529,6 +1530,8 @@ The `assemblies create` command additionally supports: or a redirected file. Other non-TTY stdin (such as an agent shell or `/dev/null`) runs the Assembly without uploads. With `--template`, stdin always requires `-i -`. - `--single-assembly` - Pass all input files to a single assembly instead of one assembly per file +- `--url-download-timeout` - Limit each URL input download to this many milliseconds (default: + `300000`, or five minutes). DNS checks and redirects share the same deadline. #### Log Levels From 0a2683b5acd7e13a9d200a68ad48837da5111fcf Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 19:02:54 +0200 Subject: [PATCH 31/35] Associate collision-test results with their actual inputs --- .../test/unit/cli/download-layout.test.ts | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 521f418a..cf68bbea 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -300,6 +300,14 @@ it.each([ const multiInput = path.join(path.dirname(inputPath), 'report.pdf.docx') await writeFile(singleInput, 'single-result input') await writeFile(multiInput, 'multi-result input') + const { stat: originalStat } = fs.promises + vi.spyOn(fs.promises, 'stat').mockImplementation(async (target, ...args) => { + // Input discovery can queue jobs in a different order from the CLI arguments. + if (target === path.join(outputDir, path.basename(fileFirst ? singleInput : multiInput))) { + await new Promise((resolve) => setTimeout(resolve, 60)) + } + return await originalStat(target, ...args) + }) const single = { output: [{ name: 'report.pdf', url: 'https://downloads.test/pdf' }] } const multi = { output: [ @@ -307,13 +315,15 @@ it.each([ { name: 'page2.jpg', url: 'https://downloads.test/page2' }, ], } - vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) - .mockImplementationOnce(async () => { - // Finish the second job first unless the CLI is explicitly asked to run serially. - await new Promise((resolve) => setTimeout(resolve, 60)) - return { ok: 'ASSEMBLY_COMPLETED', results: fileFirst ? single : multi } - }) - .mockResolvedValueOnce({ ok: 'ASSEMBLY_COMPLETED', results: fileFirst ? multi : single }) + vi.mocked(Transloadit.prototype.createAssembly).mockImplementation(async ({ files }) => { + const input = Object.values(files ?? {})[0] + expect([singleInput, multiInput]).toContain(input) + return { assembly_id: input === singleInput ? 'single' : 'multi' } + }) + vi.mocked(Transloadit.prototype.awaitAssemblyCompletion).mockImplementation(async (id) => ({ + ok: 'ASSEMBLY_COMPLETED', + results: id === 'single' ? single : multi, + })) nock('https://downloads.test') .get('/pdf') .reply(200, 'pdf bytes') @@ -332,10 +342,12 @@ it.each([ '1', ]), ).toBe(0) + const fileWasFirst = (await stat(path.join(outputDir, 'report.pdf'))).isFile() + expect(fileWasFirst).toBe(!fileFirst) expect( - await readFile(path.join(outputDir, fileFirst ? 'report.pdf' : 'report__1.pdf'), 'utf8'), + await readFile(path.join(outputDir, fileWasFirst ? 'report.pdf' : 'report__1.pdf'), 'utf8'), ).toBe('pdf bytes') - const inputDir = path.join(outputDir, fileFirst ? 'report.pdf__1' : 'report.pdf') + const inputDir = path.join(outputDir, fileWasFirst ? 'report.pdf__1' : 'report.pdf') expect(await readFile(path.join(inputDir, 'page1.jpg'), 'utf8')).toBe('first page') expect(await readFile(path.join(inputDir, 'page2.jpg'), 'utf8')).toBe('second page') }) From d8944ec8789cfe57a6e36148e8c997e32a4fb1ae Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 19:04:05 +0200 Subject: [PATCH 32/35] Use an explicit promise in the collision fixture --- packages/node/test/unit/cli/download-layout.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index cf68bbea..1b74ffd1 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -315,10 +315,10 @@ it.each([ { name: 'page2.jpg', url: 'https://downloads.test/page2' }, ], } - vi.mocked(Transloadit.prototype.createAssembly).mockImplementation(async ({ files }) => { + vi.mocked(Transloadit.prototype.createAssembly).mockImplementation(({ files }) => { const input = Object.values(files ?? {})[0] expect([singleInput, multiInput]).toContain(input) - return { assembly_id: input === singleInput ? 'single' : 'multi' } + return Promise.resolve({ assembly_id: input === singleInput ? 'single' : 'multi' }) }) vi.mocked(Transloadit.prototype.awaitAssemblyCompletion).mockImplementation(async (id) => ({ ok: 'ASSEMBLY_COMPLETED', From 003ec3fc2b2b1c87049ce37d354803c5682fe168 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 22:40:41 +0200 Subject: [PATCH 33/35] Preserve CLI layouts and directory freshness skips --- .changeset/assembly-download-paths.md | 16 +- packages/node/README.md | 16 +- packages/node/src/cli/commands/assemblies.ts | 50 +++-- .../test/e2e/cli/assemblies-create.test.ts | 13 +- .../test/unit/cli/assemblies-create.test.ts | 35 +++- .../test/unit/cli/download-layout.test.ts | 191 ++++++++++++++---- packages/transloadit/README.md | 16 +- 7 files changed, 234 insertions(+), 103 deletions(-) diff --git a/.changeset/assembly-download-paths.md b/.changeset/assembly-download-paths.md index 20f481a4..e5c2ce57 100644 --- a/.changeset/assembly-download-paths.md +++ b/.changeset/assembly-download-paths.md @@ -1,17 +1,11 @@ --- -'@transloadit/node': major +'@transloadit/node': minor '@transloadit/mcp-server': patch -'transloadit': major +'transloadit': minor --- -Keep CLI download filenames and HLS directories aligned with Assembly results, omit input parent folders from output paths, and support HTTP(S) URL inputs for Assembly and prompt commands. +Fix CLI downloads to use Assembly result names and extensions, including bundled outputs and type-changing Steps. Preserve the existing input-relative directories and `//…` layout for ordinary multi-result downloads, along with time-based skipping for directory reruns. Use `--reprocess-stale` to process unchanged inputs again. -SDK-breaking: CLI directory downloads use Assembly result names and omit input parent folders. Directory reruns process inputs again and preserve existing files with collision suffixes. Update scripts to use the result paths; use an explicit output file when mtime-based skipping is required. +Fix HLS downloads by retaining the result directories referenced by playlists, keeping independent source bundles separate, and publishing linked watch updates together. Reserve download paths using the destination root’s case and Unicode naming rules, and avoid file/directory collisions and descendant symlinks. -Independent HLS bundles returned by the same Step get separate source folders so their playlist and segment paths can coexist. - -Reserve result paths with filesystem case sensitivity and file/directory conflicts in mind. Stage linked watch downloads before publishing them together, and validate local steps and output options before fetching URL inputs. - -Directory runs process inputs again to discover the result filenames instead of guessing freshness from upload names. Watch updates reuse their own result paths, while existing unrelated files and outputs from other inputs remain separate. - -URL input downloads have a five-minute deadline per input, configurable in milliseconds with `--url-download-timeout` on `assemblies create` and `run`. +Add public HTTP(S) URL inputs to `assemblies create` and `run` using temporary uploads that are cleaned up afterward. Each download has a five-minute deadline, configurable in milliseconds with `--url-download-timeout`. Watch mode requires local inputs. diff --git a/packages/node/README.md b/packages/node/README.md index 9d175c60..46ece905 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -1516,14 +1516,14 @@ The `assemblies create` command additionally supports: Without uploads (for example, `/vimeo/import`, `/http/import`, or prompt-only `/image/generate` Steps), results are saved as `//`, with no input folder, including with `--single-assembly`. - With one local input, filenames come from the Assembly results. Multiple result Steps get their - own folders; the input’s parent folders are omitted. Directory inputs retain subfolders relative - to that directory. HLS results retain the directories in `meta.relative_path`, so playlists can - find their renditions and segments. Multiple HLS bundles in one Step get separate folders named - after their sources. On collisions, a linked bundle moves together to a new folder, preserving - playlist references. - Watch updates refresh their existing result files. Other directory runs process the inputs again, - since result filenames are unknown before processing; file outputs keep mtime-based skipping. + Downloaded filenames come from the Assembly results, including their extensions. Ordinary + downloads retain the existing input-relative directories and `//…` layout for + multiple results. Directory inputs retain subfolders relative to that directory. Existing + time-based checks skip inputs with newer outputs; use `--reprocess-stale` to process them again. + HLS results retain the directories in `meta.relative_path`, so playlists can find their + renditions and segments. Multiple HLS bundles in one Step get separate source folders. On + collisions, a linked bundle moves together to a new folder, preserving playlist references. + Watch updates refresh their existing result files. - `--input, -i` - Provide a local upload or a public HTTP(S) URL, or use `-i -` to read stdin explicitly. URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 00c81770..6281be30 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -518,7 +518,7 @@ async function createExistingPathOutputPlan(outputPath: string | undefined): Pro return createOutputPlan(outputPath, stats?.mtime ?? new Date(0)) } -function dirProvider(output: string): OutputPlanProvider { +function dirProvider(output: string, urlInputPaths?: Set): OutputPlanProvider { const namesByInput = new Map() const usedNames = new Set() return async (inpath, indir) => { @@ -532,7 +532,9 @@ function dirProvider(output: string): OutputPlanProvider { const inputKey = path.resolve(inpath) let relpath = namesByInput.get(inputKey) if (relpath == null) { - const requestedPath = indir == null ? path.basename(inpath) : path.relative(indir, inpath) + const requestedPath = urlInputPaths?.has(inputKey) + ? path.basename(inpath) + : path.relative(indir ?? process.cwd(), inpath).replace(/^(\.\.\/)+/, '') const parsed = path.parse(requestedPath) relpath = await ensureUniqueCounterValue({ initialValue: requestedPath, @@ -1091,6 +1093,7 @@ async function resolveResultDownloadTargets({ inPath, inputs, normalizedResults, + outputMode, outputPath, outputRoot, outputRootIsDirectory, @@ -1102,6 +1105,7 @@ async function resolveResultDownloadTargets({ inPath: string | null inputs: string[] normalizedResults: NormalizedAssemblyResults + outputMode?: 'directory' | 'file' outputPath: string | null outputRoot: string outputRootIsDirectory: boolean @@ -1110,24 +1114,27 @@ async function resolveResultDownloadTargets({ singleAssembly?: boolean }): Promise { const { allFiles, entries } = normalizedResults - // Keep single-result jobs flat for multi-file CLI compatibility. Input groups are only - // needed for multi-file results; reservations distinguish colliding result names. + const linked = allFiles.some((file) => getResultRelativePath(file) != null) + // Ordinary CLI outputs retain their input/Step layout. HLS uses linked result paths so + // playlist references remain reachable; moving those formerly broken outputs is intentional. + const keepInputStepLayout = + outputMode !== 'directory' && inPath != null && allFiles.length > 1 && !linked const shouldGroupByInput = !singleAssembly && inPath != null && - (hasDirectoryInput || (inputs.length > 1 && allFiles.length > 1)) + (hasDirectoryInput || + (inputs.length > 1 && (allFiles.length > 1 || outputMode === 'directory'))) const resolveDirectoryBaseDir = (): string => { - if (hasDirectoryInput && outputPath != null) { - return allFiles.length === 1 && entries.length === 1 + if (outputPath != null && (keepInputStepLayout || hasDirectoryInput)) { + return allFiles.length === 1 && outputMode !== 'directory' ? path.dirname(outputPath) : path.join(path.dirname(outputPath), path.parse(outputPath).name) } - - if (!shouldGroupByInput || inPath == null) { - return outputRoot + if (!linked && outputMode !== 'directory' && inPath != null && outputPath != null) { + return path.dirname(outputPath) } - + if (!shouldGroupByInput || inPath == null) return outputRoot return path.join(outputRoot, path.parse(path.basename(outputPath ?? inPath)).name) } @@ -1173,7 +1180,7 @@ async function resolveResultDownloadTargets({ return await buildDirectoryDownloadTargets({ allFiles, baseDir, - groupByStep: inPath == null || entries.length > 1, + groupByStep: keepInputStepLayout || inPath == null || entries.length > 1, owner, outputRoot, reservedPaths, @@ -1247,6 +1254,7 @@ async function materializeAssemblyResults({ isCurrent, inputs, normalizedResults, + outputMode, outputPath, outputRoot, outputRootIsDirectory, @@ -1261,6 +1269,7 @@ async function materializeAssemblyResults({ isCurrent: () => boolean inputs: string[] normalizedResults: NormalizedAssemblyResults + outputMode?: 'directory' | 'file' outputPath: string | null outputRoot: string | null outputRootIsDirectory: boolean @@ -1281,6 +1290,7 @@ async function materializeAssemblyResults({ inPath, inputs, normalizedResults, + outputMode, outputPath, outputRoot, outputRootIsDirectory, @@ -1759,6 +1769,7 @@ const DEFAULT_CONCURRENCY = 5 const DEFAULT_URL_DOWNLOAD_TIMEOUT_MS = 5 * 60 * 1000 interface PreparedCreateOptions { + urlInputPaths?: Set effectiveStepsData: StepsInput | undefined hasDirectoryInput: boolean outstat: StatLike | undefined @@ -1847,6 +1858,7 @@ export async function create( const preparedInputs: Array>> = [] const localInputs: string[] = [] + const urlInputPaths = new Set() try { for (const [index, input] of options.inputs.entries()) { if (!isUrlInput(input)) { @@ -1866,6 +1878,7 @@ export async function create( const localPath = prepared.files[field] if (localPath == null) throw new Error('URL input could not be prepared') localInputs.push(localPath) + urlInputPaths.add(path.resolve(localPath)) } return await createWithLocalInputs( outputctl, @@ -1874,7 +1887,7 @@ export async function create( ...options, inputs: localInputs, }, - preparedOptions, + { ...preparedOptions, urlInputPaths }, ) } finally { await Promise.all( @@ -1889,6 +1902,7 @@ async function createWithLocalInputs( { template, fields, + outputMode, watch: watchOption, recursive, inputs, @@ -1925,7 +1939,7 @@ async function createWithLocalInputs( resolvedOutput == null ? nullProvider() : outstat?.isDirectory() - ? dirProvider(resolvedOutput) + ? dirProvider(resolvedOutput, preparedOptions.urlInputPaths) : fileProvider(resolvedOutput) const emitter = makeJobEmitter(inputs, { @@ -1934,10 +1948,7 @@ async function createWithLocalInputs( recursive, watch: watchOption, singleAssembly, - // Directory plans use upload names before the Assembly's result names are known. - // Inferring freshness from those names can skip unrelated results. File outputs retain - // mtime-based skipping; directory caching would require a persistent result manifest. - reprocessStale: reprocessStale || outstat?.isDirectory(), + reprocessStale, }) // Use p-queue for concurrency management @@ -2143,8 +2154,9 @@ async function createWithLocalInputs( hasDirectoryInput: singleAssemblyMode ? false : hasDirectoryInput, inPath, isCurrent: () => !isSupersededWatchJob(outputPlan?.path ?? null, outputToken), - inputs: singleAssemblyMode ? inputPaths : inputs, + inputs: singleAssemblyMode || outputMode === 'directory' ? inputPaths : inputs, normalizedResults, + outputMode, outputPath: outputPlan?.path ?? null, outputRoot: resolvedOutput ?? null, outputRootIsDirectory, diff --git a/packages/node/test/e2e/cli/assemblies-create.test.ts b/packages/node/test/e2e/cli/assemblies-create.test.ts index 636a20b3..37054836 100644 --- a/packages/node/test/e2e/cli/assemblies-create.test.ts +++ b/packages/node/test/e2e/cli/assemblies-create.test.ts @@ -400,7 +400,7 @@ describeLive('assemblies', { retry: 1 }, () => { ) it( - 'should reprocess directory outputs to discover result filenames', + 'should not reprocess inputs that are older than their directory output', testCase(async (client) => { const infiles = await Promise.all(['in1.jpg', 'in2.jpg', 'in3.jpg'].map(imgPromise)) const steps = await stepsPromise() @@ -413,19 +413,16 @@ describeLive('assemblies', { retry: 1 }, () => { output: 'out', }) + const inputTime = new Date(Date.now() - 60_000) + await fsp.utimes(infiles[0] as string, inputTime, inputTime) const output2 = new OutputCtl() const { results } = await assembliesCreate(output2, client, { steps, inputs: infiles, output: 'out', }) - expect(results).to.have.lengthOf(3) - expect(await fsp.readdir('out')).to.deep.equal([ - 'in1.jpg', - 'in1__1.jpg', - 'in2.jpg', - 'in3.jpg', - ]) + expect(results).to.have.lengthOf(2) + expect(await fsp.readdir('out')).to.deep.equal(['in1.jpg', 'in2.jpg', 'in3.jpg']) }), 180_000, ) diff --git a/packages/node/test/unit/cli/assemblies-create.test.ts b/packages/node/test/unit/cli/assemblies-create.test.ts index 5a0afb5b..c0017dd3 100644 --- a/packages/node/test/unit/cli/assemblies-create.test.ts +++ b/packages/node/test/unit/cli/assemblies-create.test.ts @@ -21,6 +21,10 @@ async function createTempDir(prefix: string): Promise { return tempDir } +function getLegacyRelativeInputPath(inputPath: string): string { + return path.relative(process.cwd(), inputPath).replace(/^(\.\.\/)+/, '') +} + async function collectRelativeFiles(rootDir: string, currentDir = rootDir): Promise { const entries = await readdir(currentDir, { withFileTypes: true }) const files: string[] = [] @@ -318,20 +322,22 @@ describe('assemblies create', () => { }, }), ) + const mappedInput = getLegacyRelativeInputPath(inputPath) + const baseDir = path.join(path.dirname(mappedInput), 'video') expect(await collectRelativeFiles(outputDir)).toEqual([ - path.join('captioned', 'video.mp4'), - path.join('transcript', 'video.srt'), - path.join('transcription', 'video.json'), + path.join(baseDir, 'captioned', 'video.mp4'), + path.join(baseDir, 'transcript', 'video.srt'), + path.join(baseDir, 'transcription', 'video.json'), ]) - expect(await readFile(path.join(outputDir, 'captioned', 'video.mp4'), 'utf8')).toBe( + expect(await readFile(path.join(outputDir, baseDir, 'captioned', 'video.mp4'), 'utf8')).toBe( 'captioned video', ) - expect(await readFile(path.join(outputDir, 'transcript', 'video.srt'), 'utf8')).toBe( + expect(await readFile(path.join(outputDir, baseDir, 'transcript', 'video.srt'), 'utf8')).toBe( 'subtitle text', ) - expect(await readFile(path.join(outputDir, 'transcription', 'video.json'), 'utf8')).toBe( - '{"words":[]}', - ) + expect( + await readFile(path.join(outputDir, baseDir, 'transcription', 'video.json'), 'utf8'), + ).toBe('{"words":[]}') expect(downloads.isDone()).toBe(true) }) @@ -1502,7 +1508,7 @@ describe('assemblies create', () => { expect(await readFile(path.join(outputDir, 'thumb__1.jpg'), 'utf8')).toBe('second-thumb') }) - it('uses result filenames directly for single-step directory outputs', async () => { + it('preserves input and Step folders for generic multi-result directory outputs', async () => { vi.spyOn(console, 'error').mockImplementation(() => {}) const tempDir = await createTempDir('transloadit-legacy-outdir-') @@ -1545,7 +1551,12 @@ describe('assemblies create', () => { } as never, ) - expect(await collectRelativeFiles(outputDir)).toEqual(['one.jpg', 'two.jpg']) + const mappedInput = getLegacyRelativeInputPath(inputPath) + const baseDir = path.join(path.dirname(mappedInput), 'clip') + expect(await collectRelativeFiles(outputDir)).toEqual([ + path.join(baseDir, 'thumbs', 'one.jpg'), + path.join(baseDir, 'thumbs', 'two.jpg'), + ]) }) it('uses the actual result filename for single-result directory outputs', async () => { @@ -1628,7 +1639,9 @@ describe('assemblies create', () => { }, }) - expect(await collectRelativeFiles(outputDir)).toEqual(['input.txt']) + expect(await collectRelativeFiles(outputDir)).toEqual([ + path.join(path.dirname(getLegacyRelativeInputPath(inputPath)), 'input.txt'), + ]) }) it('does not create an empty output file when assembly creation fails', async () => { diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 1b74ffd1..be1e9d93 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -32,6 +32,10 @@ vi.mock('node:dns/promises', () => ({ const tempDirs: string[] = [] +function mappedOutputPath(outputDir: string, inputPath: string): string { + return path.join(outputDir, path.relative(process.cwd(), inputPath).replace(/^(\.\.\/)+/, '')) +} + async function fixture(results: Record): Promise<{ inputPath: string outputDir: string @@ -87,6 +91,69 @@ afterEach(async () => { await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))) }) +it.each([ + false, + true, +])('skips newer directory outputs before starting an Assembly (directory input: %s)', async (directoryInput) => { + const { inputPath, outputDir, run } = await fixture({ + resized: [{ name: 'logo.png', url: 'https://downloads.test/result' }], + }) + const inputTime = new Date(Date.now() - 60_000) + await utimes(inputPath, inputTime, inputTime) + const args = ['-i', directoryInput ? path.dirname(inputPath) : inputPath] + nock('https://downloads.test').get('/result').twice().reply(200, 'existing result') + expect(await run(args)).toBe(0) + vi.mocked(Transloadit.prototype.createAssembly).mockClear() + + expect(await run(args)).toBe(0) + expect(Transloadit.prototype.createAssembly).not.toHaveBeenCalled() + expect( + await readFile( + directoryInput ? path.join(outputDir, 'logo.png') : mappedOutputPath(outputDir, inputPath), + 'utf8', + ), + ).toBe('existing result') + expect(await run([...args, '--reprocess-stale'])).toBe(0) + expect(Transloadit.prototype.createAssembly).toHaveBeenCalledTimes(1) +}) + +it.each([ + { name: 'frame.jpg', bytes: 'JPEG bytes' }, + { name: 'clip.json', bytes: '{"words":[]}' }, +])('saves converted video results with their own extension ($name)', async ({ name, bytes }) => { + const { outputDir, run, tempDir } = await fixture({ + converted: [{ name, url: 'https://downloads.test/result' }], + }) + const inputPath = path.join(tempDir, 'inputs', 'clip.mp4') + await writeFile(inputPath, 'video bytes') + nock('https://downloads.test').get('/result').reply(200, bytes) + + expect(await run(['-i', inputPath])).toBe(0) + const mappedInput = mappedOutputPath(outputDir, inputPath) + expect(await readFile(path.join(path.dirname(mappedInput), name), 'utf8')).toBe(bytes) + await expect(stat(mappedInput)).rejects.toMatchObject({ code: 'ENOENT' }) +}) + +it('keeps input and Step folders for type-changing multi-result downloads', async () => { + const { outputDir, run, tempDir } = await fixture({ + thumbs: [{ name: 'frame.jpg', url: 'https://downloads.test/thumb' }], + transcript: [{ name: 'clip.json', url: 'https://downloads.test/json' }], + }) + const inputPath = path.join(tempDir, 'inputs', 'clip.mp4') + await writeFile(inputPath, 'video bytes') + nock('https://downloads.test') + .get('/thumb') + .reply(200, 'JPEG bytes') + .get('/json') + .reply(200, '{"words":[]}') + + expect(await run(['-i', inputPath])).toBe(0) + const inputOutput = mappedOutputPath(outputDir, inputPath) + const baseDir = path.join(path.dirname(inputOutput), 'clip') + expect(await readFile(path.join(baseDir, 'thumbs', 'frame.jpg'), 'utf8')).toBe('JPEG bytes') + expect(await readFile(path.join(baseDir, 'transcript', 'clip.json'), 'utf8')).toBe('{"words":[]}') +}) + it('keeps every HLS playlist reference reachable from its downloaded playlist', async () => { const master = '#EXTM3U\nvariant/stream.m3u8\n' const variant = '#EXTM3U\n#EXTINF:1,\nsegment.ts\n#EXT-X-ENDLIST\n' @@ -300,10 +367,11 @@ it.each([ const multiInput = path.join(path.dirname(inputPath), 'report.pdf.docx') await writeFile(singleInput, 'single-result input') await writeFile(multiInput, 'multi-result input') + const resultDir = path.dirname(mappedOutputPath(outputDir, inputPath)) const { stat: originalStat } = fs.promises vi.spyOn(fs.promises, 'stat').mockImplementation(async (target, ...args) => { // Input discovery can queue jobs in a different order from the CLI arguments. - if (target === path.join(outputDir, path.basename(fileFirst ? singleInput : multiInput))) { + if (target === mappedOutputPath(outputDir, fileFirst ? singleInput : multiInput)) { await new Promise((resolve) => setTimeout(resolve, 60)) } return await originalStat(target, ...args) @@ -342,12 +410,12 @@ it.each([ '1', ]), ).toBe(0) - const fileWasFirst = (await stat(path.join(outputDir, 'report.pdf'))).isFile() + const fileWasFirst = (await stat(path.join(resultDir, 'report.pdf'))).isFile() expect(fileWasFirst).toBe(!fileFirst) expect( - await readFile(path.join(outputDir, fileWasFirst ? 'report.pdf' : 'report__1.pdf'), 'utf8'), + await readFile(path.join(resultDir, fileWasFirst ? 'report.pdf' : 'report__1.pdf'), 'utf8'), ).toBe('pdf bytes') - const inputDir = path.join(outputDir, fileWasFirst ? 'report.pdf__1' : 'report.pdf') + const inputDir = path.join(resultDir, fileWasFirst ? 'report.pdf__1' : 'report.pdf', 'output') expect(await readFile(path.join(inputDir, 'page1.jpg'), 'utf8')).toBe('first page') expect(await readFile(path.join(inputDir, 'page2.jpg'), 'utf8')).toBe('second page') }) @@ -426,12 +494,17 @@ it('preserves every independent Assembly result when file and directory inputs o expect(await run(['-i', inputPath, '-i', path.dirname(inputPath)])).toBe(0) expect(count).toBe(2) - const files = await readdir(path.join(outputDir, 'logo')) + const resultDir = path.join( + path.dirname(mappedOutputPath(outputDir, inputPath)), + 'logo', + 'images', + ) + const files = await readdir(resultDir) expect(files).toEqual(['a.png', 'a__1.png', 'b.png', 'b__1.png']) expect( ( await Promise.all( - files.map(async (file) => await readFile(path.join(outputDir, 'logo', file), 'utf8')), + files.map(async (file) => await readFile(path.join(resultDir, file), 'utf8')), ) ).sort(), ).toEqual(['a1', 'a2', 'b1', 'b2']) @@ -498,6 +571,7 @@ it.each([ normalizationSensitive, }) => { const { inputPath, outputDir, run } = await fixture({}) + const resultDir = path.dirname(mappedOutputPath(outputDir, inputPath)) const secondInput = path.join(path.dirname(inputPath), 'second.png') await writeFile(secondInput, 'second input') vi.mocked(Transloadit.prototype.awaitAssemblyCompletion) @@ -547,11 +621,11 @@ it.each([ expect(await run(['-i', inputPath, '-i', secondInput, '--concurrency', '2'])).toBe(0) const { name, ext } = path.parse(secondName) const secondOutput = `${name}__1${ext}` - expect((await readdir(outputDir)).map((value) => value.normalize('NFC')).sort()).toEqual( + expect((await readdir(resultDir)).map((value) => value.normalize('NFC')).sort()).toEqual( [firstName, secondOutput].map((value) => value.normalize('NFC')).sort(), ) - expect(await readFile(path.join(outputDir, firstName), 'utf8')).toBe('first result') - expect(await readFile(path.join(outputDir, secondOutput), 'utf8')).toBe('second result') + expect(await readFile(path.join(resultDir, firstName), 'utf8')).toBe('first result') + expect(await readFile(path.join(resultDir, secondOutput), 'utf8')).toBe('second result') }) it.each([ @@ -671,14 +745,15 @@ it.each([false, true])('uses the bundle result name (single Assembly: %s)', asyn const { inputPath, outputDir, run } = await fixture({ bundled: [{ name: 'archive.zip', url: 'https://downloads.test/archive' }], }) + const resultDir = single ? outputDir : path.dirname(mappedOutputPath(outputDir, inputPath)) nock('https://downloads.test').get('/archive').reply(200, 'archive bytes') expect(await run(['-i', inputPath, ...(single ? ['--single-assembly'] : [])])).toBe(0) - expect(await readdir(outputDir)).toEqual(['archive.zip']) - expect(await readFile(path.join(outputDir, 'archive.zip'), 'utf8')).toBe('archive bytes') + expect(await readdir(resultDir)).toEqual(['archive.zip']) + expect(await readFile(path.join(resultDir, 'archive.zip'), 'utf8')).toBe('archive bytes') }) -it('leaves the input folders out of a multi-step download', async () => { +it('keeps existing parent and input folders for a multi-step download', async () => { const { inputPath, outputDir, run } = await fixture({ image: [{ name: 'result.png', url: 'https://downloads.test/image' }], text: [{ name: 'result.txt', url: 'https://downloads.test/text' }], @@ -690,9 +765,10 @@ it('leaves the input folders out of a multi-step download', async () => { .reply(200, 'text bytes') expect(await run(['-i', inputPath])).toBe(0) - expect(await readdir(outputDir)).toEqual(['image', 'text']) - expect(await readFile(path.join(outputDir, 'image', 'result.png'), 'utf8')).toBe('image bytes') - expect(await readFile(path.join(outputDir, 'text', 'result.txt'), 'utf8')).toBe('text bytes') + const resultDir = path.join(path.dirname(mappedOutputPath(outputDir, inputPath)), 'logo') + expect(await readdir(resultDir)).toEqual(['image', 'text']) + expect(await readFile(path.join(resultDir, 'image', 'result.png'), 'utf8')).toBe('image bytes') + expect(await readFile(path.join(resultDir, 'text', 'result.txt'), 'utf8')).toBe('text bytes') }) it.each([ @@ -813,16 +889,19 @@ it('rejects a relative result path through a symlink outside the output director expect(await readdir(outside)).toEqual([]) }) -it('keeps single-result downloads flat for multiple standalone inputs', async () => { - const { inputPath, outputDir, run, tempDir } = await fixture({ +it('keeps single-result downloads in their mapped parent for multiple standalone inputs', async () => { + const { inputPath, outputDir, run } = await fixture({ resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], }) - const secondInput = path.join(tempDir, 'second.png') + const secondInput = path.join(path.dirname(inputPath), 'second.png') await writeFile(secondInput, 'second input') nock('https://downloads.test').get('/result').twice().reply(200, 'result bytes') expect(await run(['-i', inputPath, '-i', secondInput])).toBe(0) - expect(await readdir(outputDir)).toEqual(['result.png', 'result__1.png']) + expect(await readdir(path.dirname(mappedOutputPath(outputDir, inputPath)))).toEqual([ + 'result.png', + 'result__1.png', + ]) }) it('accepts inputs with the same basename in different folders', async () => { @@ -834,10 +913,21 @@ it('accepts inputs with the same basename in different folders', async () => { nock('https://downloads.test').get('/result').twice().reply(200, 'result bytes') expect(await run(['-i', inputPath, '-i', secondInput])).toBe(0) - expect(await readdir(outputDir)).toEqual(['result.png', 'result__1.png']) + expect( + await readFile( + path.join(path.dirname(mappedOutputPath(outputDir, inputPath)), 'result.png'), + 'utf8', + ), + ).toBe('result bytes') + expect( + await readFile( + path.join(path.dirname(mappedOutputPath(outputDir, secondInput)), 'result.png'), + 'utf8', + ), + ).toBe('result bytes') }) -it('does not mistake an input-named file for a current result with a different filename', async () => { +it('does not skip inputs for unrelated files outside their mapped output folder', async () => { const { inputPath, outputDir, run } = await fixture({ bundled: [{ name: 'archive.zip', url: 'https://downloads.test/result' }], }) @@ -847,20 +937,31 @@ it('does not mistake an input-named file for a current result with a different f nock('https://downloads.test').get('/result').reply(200, 'result bytes') expect(await run(['-i', inputPath])).toBe(0) - expect(await readFile(path.join(outputDir, 'archive.zip'), 'utf8')).toBe('result bytes') + expect( + await readFile( + path.join(path.dirname(mappedOutputPath(outputDir, inputPath)), 'archive.zip'), + 'utf8', + ), + ).toBe('result bytes') }) it('allows an explicitly selected output root that is a symlink', async () => { const { inputPath, outputDir, run, tempDir } = await fixture({ resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], }) + const resultDir = path.dirname(mappedOutputPath(outputDir, inputPath)) const destination = path.join(tempDir, 'destination') await mkdir(destination) await symlink(destination, outputDir) nock('https://downloads.test').get('/result').reply(200, 'result bytes') expect(await run(['-i', inputPath], outputDir)).toBe(0) - expect(await readFile(path.join(destination, 'result.png'), 'utf8')).toBe('result bytes') + expect( + await readFile( + path.join(destination, path.relative(outputDir, resultDir), 'result.png'), + 'utf8', + ), + ).toBe('result bytes') }) it('reserves concurrent linked downloads together when input stems match', async () => { @@ -923,9 +1024,10 @@ it.each([ const { inputPath, outputDir } = await fixture({ resized: [{ name: 'result.png', url: 'https://downloads.test/result' }], }) + const resultDir = path.dirname(mappedOutputPath(outputDir, inputPath)) if (existing) { - await mkdir(outputDir) - await writeFile(path.join(outputDir, 'result.png'), 'unrelated file') + await mkdir(resultDir, { recursive: true }) + await writeFile(path.join(resultDir, 'result.png'), 'unrelated file') } class FakeWatcher extends EventEmitter { close(): void { @@ -961,16 +1063,16 @@ it.each([ stepsData: { resized: { robot: '/image/resize', use: ':original', width: 16 } }, }) await firstCompletion - if (removeConflict) await unlink(path.join(outputDir, 'result.png')) + if (removeConflict) await unlink(path.join(resultDir, 'result.png')) watcher.emit('change', 'update', inputPath) const result = await processing expect(result.hasFailures).toBe(false) expect(completions).toBe(2) - expect(await readdir(outputDir)).toEqual(expectedFiles) - expect(await readFile(path.join(outputDir, expectedName), 'utf8')).toBe('updated result') + expect(await readdir(resultDir)).toEqual(expectedFiles) + expect(await readFile(path.join(resultDir, expectedName), 'utf8')).toBe('updated result') if (existing && !removeConflict) - expect(await readFile(path.join(outputDir, 'result.png'), 'utf8')).toBe('unrelated file') + expect(await readFile(path.join(resultDir, 'result.png'), 'utf8')).toBe('unrelated file') }) it('keeps a watched HLS Step in its selected folder after the original conflict disappears', async () => { @@ -1171,7 +1273,13 @@ it('accepts a standalone input and a directory containing the same basename', as nock('https://downloads.test').get('/result').twice().reply(200, 'result bytes') expect(await run(['-i', inputPath, '-i', directoryInput])).toBe(0) - expect(await readdir(outputDir)).toEqual(['result.png', 'result__1.png']) + expect(await readFile(path.join(outputDir, 'result.png'), 'utf8')).toBe('result bytes') + expect( + await readFile( + path.join(path.dirname(mappedOutputPath(outputDir, inputPath)), 'result.png'), + 'utf8', + ), + ).toBe('result bytes') }) it('fails promptly when a regular file blocks an ordinary Step directory', async () => { @@ -1179,12 +1287,13 @@ it('fails promptly when a regular file blocks an ordinary Step directory', async thumbs: [{ name: 'result.png', url: 'https://downloads.test/image' }], text: [{ name: 'result.txt', url: 'https://downloads.test/text' }], }) - await mkdir(outputDir) - await writeFile(path.join(outputDir, 'thumbs'), 'existing file') + const resultDir = path.join(path.dirname(mappedOutputPath(outputDir, inputPath)), 'logo') + await mkdir(resultDir, { recursive: true }) + await writeFile(path.join(resultDir, 'thumbs'), 'existing file') const lstat = fs.promises.lstat let probes = 0 vi.spyOn(fs.promises, 'lstat').mockImplementation(async (...args) => { - probes += 1 + if (String(args[0]).startsWith(resultDir)) probes += 1 // Bound the broken implementation so a failed regression does not hang the test worker. if (probes > 30) throw new Error('Filesystem probe budget exceeded') return await lstat(...args) @@ -1192,11 +1301,12 @@ it('fails promptly when a regular file blocks an ordinary Step directory', async expect(await run(['-i', inputPath])).toBe(1) expect(probes).toBeLessThan(15) - expect(await readFile(path.join(outputDir, 'thumbs'), 'utf8')).toBe('existing file') + expect(await readFile(path.join(resultDir, 'thumbs'), 'utf8')).toBe('existing file') }) it('discards a slow watch download when a newer result has already been saved', async () => { const { inputPath, outputDir } = await fixture({}) + const resultDir = path.dirname(mappedOutputPath(outputDir, inputPath)) class FakeWatcher extends EventEmitter { close(): void { this.emit('close') @@ -1253,7 +1363,7 @@ it('discards a slow watch download when a newer result has already been saved', expect((await processing).hasFailures).toBe(false) expect(count).toBe(2) - expect(await readFile(path.join(outputDir, 'result.png'), 'utf8')).toBe('newer result') + expect(await readFile(path.join(resultDir, 'result.png'), 'utf8')).toBe('newer result') }) it('moves a linked Step together when an existing file occupies a rendition directory', async () => { @@ -1284,6 +1394,11 @@ it('moves a linked Step together when an existing file occupies a rendition dire it('reuses watched filenames when the Assembly returns its results in a different order', async () => { const { inputPath, outputDir } = await fixture({}) + const resultDir = path.join( + path.dirname(mappedOutputPath(outputDir, inputPath)), + 'logo', + 'resized', + ) class FakeWatcher extends EventEmitter { close(): void { this.emit('close') @@ -1323,7 +1438,7 @@ it('reuses watched filenames when the Assembly returns its results in a differen }) expect(result.hasFailures).toBe(false) - expect(await readdir(outputDir)).toEqual(['a.png', 'b.png']) - expect(await readFile(path.join(outputDir, 'a.png'), 'utf8')).toBe('new a') - expect(await readFile(path.join(outputDir, 'b.png'), 'utf8')).toBe('new b') + expect(await readdir(resultDir)).toEqual(['a.png', 'b.png']) + expect(await readFile(path.join(resultDir, 'a.png'), 'utf8')).toBe('new a') + expect(await readFile(path.join(resultDir, 'b.png'), 'utf8')).toBe('new b') }) diff --git a/packages/transloadit/README.md b/packages/transloadit/README.md index 9d175c60..46ece905 100644 --- a/packages/transloadit/README.md +++ b/packages/transloadit/README.md @@ -1516,14 +1516,14 @@ The `assemblies create` command additionally supports: Without uploads (for example, `/vimeo/import`, `/http/import`, or prompt-only `/image/generate` Steps), results are saved as `//`, with no input folder, including with `--single-assembly`. - With one local input, filenames come from the Assembly results. Multiple result Steps get their - own folders; the input’s parent folders are omitted. Directory inputs retain subfolders relative - to that directory. HLS results retain the directories in `meta.relative_path`, so playlists can - find their renditions and segments. Multiple HLS bundles in one Step get separate folders named - after their sources. On collisions, a linked bundle moves together to a new folder, preserving - playlist references. - Watch updates refresh their existing result files. Other directory runs process the inputs again, - since result filenames are unknown before processing; file outputs keep mtime-based skipping. + Downloaded filenames come from the Assembly results, including their extensions. Ordinary + downloads retain the existing input-relative directories and `//…` layout for + multiple results. Directory inputs retain subfolders relative to that directory. Existing + time-based checks skip inputs with newer outputs; use `--reprocess-stale` to process them again. + HLS results retain the directories in `meta.relative_path`, so playlists can find their + renditions and segments. Multiple HLS bundles in one Step get separate source folders. On + collisions, a linked bundle moves together to a new folder, preserving playlist references. + Watch updates refresh their existing result files. - `--input, -i` - Provide a local upload or a public HTTP(S) URL, or use `-i -` to read stdin explicitly. URLs are downloaded to temporary files for upload and cleaned up after processing; `--watch` requires local inputs. When `--steps` is used without `-i`, stdin is read only when it is a pipe From e8a81435e96d43972f94b9290a2030720ea89ecd Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 23:10:45 +0200 Subject: [PATCH 34/35] Preserve CLI layout and freshness compatibility --- packages/node/src/cli/commands/assemblies.ts | 42 ++++-- packages/node/src/cli/intentRuntime.ts | 8 +- .../test/unit/cli/download-layout.test.ts | 120 ++++++++++++++++++ 3 files changed, 158 insertions(+), 12 deletions(-) diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index 6281be30..d2398d30 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -518,7 +518,7 @@ async function createExistingPathOutputPlan(outputPath: string | undefined): Pro return createOutputPlan(outputPath, stats?.mtime ?? new Date(0)) } -function dirProvider(output: string, urlInputPaths?: Set): OutputPlanProvider { +function dirProvider(output: string, transientInputPaths?: Set): OutputPlanProvider { const namesByInput = new Map() const usedNames = new Set() return async (inpath, indir) => { @@ -532,7 +532,7 @@ function dirProvider(output: string, urlInputPaths?: Set): OutputPlanPro const inputKey = path.resolve(inpath) let relpath = namesByInput.get(inputKey) if (relpath == null) { - const requestedPath = urlInputPaths?.has(inputKey) + const requestedPath = transientInputPaths?.has(inputKey) ? path.basename(inpath) : path.relative(indir ?? process.cwd(), inpath).replace(/^(\.\.\/)+/, '') const parsed = path.parse(requestedPath) @@ -1628,7 +1628,8 @@ function dismissStaleJobs(jobEmitter: EventEmitter): MyEventEmitter { jobEmitter.on('end', () => Promise.all(pendingChecks).then(() => emitter.emit('end'))) jobEmitter.on('error', (err: Error) => emitter.emit('error', err)) jobEmitter.on('job', (job: Job) => { - if (job.inputPath == null || job.out == null) { + // An older download can finish after a watch change and have a misleading newer mtime. + if (job.watchEvent || job.inputPath == null || job.out == null) { emitter.emit('job', job) return } @@ -1756,6 +1757,8 @@ export interface AssembliesCreateOptions { watch?: boolean recursive?: boolean inputs: string[] + /** Prepared URL/base64 uploads whose temporary parent paths must not enter the output layout. */ + transientInputPaths?: string[] output?: string | null del?: boolean reprocessStale?: boolean @@ -1769,7 +1772,7 @@ const DEFAULT_CONCURRENCY = 5 const DEFAULT_URL_DOWNLOAD_TIMEOUT_MS = 5 * 60 * 1000 interface PreparedCreateOptions { - urlInputPaths?: Set + transientInputPaths?: Set effectiveStepsData: StepsInput | undefined hasDirectoryInput: boolean outstat: StatLike | undefined @@ -1782,7 +1785,15 @@ function isUrlInput(input: string): boolean { async function prepareCreateOptions( outputctl: IOutputCtl, - { steps, stepsData, outputMode, output, inputs, singleAssembly }: AssembliesCreateOptions, + { + steps, + stepsData, + outputMode, + output, + inputs, + singleAssembly, + transientInputPaths, + }: AssembliesCreateOptions, ): Promise { // Quick fix for https://github.com/transloadit/transloadify/issues/13 // Only default to stdout when output is undefined (not provided), not when explicitly null @@ -1838,7 +1849,16 @@ async function prepareCreateOptions( }), ) const hasDirectoryInput = inputStats.some((stat) => stat?.isDirectory() === true) - return { effectiveStepsData, hasDirectoryInput, outstat, resolvedOutput } + return { + effectiveStepsData, + hasDirectoryInput, + outstat, + resolvedOutput, + transientInputPaths: + transientInputPaths == null + ? undefined + : new Set(transientInputPaths.map((inputPath) => path.resolve(inputPath))), + } } // --- Main assembly create function --- @@ -1858,7 +1878,7 @@ export async function create( const preparedInputs: Array>> = [] const localInputs: string[] = [] - const urlInputPaths = new Set() + const transientInputPaths = preparedOptions.transientInputPaths ?? new Set() try { for (const [index, input] of options.inputs.entries()) { if (!isUrlInput(input)) { @@ -1878,7 +1898,7 @@ export async function create( const localPath = prepared.files[field] if (localPath == null) throw new Error('URL input could not be prepared') localInputs.push(localPath) - urlInputPaths.add(path.resolve(localPath)) + transientInputPaths.add(path.resolve(localPath)) } return await createWithLocalInputs( outputctl, @@ -1887,7 +1907,7 @@ export async function create( ...options, inputs: localInputs, }, - { ...preparedOptions, urlInputPaths }, + { ...preparedOptions, transientInputPaths }, ) } finally { await Promise.all( @@ -1939,7 +1959,7 @@ async function createWithLocalInputs( resolvedOutput == null ? nullProvider() : outstat?.isDirectory() - ? dirProvider(resolvedOutput, preparedOptions.urlInputPaths) + ? dirProvider(resolvedOutput, preparedOptions.transientInputPaths) : fileProvider(resolvedOutput) const emitter = makeJobEmitter(inputs, { @@ -2154,7 +2174,7 @@ async function createWithLocalInputs( hasDirectoryInput: singleAssemblyMode ? false : hasDirectoryInput, inPath, isCurrent: () => !isSupersededWatchJob(outputPlan?.path ?? null, outputToken), - inputs: singleAssemblyMode || outputMode === 'directory' ? inputPaths : inputs, + inputs: singleAssemblyMode ? inputPaths : inputs, normalizedResults, outputMode, outputPath: outputPlan?.path ?? null, diff --git a/packages/node/src/cli/intentRuntime.ts b/packages/node/src/cli/intentRuntime.ts index 013e651d..3063d75b 100644 --- a/packages/node/src/cli/intentRuntime.ts +++ b/packages/node/src/cli/intentRuntime.ts @@ -33,6 +33,7 @@ export interface PreparedIntentInputs { cleanup: Array<() => Promise> hasTransientInputs: boolean inputs: string[] + transientInputPaths?: string[] } export interface IntentSingleStepExecutionDefinition { @@ -252,6 +253,7 @@ export async function prepareIntentInputs({ cleanup: prepared.cleanup, hasTransientInputs: true, inputs, + transientInputPaths: Object.values(prepared.files), } } @@ -767,9 +769,13 @@ abstract class GeneratedFileIntentCommandBase extends GeneratedIntentCommandBase } const effectiveOutputPath = this.getEffectiveOutputPath(rawValues) + const createOptions = this.getCreateOptions(effectivePreparedInputs.inputs) + if (effectivePreparedInputs.transientInputPaths != null) { + createOptions.transientInputPaths = effectivePreparedInputs.transientInputPaths + } return await executeIntentCommand({ client: this.client, - createOptions: this.getCreateOptions(effectivePreparedInputs.inputs), + createOptions, definition: this.getIntentDefinition(), output: this.output, outputMode: this.getEffectiveOutputMode(rawValues, effectiveOutputPath), diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index be1e9d93..92bf3c92 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -23,6 +23,7 @@ import { afterEach, beforeEach, expect, it, vi } from 'vitest' import { tryCatch } from '../../../src/alphalib/tryCatch.ts' import { AssembliesCreateCommand } from '../../../src/cli/commands/assemblies.ts' +import { intentCommands } from '../../../src/cli/intentCommands.ts' import OutputCtl from '../../../src/cli/OutputCtl.ts' import { Transloadit } from '../../../src/Transloadit.ts' @@ -154,6 +155,125 @@ it('keeps input and Step folders for type-changing multi-result downloads', asyn expect(await readFile(path.join(baseDir, 'transcript', 'clip.json'), 'utf8')).toBe('{"words":[]}') }) +it('keeps per-input folders for multiple video thumbs inputs', async () => { + const { inputPath, outputDir } = await fixture({ + thumbs: [{ name: 'frame.jpg', url: 'https://downloads.test/frame' }], + }) + const secondInput = path.join(path.dirname(inputPath), 'second.mp4') + await writeFile(secondInput, 'second video') + nock('https://downloads.test').get('/frame').twice().reply(200, 'JPEG bytes') + + expect( + await Cli.from(intentCommands).run([ + 'video', + 'thumbs', + '-i', + inputPath, + '-i', + secondInput, + '-o', + `${outputDir}/`, + ]), + ).toBe(0) + expect(await readFile(path.join(outputDir, 'logo', 'frame.jpg'), 'utf8')).toBe('JPEG bytes') + expect(await readFile(path.join(outputDir, 'second', 'frame.jpg'), 'utf8')).toBe('JPEG bytes') +}) + +it.each(['URL', 'base64'])('omits staging folders from %s intent downloads', async (kind) => { + const { outputDir } = await fixture({ + preview: [{ name: 'preview.png', url: 'https://downloads.test/preview' }], + }) + const args = + kind === 'URL' + ? ['-i', 'https://inputs.test/source.pdf'] + : ['--input-base64', Buffer.from('PDF bytes').toString('base64')] + if (kind === 'URL') nock('https://inputs.test').get('/source.pdf').reply(200, 'PDF bytes') + nock('https://downloads.test').get('/preview').reply(200, 'PNG bytes') + + expect( + await Cli.from(intentCommands).run(['preview', 'generate', ...args, '-o', `${outputDir}/`]), + ).toBe(0) + expect(await readdir(outputDir)).toEqual(['preview.png']) + expect(await readFile(path.join(outputDir, 'preview.png'), 'utf8')).toBe('PNG bytes') +}) + +it('processes a watch change when an older download finishes during its output check', async () => { + const { inputPath, outputDir } = await fixture({ + resized: [{ name: 'logo.png', url: 'https://downloads.test/result' }], + }) + class FakeWatcher extends EventEmitter { + close(): void { + this.emit('close') + } + } + const watcher = new FakeWatcher() + let resolveWatcherReady: (() => void) | undefined + const watcherReady = new Promise((resolve) => { + resolveWatcherReady = resolve + }) + vi.doMock('node-watch', () => ({ + default: () => { + resolveWatcherReady?.() + return watcher + }, + })) + vi.resetModules() + const { create } = await import('../../../src/cli/commands/assemblies.ts') + let finishDownload: (() => void) | undefined + const downloadBody = new Promise((resolve) => { + finishDownload = () => resolve('older result') + }) + let resolveDownloadStarted: (() => void) | undefined + const downloadStarted = new Promise((resolve) => { + resolveDownloadStarted = resolve + }) + nock('https://downloads.test') + .get('/result') + .reply(200, async () => { + resolveDownloadStarted?.() + return await downloadBody + }) + .get('/result') + .reply(200, 'newer result') + let resolveFirstCompletion: (() => void) | undefined + const firstCompletion = new Promise((resolve) => { + resolveFirstCompletion = resolve + }) + const output = new OutputCtl() + vi.spyOn(output, 'debug').mockImplementation((message) => { + if (String(message).startsWith('COMPLETED')) resolveFirstCompletion?.() + }) + const originalStat = fs.promises.stat + let changed = false + vi.spyOn(fs.promises, 'stat').mockImplementation(async (target, ...args) => { + const checkingChangedOutput = changed && target === mappedOutputPath(outputDir, inputPath) + if (checkingChangedOutput) { + finishDownload?.() + await firstCompletion + } + const stats = await originalStat(target, ...args) + if (checkingChangedOutput) setImmediate(() => watcher.close()) + return stats + }) + + const processing = create(output, new Transloadit({ authKey: 'key', authSecret: 'secret' }), { + inputs: [inputPath], + output: `${outputDir}/`, + watch: true, + stepsData: { resized: { robot: '/image/resize', use: ':original', width: 16 } }, + }) + await Promise.all([watcherReady, downloadStarted]) + await writeFile(inputPath, 'changed input') + const inputTime = new Date(Date.now() - 30_000) + await utimes(inputPath, inputTime, inputTime) + changed = true + watcher.emit('change', 'update', inputPath) + + expect((await processing).hasFailures).toBe(false) + expect(Transloadit.prototype.createAssembly).toHaveBeenCalledTimes(2) + expect(await readFile(mappedOutputPath(outputDir, inputPath), 'utf8')).toBe('newer result') +}) + it('keeps every HLS playlist reference reachable from its downloaded playlist', async () => { const master = '#EXTM3U\nvariant/stream.m3u8\n' const variant = '#EXTM3U\n#EXTINF:1,\nsegment.ts\n#EXT-X-ENDLIST\n' From 1d5bb2aa213c3500c55a10b37dbd1d4a4bf4dac4 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Fri, 23 Oct 2026 23:19:34 +0200 Subject: [PATCH 35/35] Refresh mapped downloads before directory freshness checks --- packages/node/src/cli/commands/assemblies.ts | 22 +++++++++++-- .../test/unit/cli/download-layout.test.ts | 33 +++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/packages/node/src/cli/commands/assemblies.ts b/packages/node/src/cli/commands/assemblies.ts index d2398d30..c4e5adbe 100644 --- a/packages/node/src/cli/commands/assemblies.ts +++ b/packages/node/src/cli/commands/assemblies.ts @@ -712,6 +712,7 @@ async function ensureUniquePath( reservedPaths: DownloadPathReservations, owner: string, directory = false, + overwriteExistingFile = false, ): Promise { const selectedPath = reservedPaths.pathsByOwner.get(owner) if (selectedPath != null) return selectedPath @@ -719,7 +720,13 @@ async function ensureUniquePath( return await ensureUniqueCounterValue({ initialValue: targetPath, isTaken: async (candidate) => - await isDownloadPathTaken(candidate, reservedPaths, owner, directory), + await isDownloadPathTaken( + candidate, + reservedPaths, + owner, + directory, + overwriteExistingFile && candidate === targetPath, + ), reserve: (candidate) => { reserveDownloadPath(candidate, owner, reservedPaths, directory) reservedPaths.pathsByOwner.set(owner, candidate) @@ -849,6 +856,7 @@ async function isDownloadPathTaken( reservedPaths: DownloadPathReservations, owner: string, directory = false, + overwriteExistingFile = false, ): Promise { const key = downloadPathKey(target, reservedPaths) if (directory && reservedPaths.directories.has(key)) return false @@ -860,7 +868,9 @@ async function isDownloadPathTaken( const [error, stats] = await tryCatch(fsp.lstat(target)) if (isErrnoException(error) && error.code === 'ENOTDIR') return true if (error && (!isErrnoException(error) || error.code !== 'ENOENT')) throw error - return error == null && (!directory || !stats?.isDirectory()) + if (error != null) return false + if (directory) return !stats?.isDirectory() + return !overwriteExistingFile || !stats?.isFile() } async function prepareDownloadDirectory( @@ -965,6 +975,7 @@ async function buildDirectoryDownloadTargets({ groupByStep, owner, outputRoot, + overwritePath, reservedPaths, }: { allFiles: NormalizedAssemblyResultFile[] @@ -972,6 +983,7 @@ async function buildDirectoryDownloadTargets({ groupByStep: boolean owner: string outputRoot: string + overwritePath?: string reservedPaths: DownloadPathReservations }): Promise { await prepareDownloadDirectory(outputRoot, baseDir, reservedPaths) @@ -1062,6 +1074,8 @@ async function buildDirectoryDownloadTargets({ targetPath, reservedPaths, JSON.stringify([groupOwner, name, occurrence]), + false, + overwritePath != null && path.resolve(targetPath) === path.resolve(overwritePath), ) await assertNoOutputSymlinks(outputRoot, selectedPath) targets.push({ @@ -1183,6 +1197,10 @@ async function resolveResultDownloadTargets({ groupByStep: keepInputStepLayout || inPath == null || entries.length > 1, owner, outputRoot, + // The mapped single-file destination is also the historical freshness marker. Updating it + // lets an unchanged rerun skip processing; differently named results retain collision suffixes. + overwritePath: + inPath != null && allFiles.length === 1 && !linked ? (outputPath ?? undefined) : undefined, reservedPaths, }) } diff --git a/packages/node/test/unit/cli/download-layout.test.ts b/packages/node/test/unit/cli/download-layout.test.ts index 92bf3c92..718dcf62 100644 --- a/packages/node/test/unit/cli/download-layout.test.ts +++ b/packages/node/test/unit/cli/download-layout.test.ts @@ -118,6 +118,39 @@ it.each([ expect(Transloadit.prototype.createAssembly).toHaveBeenCalledTimes(1) }) +it.each([ + false, + true, +])('refreshes mapped outputs and resumes skipping after an input change (directory input: %s)', async (directoryInput) => { + const { inputPath, outputDir, run } = await fixture({ + resized: [{ name: 'logo.png', url: 'https://downloads.test/result' }], + }) + const args = ['-i', directoryInput ? path.dirname(inputPath) : inputPath] + const outputPath = directoryInput + ? path.join(outputDir, 'logo.png') + : mappedOutputPath(outputDir, inputPath) + nock('https://downloads.test') + .get('/result') + .reply(200, 'original result') + .get('/result') + .twice() + .reply(200, 'updated result') + expect(await run(args)).toBe(0) + const outputTime = new Date(Date.now() - 60_000) + await utimes(outputPath, outputTime, outputTime) + await writeFile(inputPath, 'changed input') + const inputTime = new Date(Date.now() - 30_000) + await utimes(inputPath, inputTime, inputTime) + + expect(await run(args)).toBe(0) + expect(await run(args)).toBe(0) + expect(Transloadit.prototype.createAssembly).toHaveBeenCalledTimes(2) + expect(await readFile(outputPath, 'utf8')).toBe('updated result') + expect(await run([...args, '--reprocess-stale'])).toBe(0) + expect(Transloadit.prototype.createAssembly).toHaveBeenCalledTimes(3) + expect(await readdir(path.dirname(outputPath))).toEqual(['logo.png']) +}) + it.each([ { name: 'frame.jpg', bytes: 'JPEG bytes' }, { name: 'clip.json', bytes: '{"words":[]}' },