diff --git a/.talismanrc b/.talismanrc index b03f6eed4..eeae15c85 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,4 +1,4 @@ fileignoreconfig: - filename: pnpm-lock.yaml - checksum: 7ec6345eb15ed0be001753ee49733421a8a07096dc8a18465cdad1b82562fed8 + checksum: 1215ef6c0dfe2200186bb5d842d7fce7ee7d7ea5954224a38553e9c574b9f9da version: '1.0' diff --git a/packages/contentstack-asset-management/package.json b/packages/contentstack-asset-management/package.json index 9b0439ca7..ad91e9556 100644 --- a/packages/contentstack-asset-management/package.json +++ b/packages/contentstack-asset-management/package.json @@ -30,7 +30,8 @@ ], "license": "MIT", "dependencies": { - "@contentstack/cli-utilities": "~2.0.0-beta.9" + "@contentstack/cli-utilities": "~2.0.0-beta.9", + "lodash": "^4.17.21" }, "oclif": { "commands": "./lib/commands", @@ -42,6 +43,7 @@ }, "devDependencies": { "@types/chai": "^4.3.11", + "@types/lodash": "^4.17.0", "@types/mocha": "^10.0.6", "@types/node": "^20.17.50", "@types/sinon": "^17.0.4", diff --git a/packages/contentstack-asset-management/src/constants/index.ts b/packages/contentstack-asset-management/src/constants/index.ts index 9dbc66186..ec288b72a 100644 --- a/packages/contentstack-asset-management/src/constants/index.ts +++ b/packages/contentstack-asset-management/src/constants/index.ts @@ -4,6 +4,8 @@ export const FALLBACK_AM_CHUNK_FILE_SIZE_MB = 1; export const FALLBACK_AM_API_CONCURRENCY = 5; /** @deprecated Use FALLBACK_AM_API_CONCURRENCY */ export const DEFAULT_AM_API_CONCURRENCY = FALLBACK_AM_API_CONCURRENCY; +export const FALLBACK_AM_API_PAGE_SIZE = 100; +export const FALLBACK_AM_API_FETCH_CONCURRENCY = 5; /** Fallback strip lists when import options omit `fieldsImportInvalidKeys` / `assetTypesImportInvalidKeys`. */ export const FALLBACK_FIELDS_IMPORT_INVALID_KEYS = [ diff --git a/packages/contentstack-asset-management/src/export/asset-types.ts b/packages/contentstack-asset-management/src/export/asset-types.ts index 50487195e..7b171a25f 100644 --- a/packages/contentstack-asset-management/src/export/asset-types.ts +++ b/packages/contentstack-asset-management/src/export/asset-types.ts @@ -18,7 +18,7 @@ export default class ExportAssetTypes extends CSAssetsExportAdapter { log.debug('Starting shared asset types export process...', this.exportContext.context); - const assetTypesData = await this.getWorkspaceAssetTypes(spaceUid); + const assetTypesData = await this.getWorkspaceAssetTypes(spaceUid, this.apiPageSize, this.apiFetchConcurrency); const items = getArrayFromResponse(assetTypesData, 'asset_types'); const dir = this.getAssetTypesDir(); if (items.length === 0) { diff --git a/packages/contentstack-asset-management/src/export/assets.ts b/packages/contentstack-asset-management/src/export/assets.ts index 6cc1129a3..8fdad2039 100644 --- a/packages/contentstack-asset-management/src/export/assets.ts +++ b/packages/contentstack-asset-management/src/export/assets.ts @@ -1,20 +1,31 @@ import { resolve as pResolve } from 'node:path'; import { Readable } from 'node:stream'; import { mkdir, writeFile } from 'node:fs/promises'; -import { configHandler, log } from '@contentstack/cli-utilities'; +import chunk from 'lodash/chunk'; +import { configHandler, log, FsUtility } from '@contentstack/cli-utilities'; import type { CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api'; import type { ExportContext } from '../types/export-types'; import { CSAssetsExportAdapter } from './base'; -import { getAssetItems, writeStreamToFile } from '../utils/export-helpers'; -import { runInBatches } from '../utils/concurrent-batch'; +import { writeStreamToFile } from '../utils/export-helpers'; +import { forEachChunkedJsonStore } from '../utils/chunked-json-reader'; +import { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from '../utils/retry'; +import type { CustomPromiseHandler } from '../utils/cs-assets-api-adapter'; import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +const ASSET_META_KEYS = ['uid', 'url', 'filename', 'file_name', 'parent_uid']; + +type AssetRecord = { uid?: string; _uid?: string; url?: string; filename?: string; file_name?: string }; + export default class ExportAssets extends CSAssetsExportAdapter { constructor(apiConfig: CSAssetsAPIConfig, exportContext: ExportContext) { super(apiConfig, exportContext); } + private isDownloadable(asset: AssetRecord): boolean { + return Boolean(asset?.url && (asset?.uid ?? asset?._uid)); + } + async start(workspace: LinkedWorkspace, spaceDir: string): Promise { await this.init(); @@ -25,113 +36,142 @@ export default class ExportAssets extends CSAssetsExportAdapter { await mkdir(assetsDir, { recursive: true }); log.debug(`Assets directory ready: ${assetsDir}`, this.exportContext.context); - log.debug(`Fetching folders and assets for space ${workspace.space_uid}`, this.exportContext.context); - - const [folders, assetsData] = await Promise.all([ - this.getWorkspaceFolders(workspace.space_uid, workspace.uid), - this.getWorkspaceAssets(workspace.space_uid, workspace.uid), + // Stream asset metadata straight to chunked JSON as pages arrive — never hold the full set in + // memory. The writer is created lazily so an empty space writes an empty index instead of chunks. + let fsWriter: FsUtility | undefined; + let totalStreamed = 0; + let downloadableCount = 0; + const onPage = (items: unknown[]) => { + if (items.length === 0) return; + if (!fsWriter) fsWriter = this.createChunkedJsonWriter(assetsDir, 'assets.json', 'assets', ASSET_META_KEYS); + fsWriter.writeIntoFile(items as Record[], { mapKeyVal: true }); + totalStreamed += items.length; + for (const asset of items as AssetRecord[]) if (this.isDownloadable(asset)) downloadableCount += 1; + }; + + log.debug(`Fetching folders and streaming assets for space ${workspace.space_uid}`, this.exportContext.context); + const [folders] = await Promise.all([ + this.getWorkspaceFolders(workspace.space_uid, workspace.uid, this.apiPageSize, this.apiFetchConcurrency), + this.streamWorkspaceAssets(workspace.space_uid, workspace.uid, onPage, this.apiPageSize, this.apiFetchConcurrency), ]); - const assetItems = getAssetItems(assetsData); - const downloadableCount = assetItems.filter((asset) => Boolean(asset.url && (asset.uid ?? asset._uid))).length; + if (fsWriter) fsWriter.completeFile(true); + else await this.writeEmptyChunkedJson(assetsDir, 'assets.json'); + log.debug(`Wrote chunked assets metadata (${totalStreamed} item(s)) under ${assetsDir}`, this.exportContext.context); + // Per-space total: 1 folder write + 1 metadata write + N per-asset downloads. - // The shared module-level total is just a placeholder before this point; update - // it now so the multibar row shows real progress as downloads tick in. this.progressOrParent?.updateProcessTotal?.(this.processName, 2 + downloadableCount); await writeFile(pResolve(assetsDir, 'folders.json'), JSON.stringify(folders, null, 2)); this.tick(true, `folders: ${workspace.space_uid}`, null); log.debug(`Wrote folders.json for space ${workspace.space_uid}`, this.exportContext.context); - log.debug( - assetItems.length === 0 - ? `No assets for space ${workspace.space_uid}, wrote empty assets.json` - : `Writing ${assetItems.length} assets metadata for space ${workspace.space_uid}`, - this.exportContext.context, - ); - await this.writeItemsToChunkedJson( - assetsDir, - 'assets.json', - 'assets', - ['uid', 'url', 'filename', 'file_name', 'parent_uid'], - assetItems, - ); - log.debug( - `Finished writing chunked assets metadata (${assetItems.length} item(s)) under ${assetsDir}`, - this.exportContext.context, - ); log.info( - assetItems.length === 0 + totalStreamed === 0 ? `Wrote empty asset metadata for space ${workspace.space_uid}` - : `Wrote ${assetItems.length} asset metadata record(s) for space ${workspace.space_uid}`, + : `Wrote ${totalStreamed} asset metadata record(s) for space ${workspace.space_uid}`, this.exportContext.context, ); - this.tick(true, `metadata: ${workspace.space_uid} (${assetItems.length})`, null); + this.tick(true, `metadata: ${workspace.space_uid} (${totalStreamed})`, null); log.debug(`Starting binary downloads for space ${workspace.space_uid}`, this.exportContext.context); - await this.downloadWorkspaceAssets(assetsData, assetsDir, workspace.space_uid); + await this.downloadWorkspaceAssets(assetsDir, workspace.space_uid, downloadableCount); } - private async downloadWorkspaceAssets(assetsData: unknown, assetsDir: string, spaceUid: string): Promise { - const items = getAssetItems(assetsData); - if (items.length === 0) { - log.info(`No asset files to download for space ${spaceUid}`, this.exportContext.context); - log.debug('No assets to download', this.exportContext.context); - return; - } - - this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].DOWNLOADING); - log.info(`Downloading asset files for space ${spaceUid} (${items.length} in metadata)`, this.exportContext.context); - log.debug(`Downloading ${items.length} asset file(s) for space ${spaceUid}...`, this.exportContext.context); + /** + * Download asset binaries by reading the just-written chunked `assets.json` back from disk + * (one chunk at a time), so we never re-materialize the whole asset list in memory. + */ + private async downloadWorkspaceAssets(assetsDir: string, spaceUid: string, expectedDownloads: number): Promise { const filesDir = pResolve(assetsDir, 'files'); await mkdir(filesDir, { recursive: true }); - log.debug(`Asset files directory ready: ${filesDir}`, this.exportContext.context); const securedAssets = this.exportContext.securedAssets ?? false; const authtoken = securedAssets ? configHandler.get('authtoken') : null; log.debug( - `Asset downloads: securedAssets=${securedAssets}, concurrency=${this.downloadAssetsBatchConcurrency}`, + `Asset downloads: securedAssets=${securedAssets}, concurrency=${this.downloadAssetsBatchConcurrency}, expected=${expectedDownloads}`, this.exportContext.context, ); + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].DOWNLOADING); + let downloadOk = 0; let downloadFail = 0; - const validItems = items.filter((asset) => Boolean(asset.url && (asset.uid ?? asset._uid))); - const skipped = items.length - validItems.length; - if (skipped > 0) { - log.debug( - `Skipping ${skipped} asset row(s) without url or uid (${validItems.length} file download(s) scheduled)`, + await forEachChunkedJsonStore( + assetsDir, + 'assets.json', + { + context: this.exportContext.context, + chunkReadLogLabel: 'assets', + onOpenError: (err) => log.debug(`Could not open assets.json for download: ${err}`, this.exportContext.context), + onEmptyIndexer: () => log.info(`No asset files to download for space ${spaceUid}`, this.exportContext.context), + }, + async (records) => { + const valid = records.filter((asset) => this.isDownloadable(asset)); + if (valid.length === 0) return; + const apiBatches = chunk(valid, this.downloadAssetsBatchConcurrency); + const promisifyHandler: CustomPromiseHandler = async ({ index, batchIndex }) => { + const asset = apiBatches[batchIndex][index] as AssetRecord; + const uid = (asset.uid ?? asset._uid) as string; + const url = asset.url as string; + const filename = asset.filename ?? asset.file_name ?? 'asset'; + if (!url || !uid) return; + try { + const separator = url.includes('?') ? '&' : '?'; + const downloadUrl = securedAssets && authtoken ? `${url}${separator}authtoken=${authtoken}` : url; + // Binary GET is idempotent — retry transient failures with backoff. + const response = await withRetry( + async () => { + let resp: Response; + try { + resp = await fetch(downloadUrl); + } catch (e) { + throw new RetryableHttpError(`download network error: ${(e as Error)?.message ?? String(e)}`); + } + if (!resp.ok) { + if (isRetryableStatus(resp.status)) { + throw new RetryableHttpError(`HTTP ${resp.status}`, resp.status, parseRetryAfterMs(resp.headers.get('retry-after'))); + } + throw new Error(`HTTP ${resp.status}`); + } + return resp; + }, + { context: this.exportContext.context, label: `download ${filename}` }, + ); + const body = response.body; + if (!body) throw new Error('No response body'); + const nodeStream = Readable.fromWeb(body as Parameters[0]); + const assetFolderPath = pResolve(filesDir, uid); + await mkdir(assetFolderPath, { recursive: true }); + const filePath = pResolve(assetFolderPath, filename); + await writeStreamToFile(nodeStream, filePath); + downloadOk += 1; + // Per-asset tick so the per-space progress bar moves in real time. + this.tick(true, `asset: ${filename}`, null); + log.debug(`Downloaded asset ${uid} → ${filePath}`, this.exportContext.context); + } catch (e) { + downloadFail += 1; + const err = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED; + this.tick(false, `asset: ${filename}`, err); + log.debug(`Failed to download asset ${uid}: ${e}`, this.exportContext.context); + } + }; + + await this.makeConcurrentCall({ apiBatches, module: 'asset downloads' }, promisifyHandler); + }, + ); + + // Completeness check: a chunk that fails to read back is skipped (logged at debug) by + // forEachChunkedJsonStore, which would silently drop those downloads. Reconcile attempts + // (ok + failed) against what streaming counted as downloadable. + const attempted = downloadOk + downloadFail; + if (attempted < expectedDownloads) { + log.warn( + `Asset downloads for space ${spaceUid} incomplete: expected ${expectedDownloads}, attempted ${attempted}` + + ` — ${expectedDownloads - attempted} asset(s) were never read back for download.`, this.exportContext.context, ); } - await runInBatches(validItems, this.downloadAssetsBatchConcurrency, async (asset) => { - const uid = asset.uid ?? asset._uid; - const url = asset.url; - const filename = asset.filename ?? asset.file_name ?? 'asset'; - if (!url || !uid) return; - try { - const separator = url.includes('?') ? '&' : '?'; - const downloadUrl = securedAssets && authtoken ? `${url}${separator}authtoken=${authtoken}` : url; - const response = await fetch(downloadUrl); - if (!response.ok) throw new Error(`HTTP ${response.status}`); - const body = response.body; - if (!body) throw new Error('No response body'); - const nodeStream = Readable.fromWeb(body as Parameters[0]); - const assetFolderPath = pResolve(filesDir, uid); - await mkdir(assetFolderPath, { recursive: true }); - const filePath = pResolve(assetFolderPath, filename); - await writeStreamToFile(nodeStream, filePath); - downloadOk += 1; - // Per-asset tick so the per-space progress bar moves in real time. - this.tick(true, `asset: ${filename}`, null); - log.debug(`Downloaded asset ${uid} → ${filePath}`, this.exportContext.context); - } catch (e) { - downloadFail += 1; - const err = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED; - this.tick(false, `asset: ${filename}`, err); - log.debug(`Failed to download asset ${uid}: ${e}`, this.exportContext.context); - } - }); log.info( downloadFail === 0 diff --git a/packages/contentstack-asset-management/src/export/base.ts b/packages/contentstack-asset-management/src/export/base.ts index b9721685c..13e4016e1 100644 --- a/packages/contentstack-asset-management/src/export/base.ts +++ b/packages/contentstack-asset-management/src/export/base.ts @@ -5,7 +5,7 @@ import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack import type { CSAssetsAPIConfig } from '../types/cs-assets-api'; import type { ExportContext } from '../types/export-types'; import { CSAssetsAdapter } from '../utils/cs-assets-api-adapter'; -import { CS_ASSETS_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY, FALLBACK_AM_CHUNK_FILE_SIZE_MB } from '../constants/index'; +import { CS_ASSETS_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY, FALLBACK_AM_API_FETCH_CONCURRENCY, FALLBACK_AM_API_PAGE_SIZE, FALLBACK_AM_CHUNK_FILE_SIZE_MB } from '../constants/index'; export type { ExportContext }; @@ -82,6 +82,14 @@ export class CSAssetsExportAdapter extends CSAssetsAdapter { return this.exportContext.downloadAssetsConcurrency ?? this.apiConcurrency; } + protected get apiPageSize(): number { + return this.exportContext.pageSize ?? FALLBACK_AM_API_PAGE_SIZE; + } + + protected get apiFetchConcurrency(): number { + return this.exportContext.fetchConcurrency ?? FALLBACK_AM_API_FETCH_CONCURRENCY; + } + protected getAssetTypesDir(): string { return pResolve(this.exportContext.spacesRootPath, 'asset_types'); } @@ -90,6 +98,25 @@ export class CSAssetsExportAdapter extends CSAssetsAdapter { return pResolve(this.exportContext.spacesRootPath, 'fields'); } + /** Build a chunked-JSON writer for incremental (streaming) writes. Caller must `completeFile(true)`. */ + protected createChunkedJsonWriter(dir: string, indexFileName: string, moduleName: string, metaPickKeys: string[]): FsUtility { + const chunkMb = this.exportContext.chunkFileSizeMb ?? FALLBACK_AM_CHUNK_FILE_SIZE_MB; + return new FsUtility({ + basePath: dir, + indexFileName, + chunkFileSize: chunkMb, + moduleName, + fileExt: 'json', + metaPickKeys, + keepMetadata: true, + }); + } + + /** Write an empty index file (matches FsUtility's layout for a zero-record store). */ + protected async writeEmptyChunkedJson(dir: string, indexFileName: string): Promise { + await writeFile(pResolve(dir, indexFileName), '{}'); + } + protected async writeItemsToChunkedJson( dir: string, indexFileName: string, @@ -98,19 +125,10 @@ export class CSAssetsExportAdapter extends CSAssetsAdapter { items: unknown[], ): Promise { if (items.length === 0) { - await writeFile(pResolve(dir, indexFileName), '{}'); + await this.writeEmptyChunkedJson(dir, indexFileName); return; } - const chunkMb = this.exportContext.chunkFileSizeMb ?? FALLBACK_AM_CHUNK_FILE_SIZE_MB; - const fs = new FsUtility({ - basePath: dir, - indexFileName, - chunkFileSize: chunkMb, - moduleName, - fileExt: 'json', - metaPickKeys, - keepMetadata: true, - }); + const fs = this.createChunkedJsonWriter(dir, indexFileName, moduleName, metaPickKeys); fs.writeIntoFile(items as Record[], { mapKeyVal: true }); fs.completeFile(true); } diff --git a/packages/contentstack-asset-management/src/export/fields.ts b/packages/contentstack-asset-management/src/export/fields.ts index c1ca623f8..a4cfe8460 100644 --- a/packages/contentstack-asset-management/src/export/fields.ts +++ b/packages/contentstack-asset-management/src/export/fields.ts @@ -18,7 +18,7 @@ export default class ExportFields extends CSAssetsExportAdapter { log.debug('Starting shared fields export process...', this.exportContext.context); - const fieldsData = await this.getWorkspaceFields(spaceUid); + const fieldsData = await this.getWorkspaceFields(spaceUid, this.apiPageSize, this.apiFetchConcurrency); const items = getArrayFromResponse(fieldsData, 'fields'); const dir = this.getFieldsDir(); if (items.length === 0) { diff --git a/packages/contentstack-asset-management/src/export/spaces.ts b/packages/contentstack-asset-management/src/export/spaces.ts index 3a3459c3f..6d3b0ab13 100644 --- a/packages/contentstack-asset-management/src/export/spaces.ts +++ b/packages/contentstack-asset-management/src/export/spaces.ts @@ -79,6 +79,8 @@ export class ExportSpaces { chunkFileSizeMb, apiConcurrency: this.options.apiConcurrency, downloadAssetsConcurrency: this.options.downloadAssetsConcurrency, + pageSize: this.options.pageSize, + fetchConcurrency: this.options.fetchConcurrency, }; const sharedFieldsDir = pResolve(spacesRootPath, 'fields'); diff --git a/packages/contentstack-asset-management/src/import-setup/import-setup-asset-mappers.ts b/packages/contentstack-asset-management/src/import-setup/import-setup-asset-mappers.ts index 5ea914e95..dca54810b 100644 --- a/packages/contentstack-asset-management/src/import-setup/import-setup-asset-mappers.ts +++ b/packages/contentstack-asset-management/src/import-setup/import-setup-asset-mappers.ts @@ -4,7 +4,7 @@ import { join, resolve } from 'node:path'; import { formatError, log } from '@contentstack/cli-utilities'; -import { IMPORT_ASSETS_MAPPER_FILES, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +import { FALLBACK_AM_API_FETCH_CONCURRENCY, FALLBACK_AM_API_PAGE_SIZE, IMPORT_ASSETS_MAPPER_FILES, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; import type { CSAssetsAPIConfig, ImportContext } from '../types/cs-assets-api'; import type { AssetMapperImportSetupResult, RunAssetMapperImportSetupParams } from '../types/import-setup-asset-mapper'; import ImportAssets from '../import/assets'; @@ -25,7 +25,7 @@ export default class ImportSetupAssetMappers extends AssetManagementImportSetupA private async fetchExistingSpaceUidsInOrg(apiConfig: CSAssetsAPIConfig): Promise> { const adapter = new CSAssetsAdapter(apiConfig); await adapter.init(); - const { spaces } = await adapter.listSpaces(); + const { spaces } = await adapter.listSpaces(FALLBACK_AM_API_PAGE_SIZE, FALLBACK_AM_API_FETCH_CONCURRENCY); const uids = new Set(); for (const s of spaces) { if (s.uid) { diff --git a/packages/contentstack-asset-management/src/import/spaces.ts b/packages/contentstack-asset-management/src/import/spaces.ts index 457c1f36d..faec9d13d 100644 --- a/packages/contentstack-asset-management/src/import/spaces.ts +++ b/packages/contentstack-asset-management/src/import/spaces.ts @@ -10,7 +10,7 @@ import type { ImportSpacesOptions, SpaceMapping, } from '../types/cs-assets-api'; -import { CS_ASSETS_MAIN_PROCESS_NAME, PROCESS_NAMES, getSpaceProcessName } from '../constants/index'; +import { CS_ASSETS_MAIN_PROCESS_NAME, FALLBACK_AM_API_PAGE_SIZE, FALLBACK_AM_API_FETCH_CONCURRENCY, PROCESS_NAMES, getSpaceProcessName } from '../constants/index'; import { CSAssetsAdapter } from '../utils/cs-assets-api-adapter'; import ImportAssetTypes from './asset-types'; import ImportFields from './fields'; @@ -112,7 +112,7 @@ export class ImportSpaces { try { const adapterForList = new CSAssetsAdapter(apiConfig); await adapterForList.init(); - const { spaces } = await adapterForList.listSpaces(); + const { spaces } = await adapterForList.listSpaces(FALLBACK_AM_API_PAGE_SIZE, FALLBACK_AM_API_FETCH_CONCURRENCY); for (const s of spaces) { if (s.uid) existingSpaceUids.add(s.uid); } diff --git a/packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts b/packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts index 9e50ae94b..eda828159 100644 --- a/packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts +++ b/packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts @@ -8,8 +8,10 @@ import type { ExportContext } from '../types/export-types'; import ExportAssetTypes from '../export/asset-types'; import ExportFields from '../export/fields'; import { CSAssetsExportAdapter } from '../export/base'; +import chunk from 'lodash/chunk'; import { getAssetItems, writeStreamToFile } from '../utils/export-helpers'; -import { runInBatches } from '../utils/concurrent-batch'; +import { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from '../utils/retry'; +import type { CustomPromiseHandler } from '../utils/cs-assets-api-adapter'; const DEFAULT_ASSET_BATCH_SIZE = 100; const SEARCH_PAGE_LIMIT = 100; @@ -223,15 +225,34 @@ class QueryExportWorkspaceAdapter extends CSAssetsExportAdapter { const securedAssets = this.exportContext.securedAssets ?? false; const authtoken = securedAssets ? configHandler.get('authtoken') : null; - await runInBatches(downloadable, this.downloadAssetsBatchConcurrency, async (asset) => { + const apiBatches = chunk(downloadable, this.downloadAssetsBatchConcurrency); + const promisifyHandler: CustomPromiseHandler = async ({ index, batchIndex }) => { + const asset = apiBatches[batchIndex][index]; const uid = String(asset.uid ?? asset._uid); const url = String(asset.url); const filename = String(asset.filename ?? asset.file_name ?? 'asset'); try { const separator = url.includes('?') ? '&' : '?'; const downloadUrl = securedAssets && authtoken ? `${url}${separator}authtoken=${authtoken}` : url; - const response = await fetch(downloadUrl); - if (!response.ok) throw new Error(`HTTP ${response.status}`); + // Binary GET is idempotent — retry transient failures with backoff. + const response = await withRetry( + async () => { + let resp: Response; + try { + resp = await fetch(downloadUrl); + } catch (e) { + throw new RetryableHttpError(`download network error: ${(e as Error)?.message ?? String(e)}`); + } + if (!resp.ok) { + if (isRetryableStatus(resp.status)) { + throw new RetryableHttpError(`HTTP ${resp.status}`, resp.status, parseRetryAfterMs(resp.headers.get('retry-after'))); + } + throw new Error(`HTTP ${resp.status}`); + } + return resp; + }, + { context: this.exportContext.context, label: `download ${filename}` }, + ); const body = response.body; if (!body) throw new Error('No response body'); const nodeStream = Readable.fromWeb(body as Parameters[0]); @@ -241,6 +262,8 @@ class QueryExportWorkspaceAdapter extends CSAssetsExportAdapter { } catch (e) { log.debug(`Failed to download asset ${uid} in space ${spaceUid}: ${e}`, this.exportContext.context); } - }); + }; + + await this.makeConcurrentCall({ apiBatches, module: 'asset downloads' }, promisifyHandler); } } diff --git a/packages/contentstack-asset-management/src/types/cs-assets-api.ts b/packages/contentstack-asset-management/src/types/cs-assets-api.ts index 96dbac1bd..56e45bfff 100644 --- a/packages/contentstack-asset-management/src/types/cs-assets-api.ts +++ b/packages/contentstack-asset-management/src/types/cs-assets-api.ts @@ -113,6 +113,10 @@ export type CSAssetsAPIConfig = { headers?: Record; /** Optional context for logging (e.g. exportConfig.context) */ context?: Record; + /** Max retry attempts for transient read failures (network/429/5xx). Default 3. */ + retries?: number; + /** Base backoff (ms) for retries; actual delay grows exponentially with jitter. Default 500. */ + retryBaseDelayMs?: number; }; // --------------------------------------------------------------------------- @@ -167,12 +171,19 @@ export type SearchAssetsResponse = { export interface ICSAssetsAdapter { init(): Promise; - listSpaces(): Promise; + listSpaces(pageSize?: number, fetchConcurrency?: number): Promise; getSpace(spaceUid: string): Promise; - getWorkspaceFields(spaceUid: string): Promise; - getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise; - getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise; - getWorkspaceAssetTypes(spaceUid: string): Promise; + getWorkspaceFields(spaceUid: string, pageSize?: number, fetchConcurrency?: number): Promise; + getWorkspaceAssets(spaceUid: string, workspaceUid?: string, pageSize?: number, fetchConcurrency?: number): Promise; + streamWorkspaceAssets( + spaceUid: string, + workspaceUid: string | undefined, + onPage: (items: unknown[]) => void | Promise, + pageSize?: number, + fetchConcurrency?: number, + ): Promise; + getWorkspaceFolders(spaceUid: string, workspaceUid?: string, pageSize?: number, fetchConcurrency?: number): Promise; + getWorkspaceAssetTypes(spaceUid: string, pageSize?: number, fetchConcurrency?: number): Promise; searchAssets(params: SearchAssetsParams): Promise; bulkDeleteAssets( spaceUid: string, @@ -233,6 +244,8 @@ export type AssetManagementExportOptions = { * Max parallel asset file downloads per workspace. */ downloadAssetsConcurrency?: number; + pageSize?: number; + fetchConcurrency?: number; }; // --------------------------------------------------------------------------- diff --git a/packages/contentstack-asset-management/src/types/export-types.ts b/packages/contentstack-asset-management/src/types/export-types.ts index 865302a60..3e21682b4 100644 --- a/packages/contentstack-asset-management/src/types/export-types.ts +++ b/packages/contentstack-asset-management/src/types/export-types.ts @@ -5,6 +5,8 @@ export type ExportContext = { chunkFileSizeMb?: number; apiConcurrency?: number; downloadAssetsConcurrency?: number; + pageSize?: number; + fetchConcurrency?: number; }; /** diff --git a/packages/contentstack-asset-management/src/utils/concurrent-batch.ts b/packages/contentstack-asset-management/src/utils/concurrent-batch.ts index dcd916c4b..727c6f0ff 100644 --- a/packages/contentstack-asset-management/src/utils/concurrent-batch.ts +++ b/packages/contentstack-asset-management/src/utils/concurrent-batch.ts @@ -1,3 +1,12 @@ +/** + * Fault-tolerant batched concurrency for the import side (uploads, folder/asset-type/ + * field creation). `runInBatches` runs work in batches of `concurrency`, settling each + * batch (`Promise.allSettled`) before the next so one failure doesn't abort the batch. + * + * NOTE: the export side (pagination + downloads) uses the legacy-style `makeConcurrentCall` + * on `CSAssetsAdapter` instead; do not route export work through here. + */ + /** * Split an array into chunks of at most `size` elements. */ diff --git a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts index f76fa4106..e10761ac2 100644 --- a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts +++ b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts @@ -1,7 +1,11 @@ import { readFileSync } from 'node:fs'; import { basename } from 'node:path'; +import chunk from 'lodash/chunk'; import { HttpClient, log, authenticationHandler, handleAndLogError } from '@contentstack/cli-utilities'; +import { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from './retry'; +import { FALLBACK_AM_API_FETCH_CONCURRENCY, FALLBACK_AM_API_PAGE_SIZE } from '../constants/index'; + import type { CSAssetsAPIConfig, AssetTypesResponse, @@ -51,6 +55,42 @@ export const DEFAULT_SEARCH_ASSET_FIELDS = [ '_asset_scan_status', ] as const; +/** + * Concurrency model ported from the legacy `contentstack-export` package + * (`src/export/modules/base-class.ts`). `makeConcurrentCall` runs work in + * batches of `concurrencyLimit`, settling each batch before the next and + * throttling between batches. Transport differs from legacy: `makeAPICall` + * dispatches to this adapter's HttpClient (`getSpaceLevel`) instead of the SDK. + */ +export type ApiModuleType = 'paginated-collection'; + +export type ApiOptions = { + uid?: string; + url?: string; + module: ApiModuleType; + queryParam?: Record; + resolve: (value: any) => void; + reject: (error: any) => void; + additionalInfo?: Record; +}; + +export type EnvType = { + module: string; + /** Pre-chunked work: each inner array runs in parallel, the outer array runs sequentially. */ + apiBatches: any[][]; + apiParams?: ApiOptions; +}; + +export type CustomPromiseHandlerInput = { + index: number; + batchIndex: number; + element?: any; + apiParams?: ApiOptions; + isLastRequest: boolean; +}; + +export type CustomPromiseHandler = (input: CustomPromiseHandlerInput) => Promise; + export class CSAssetsAdapter implements ICSAssetsAdapter { private readonly config: CSAssetsAPIConfig; private readonly apiClient: HttpClient; @@ -144,26 +184,38 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { log.debug(`GET ${fullPath}`, this.config.context); try { - const response = await this.apiClient.get(fullPath); - if (response.status < 200 || response.status >= 300) { - const bodySnippet = this.formatResponseBodyForError(response.data); - throw this.normalizeAmGetFailure({ - path, - fullPath, - status: response.status, - bodySnippet: bodySnippet || undefined, - }); - } - return response.data as T; + // GETs are idempotent, so retry transient failures (network / 429 / 5xx) with backoff. + return await withRetry( + async () => { + let response: Awaited>; + try { + response = await this.apiClient.get(fullPath); + } catch (netErr) { + // Transport-level rejection (connection reset, timeout, DNS) — transient. + throw new RetryableHttpError(`network error: ${(netErr as Error)?.message ?? String(netErr)}`); + } + if (response.status < 200 || response.status >= 300) { + if (isRetryableStatus(response.status)) { + const retryAfter = parseRetryAfterMs((response as { headers?: Record })?.headers?.['retry-after']); + throw new RetryableHttpError(`GET ${fullPath} → ${response.status}`, response.status, retryAfter); + } + // Terminal (e.g. 4xx): normalize and propagate without retrying. + const bodySnippet = this.formatResponseBodyForError(response.data); + throw this.normalizeAmGetFailure({ path, fullPath, status: response.status, bodySnippet: bodySnippet || undefined }); + } + return response.data as T; + }, + { retries: this.config.retries, baseDelayMs: this.config.retryBaseDelayMs, context: this.config.context, label: `GET ${path}` }, + ); } catch (error) { + if (error instanceof RetryableHttpError) { + // Retries exhausted on a transient failure — surface a normalized error to the caller. + throw this.normalizeAmGetFailure({ path, fullPath, status: error.status, cause: error }); + } if (error instanceof Error && error.message.includes('CS Assets API GET failed')) { throw error; } - throw this.normalizeAmGetFailure({ - path, - fullPath, - cause: error, - }); + throw this.normalizeAmGetFailure({ path, fullPath, cause: error }); } } @@ -189,11 +241,17 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { } } - async listSpaces(): Promise { + async listSpaces(pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY): Promise { log.debug('Fetching all spaces in org', this.config.context); - const result = await this.getSpaceLevel('', '/api/spaces', {}); - log.debug(`Fetched ${result?.count ?? result?.spaces?.length ?? '?'} space(s)`, this.config.context); - return result; + const items = await this.fetchAllPages( + '', + '/api/spaces', + 'spaces', + pageSize, + fetchConcurrency, + ); + log.debug(`Fetched ${items.length} space(s)`, this.config.context); + return { spaces: items as Space[], count: items.length }; } async getSpace(spaceUid: string): Promise { @@ -207,11 +265,15 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { return result; } - async getWorkspaceFields(spaceUid: string): Promise { + async getWorkspaceFields( + spaceUid: string, + pageSize = FALLBACK_AM_API_PAGE_SIZE, + fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY, + ): Promise { log.debug(`Fetching fields for space: ${spaceUid}`, this.config.context); - const result = await this.getSpaceLevel(spaceUid, '/api/fields', {}); - log.debug(`Fetched fields (count: ${result?.count ?? '?'})`, this.config.context); - return result; + const items = await this.fetchAllPages(spaceUid, '/api/fields', 'fields', pageSize, fetchConcurrency, {}); + log.debug(`Fetched fields (count: ${items.length})`, this.config.context); + return { fields: items, count: items.length } as FieldsResponse; } /** @@ -230,31 +292,238 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { return result; } - async getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise { - return this.getWorkspaceCollection( + + /** + * Core pagination: read the total `count` from page 0, then drive the remaining pages through + * {@link makeConcurrentCall}. Every page (including page 0) is handed to `onPage` — writes are + * serialized through a promise chain so a streaming sink (e.g. FsUtility) is never called + * reentrantly while pages fetch concurrently. Returns the number of items seen. + * + * Peak memory is bounded by the sink: the array wrapper holds everything, but a disk-writing + * sink keeps only the in-flight pages (~concurrency × pageSize). + */ + private async paginate( + spaceUid: string, + path: string, + itemsKey: string, + pageSize: number, + concurrency: number, + baseParams: Record, + onPage: (items: unknown[]) => void | Promise, + ): Promise { + const first = await this.getSpaceLevel>(spaceUid, path, { + ...baseParams, limit: String(pageSize), skip: '0', + }); + + const total: number = Number(first?.count ?? 0); + const firstItems: unknown[] = Array.isArray(first?.[itemsKey]) ? (first[itemsKey] as unknown[]) : []; + + let collected = 0; + let writeFailures = 0; + let writeChain: Promise = Promise.resolve(); + const enqueue = (items: unknown[]) => { + collected += items.length; + // Each link catches its own error so a single failed sink write doesn't skip the queued ones. + writeChain = writeChain.then(async () => { + try { + await onPage(items); + } catch (e) { + writeFailures += 1; + log.warn(`Failed to persist a page of ${itemsKey} (${path}): ${(e as Error)?.message ?? e}`, this.config.context); + } + }); + }; + + enqueue(firstItems); + + if (firstItems.length < total) { + // Remaining skip offsets (page 0 already fetched), pre-chunked into batches of `concurrency`. + const skips: string[] = Array.from( + { length: Math.ceil(total / pageSize) - 1 }, + (_, i) => String((i + 1) * pageSize), + ); + const apiBatches = chunk(skips, concurrency); + + let failedPages = 0; + const onSuccess = ({ response }: any) => { + const items = Array.isArray(response?.[itemsKey]) ? (response[itemsKey] as unknown[]) : []; + enqueue(items); + }; + const onReject = ({ error }: any) => { + // A failed page is skipped (Promise.allSettled); surface it loudly rather than silently dropping data. + failedPages += 1; + log.warn(`Failed to fetch a page of ${itemsKey} (${path}): ${error?.message ?? error}`, this.config.context); + }; + + await this.makeConcurrentCall({ + module: itemsKey, + apiBatches, + apiParams: { + module: 'paginated-collection', + resolve: onSuccess, + reject: onReject, + queryParam: { ...baseParams, limit: String(pageSize) }, + additionalInfo: { spaceUid, path, itemsKey }, + }, + }); + + // Completeness check: the export "succeeding" with silently-missing pages is the worst failure + // mode for a backup/migration, so reconcile what we saw against the server's reported total. + if (collected !== total) { + log.warn( + `Incomplete pagination for ${itemsKey} (${path}): expected ${total}, collected ${collected}` + + (failedPages > 0 ? ` — ${failedPages} page request(s) failed.` : '.'), + this.config.context, + ); + } + } + + await writeChain; // flush any queued sink writes before returning + if (writeFailures > 0) { + log.warn( + `${writeFailures} page(s) of ${itemsKey} (${path}) failed to persist — output may be incomplete.`, + this.config.context, + ); + } + return collected; + } + + /** + * Fetch all pages of a paginated collection into an in-memory array. Use for small collections + * (spaces/folders/fields/asset-types); for potentially large asset sets prefer + * {@link streamWorkspaceAssets}, which streams to a sink instead of accumulating. + */ + private async fetchAllPages( + spaceUid: string, + path: string, + itemsKey: string, + pageSize: number, + concurrency: number, + baseParams: Record = {}, + ): Promise { + const out: unknown[] = []; + await this.paginate(spaceUid, path, itemsKey, pageSize, concurrency, baseParams, (items) => { + out.push(...items); + }); + return out; + } + + /** + * Stream a workspace's assets page-by-page to `onPage` (e.g. an incremental chunked-JSON writer) + * instead of buffering the whole set. Returns the number of asset records streamed. + */ + async streamWorkspaceAssets( + spaceUid: string, + workspaceUid: string | undefined, + onPage: (items: unknown[]) => void | Promise, + pageSize = FALLBACK_AM_API_PAGE_SIZE, + fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY, + ): Promise { + const baseParams: Record = workspaceUid ? { workspace: workspaceUid } : {}; + return this.paginate( + spaceUid, + `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, + 'assets', + pageSize, + fetchConcurrency, + baseParams, + onPage, + ); + } + + /** + * Run pre-batched API work with bounded concurrency: each inner array of `apiBatches` + * runs in parallel (`Promise.allSettled`), and batches run sequentially. Either invokes a + * `promisifyHandler` per element, or — for paginated GETs — injects each element as `skip` + * and dispatches through {@link makeAPICall}. Adapted from legacy export's `makeConcurrentCall`. + * + * Callers pre-chunk the work (`chunk(items, concurrency)`), so this never derives batches itself. + */ + async makeConcurrentCall(env: EnvType, promisifyHandler?: CustomPromiseHandler): Promise { + const { module, apiBatches, apiParams } = env; + if (!apiBatches?.length) return; + + for (let batchIndex = 0; batchIndex < apiBatches.length; batchIndex++) { + const currentBatch = apiBatches[batchIndex]; + const allPromise: Array> = []; + + for (let index = 0; index < currentBatch.length; index++) { + const element = currentBatch[index]; + const isLastRequest = batchIndex === apiBatches.length - 1 && index === currentBatch.length - 1; + + if (promisifyHandler) { + allPromise.push(promisifyHandler({ apiParams, element, isLastRequest, index, batchIndex })); + } else if (apiParams?.queryParam) { + // Mutated in place per iteration; makeAPICall snapshots it synchronously (see below). + apiParams.queryParam.skip = element; + allPromise.push(this.makeAPICall(apiParams, isLastRequest)); + } + } + + await Promise.allSettled(allPromise); + log.debug(`Batch ${batchIndex + 1}/${apiBatches.length} of ${module} complete`, this.config.context); + } + } + + /** + * Dispatch a single API call for {@link makeConcurrentCall}. Transport adapted from + * legacy's SDK calls to this adapter's HttpClient. `queryParam` is snapshotted + * synchronously (the caller mutates `skip` in place between iterations). + */ + makeAPICall( + { module: moduleName, reject, resolve, additionalInfo, queryParam = {} }: ApiOptions, + isLastRequest = false, + ): Promise { + switch (moduleName) { + case 'paginated-collection': { + const { spaceUid = '', path = '' } = (additionalInfo ?? {}) as { spaceUid?: string; path?: string }; + const params = { ...queryParam }; + return this.getSpaceLevel>(spaceUid, path, params) + .then((response: any) => resolve({ response, isLastRequest, additionalInfo })) + .catch((error: Error) => reject({ error, isLastRequest, additionalInfo })); + } + default: + return Promise.resolve(); + } + } + + async getWorkspaceAssets(spaceUid: string, workspaceUid?: string, pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY): Promise { + const baseParams: Record = workspaceUid ? { workspace: workspaceUid } : {}; + const items = await this.fetchAllPages( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, 'assets', - workspaceUid ? { workspace: workspaceUid } : {}, + pageSize, + fetchConcurrency, + baseParams, ); + return { assets: items, count: items.length }; } - async getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise { - return this.getWorkspaceCollection( + async getWorkspaceFolders(spaceUid: string, workspaceUid?: string, pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY): Promise { + const baseParams: Record = workspaceUid ? { workspace: workspaceUid } : {}; + const items = await this.fetchAllPages( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/folders`, 'folders', - workspaceUid ? { workspace: workspaceUid } : {}, + pageSize, + fetchConcurrency, + baseParams, ); + return { folders: items, count: items.length }; } - async getWorkspaceAssetTypes(spaceUid: string): Promise { + async getWorkspaceAssetTypes( + spaceUid: string, + pageSize = FALLBACK_AM_API_PAGE_SIZE, + fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY, + ): Promise { log.debug(`Fetching asset types for space: ${spaceUid}`, this.config.context); - const result = await this.getSpaceLevel(spaceUid, '/api/asset_types', { + const items = await this.fetchAllPages(spaceUid, '/api/asset_types', 'asset_types', pageSize, fetchConcurrency, { include_fields: 'true', }); - log.debug(`Fetched asset types (count: ${result?.count ?? '?'})`, this.config.context); - return result; + log.debug(`Fetched asset types (count: ${items.length})`, this.config.context); + return { asset_types: items, count: items.length } as AssetTypesResponse; } /** @@ -284,7 +553,8 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { `Searching assets (skip=${skip}, limit=${limit}, uids=${assetUIDs.length}, spaces=${spaces.length})`, this.config.context, ); - return this.postJson('/api/search', body); + // Search is a read — safe to retry transient failures. + return this.postJson('/api/search', body, {}, { retry: true }); } // --------------------------------------------------------------------------- @@ -309,18 +579,32 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { }; } - private async postJson(path: string, body: unknown, extraHeaders: Record = {}): Promise { + /** + * POST a JSON body. Pass `{ retry: true }` ONLY for idempotent reads (e.g. /api/search) — never + * for writes (create/bulk), which could double-apply on retry. + */ + private async postJson( + path: string, + body: unknown, + extraHeaders: Record = {}, + opts: { retry?: boolean } = {}, + ): Promise { const baseUrl = this.config.baseURL?.replace(/\/$/, '') ?? ''; const headers = await this.getPostHeaders({ 'Content-Type': 'application/json', ...extraHeaders }); log.debug(`POST ${path}`, this.config.context); - try { - const response = await fetch(`${baseUrl}${path}`, { - method: 'POST', - headers, - body: JSON.stringify(body), - }); + const doPost = async (): Promise => { + let response: Response; + try { + response = await fetch(`${baseUrl}${path}`, { method: 'POST', headers, body: JSON.stringify(body) }); + } catch (netErr) { + if (opts.retry) throw new RetryableHttpError(`POST ${path} network error: ${(netErr as Error)?.message ?? String(netErr)}`); + throw netErr; + } if (!response.ok) { + if (opts.retry && isRetryableStatus(response.status)) { + throw new RetryableHttpError(`POST ${path} → ${response.status}`, response.status, parseRetryAfterMs(response.headers.get('retry-after'))); + } const text = await response.text().catch(() => ''); const bodySnippet = this.formatResponseBodyForError(text); throw new Error( @@ -330,7 +614,21 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { ); } return response.json() as Promise; + }; + + try { + return opts.retry + ? await withRetry(doPost, { + retries: this.config.retries, + baseDelayMs: this.config.retryBaseDelayMs, + context: this.config.context, + label: `POST ${path}`, + }) + : await doPost(); } catch (error) { + if (error instanceof RetryableHttpError) { + throw new Error(`CS Assets API POST failed: path ${path} (status ${error.status ?? 'network'}) - ${error.message}`); + } if (error instanceof Error && error.message.includes('CS Assets API POST failed')) { throw error; } diff --git a/packages/contentstack-asset-management/src/utils/index.ts b/packages/contentstack-asset-management/src/utils/index.ts index dca27cade..b78e6bf0a 100644 --- a/packages/contentstack-asset-management/src/utils/index.ts +++ b/packages/contentstack-asset-management/src/utils/index.ts @@ -8,5 +8,6 @@ export { writeStreamToFile, } from './export-helpers'; export { chunkArray, runInBatches } from './concurrent-batch'; +export { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from './retry'; export { detectAssetManagementExportFromContentDir } from './detect-asset-management-export'; export type { AssetManagementExportFlags } from '../types/asset-management-export-flags'; diff --git a/packages/contentstack-asset-management/src/utils/retry.ts b/packages/contentstack-asset-management/src/utils/retry.ts new file mode 100644 index 000000000..1aed4cde9 --- /dev/null +++ b/packages/contentstack-asset-management/src/utils/retry.ts @@ -0,0 +1,87 @@ +import { log } from '@contentstack/cli-utilities'; + +export const DEFAULT_RETRIES = 3; +export const DEFAULT_RETRY_BASE_DELAY_MS = 500; +/** Hard ceiling on any single backoff — caps both exponential growth and a server-supplied Retry-After. */ +export const MAX_RETRY_BACKOFF_MS = 30_000; + +export type RetryOptions = { + /** Max retry attempts after the initial try (default 3). */ + retries?: number; + /** Base backoff in ms; actual delay is baseDelayMs * 2^attempt + jitter (default 500). */ + baseDelayMs?: number; + context?: Record; + /** Short label for retry log lines (e.g. "GET /api/fields"). */ + label?: string; +}; + +/** + * Error that marks an operation as worth retrying (transient network failure, 429, or 5xx). + * Anything that is NOT a RetryableHttpError is treated as terminal by {@link withRetry}. + */ +export class RetryableHttpError extends Error { + readonly status?: number; + readonly retryAfterMs?: number; + + constructor(message: string, status?: number, retryAfterMs?: number) { + super(message); + this.name = 'RetryableHttpError'; + this.status = status; + this.retryAfterMs = retryAfterMs; + } +} + +/** Transient HTTP statuses worth retrying. */ +export function isRetryableStatus(status: number): boolean { + return status === 429 || status >= 500; +} + +/** Parse a Retry-After header (delta-seconds or HTTP date) into milliseconds, or undefined. */ +export function parseRetryAfterMs(headerValue: string | null | undefined): number | undefined { + if (!headerValue) return undefined; + const seconds = Number(headerValue); + if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000); + const dateMs = Date.parse(headerValue); + if (!Number.isNaN(dateMs)) return Math.max(0, dateMs - Date.now()); + return undefined; +} + +function sleep(ms: number): Promise { + // eslint-disable-next-line no-promise-executor-return + return new Promise((resolve) => setTimeout(resolve, ms <= 0 ? 0 : ms)); +} + +/** + * Run `fn`, retrying only when it throws a {@link RetryableHttpError}. Uses exponential backoff + * (`baseDelayMs * 2^attempt`) plus jitter, or the error's `retryAfterMs` when present. Terminal + * errors (anything that isn't a RetryableHttpError) propagate immediately, as does the last + * RetryableHttpError once attempts are exhausted. + * + * Wrap sites are responsible for classifying: throw RetryableHttpError for network errors / 429 / + * 5xx, and throw a plain error for non-retryable failures (e.g. 4xx). Only idempotent reads should + * be wrapped — never non-idempotent writes (uploads/creates), which could double-apply. + */ +export async function withRetry(fn: () => Promise, opts: RetryOptions = {}): Promise { + const retries = opts.retries ?? DEFAULT_RETRIES; + const baseDelayMs = opts.baseDelayMs ?? DEFAULT_RETRY_BASE_DELAY_MS; + let attempt = 0; + + // eslint-disable-next-line no-constant-condition + while (true) { + try { + return await fn(); + } catch (error) { + if (!(error instanceof RetryableHttpError) || attempt >= retries) throw error; + const jitter = Math.floor(Math.random() * baseDelayMs); + const backoff = baseDelayMs * 2 ** attempt + jitter; + // Clamp so a hostile/broken server's Retry-After (or runaway exponential) can't stall the export. + const delay = Math.min(error.retryAfterMs ?? backoff, MAX_RETRY_BACKOFF_MS); + attempt += 1; + log.debug( + `Retry ${attempt}/${retries} in ${delay}ms${opts.label ? ` (${opts.label})` : ''}: ${error.message}`, + opts.context, + ); + await sleep(delay); + } + } +} diff --git a/packages/contentstack-asset-management/test/unit/export/assets.test.ts b/packages/contentstack-asset-management/test/unit/export/assets.test.ts index f6a4bc61e..ae930da8b 100644 --- a/packages/contentstack-asset-management/test/unit/export/assets.test.ts +++ b/packages/contentstack-asset-management/test/unit/export/assets.test.ts @@ -4,38 +4,28 @@ import { configHandler } from '@contentstack/cli-utilities'; import ExportAssets from '../../../src/export/assets'; import { CSAssetsExportAdapter } from '../../../src/export/base'; +import * as chunkedJsonReader from '../../../src/utils/chunked-json-reader'; +import * as retryModule from '../../../src/utils/retry'; import type { CSAssetsAPIConfig, LinkedWorkspace } from '../../../src/types/cs-assets-api'; import type { ExportContext } from '../../../src/types/export-types'; const foldersData = [{ uid: 'folder-1', name: 'Images' }]; -const assetsResponseWithItems = { - items: [ - { uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'image.png' }, - { uid: 'a2', url: 'https://cdn.example.com/a2.pdf', file_name: 'doc.pdf' }, - ], -}; -const emptyAssetsResponse = { items: [] as any[] }; +const assetItems = [ + { uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'image.png' }, + { uid: 'a2', url: 'https://cdn.example.com/a2.pdf', file_name: 'doc.pdf' }, +]; +const ASSET_META_KEYS = ['uid', 'url', 'filename', 'file_name', 'parent_uid']; describe('ExportAssets', () => { - const apiConfig: CSAssetsAPIConfig = { - baseURL: 'https://am.example.com', - headers: { organization_uid: 'org-1' }, - }; - - const exportContext: ExportContext = { - spacesRootPath: '/tmp/export/spaces', - }; - - const workspace: LinkedWorkspace = { - uid: 'ws-1', - space_uid: 'space-uid-1', - is_default: true, - }; - + const apiConfig: CSAssetsAPIConfig = { baseURL: 'https://am.example.com', headers: { organization_uid: 'org-1' } }; + const exportContext: ExportContext = { spacesRootPath: '/tmp/export/spaces' }; + const workspace: LinkedWorkspace = { uid: 'ws-1', space_uid: 'space-uid-1', is_default: true }; const spaceDir = '/tmp/export/spaces/space-uid-1'; let fetchStub: sinon.SinonStub; + let writerStub: { writeIntoFile: sinon.SinonStub; completeFile: sinon.SinonStub }; + let createWriterStub: sinon.SinonStub; const makeFetchResponse = () => { const webStream = new ReadableStream({ @@ -47,11 +37,39 @@ describe('ExportAssets', () => { return { ok: true, status: 200, body: webStream }; }; + /** + * Wire the streaming flow without real pagination or disk: `streamWorkspaceAssets` feeds `items` + * through the `onPage` sink, and the download read-back (`forEachChunkedJsonStore`) yields the + * same `items` back as one chunk (or signals empty). + */ + const wireStreaming = (items: Array>) => { + sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData as any); + sinon + .stub(ExportAssets.prototype, 'streamWorkspaceAssets') + .callsFake(async (_s: string, _ws: string | undefined, onPage: (i: unknown[]) => void | Promise) => { + await onPage(items); + return items.length; + }); + sinon + .stub(chunkedJsonReader, 'forEachChunkedJsonStore') + .callsFake(async (_base: string, _idx: string, opts: any, onChunk: (records: unknown[]) => Promise) => { + if (items.length === 0) { + opts.onEmptyIndexer(); + return; + } + await onChunk(items); + }); + }; + beforeEach(() => { sinon.stub(CSAssetsExportAdapter.prototype, 'init' as any).resolves(); - sinon.stub(CSAssetsExportAdapter.prototype, 'writeItemsToChunkedJson' as any).resolves(); sinon.stub(CSAssetsExportAdapter.prototype, 'tick' as any); sinon.stub(CSAssetsExportAdapter.prototype, 'updateStatus' as any); + sinon.stub(CSAssetsExportAdapter.prototype, 'writeEmptyChunkedJson' as any).resolves(); + writerStub = { writeIntoFile: sinon.stub(), completeFile: sinon.stub() }; + createWriterStub = sinon.stub(CSAssetsExportAdapter.prototype, 'createChunkedJsonWriter' as any).returns(writerStub); + // Run the retry wrapper inline (single attempt, no backoff) so tests don't wait on real delays. + sinon.stub(retryModule, 'withRetry').callsFake(async (fn: () => Promise) => fn()); fetchStub = sinon.stub(globalThis, 'fetch'); }); @@ -59,7 +77,7 @@ describe('ExportAssets', () => { sinon.restore(); }); - describe('start method', () => { + describe('concurrency config', () => { it('should use fallback download concurrency when not configured', () => { const exporter = new ExportAssets(apiConfig, exportContext); expect((exporter as any).downloadAssetsBatchConcurrency).to.equal(5); @@ -69,38 +87,45 @@ describe('ExportAssets', () => { const exporter = new ExportAssets(apiConfig, { ...exportContext, downloadAssetsConcurrency: 2 }); expect((exporter as any).downloadAssetsBatchConcurrency).to.equal(2); }); + }); - it('should fetch folders and assets using the workspace space_uid', async () => { - const foldersStub = sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - const assetsStub = sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(emptyAssetsResponse); - + describe('start method', () => { + it('should fetch folders and stream assets using the workspace space_uid', async () => { + wireStreaming([]); const exporter = new ExportAssets(apiConfig, exportContext); await exporter.start(workspace, spaceDir); + const foldersStub = ExportAssets.prototype.getWorkspaceFolders as sinon.SinonStub; + const streamStub = ExportAssets.prototype.streamWorkspaceAssets as sinon.SinonStub; expect(foldersStub.firstCall.args[0]).to.equal(workspace.space_uid); - expect(assetsStub.firstCall.args[0]).to.equal(workspace.space_uid); + expect(streamStub.firstCall.args[0]).to.equal(workspace.space_uid); }); - it('should write chunked assets metadata with correct args', async () => { - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(assetsResponseWithItems); + it('should stream asset metadata into a chunked-JSON writer', async () => { + wireStreaming(assetItems); fetchStub.callsFake(async () => makeFetchResponse() as any); const exporter = new ExportAssets(apiConfig, exportContext); await exporter.start(workspace, spaceDir); - const writeStub = (CSAssetsExportAdapter.prototype as any).writeItemsToChunkedJson as sinon.SinonStub; - const args = writeStub.firstCall.args; - expect(args[1]).to.equal('assets.json'); - expect(args[2]).to.equal('assets'); - expect(args[3]).to.deep.equal(['uid', 'url', 'filename', 'file_name', 'parent_uid']); - expect(args[4]).to.have.length(2); + expect(createWriterStub.firstCall.args[1]).to.equal('assets.json'); + expect(createWriterStub.firstCall.args[2]).to.equal('assets'); + expect(createWriterStub.firstCall.args[3]).to.deep.equal(ASSET_META_KEYS); + expect(writerStub.writeIntoFile.firstCall.args[0]).to.have.length(2); + expect(writerStub.completeFile.calledOnceWith(true)).to.be.true; }); - it('should not attempt any downloads when the asset list is empty', async () => { - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(emptyAssetsResponse); + it('should write an empty index (no writer) when there are no assets', async () => { + wireStreaming([]); + const exporter = new ExportAssets(apiConfig, exportContext); + await exporter.start(workspace, spaceDir); + expect(createWriterStub.called).to.be.false; + expect((CSAssetsExportAdapter.prototype as any).writeEmptyChunkedJson.calledOnce).to.be.true; + }); + + it('should not attempt any downloads when the asset list is empty', async () => { + wireStreaming([]); const exporter = new ExportAssets(apiConfig, exportContext); await exporter.start(workspace, spaceDir); @@ -110,9 +135,8 @@ describe('ExportAssets', () => { expect(assetTicks).to.have.length(0); }); - it('should tick per failed asset with success=false and the error message on download failure', async () => { - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(assetsResponseWithItems); + it('should tick per failed asset with success=false and the error on download failure', async () => { + wireStreaming(assetItems); fetchStub.rejects(new Error('network failure')); const exporter = new ExportAssets(apiConfig, exportContext); @@ -120,17 +144,15 @@ describe('ExportAssets', () => { const tickStub = (CSAssetsExportAdapter.prototype as any).tick as sinon.SinonStub; const assetTicks = tickStub.getCalls().filter((c) => String(c.args[1]).startsWith('asset:')); - // Per-asset tick: one failure entry per attempted download. expect(assetTicks.length).to.be.greaterThan(0); for (const t of assetTicks) { expect(t.args[0]).to.be.false; - expect(t.args[2]).to.equal('network failure'); + expect(String(t.args[2])).to.include('network failure'); } }); it('should tick per asset with success=true and null error on successful downloads', async () => { - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(assetsResponseWithItems); + wireStreaming(assetItems); fetchStub.callsFake(async () => makeFetchResponse() as any); const exporter = new ExportAssets(apiConfig, exportContext); @@ -138,8 +160,7 @@ describe('ExportAssets', () => { const tickStub = (CSAssetsExportAdapter.prototype as any).tick as sinon.SinonStub; const assetTicks = tickStub.getCalls().filter((c) => String(c.args[1]).startsWith('asset:')); - // One successful tick per asset in the workspace. - expect(assetTicks).to.have.length(assetsResponseWithItems.items.length); + expect(assetTicks).to.have.length(assetItems.length); for (const t of assetTicks) { expect(t.args[0]).to.be.true; expect(t.args[2]).to.be.null; @@ -147,16 +168,11 @@ describe('ExportAssets', () => { }); it('should skip assets that have neither a url nor a uid', async () => { - const incompleteAssets = { - items: [ - { uid: 'a1', url: null as any }, - { url: 'https://cdn.example.com/a2.png', filename: 'img.png' }, - { uid: null as any, url: null as any }, - ], - }; - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(incompleteAssets); - + wireStreaming([ + { uid: 'a1', url: null }, + { url: 'https://cdn.example.com/a2.png', filename: 'img.png' }, + { uid: null, url: null }, + ] as any); const exporter = new ExportAssets(apiConfig, exportContext); await exporter.start(workspace, spaceDir); @@ -164,11 +180,7 @@ describe('ExportAssets', () => { }); it('should process assets that have _uid instead of uid without skipping them', async () => { - const assetsWithUnderscoreUid = { - items: [{ _uid: 'a-uid', url: 'https://cdn.example.com/a.png', filename: 'a.png' }], - }; - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(assetsWithUnderscoreUid); + wireStreaming([{ _uid: 'a-uid', url: 'https://cdn.example.com/a.png', filename: 'a.png' }] as any); fetchStub.callsFake(async () => makeFetchResponse() as any); const exporter = new ExportAssets(apiConfig, exportContext); @@ -179,69 +191,47 @@ describe('ExportAssets', () => { const assetTicks = tickStub.getCalls().filter((c) => String(c.args[1]).startsWith('asset:')); expect(assetTicks).to.have.length(1); expect(assetTicks[0].args[0]).to.be.true; - expect(assetTicks[0].args[2]).to.be.null; }); it('should download assets that use file_name, and fall back to "asset" when both names are absent', async () => { - const assetsNoFilename = { - items: [ - { uid: 'a1', url: 'https://cdn.example.com/a1.pdf', file_name: 'named.pdf' }, - { uid: 'a2', url: 'https://cdn.example.com/a2.bin' }, - ], - }; - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(assetsNoFilename); + wireStreaming([ + { uid: 'a1', url: 'https://cdn.example.com/a1.pdf', file_name: 'named.pdf' }, + { uid: 'a2', url: 'https://cdn.example.com/a2.bin' }, + ] as any); fetchStub.callsFake(async () => makeFetchResponse() as any); const exporter = new ExportAssets(apiConfig, exportContext); await exporter.start(workspace, spaceDir); expect(fetchStub.callCount).to.equal(2); - expect(fetchStub.firstCall.args[0]).to.equal('https://cdn.example.com/a1.pdf'); - expect(fetchStub.secondCall.args[0]).to.equal('https://cdn.example.com/a2.bin'); - const tickStub = (CSAssetsExportAdapter.prototype as any).tick as sinon.SinonStub; - const assetTicks = tickStub.getCalls().filter((c) => String(c.args[1]).startsWith('asset:')); - expect(assetTicks).to.have.length(2); - for (const t of assetTicks) expect(t.args[0]).to.be.true; + const urls = fetchStub.getCalls().map((c) => c.args[0]).sort(); + expect(urls).to.deep.equal(['https://cdn.example.com/a1.pdf', 'https://cdn.example.com/a2.bin']); }); it('should append authtoken to URL when securedAssets is true', async () => { sinon.stub(configHandler, 'get').returns('my-auth-token'); - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves({ - items: [{ uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'img.png' }], - }); + wireStreaming([{ uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'img.png' }] as any); fetchStub.callsFake(async () => makeFetchResponse() as any); - const securedContext: typeof exportContext = { ...exportContext, securedAssets: true }; - const exporter = new ExportAssets(apiConfig, securedContext); + const exporter = new ExportAssets(apiConfig, { ...exportContext, securedAssets: true }); await exporter.start(workspace, spaceDir); - const downloadUrl = fetchStub.firstCall.args[0] as string; - expect(downloadUrl).to.include('authtoken=my-auth-token'); + expect(String(fetchStub.firstCall.args[0])).to.include('authtoken=my-auth-token'); }); it('should use "&" separator when URL already contains "?"', async () => { sinon.stub(configHandler, 'get').returns('my-token'); - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves({ - items: [{ uid: 'a1', url: 'https://cdn.example.com/a1?v=1', filename: 'img.png' }], - }); + wireStreaming([{ uid: 'a1', url: 'https://cdn.example.com/a1?v=1', filename: 'img.png' }] as any); fetchStub.callsFake(async () => makeFetchResponse() as any); - const securedContext: typeof exportContext = { ...exportContext, securedAssets: true }; - const exporter = new ExportAssets(apiConfig, securedContext); + const exporter = new ExportAssets(apiConfig, { ...exportContext, securedAssets: true }); await exporter.start(workspace, spaceDir); - const downloadUrl = fetchStub.firstCall.args[0] as string; - expect(downloadUrl).to.include('?v=1&authtoken='); + expect(String(fetchStub.firstCall.args[0])).to.include('?v=1&authtoken='); }); it('should tick with success=false and the HTTP status code on non-ok response', async () => { - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves({ - items: [{ uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'img.png' }], - }); + wireStreaming([{ uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'img.png' }] as any); fetchStub.resolves({ ok: false, status: 403, body: null } as any); const exporter = new ExportAssets(apiConfig, exportContext); @@ -251,14 +241,11 @@ describe('ExportAssets', () => { const assetTicks = tickStub.getCalls().filter((c) => String(c.args[1]).startsWith('asset:')); expect(assetTicks).to.have.length(1); expect(assetTicks[0].args[0]).to.be.false; - expect(assetTicks[0].args[2]).to.include('403'); + expect(String(assetTicks[0].args[2])).to.include('403'); }); it('should tick with success=false and "No response body" when body is null', async () => { - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves({ - items: [{ uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'img.png' }], - }); + wireStreaming([{ uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'img.png' }] as any); fetchStub.resolves({ ok: true, status: 200, body: null } as any); const exporter = new ExportAssets(apiConfig, exportContext); diff --git a/packages/contentstack-asset-management/test/unit/export/base.test.ts b/packages/contentstack-asset-management/test/unit/export/base.test.ts index 08993eddf..22f41d67e 100644 --- a/packages/contentstack-asset-management/test/unit/export/base.test.ts +++ b/packages/contentstack-asset-management/test/unit/export/base.test.ts @@ -35,6 +35,12 @@ class TestAdapter extends CSAssetsExportAdapter { public get spacesRootPathPublic() { return this.spacesRootPath; } + public get apiPageSizePublic() { + return this.apiPageSize; + } + public get apiFetchConcurrencyPublic() { + return this.apiFetchConcurrency; + } } describe('CSAssetsExportAdapter (base)', () => { @@ -192,6 +198,30 @@ describe('CSAssetsExportAdapter (base)', () => { }); }); + describe('apiPageSize', () => { + it('should return FALLBACK_AM_API_PAGE_SIZE (100) when pageSize is not set in exportContext', () => { + const adapter = new TestAdapter(apiConfig, exportContext); + expect(adapter.apiPageSizePublic).to.equal(100); + }); + + it('should return the configured pageSize when set in exportContext', () => { + const adapter = new TestAdapter(apiConfig, { ...exportContext, pageSize: 50 }); + expect(adapter.apiPageSizePublic).to.equal(50); + }); + }); + + describe('apiFetchConcurrency', () => { + it('should return FALLBACK_AM_API_FETCH_CONCURRENCY (5) when fetchConcurrency is not set', () => { + const adapter = new TestAdapter(apiConfig, exportContext); + expect(adapter.apiFetchConcurrencyPublic).to.equal(5); + }); + + it('should return the configured fetchConcurrency when set in exportContext', () => { + const adapter = new TestAdapter(apiConfig, { ...exportContext, fetchConcurrency: 10 }); + expect(adapter.apiFetchConcurrencyPublic).to.equal(10); + }); + }); + describe('writeItemsToChunkedJson', () => { it('should write {} to an empty file when items array is empty', async () => { const os = require('node:os'); diff --git a/packages/contentstack-asset-management/test/unit/import/asset-types.test.ts b/packages/contentstack-asset-management/test/unit/import/asset-types.test.ts new file mode 100644 index 000000000..c3252526d --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/import/asset-types.test.ts @@ -0,0 +1,186 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { FsUtility } from '@contentstack/cli-utilities'; + +import ImportAssetTypes from '../../../src/import/asset-types'; +import { CSAssetsImportAdapter } from '../../../src/import/base'; +import type { CSAssetsAPIConfig, ImportContext } from '../../../src/types/cs-assets-api'; + +describe('ImportAssetTypes', () => { + const apiConfig: CSAssetsAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + const importContext: ImportContext = { + spacesRootPath: '/tmp/import/spaces', + apiKey: 'api-key-1', + host: 'https://api.contentstack.io/v3', + org_uid: 'org-1', + }; + + let tickStub: sinon.SinonStub; + + beforeEach(() => { + sinon.stub(CSAssetsImportAdapter.prototype, 'init' as any).resolves(); + tickStub = sinon.stub(CSAssetsImportAdapter.prototype, 'tick' as any); + sinon.stub(CSAssetsImportAdapter.prototype, 'updateStatus' as any); + sinon.stub(CSAssetsImportAdapter.prototype, 'getAssetTypesDir' as any).returns('/tmp/import/spaces/asset_types'); + }); + + afterEach(() => sinon.restore()); + + const stubExistingAssetTypes = (assetTypes: any[]) => { + sinon.stub(CSAssetsImportAdapter.prototype, 'getWorkspaceAssetTypes' as any) + .resolves({ asset_types: assetTypes }); + }; + + const stubChunks = (records: Record[]) => { + const indexer = records.length > 0 ? { '0': true } : {}; + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => indexer); + if (records.length > 0) { + const chunk = Object.fromEntries(records.map((r) => [(r.uid as string), r])); + sinon.stub(FsUtility.prototype, 'readChunkFiles' as any).get(() => ({ + next: sinon.stub().resolves(chunk), + })); + } + }; + + describe('when index file does not exist', () => { + it('ticks once and returns without calling createAssetType', async () => { + sinon.stub(require('node:fs'), 'existsSync').returns(false); + stubExistingAssetTypes([]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.callCount).to.equal(1); + expect(tickStub.firstCall.args[0]).to.equal(true); + }); + }); + + describe('when asset types exist in the export', () => { + beforeEach(() => { + sinon.stub(require('node:fs'), 'existsSync').returns(true); + }); + + it('creates a new asset type that does not exist in the target org', async () => { + const newType = { uid: 'type-new', label: 'New Type' }; + stubExistingAssetTypes([]); + stubChunks([newType]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(1); + const payload = createStub.firstCall.args[0]; + expect(payload.label).to.equal('New Type'); + }); + + it('skips asset types with is_system=true', async () => { + stubExistingAssetTypes([]); + stubChunks([{ uid: 'sys-type', is_system: true, label: 'System Type' }]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + const tickArgs = tickStub.lastCall.args[1] as string; + expect(tickArgs).to.include('skipped'); + }); + + it('skips (no create) when uid already exists in target with matching definition', async () => { + const existing = { uid: 'type-1', label: 'Type One', created_at: '2024-01-01' }; + const exported = { uid: 'type-1', label: 'Type One', created_at: '2024-01-01' }; + stubExistingAssetTypes([existing]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.lastCall.args[1]).to.include('skipped'); + }); + + it('skips (no create) when uid exists with a different definition in target', async () => { + const existing = { uid: 'type-1', label: 'Old Label' }; + const exported = { uid: 'type-1', label: 'New Label' }; + stubExistingAssetTypes([existing]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.lastCall.args[1]).to.include('skipped'); + }); + + it('strips invalid keys (created_at, updated_at, is_system) from the POST payload', async () => { + const exported = { + uid: 'type-clean', + label: 'Clean Type', + created_at: '2024-01-01', + updated_at: '2024-06-01', + is_system: false, + created_by: 'user-1', + updated_by: 'user-2', + }; + stubExistingAssetTypes([]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + const payload = createStub.firstCall.args[0]; + expect(payload).to.not.have.property('created_at'); + expect(payload).to.not.have.property('updated_at'); + expect(payload).to.not.have.property('is_system'); + expect(payload).to.not.have.property('created_by'); + expect(payload).to.not.have.property('updated_by'); + expect(payload.label).to.equal('Clean Type'); + }); + + it('handles createAssetType failure: increments failure count, final tick reflects failure', async () => { + stubExistingAssetTypes([]); + stubChunks([{ uid: 'type-bad', label: 'Bad Type' }]); + sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).rejects(new Error('API error')); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + const lastTickArgs = tickStub.lastCall.args; + expect(lastTickArgs[0]).to.equal(false); + expect(lastTickArgs[1]).to.include('1 failed'); + }); + + it('handles getWorkspaceAssetTypes failure: proceeds as if no existing types', async () => { + sinon.stub(CSAssetsImportAdapter.prototype, 'getWorkspaceAssetTypes' as any) + .rejects(new Error('API unavailable')); + stubChunks([{ uid: 'type-new', label: 'New Type' }]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(1); + }); + + it('final tick is success=true when all creates succeed', async () => { + stubExistingAssetTypes([]); + stubChunks([{ uid: 'type-ok', label: 'OK Type' }]); + sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + const lastTickArgs = tickStub.lastCall.args; + expect(lastTickArgs[0]).to.equal(true); + expect(lastTickArgs[1]).to.include('1 created'); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/import/assets.test.ts b/packages/contentstack-asset-management/test/unit/import/assets.test.ts new file mode 100644 index 000000000..d60a758ae --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/import/assets.test.ts @@ -0,0 +1,239 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import * as os from 'os'; +import * as path from 'path'; +import * as fsReal from 'fs'; +import { FsUtility } from '@contentstack/cli-utilities'; + +import ImportAssets from '../../../src/import/assets'; +import { CSAssetsImportAdapter } from '../../../src/import/base'; +import type { CSAssetsAPIConfig, ImportContext } from '../../../src/types/cs-assets-api'; + +describe('ImportAssets', () => { + const apiConfig: CSAssetsAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + const importContext: ImportContext = { + spacesRootPath: '/tmp/import/spaces', + apiKey: 'api-key-1', + host: 'https://api.contentstack.io/v3', + org_uid: 'org-1', + }; + + let tickStub: sinon.SinonStub; + + beforeEach(() => { + sinon.stub(CSAssetsImportAdapter.prototype, 'init' as any).resolves(); + tickStub = sinon.stub(CSAssetsImportAdapter.prototype, 'tick' as any); + sinon.stub(CSAssetsImportAdapter.prototype, 'updateStatus' as any); + }); + + afterEach(() => sinon.restore()); + + const makeSpaceDir = () => { + const dir = path.join(os.tmpdir(), `am-test-${Date.now()}`); + fsReal.mkdirSync(path.join(dir, 'assets'), { recursive: true }); + return dir; + }; + + const stubAssetChunks = (assets: Record[]) => { + const indexer = assets.length > 0 ? { '0': true } : {}; + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => indexer); + if (assets.length > 0) { + const chunk = Object.fromEntries(assets.map((a) => [(a.uid as string), a])); + sinon.stub(FsUtility.prototype, 'readChunkFiles' as any).get(() => ({ + next: sinon.stub().resolves(chunk), + })); + } + sinon.stub(FsUtility.prototype, 'getPlainMeta').returns( + assets.length > 0 ? { 'chunk0': assets.map((a) => a.uid) } : {}, + ); + }; + + describe('buildIdentityMappersFromExport', () => { + it('returns empty maps when no assets.json index exists in spaceDir', async () => { + const spaceDir = makeSpaceDir(); + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.buildIdentityMappersFromExport(spaceDir); + + expect(result.uidMap).to.deep.equal({}); + expect(result.urlMap).to.deep.equal({}); + }); + + it('builds identity uid and url maps from chunked assets', async () => { + const spaceDir = makeSpaceDir(); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + stubAssetChunks([ + { uid: 'asset-1', url: 'https://cdn.example.com/asset-1.png' }, + { uid: 'asset-2', url: 'https://cdn.example.com/asset-2.png' }, + ]); + + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.buildIdentityMappersFromExport(spaceDir); + + expect(result.uidMap).to.deep.equal({ 'asset-1': 'asset-1', 'asset-2': 'asset-2' }); + expect(result.urlMap).to.deep.equal({ + 'https://cdn.example.com/asset-1.png': 'https://cdn.example.com/asset-1.png', + 'https://cdn.example.com/asset-2.png': 'https://cdn.example.com/asset-2.png', + }); + }); + + it('handles assets with missing uid gracefully: only url is added to urlMap', async () => { + const spaceDir = makeSpaceDir(); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => ({ '0': true })); + sinon.stub(FsUtility.prototype, 'readChunkFiles' as any).get(() => ({ + next: sinon.stub().resolves({ + 'asset-no-url': { uid: 'asset-no-url' }, + }), + })); + sinon.stub(FsUtility.prototype, 'getPlainMeta').returns({}); + + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.buildIdentityMappersFromExport(spaceDir); + + expect(result.uidMap).to.have.key('asset-no-url'); + expect(result.urlMap).to.deep.equal({}); + }); + }); + + describe('start', () => { + it('returns empty maps and ticks once for an empty space (no folders, no assets)', async () => { + const spaceDir = makeSpaceDir(); + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.start('new-space-uid', spaceDir); + + expect(result.uidMap).to.deep.equal({}); + expect(result.urlMap).to.deep.equal({}); + expect(tickStub.callCount).to.equal(1); + expect(tickStub.firstCall.args[0]).to.equal(true); + }); + + it('creates root-level folders and maps their uids', async () => { + const spaceDir = makeSpaceDir(); + const folders = [{ uid: 'folder-old', title: 'My Folder' }]; + fsReal.writeFileSync( + path.join(spaceDir, 'assets', 'folders.json'), + JSON.stringify({ folders }), + ); + stubAssetChunks([]); + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => ({})); + const createFolderStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createFolder' as any) + .resolves({ folder: { uid: 'folder-new' } }); + + const importer = new ImportAssets(apiConfig, importContext); + await importer.start('space-uid', spaceDir); + + expect(createFolderStub.callCount).to.equal(1); + const createArgs = createFolderStub.firstCall.args; + expect(createArgs[0]).to.equal('space-uid'); + expect(createArgs[1].title).to.equal('My Folder'); + }); + + it('imports nested folders in multi-pass: child waits for parent to be created', async () => { + const spaceDir = makeSpaceDir(); + const folders = [ + { uid: 'child-folder', title: 'Child', parent_uid: 'parent-folder' }, + { uid: 'parent-folder', title: 'Parent' }, + ]; + fsReal.writeFileSync( + path.join(spaceDir, 'assets', 'folders.json'), + JSON.stringify({ folders }), + ); + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => ({})); + let callOrder: string[] = []; + const createFolderStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createFolder' as any) + .callsFake(async (_spaceUid: string, payload: any) => { + callOrder.push(payload.title); + return { folder: { uid: `new-${payload.title.toLowerCase()}` } }; + }); + + const importer = new ImportAssets(apiConfig, importContext); + await importer.start('space-uid', spaceDir); + + expect(createFolderStub.callCount).to.equal(2); + expect(callOrder[0]).to.equal('Parent'); + expect(callOrder[1]).to.equal('Child'); + }); + + it('uploads assets: calls uploadAsset and builds uidMap and urlMap', async () => { + const spaceDir = makeSpaceDir(); + const assetUid = 'asset-old-uid'; + const assetFilename = 'photo.png'; + fsReal.mkdirSync(path.join(spaceDir, 'assets', 'files', assetUid), { recursive: true }); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'files', assetUid, assetFilename), 'fake-content'); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + stubAssetChunks([{ uid: assetUid, url: 'https://old-cdn.com/photo.png', filename: assetFilename }]); + + const uploadStub = sinon.stub(CSAssetsImportAdapter.prototype, 'uploadAsset' as any) + .resolves({ asset: { uid: 'asset-new-uid', url: 'https://new-cdn.com/photo.png' } }); + + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.start('space-uid', spaceDir); + + expect(uploadStub.callCount).to.equal(1); + expect(result.uidMap[assetUid]).to.equal('asset-new-uid'); + expect(result.urlMap['https://old-cdn.com/photo.png']).to.equal('https://new-cdn.com/photo.png'); + }); + + it('skips an asset and ticks false when the file is not found on disk', async () => { + const spaceDir = makeSpaceDir(); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + stubAssetChunks([{ uid: 'missing-asset', url: 'https://cdn.com/x.png', filename: 'x.png' }]); + const uploadStub = sinon.stub(CSAssetsImportAdapter.prototype, 'uploadAsset' as any).resolves(); + + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.start('space-uid', spaceDir); + + expect(uploadStub.callCount).to.equal(0); + expect(result.uidMap).to.deep.equal({}); + const failTick = tickStub.getCalls().find((c) => c.args[0] === false && c.args[2]); + expect(failTick).to.exist; + }); + + it('handles uploadAsset failure gracefully: continues, ticks false, omits from maps', async () => { + const spaceDir = makeSpaceDir(); + const assetUid = 'asset-fail'; + const filename = 'fail.png'; + fsReal.mkdirSync(path.join(spaceDir, 'assets', 'files', assetUid), { recursive: true }); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'files', assetUid, filename), 'data'); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + stubAssetChunks([{ uid: assetUid, url: 'https://cdn.com/fail.png', filename }]); + + sinon.stub(CSAssetsImportAdapter.prototype, 'uploadAsset' as any).rejects(new Error('upload failed')); + + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.start('space-uid', spaceDir); + + expect(result.uidMap).to.deep.equal({}); + const failTick = tickStub.getCalls().find((c) => c.args[0] === false); + expect(failTick).to.exist; + }); + + it('maps asset parent_uid to the new folder uid when parent was imported', async () => { + const spaceDir = makeSpaceDir(); + fsReal.writeFileSync( + path.join(spaceDir, 'assets', 'folders.json'), + JSON.stringify({ folders: [{ uid: 'old-folder', title: 'Folder A' }] }), + ); + const assetUid = 'asset-in-folder'; + const filename = 'file.png'; + fsReal.mkdirSync(path.join(spaceDir, 'assets', 'files', assetUid), { recursive: true }); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'files', assetUid, filename), 'data'); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + stubAssetChunks([{ uid: assetUid, parent_uid: 'old-folder', filename }]); + + sinon.stub(CSAssetsImportAdapter.prototype, 'createFolder' as any) + .resolves({ folder: { uid: 'new-folder-uid' } }); + const uploadStub = sinon.stub(CSAssetsImportAdapter.prototype, 'uploadAsset' as any) + .resolves({ asset: { uid: 'new-asset-uid' } }); + + const importer = new ImportAssets(apiConfig, importContext); + await importer.start('space-uid', spaceDir); + + const uploadArgs = uploadStub.firstCall.args; + expect(uploadArgs[2].parent_uid).to.equal('new-folder-uid'); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/import/base.test.ts b/packages/contentstack-asset-management/test/unit/import/base.test.ts new file mode 100644 index 000000000..ddbab234e --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/import/base.test.ts @@ -0,0 +1,224 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import { CSAssetsImportAdapter } from '../../../src/import/base'; +import type { CSAssetsAPIConfig, ImportContext } from '../../../src/types/cs-assets-api'; + +class TestImportAdapter extends CSAssetsImportAdapter { + public callCreateNestedProgress(name: string) { return this.createNestedProgress(name); } + public callTick(success: boolean, name: string, error: string | null, processName?: string) { + return this.tick(success, name, error, processName); + } + public callUpdateStatus(msg: string, processName?: string) { return this.updateStatus(msg, processName); } + public callCompleteProcess(name: string, success: boolean) { return this.completeProcess(name, success); } + public get progressOrParentPublic() { return this.progressOrParent; } + public get spacesRootPathPublic() { return this.spacesRootPath; } + public get apiConcurrencyPublic() { return this.apiConcurrency; } + public get uploadBatchPublic() { return this.uploadAssetsBatchConcurrency; } + public get foldersBatchPublic() { return this.importFoldersBatchConcurrency; } + public getAssetTypesDirPublic() { return this.getAssetTypesDir(); } + public getFieldsDirPublic() { return this.getFieldsDir(); } +} + +describe('CSAssetsImportAdapter (base)', () => { + const apiConfig: CSAssetsAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + const importContext: ImportContext = { + spacesRootPath: '/tmp/import/spaces', + apiKey: 'api-key-1', + host: 'https://api.contentstack.io/v3', + org_uid: 'org-1', + }; + + beforeEach(() => { + sinon.stub(CSAssetsImportAdapter.prototype, 'init' as any).resolves(); + }); + afterEach(() => sinon.restore()); + + describe('setParentProgressManager / progressOrParent', () => { + it('returns null when no progress manager is set', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + expect(adapter.progressOrParentPublic).to.be.null; + }); + + it('returns the parent manager after setParentProgressManager', () => { + const fakeParent = { tick: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + expect(adapter.progressOrParentPublic).to.equal(fakeParent); + }); + + it('returns progressManager when parentProgressManager is not set', () => { + sinon.stub(configHandler, 'get').returns({}); + const fakeProgress = { tick: sinon.stub() } as any; + sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress); + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.callCreateNestedProgress('test-module'); + expect(adapter.progressOrParentPublic).to.equal(fakeProgress); + }); + }); + + describe('setProcessName', () => { + it('overrides the processName used in tick calls', () => { + const fakeParent = { tick: sinon.stub(), updateStatus: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + adapter.setProcessName('custom-process'); + adapter.callTick(true, 'item', null); + expect(fakeParent.tick.firstCall.args[3]).to.equal('custom-process'); + }); + }); + + describe('createNestedProgress', () => { + it('creates a CLIProgressManager when no parent is set', () => { + sinon.stub(configHandler, 'get').returns({ showConsoleLogs: true }); + const fakeProgress = { tick: sinon.stub() } as any; + const createNestedStub = sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress); + const adapter = new TestImportAdapter(apiConfig, importContext); + const result = adapter.callCreateNestedProgress('my-module'); + expect(createNestedStub.firstCall.args[0]).to.equal('my-module'); + expect(result).to.equal(fakeProgress); + }); + + it('returns parent directly when parentProgressManager is set', () => { + const fakeParent = { tick: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + const result = adapter.callCreateNestedProgress('ignored'); + expect(result).to.equal(fakeParent); + }); + + it('defaults showConsoleLogs to false when log config is missing', () => { + sinon.stub(configHandler, 'get').returns(null); + const fakeProgress = { tick: sinon.stub() } as any; + const createNestedStub = sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress); + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.callCreateNestedProgress('test'); + expect(createNestedStub.firstCall.args[1]).to.be.false; + }); + }); + + describe('tick', () => { + it('forwards success, itemName, error to progress manager tick', () => { + const fakeParent = { tick: sinon.stub(), updateStatus: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + adapter.callTick(true, 'my-item', 'some-error'); + expect(fakeParent.tick.firstCall.args[0]).to.equal(true); + expect(fakeParent.tick.firstCall.args[1]).to.equal('my-item'); + expect(fakeParent.tick.firstCall.args[2]).to.equal('some-error'); + }); + + it('uses explicit processName override when provided', () => { + const fakeParent = { tick: sinon.stub(), updateStatus: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + adapter.callTick(false, 'item', null, 'override-process'); + expect(fakeParent.tick.firstCall.args[3]).to.equal('override-process'); + }); + + it('does not throw when progressOrParent is null', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + expect(() => adapter.callTick(true, 'item', null)).to.not.throw(); + }); + }); + + describe('updateStatus', () => { + it('forwards status message to progress manager', () => { + const fakeParent = { tick: sinon.stub(), updateStatus: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + adapter.callUpdateStatus('Importing...'); + expect(fakeParent.updateStatus.firstCall.args[0]).to.equal('Importing...'); + }); + + it('does not throw when progressOrParent is null', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + expect(() => adapter.callUpdateStatus('msg')).to.not.throw(); + }); + }); + + describe('completeProcess', () => { + it('calls completeProcess on progressManager when no parent is set', () => { + sinon.stub(configHandler, 'get').returns({}); + const fakeProgress = { tick: sinon.stub(), completeProcess: sinon.stub() } as any; + sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress); + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.callCreateNestedProgress('test'); + adapter.callCompleteProcess('test-process', true); + expect(fakeProgress.completeProcess.firstCall.args).to.deep.equal(['test-process', true]); + }); + + it('does NOT call completeProcess when parentProgressManager is set', () => { + const fakeParent = { tick: sinon.stub(), completeProcess: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + adapter.callCompleteProcess('test-process', true); + expect(fakeParent.completeProcess.callCount).to.equal(0); + }); + }); + + describe('path and concurrency getters', () => { + it('spacesRootPath returns the value from importContext', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + expect(adapter.spacesRootPathPublic).to.equal('/tmp/import/spaces'); + }); + + it('apiConcurrency defaults to FALLBACK_AM_API_CONCURRENCY (5) when not set', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + expect(adapter.apiConcurrencyPublic).to.equal(5); + }); + + it('apiConcurrency uses importContext.apiConcurrency when set', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, apiConcurrency: 10 }); + expect(adapter.apiConcurrencyPublic).to.equal(10); + }); + + it('uploadAssetsBatchConcurrency falls back to apiConcurrency when uploadAssetsConcurrency not set', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, apiConcurrency: 8 }); + expect(adapter.uploadBatchPublic).to.equal(8); + }); + + it('uploadAssetsBatchConcurrency uses uploadAssetsConcurrency when set', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, uploadAssetsConcurrency: 3 }); + expect(adapter.uploadBatchPublic).to.equal(3); + }); + + it('importFoldersBatchConcurrency falls back to apiConcurrency when not set', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, apiConcurrency: 6 }); + expect(adapter.foldersBatchPublic).to.equal(6); + }); + + it('importFoldersBatchConcurrency uses importFoldersConcurrency when set', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, importFoldersConcurrency: 2 }); + expect(adapter.foldersBatchPublic).to.equal(2); + }); + + it('getAssetTypesDir defaults to spacesRootPath/asset_types', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + const expected = require('node:path').join('/tmp/import/spaces', 'asset_types'); + expect(adapter.getAssetTypesDirPublic()).to.equal(expected); + }); + + it('getAssetTypesDir uses custom assetTypesDir when set in importContext', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, assetTypesDir: 'custom_at' }); + const expected = require('node:path').join('/tmp/import/spaces', 'custom_at'); + expect(adapter.getAssetTypesDirPublic()).to.equal(expected); + }); + + it('getFieldsDir defaults to spacesRootPath/fields', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + const expected = require('node:path').join('/tmp/import/spaces', 'fields'); + expect(adapter.getFieldsDirPublic()).to.equal(expected); + }); + + it('getFieldsDir uses custom fieldsDir when set in importContext', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, fieldsDir: 'custom_fields' }); + const expected = require('node:path').join('/tmp/import/spaces', 'custom_fields'); + expect(adapter.getFieldsDirPublic()).to.equal(expected); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/import/fields.test.ts b/packages/contentstack-asset-management/test/unit/import/fields.test.ts new file mode 100644 index 000000000..c45ef17c8 --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/import/fields.test.ts @@ -0,0 +1,186 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { FsUtility } from '@contentstack/cli-utilities'; + +import ImportFields from '../../../src/import/fields'; +import { CSAssetsImportAdapter } from '../../../src/import/base'; +import type { CSAssetsAPIConfig, ImportContext } from '../../../src/types/cs-assets-api'; + +describe('ImportFields', () => { + const apiConfig: CSAssetsAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + const importContext: ImportContext = { + spacesRootPath: '/tmp/import/spaces', + apiKey: 'api-key-1', + host: 'https://api.contentstack.io/v3', + org_uid: 'org-1', + }; + + let tickStub: sinon.SinonStub; + + beforeEach(() => { + sinon.stub(CSAssetsImportAdapter.prototype, 'init' as any).resolves(); + tickStub = sinon.stub(CSAssetsImportAdapter.prototype, 'tick' as any); + sinon.stub(CSAssetsImportAdapter.prototype, 'updateStatus' as any); + sinon.stub(CSAssetsImportAdapter.prototype, 'getFieldsDir' as any).returns('/tmp/import/spaces/fields'); + }); + + afterEach(() => sinon.restore()); + + const stubExistingFields = (fields: any[]) => { + sinon.stub(CSAssetsImportAdapter.prototype, 'getWorkspaceFields' as any) + .resolves({ fields }); + }; + + const stubChunks = (records: Record[]) => { + const indexer = records.length > 0 ? { '0': true } : {}; + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => indexer); + if (records.length > 0) { + const chunk = Object.fromEntries(records.map((r) => [(r.uid as string), r])); + sinon.stub(FsUtility.prototype, 'readChunkFiles' as any).get(() => ({ + next: sinon.stub().resolves(chunk), + })); + } + }; + + describe('when index file does not exist', () => { + it('ticks once and returns without calling createField', async () => { + sinon.stub(require('node:fs'), 'existsSync').returns(false); + stubExistingFields([]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.callCount).to.equal(1); + expect(tickStub.firstCall.args[0]).to.equal(true); + }); + }); + + describe('when fields exist in the export', () => { + beforeEach(() => { + sinon.stub(require('node:fs'), 'existsSync').returns(true); + }); + + it('creates a new field that does not exist in the target org', async () => { + const newField = { uid: 'field-new', label: 'New Field', type: 'text' }; + stubExistingFields([]); + stubChunks([newField]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(1); + const payload = createStub.firstCall.args[0]; + expect(payload.label).to.equal('New Field'); + }); + + it('skips fields with is_system=true', async () => { + stubExistingFields([]); + stubChunks([{ uid: 'sys-field', is_system: true, label: 'System Field' }]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.lastCall.args[1]).to.include('skipped'); + }); + + it('silently skips (no create) when uid exists with matching definition after stripping invalid keys', async () => { + const existing = { uid: 'field-1', label: 'Field One', created_at: '2024-01-01', asset_types_count: 3 }; + const exported = { uid: 'field-1', label: 'Field One', created_at: '2024-01-01', asset_types_count: 5 }; + stubExistingFields([existing]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.lastCall.args[1]).to.include('skipped'); + }); + + it('skips (no create) when uid exists with a different definition', async () => { + const existing = { uid: 'field-1', label: 'Old Label', type: 'text' }; + const exported = { uid: 'field-1', label: 'New Label', type: 'text' }; + stubExistingFields([existing]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.lastCall.args[1]).to.include('skipped'); + }); + + it('strips invalid keys (created_at, updated_at, is_system, asset_types_count) from POST payload', async () => { + const exported = { + uid: 'field-clean', + label: 'Clean Field', + created_at: '2024-01-01', + updated_at: '2024-06-01', + is_system: false, + asset_types_count: 10, + created_by: 'user-1', + updated_by: 'user-2', + }; + stubExistingFields([]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + const payload = createStub.firstCall.args[0]; + expect(payload).to.not.have.property('created_at'); + expect(payload).to.not.have.property('updated_at'); + expect(payload).to.not.have.property('is_system'); + expect(payload).to.not.have.property('asset_types_count'); + expect(payload.label).to.equal('Clean Field'); + }); + + it('handles createField failure: final tick reflects failure count', async () => { + stubExistingFields([]); + stubChunks([{ uid: 'field-bad', label: 'Bad Field' }]); + sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).rejects(new Error('API error')); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + const lastTickArgs = tickStub.lastCall.args; + expect(lastTickArgs[0]).to.equal(false); + expect(lastTickArgs[1]).to.include('1 failed'); + }); + + it('handles getWorkspaceFields failure: proceeds as if no existing fields', async () => { + sinon.stub(CSAssetsImportAdapter.prototype, 'getWorkspaceFields' as any) + .rejects(new Error('API unavailable')); + stubChunks([{ uid: 'field-new', label: 'New Field' }]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(1); + }); + + it('final tick is success=true when all creates succeed and none fail', async () => { + stubExistingFields([]); + stubChunks([{ uid: 'field-ok', label: 'OK Field' }]); + sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + const lastTickArgs = tickStub.lastCall.args; + expect(lastTickArgs[0]).to.equal(true); + expect(lastTickArgs[1]).to.include('1 created'); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/import/spaces.test.ts b/packages/contentstack-asset-management/test/unit/import/spaces.test.ts index 5cff18b66..d77f7e943 100644 --- a/packages/contentstack-asset-management/test/unit/import/spaces.test.ts +++ b/packages/contentstack-asset-management/test/unit/import/spaces.test.ts @@ -30,7 +30,10 @@ describe('ImportSpaces', () => { }; beforeEach(() => { - sinon.stub(configHandler, 'get').returns({ showConsoleLogs: false }); + sinon.stub(configHandler, 'get').callsFake((key: string) => { + if (key === 'log') return { showConsoleLogs: false }; + return undefined; + }); sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress as any); // init and listSpaces live on AssetManagementAdapter (the common base). // Stubbing the base once covers both the adapter used for listSpaces and ImportWorkspace. @@ -173,4 +176,178 @@ describe('ImportSpaces', () => { expect(result.spaceUidMap).to.deep.equal({}); }); }); + + describe('bootstrap failure', () => { + it('should mark all space rows as failed and re-throw when ImportFields throws', async () => { + stubSpaceDirs(['am-space-1']); + sinon.stub(ImportWorkspace.prototype, 'start').resolves({ + oldSpaceUid: 'am-space-1', newSpaceUid: 'new-space', workspaceUid: 'main', + isDefault: false, uidMap: {}, urlMap: {}, + }); + (ImportFields.prototype.start as sinon.SinonStub).rejects(new Error('fields-bootstrap-error')); + + const importer = new ImportSpaces(baseOptions); + try { + await importer.start(); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.equal('fields-bootstrap-error'); + } + + const completeCalls = fakeProgress.completeProcess.getCalls().map((c) => c.args); + expect(completeCalls).to.deep.include([PROCESS_NAMES.AM_IMPORT_FIELDS, false]); + }); + + it('should mark all space rows as failed and re-throw when ImportAssetTypes throws', async () => { + stubSpaceDirs(['am-space-1']); + (ImportAssetTypes.prototype.start as sinon.SinonStub).rejects(new Error('at-bootstrap-error')); + + const importer = new ImportSpaces(baseOptions); + try { + await importer.start(); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.equal('at-bootstrap-error'); + } + + const completeCalls = fakeProgress.completeProcess.getCalls().map((c) => c.args); + expect(completeCalls).to.deep.include([PROCESS_NAMES.AM_IMPORT_ASSET_TYPES, false]); + }); + }); + + describe('per-space failure resilience', () => { + it('should continue importing remaining spaces when one space fails', async () => { + stubSpaceDirs(['am-space-1', 'am-space-2']); + const startStub = sinon.stub(ImportWorkspace.prototype, 'start'); + startStub.onFirstCall().rejects(new Error('space-1-error')); + startStub.onSecondCall().resolves({ + oldSpaceUid: 'am-space-2', newSpaceUid: 'new-space-2', workspaceUid: 'main', + isDefault: false, uidMap: {}, urlMap: {}, + }); + + const importer = new ImportSpaces(baseOptions); + const result = await importer.start(); + + expect(startStub.callCount).to.equal(2); + expect(result.spaceMappings).to.have.lengthOf(1); + expect(result.spaceMappings[0].oldSpaceUid).to.equal('am-space-2'); + }); + }); + + describe('backupDir mapper file writing', () => { + it('should write uid, url, and space-uid mapping files when backupDir is set', async () => { + const os = require('node:os'); + const path = require('node:path'); + const fsReal = require('node:fs'); + const tmpDir = path.join(os.tmpdir(), `import-spaces-backup-${Date.now()}`); + fsReal.mkdirSync(tmpDir, { recursive: true }); + + stubSpaceDirs(['am-space-1']); + sinon.stub(ImportWorkspace.prototype, 'start').resolves({ + oldSpaceUid: 'am-space-1', newSpaceUid: 'new-space-1', workspaceUid: 'main', + isDefault: false, + uidMap: { 'old-uid': 'new-uid' }, + urlMap: { 'old-url': 'new-url' }, + }); + + const options: ImportSpacesOptions = { ...baseOptions, backupDir: tmpDir }; + const importer = new ImportSpaces(options); + await importer.start(); + + const mapperDir = path.join(tmpDir, 'mapper', 'assets'); + expect(fsReal.existsSync(path.join(mapperDir, 'uid-mapping.json'))).to.be.true; + expect(fsReal.existsSync(path.join(mapperDir, 'url-mapping.json'))).to.be.true; + expect(fsReal.existsSync(path.join(mapperDir, 'space-uid-mapping.json'))).to.be.true; + + const uidMap = JSON.parse(fsReal.readFileSync(path.join(mapperDir, 'uid-mapping.json'), 'utf8')); + expect(uidMap).to.deep.equal({ 'old-uid': 'new-uid' }); + }); + }); + + describe('listSpaces error handling and uid filtering', () => { + it('should pass existing org space uids to ImportWorkspace when listSpaces returns spaces', async () => { + (CSAssetsAdapter.prototype.listSpaces as sinon.SinonStub).resolves({ spaces: [{ uid: 'org-space-uid' }] }); + stubSpaceDirs(['am-space-1']); + const startStub = sinon.stub(ImportWorkspace.prototype, 'start').resolves({ + oldSpaceUid: 'am-space-1', newSpaceUid: 'new-space', workspaceUid: 'main', + isDefault: false, uidMap: {}, urlMap: {}, + }); + + const importer = new ImportSpaces(baseOptions); + await importer.start(); + + expect(startStub.callCount).to.equal(1); + const existingSpaceUids: Set = startStub.firstCall.args[2]; + expect(existingSpaceUids.has('org-space-uid')).to.be.true; + }); + + it('should continue (disable reuse-by-uid) when listSpaces throws', async () => { + (CSAssetsAdapter.prototype.listSpaces as sinon.SinonStub).rejects(new Error('network error')); + stubSpaceDirs(['am-space-1']); + sinon.stub(ImportWorkspace.prototype, 'start').resolves({ + oldSpaceUid: 'am-space-1', newSpaceUid: 'new-uid', workspaceUid: 'main', + isDefault: false, uidMap: {}, urlMap: {}, + }); + + const importer = new ImportSpaces(baseOptions); + const result = await importer.start(); + + expect(result.spaceMappings).to.have.lengthOf(1); + }); + + it('should return false for a directory entry when statSync throws', async () => { + const fsMock = require('node:fs'); + const pResolve = require('node:path').resolve; + const join = require('node:path').join; + const spacesRoot = pResolve('/tmp/import', 'spaces'); + const origStatSync = fsMock.statSync.bind(fsMock); + sinon.stub(fsMock, 'readdirSync').returns(['am-bad-entry'] as any); + sinon.stub(fsMock, 'statSync').callsFake((p: string) => { + if (p === join(spacesRoot, 'am-bad-entry')) throw new Error('permission denied'); + return origStatSync(p); + }); + + const importer = new ImportSpaces(baseOptions); + const result = await importer.start(); + + expect(result.spaceMappings).to.deep.equal([]); + }); + + it('should log warning and return empty dirs when readdirSync throws', async () => { + const fsMock = require('node:fs'); + const pResolve = require('node:path').resolve; + const spacesRoot = pResolve('/tmp/import', 'spaces'); + const origReaddir = fsMock.readdirSync.bind(fsMock); + sinon.stub(fsMock, 'readdirSync').callsFake((p: string) => { + if (p === spacesRoot) throw new Error('ENOENT: no such file or directory'); + return origReaddir(p); + }); + sinon.stub(fsMock, 'statSync').returns({ isDirectory: () => true } as any); + + const importer = new ImportSpaces(baseOptions); + const result = await importer.start(); + + expect(result.spaceMappings).to.deep.equal([]); + }); + }); + + describe('setParentProgressManager', () => { + it('should use parent progress manager instead of creating a new CLIProgressManager', async () => { + const fakeParent = { + addProcess: sinon.stub().returnsThis(), + startProcess: sinon.stub().returnsThis(), + updateStatus: sinon.stub().returnsThis(), + tick: sinon.stub(), + completeProcess: sinon.stub(), + }; + stubSpaceDirs([]); + + const importer = new ImportSpaces(baseOptions); + importer.setParentProgressManager(fakeParent as any); + await importer.start(); + + expect((CLIProgressManager.createNested as sinon.SinonStub).callCount).to.equal(0); + expect(fakeParent.addProcess.callCount).to.be.greaterThan(0); + }); + }); }); diff --git a/packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts b/packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts index 7d5af5090..87e5dbe6a 100644 --- a/packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts +++ b/packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts @@ -10,7 +10,7 @@ import ExportAssetTypes from '../../../src/export/asset-types'; import ExportFields from '../../../src/export/fields'; import { CSAssetsExportAdapter } from '../../../src/export/base'; import { CSAssetsAdapter } from '../../../src/utils/cs-assets-api-adapter'; -import * as concurrentBatch from '../../../src/utils/concurrent-batch'; +import * as retryModule from '../../../src/utils/retry'; import type { CsAssetsQueryExportOptions } from '../../../src/types/cs-assets-api'; @@ -46,11 +46,32 @@ describe('CsAssetsQueryExporter', () => { ], }); sinon.stub(CSAssetsExportAdapter.prototype as any, 'writeItemsToChunkedJson').resolves(); - sinon.stub(concurrentBatch, 'runInBatches').callsFake(async (items, _concurrency, handler) => { - for (let i = 0; i < items.length; i++) { - await handler(items[i], i); + // Downloads now run through makeConcurrentCall; fake it by invoking the + // promisifyHandler synchronously over each element of every apiBatch. + sinon.stub(CSAssetsAdapter.prototype, 'makeConcurrentCall').callsFake(async (env: any, handler: any) => { + const batches = env?.apiBatches ?? []; + for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) { + for (let index = 0; index < batches[batchIndex].length; index++) { + if (handler) await handler({ index, batchIndex, isLastRequest: false }); + } } }); + // Run the download retry wrapper inline (single attempt, no backoff) and serve a fake binary + // so download attempts don't hit the network or wait on real retry delays. + sinon.stub(retryModule, 'withRetry').callsFake(async (fn: () => Promise) => fn()); + sinon.stub(globalThis, 'fetch').callsFake( + async () => + ({ + ok: true, + status: 200, + body: new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('x')); + controller.close(); + }, + }), + }) as any, + ); }); afterEach(() => { diff --git a/packages/contentstack-asset-management/test/unit/utils/chunked-json-reader.test.ts b/packages/contentstack-asset-management/test/unit/utils/chunked-json-reader.test.ts new file mode 100644 index 000000000..309b24776 --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/utils/chunked-json-reader.test.ts @@ -0,0 +1,143 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import * as os from 'os'; +import * as fsReal from 'fs'; +import * as path from 'path'; +import { FsUtility } from '@contentstack/cli-utilities'; + +import { forEachChunkedJsonStore, forEachChunkRecordsFromFs } from '../../../src/utils/chunked-json-reader'; + +describe('chunked-json-reader', () => { + afterEach(() => sinon.restore()); + + const makeFakeFs = (indexer: Record, chunks: unknown[]): FsUtility => { + let idx = 0; + return { + indexFileContent: indexer, + readChunkFiles: { next: async () => chunks[idx++] ?? null }, + getPlainMeta: () => ({}), + } as unknown as FsUtility; + }; + + describe('forEachChunkRecordsFromFs', () => { + it('does nothing when indexer is empty', async () => { + const onChunk = sinon.stub(); + await forEachChunkRecordsFromFs(makeFakeFs({}, []), { chunkReadLogLabel: 'test' }, onChunk); + expect(onChunk.callCount).to.equal(0); + }); + + it('calls onChunk with Object.values of each chunk record', async () => { + const r1 = { uid: 'uid-1', url: 'https://a.com' }; + const r2 = { uid: 'uid-2', url: 'https://b.com' }; + const collected: unknown[] = []; + await forEachChunkRecordsFromFs( + makeFakeFs({ '0': true }, [{ 'uid-1': r1, 'uid-2': r2 }]), + { chunkReadLogLabel: 'assets' }, + async (records) => { collected.push(...records); }, + ); + expect(collected).to.deep.equal([r1, r2]); + }); + + it('processes multiple chunks in order', async () => { + const order: string[] = []; + await forEachChunkRecordsFromFs( + makeFakeFs({ '0': true, '1': true }, [ + { 'uid-A': { uid: 'uid-A' } }, + { 'uid-B': { uid: 'uid-B' } }, + ]), + { chunkReadLogLabel: 'test' }, + async (records: any[]) => { order.push(...records.map((r) => r.uid)); }, + ); + expect(order).to.deep.equal(['uid-A', 'uid-B']); + }); + + it('skips a chunk when readChunkFiles.next() rejects', async () => { + const onChunk = sinon.stub(); + const fakeFs = { + indexFileContent: { '0': true }, + readChunkFiles: { next: sinon.stub().rejects(new Error('disk error')) }, + } as unknown as FsUtility; + await forEachChunkRecordsFromFs(fakeFs, { chunkReadLogLabel: 'test' }, onChunk); + expect(onChunk.callCount).to.equal(0); + }); + + it('skips null chunks returned by readChunkFiles', async () => { + const onChunk = sinon.stub(); + await forEachChunkRecordsFromFs( + makeFakeFs({ '0': true }, [null]), + { chunkReadLogLabel: 'test' }, + onChunk, + ); + expect(onChunk.callCount).to.equal(0); + }); + }); + + describe('forEachChunkedJsonStore', () => { + it('calls onOpenError and does not call onEmptyIndexer or onChunk when FsUtility constructor throws', async () => { + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => { + throw new Error('constructor error'); + }); + const onOpenError = sinon.stub(); + const onEmptyIndexer = sinon.stub(); + const onChunk = sinon.stub(); + + await forEachChunkedJsonStore( + '/nonexistent/path', + 'index.json', + { chunkReadLogLabel: 'test', onOpenError, onEmptyIndexer }, + onChunk, + ); + + expect(onOpenError.callCount).to.equal(1); + expect(onEmptyIndexer.callCount).to.equal(0); + expect(onChunk.callCount).to.equal(0); + }); + + it('calls onEmptyIndexer when the index file exists but has no entries', async () => { + const tmpDir = path.join(os.tmpdir(), `cjr-empty-${Date.now()}`); + fsReal.mkdirSync(tmpDir, { recursive: true }); + fsReal.writeFileSync(path.join(tmpDir, 'index.json'), '{}'); + + const onOpenError = sinon.stub(); + const onEmptyIndexer = sinon.stub(); + const onChunk = sinon.stub(); + + await forEachChunkedJsonStore( + tmpDir, + 'index.json', + { chunkReadLogLabel: 'test', onOpenError, onEmptyIndexer }, + onChunk, + ); + + expect(onEmptyIndexer.callCount).to.equal(1); + expect(onChunk.callCount).to.equal(0); + }); + + it('calls onChunk with records when the index has entries', async () => { + const tmpDir = path.join(os.tmpdir(), `cjr-chunks-${Date.now()}`); + fsReal.mkdirSync(tmpDir, { recursive: true }); + fsReal.writeFileSync(path.join(tmpDir, 'index.json'), '{"0": true}'); + + const record = { uid: 'field-1', name: 'My Field' }; + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => ({ '0': true })); + sinon.stub(FsUtility.prototype, 'readChunkFiles' as any).get(() => ({ + next: sinon.stub().resolves({ 'field-1': record }), + })); + + const onOpenError = sinon.stub(); + const onEmptyIndexer = sinon.stub(); + const collected: unknown[] = []; + + await forEachChunkedJsonStore( + tmpDir, + 'index.json', + { chunkReadLogLabel: 'fields', onOpenError, onEmptyIndexer }, + async (records) => { collected.push(...records); }, + ); + + expect(onOpenError.callCount).to.equal(0); + expect(onEmptyIndexer.callCount).to.equal(0); + expect(collected).to.deep.equal([record]); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/utils/concurrent-batch.test.ts b/packages/contentstack-asset-management/test/unit/utils/concurrent-batch.test.ts new file mode 100644 index 000000000..cc95ad49e --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/utils/concurrent-batch.test.ts @@ -0,0 +1,54 @@ +import { expect } from 'chai'; + +import { chunkArray, runInBatches } from '../../../src/utils/concurrent-batch'; + +describe('concurrent-batch', () => { + describe('chunkArray', () => { + it('should split an array into chunks of at most `size`', () => { + expect(chunkArray([1, 2, 3, 4, 5], 2)).to.deep.equal([[1, 2], [3, 4], [5]]); + }); + + it('should return a single chunk when size >= length', () => { + expect(chunkArray([1, 2, 3], 10)).to.deep.equal([[1, 2, 3]]); + }); + + it('should return the whole array as one chunk when size <= 0', () => { + expect(chunkArray([1, 2, 3], 0)).to.deep.equal([[1, 2, 3]]); + }); + + it('should return [] for an empty array', () => { + expect(chunkArray([], 3)).to.deep.equal([]); + }); + }); + + describe('runInBatches', () => { + it('should invoke fn for every item with the correct absolute index', async () => { + const seen: Array<{ item: string; index: number }> = []; + await runInBatches(['a', 'b', 'c'], 2, async (item, index) => { + seen.push({ item, index }); + }); + expect(seen.sort((a, b) => a.index - b.index)).to.deep.equal([ + { item: 'a', index: 0 }, + { item: 'b', index: 1 }, + { item: 'c', index: 2 }, + ]); + }); + + it('should not abort the batch when one task rejects (fault-tolerant)', async () => { + const completed: number[] = []; + await runInBatches([1, 2, 3, 4], 2, async (n) => { + if (n === 2) throw new Error('boom'); + completed.push(n); + }); + expect(completed.sort((a, b) => a - b)).to.deep.equal([1, 3, 4]); + }); + + it('should be a no-op for an empty array', async () => { + let called = false; + await runInBatches([], 5, async () => { + called = true; + }); + expect(called).to.equal(false); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts b/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts index 20f122d82..d9b86a60a 100644 --- a/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts +++ b/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts @@ -141,14 +141,27 @@ describe('CSAssetsAdapter', () => { }); describe('getWorkspaceFields', () => { - it('should GET /api/fields and return the response data', async () => { + it('should GET /api/fields (paginated) and return the merged fields', async () => { const fieldsResponse = { count: 1, relation: 'org', fields: [{ uid: 'f1' }] }; getStub.resolves({ status: 200, data: fieldsResponse }); const adapter = new CSAssetsAdapter(baseConfig); const result = await adapter.getWorkspaceFields('sp-1'); - expect(getStub.firstCall.args[0]).to.equal('/api/fields'); - expect(result).to.deep.equal(fieldsResponse); + expect(getStub.firstCall.args[0]).to.include('/api/fields'); + expect(getStub.firstCall.args[0]).to.include('skip=0'); + expect(result).to.deep.equal({ fields: [{ uid: 'f1' }], count: 1 }); + }); + + it('should fetch all fields across multiple pages', async () => { + getStub.onCall(0).resolves({ status: 200, data: { fields: [{ uid: 'f1' }, { uid: 'f2' }], count: 3 } }); + getStub.onCall(1).resolves({ status: 200, data: { fields: [{ uid: 'f3' }], count: 3 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.getWorkspaceFields('sp-1', 2, 5); + + expect(getStub.callCount).to.equal(2); + expect(getStub.secondCall.args[0]).to.include('skip=2'); + expect(result.count).to.equal(3); + expect(result.fields).to.have.lengthOf(3); }); }); @@ -182,16 +195,30 @@ describe('CSAssetsAdapter', () => { }); describe('getWorkspaceAssetTypes', () => { - it('should GET /api/asset_types?include_fields=true and return the response data', async () => { + it('should GET /api/asset_types?include_fields=true (paginated) and return the merged asset types', async () => { const atResponse = { count: 1, relation: 'org', asset_types: [{ uid: 'at1' }] }; getStub.resolves({ status: 200, data: atResponse }); const adapter = new CSAssetsAdapter(baseConfig); - const result = await adapter.getWorkspaceAssetTypes('sp-1'); + const result: unknown = await adapter.getWorkspaceAssetTypes('sp-1'); const path = getStub.firstCall.args[0] as string; expect(path).to.include('/api/asset_types'); expect(path).to.include('include_fields=true'); - expect(result).to.deep.equal(atResponse); + expect(path).to.include('skip=0'); + expect(result).to.deep.equal({ asset_types: [{ uid: 'at1' }], count: 1 }); + }); + + it('should fetch all asset types across multiple pages, preserving include_fields', async () => { + getStub.onCall(0).resolves({ status: 200, data: { asset_types: [{ uid: 'at1' }, { uid: 'at2' }], count: 3 } }); + getStub.onCall(1).resolves({ status: 200, data: { asset_types: [{ uid: 'at3' }], count: 3 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.getWorkspaceAssetTypes('sp-1', 2, 5); + + expect(getStub.callCount).to.equal(2); + expect(getStub.secondCall.args[0]).to.include('include_fields=true'); + expect(getStub.secondCall.args[0]).to.include('skip=2'); + expect(result.count).to.equal(3); + expect(result.asset_types).to.have.lengthOf(3); }); }); @@ -206,14 +233,435 @@ describe('CSAssetsAdapter', () => { expect(path).to.include('addl_fields=users'); }); - it('should return empty string and no "?" when params are empty', async () => { + it('should append pagination params (limit/skip) for paginated fields collection', async () => { getStub.resolves({ status: 200, data: { count: 0, relation: '', fields: [] } }); const adapter = new CSAssetsAdapter(baseConfig); await adapter.getWorkspaceFields('sp-1'); const path = getStub.firstCall.args[0] as string; - expect(path).to.equal('/api/fields'); - expect(path).to.not.include('?'); + expect(path).to.include('/api/fields?'); + expect(path).to.include('limit='); + expect(path).to.include('skip=0'); + }); + }); + + describe('listSpaces (paginated)', () => { + it('should return all spaces in a single page when count <= pageSize', async () => { + const spaces = [{ uid: 'sp-1' }, { uid: 'sp-2' }]; + getStub.resolves({ status: 200, data: { spaces, count: 2 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.listSpaces(100, 5); + + expect(getStub.callCount).to.equal(1); + expect(getStub.firstCall.args[0]).to.include('/api/spaces'); + expect(getStub.firstCall.args[0]).to.include('limit=100'); + expect(getStub.firstCall.args[0]).to.include('skip=0'); + expect(result.spaces).to.deep.equal(spaces); + expect(result.count).to.equal(2); + }); + + it('should issue additional page requests when total exceeds first page', async () => { + const page1 = Array.from({ length: 2 }, (_, i) => ({ uid: `sp-${i}` })); + const page2 = Array.from({ length: 1 }, (_, i) => ({ uid: `sp-${i + 2}` })); + getStub.onCall(0).resolves({ status: 200, data: { spaces: page1, count: 3 } }); + getStub.onCall(1).resolves({ status: 200, data: { spaces: page2, count: 3 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.listSpaces(2, 5); + + expect(getStub.callCount).to.equal(2); + expect(getStub.secondCall.args[0]).to.include('skip=2'); + expect(result.spaces).to.have.lengthOf(3); + expect(result.count).to.equal(3); + }); + + it('should return empty spaces when count is 0', async () => { + getStub.resolves({ status: 200, data: { spaces: [], count: 0 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.listSpaces(); + + expect(getStub.callCount).to.equal(1); + expect(result.spaces).to.deep.equal([]); + expect(result.count).to.equal(0); + }); + + it('should batch additional page requests by fetchConcurrency', async () => { + // 5 total, pageSize=1, concurrency=2 → 4 additional pages in 2 batches + const pages = Array.from({ length: 5 }, (_, i) => [{ uid: `sp-${i}` }]); + getStub.onCall(0).resolves({ status: 200, data: { spaces: pages[0], count: 5 } }); + getStub.onCall(1).resolves({ status: 200, data: { spaces: pages[1], count: 5 } }); + getStub.onCall(2).resolves({ status: 200, data: { spaces: pages[2], count: 5 } }); + getStub.onCall(3).resolves({ status: 200, data: { spaces: pages[3], count: 5 } }); + getStub.onCall(4).resolves({ status: 200, data: { spaces: pages[4], count: 5 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.listSpaces(1, 2); + + expect(getStub.callCount).to.equal(5); + expect(result.spaces).to.have.lengthOf(5); + }); + + it('should request each page with its OWN skip when pages run concurrently (no shared-mutation race)', async () => { + // total 6, pageSize 2, concurrency 5 → page0 inline, then skips [2,4] in a single concurrent batch. + // If makeAPICall did not snapshot queryParam, both concurrent calls would read the last skip. + const bySkip: Record = { + '0': { spaces: [{ uid: 's0' }, { uid: 's1' }], count: 6 }, + '2': { spaces: [{ uid: 's2' }, { uid: 's3' }], count: 6 }, + '4': { spaces: [{ uid: 's4' }, { uid: 's5' }], count: 6 }, + }; + getStub.callsFake(async (path: string) => { + const skip = new URL(`https://x${path}`).searchParams.get('skip') ?? '0'; + return { status: 200, data: bySkip[skip] }; + }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.listSpaces(2, 5); + + const uids = (result.spaces as Array<{ uid: string }>).map((s) => s.uid).sort(); + expect(uids).to.deep.equal(['s0', 's1', 's2', 's3', 's4', 's5']); + const requestedSkips = getStub.getCalls().map((c) => new URL(`https://x${c.args[0]}`).searchParams.get('skip')); + expect([...requestedSkips].sort()).to.deep.equal(['0', '2', '4']); + }); + }); + + describe('makeConcurrentCall', () => { + it('should be a no-op for empty apiBatches and never hang', async () => { + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.makeConcurrentCall({ module: 'noop', apiBatches: [] }); + expect(getStub.called).to.equal(false); + }); + + it('should invoke the promisifyHandler once per element with correct index/batchIndex', async () => { + const adapter = new CSAssetsAdapter(baseConfig); + const seen: Array<{ index: number; batchIndex: number; element: unknown; isLastRequest: boolean }> = []; + await adapter.makeConcurrentCall({ module: 'downloads', apiBatches: [['a', 'b'], ['c']] }, async (input) => { + seen.push({ + index: input.index, + batchIndex: input.batchIndex, + element: input.element, + isLastRequest: input.isLastRequest, + }); + }); + expect(seen).to.deep.equal([ + { index: 0, batchIndex: 0, element: 'a', isLastRequest: false }, + { index: 1, batchIndex: 0, element: 'b', isLastRequest: false }, + { index: 0, batchIndex: 1, element: 'c', isLastRequest: true }, + ]); + }); + }); + + describe('retry on transient failures', () => { + // retryBaseDelayMs: 0 → instant retries (no wall-clock backoff in tests). + const retryConfig: CSAssetsAPIConfig = { ...baseConfig, retryBaseDelayMs: 0 }; + + it('should retry a 429 and then succeed', async () => { + getStub.onCall(0).resolves({ status: 429, data: {} }); + getStub.onCall(1).resolves({ status: 429, data: {} }); + getStub.onCall(2).resolves({ status: 200, data: { fields: [{ uid: 'f1' }], count: 1 } }); + const adapter = new CSAssetsAdapter(retryConfig); + const result = await adapter.getWorkspaceFields('sp-1'); + + expect(getStub.callCount).to.equal(3); + expect(result.fields).to.deep.equal([{ uid: 'f1' }]); + }); + + it('should retry a 5xx and then succeed', async () => { + getStub.onCall(0).resolves({ status: 503, data: {} }); + getStub.onCall(1).resolves({ status: 200, data: { fields: [{ uid: 'f1' }], count: 1 } }); + const adapter = new CSAssetsAdapter(retryConfig); + const result = await adapter.getWorkspaceFields('sp-1'); + + expect(getStub.callCount).to.equal(2); + expect(result.fields).to.deep.equal([{ uid: 'f1' }]); + }); + + it('should NOT retry a 404 (terminal) and surface a normalized error', async () => { + getStub.resolves({ status: 404, data: { error: 'not found' } }); + const adapter = new CSAssetsAdapter(retryConfig); + let error: unknown; + try { + await adapter.getWorkspaceFields('sp-1'); + } catch (e) { + error = e; + } + expect(getStub.callCount).to.equal(1); + expect((error as Error)?.message).to.include('CS Assets API GET failed'); + }); + }); + + describe('getWorkspaceAssets (paginated)', () => { + it('should fetch all assets across multiple pages', async () => { + const page1 = [{ uid: 'a-1' }, { uid: 'a-2' }]; + const page2 = [{ uid: 'a-3' }]; + getStub.onCall(0).resolves({ status: 200, data: { assets: page1, count: 3 } }); + getStub.onCall(1).resolves({ status: 200, data: { assets: page2, count: 3 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.getWorkspaceAssets('sp-1', undefined, 2, 5) as any; + + expect(result.assets).to.have.lengthOf(3); + expect(result.count).to.equal(3); + }); + + it('should include workspace query param when workspaceUid is provided', async () => { + getStub.resolves({ status: 200, data: { assets: [], count: 0 } }); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.getWorkspaceAssets('sp-1', 'ws-main', 100, 5); + + const path = getStub.firstCall.args[0] as string; + expect(path).to.include('workspace=ws-main'); + }); + + it('should NOT include workspace param when workspaceUid is undefined', async () => { + getStub.resolves({ status: 200, data: { assets: [], count: 0 } }); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.getWorkspaceAssets('sp-1', undefined, 100, 5); + + const path = getStub.firstCall.args[0] as string; + expect(path).to.not.include('workspace='); + }); + }); + + describe('getWorkspaceFolders (paginated)', () => { + it('should fetch all folders across multiple pages', async () => { + const page1 = [{ uid: 'f-1' }]; + const page2 = [{ uid: 'f-2' }]; + getStub.onCall(0).resolves({ status: 200, data: { folders: page1, count: 2 } }); + getStub.onCall(1).resolves({ status: 200, data: { folders: page2, count: 2 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.getWorkspaceFolders('sp-1', undefined, 1, 5) as any; + + expect(result.folders).to.have.lengthOf(2); + expect(result.count).to.equal(2); + }); + + it('should include workspace param when workspaceUid is provided', async () => { + getStub.resolves({ status: 200, data: { folders: [], count: 0 } }); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.getWorkspaceFolders('sp-1', 'ws-main', 100, 5); + + const path = getStub.firstCall.args[0] as string; + expect(path).to.include('workspace=ws-main'); + }); + }); + + describe('POST methods (createSpace, createFolder, createField, createAssetType, bulkDelete, bulkMove)', () => { + let fetchStub: sinon.SinonStub; + + beforeEach(() => { + fetchStub = sinon.stub(global, 'fetch' as any); + }); + + const okJsonResponse = (data: unknown) => ({ + ok: true, + json: async () => data, + text: async () => JSON.stringify(data), + }); + + const failResponse = (status: number, body = 'error body') => ({ + ok: false, + status, + json: async () => ({}), + text: async () => body, + }); + + describe('createSpace', () => { + it('POSTs to /api/spaces and returns the created space', async () => { + const created = { space: { uid: 'new-space-uid', title: 'My Space' } }; + fetchStub.resolves(okJsonResponse(created)); + + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.createSpace({ title: 'My Space' }); + + const [url, opts] = fetchStub.firstCall.args; + expect(url).to.include('/api/spaces'); + expect(opts.method).to.equal('POST'); + expect(result).to.deep.equal(created); + }); + + it('throws when POST returns non-ok status', async () => { + fetchStub.resolves(failResponse(400, 'bad request')); + const adapter = new CSAssetsAdapter(baseConfig); + + try { + await adapter.createSpace({ title: 'Bad Space' }); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('400'); + } + }); + }); + + describe('createFolder', () => { + it('POSTs to /api/spaces/{spaceUid}/folders with space_key header', async () => { + const created = { folder: { uid: 'folder-new' } }; + fetchStub.resolves(okJsonResponse(created)); + + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.createFolder('sp-1', { title: 'Docs' }); + + const [url, opts] = fetchStub.firstCall.args; + expect(url).to.include('/api/spaces/sp-1/folders'); + expect(opts.headers['space_key']).to.equal('sp-1'); + expect(result).to.deep.equal(created); + }); + + it('URL-encodes spaceUid with special characters', async () => { + fetchStub.resolves(okJsonResponse({ folder: { uid: 'f1' } })); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.createFolder('sp uid/1', { title: 'X' }); + + const [url] = fetchStub.firstCall.args; + expect(url).to.include('sp%20uid%2F1'); + }); + }); + + describe('createField', () => { + it('POSTs to /api/fields and returns the created field', async () => { + const created = { field: { uid: 'field-1' } }; + fetchStub.resolves(okJsonResponse(created)); + + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.createField({ uid: 'field-1', label: 'My Field' } as any); + + const [url] = fetchStub.firstCall.args; + expect(url).to.include('/api/fields'); + expect(result).to.deep.equal(created); + }); + }); + + describe('createAssetType', () => { + it('POSTs to /api/asset_types and returns the created asset type', async () => { + const created = { asset_type: { uid: 'at-1' } }; + fetchStub.resolves(okJsonResponse(created)); + + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.createAssetType({ uid: 'at-1' } as any); + + const [url] = fetchStub.firstCall.args; + expect(url).to.include('/api/asset_types'); + expect(result).to.deep.equal(created); + }); + }); + + describe('bulkDeleteAssets', () => { + it('POSTs to the bulk delete endpoint with workspace query param', async () => { + fetchStub.resolves(okJsonResponse({ deleted: 2 })); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.bulkDeleteAssets('sp-1', 'ws-main', { asset_uids: ['a1', 'a2'] } as any); + + const [url, opts] = fetchStub.firstCall.args; + expect(url).to.include('/api/spaces/sp-1/assets/bulk/delete'); + expect(url).to.include('workspace=ws-main'); + expect(opts.headers['space_key']).to.equal('sp-1'); + }); + + it('uses "main" as default workspace uid', async () => { + fetchStub.resolves(okJsonResponse({})); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.bulkDeleteAssets('sp-1', undefined as any, {} as any); + + const [url] = fetchStub.firstCall.args; + expect(url).to.include('workspace=main'); + }); + }); + + describe('bulkMoveAssets', () => { + it('POSTs to the bulk-move endpoint with workspace query param', async () => { + fetchStub.resolves(okJsonResponse({ moved: 1 })); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.bulkMoveAssets('sp-1', 'ws-main', { asset_uids: ['a1'], folder_uid: 'f1' } as any); + + const [url, opts] = fetchStub.firstCall.args; + expect(url).to.include('/api/spaces/sp-1/assets/bulk-move'); + expect(url).to.include('workspace=ws-main'); + expect(opts.headers['space_key']).to.equal('sp-1'); + }); + }); + + describe('postJson error handling', () => { + it('wraps non-API errors in a consistent error message', async () => { + fetchStub.rejects(new Error('network failure')); + const adapter = new CSAssetsAdapter(baseConfig); + + try { + await adapter.createField({} as any); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('CS Assets API POST failed'); + expect(err.message).to.include('network failure'); + } + }); + }); + + describe('uploadAsset', () => { + const os = require('os'); + const path = require('path'); + const fsReal = require('fs'); + + it('reads the file, builds multipart form, and POSTs to /api/spaces/{uid}/assets', async () => { + const tmpFile = path.join(os.tmpdir(), `upload-test-${Date.now()}.png`); + fsReal.writeFileSync(tmpFile, 'fake-image-content'); + fetchStub.resolves(okJsonResponse({ asset: { uid: 'new-asset', url: 'https://cdn.com/x.png' } })); + + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.uploadAsset('sp-1', tmpFile, { title: 'My Image' }); + + const [url, opts] = fetchStub.firstCall.args; + expect(url).to.include('/api/spaces/sp-1/assets'); + expect(opts.method).to.equal('POST'); + expect(opts.headers['space_key']).to.equal('sp-1'); + expect(result).to.deep.equal({ asset: { uid: 'new-asset', url: 'https://cdn.com/x.png' } }); + + fsReal.unlinkSync(tmpFile); + }); + + it('appends description and parent_uid to the form when provided', async () => { + const tmpFile = path.join(os.tmpdir(), `upload-test-desc-${Date.now()}.png`); + fsReal.writeFileSync(tmpFile, 'data'); + const formAppendSpy = sinon.spy(FormData.prototype, 'append'); + fetchStub.resolves(okJsonResponse({ asset: { uid: 'a1', url: 'https://cdn.com/a1.png' } })); + + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.uploadAsset('sp-1', tmpFile, { + title: 'T', description: 'Desc', parent_uid: 'folder-uid', + }); + + const appendCalls = formAppendSpy.getCalls().map((c) => c.args[0]); + expect(appendCalls).to.include('description'); + expect(appendCalls).to.include('parent_uid'); + + fsReal.unlinkSync(tmpFile); + }); + + it('throws when multipart POST returns non-ok status', async () => { + const tmpFile = path.join(os.tmpdir(), `upload-fail-${Date.now()}.png`); + fsReal.writeFileSync(tmpFile, 'data'); + fetchStub.resolves(failResponse(413, 'file too large')); + + const adapter = new CSAssetsAdapter(baseConfig); + try { + await adapter.uploadAsset('sp-1', tmpFile, { title: 'Big File' }); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('413'); + } + + fsReal.unlinkSync(tmpFile); + }); + + it('wraps network errors from multipart fetch in a consistent error message', async () => { + const tmpFile = path.join(os.tmpdir(), `upload-net-${Date.now()}.png`); + fsReal.writeFileSync(tmpFile, 'data'); + fetchStub.rejects(new Error('connection reset')); + + const adapter = new CSAssetsAdapter(baseConfig); + try { + await adapter.uploadAsset('sp-1', tmpFile, { title: 'File' }); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('CS Assets API multipart POST failed'); + expect(err.message).to.include('connection reset'); + } + + fsReal.unlinkSync(tmpFile); + }); }); }); }); diff --git a/packages/contentstack-asset-management/test/unit/utils/retry.test.ts b/packages/contentstack-asset-management/test/unit/utils/retry.test.ts new file mode 100644 index 000000000..c3bd30a6c --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/utils/retry.test.ts @@ -0,0 +1,98 @@ +import { expect } from 'chai'; + +import { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from '../../../src/utils/retry'; + +describe('retry', () => { + describe('isRetryableStatus', () => { + it('treats 429 and 5xx as retryable, others not', () => { + expect(isRetryableStatus(429)).to.equal(true); + expect(isRetryableStatus(500)).to.equal(true); + expect(isRetryableStatus(503)).to.equal(true); + expect(isRetryableStatus(404)).to.equal(false); + expect(isRetryableStatus(400)).to.equal(false); + expect(isRetryableStatus(200)).to.equal(false); + }); + }); + + describe('parseRetryAfterMs', () => { + it('parses delta-seconds into ms', () => { + expect(parseRetryAfterMs('2')).to.equal(2000); + }); + + it('returns undefined for null/empty', () => { + expect(parseRetryAfterMs(null)).to.equal(undefined); + expect(parseRetryAfterMs('')).to.equal(undefined); + }); + + it('parses an HTTP date into a non-negative ms delay', () => { + const value = parseRetryAfterMs(new Date(Date.now() + 1000).toUTCString()); + expect(value).to.be.a('number'); + expect(value as number).to.be.at.least(0); + }); + }); + + describe('withRetry', () => { + it('returns immediately on success without retrying', async () => { + let calls = 0; + const result = await withRetry( + async () => { + calls += 1; + return 'ok'; + }, + { baseDelayMs: 0 }, + ); + expect(result).to.equal('ok'); + expect(calls).to.equal(1); + }); + + it('retries a RetryableHttpError up to `retries` times then rethrows', async () => { + let calls = 0; + let error: unknown; + try { + await withRetry( + async () => { + calls += 1; + throw new RetryableHttpError('boom', 503); + }, + { retries: 2, baseDelayMs: 0 }, + ); + } catch (e) { + error = e; + } + expect(calls).to.equal(3); // initial attempt + 2 retries + expect(error).to.be.instanceOf(RetryableHttpError); + }); + + it('succeeds after transient failures', async () => { + let calls = 0; + const result = await withRetry( + async () => { + calls += 1; + if (calls < 3) throw new RetryableHttpError('transient', 500); + return calls; + }, + { retries: 5, baseDelayMs: 0 }, + ); + expect(result).to.equal(3); + expect(calls).to.equal(3); + }); + + it('does NOT retry a non-RetryableHttpError (terminal)', async () => { + let calls = 0; + let error: unknown; + try { + await withRetry( + async () => { + calls += 1; + throw new Error('terminal'); + }, + { retries: 3, baseDelayMs: 0 }, + ); + } catch (e) { + error = e; + } + expect(calls).to.equal(1); + expect((error as Error).message).to.equal('terminal'); + }); + }); +}); diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index 3df8b5847..373a592a3 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -122,6 +122,8 @@ const config: DefaultConfig = { chunkFileSizeMb: 1, apiConcurrency: 5, downloadAssetsConcurrency: 5, + pageSize: 100, + fetchConcurrency: 5, }, content_types: { dirName: 'content_types', diff --git a/packages/contentstack-export/src/export/modules/assets.ts b/packages/contentstack-export/src/export/modules/assets.ts index a5a529ad9..583f9d8b6 100644 --- a/packages/contentstack-export/src/export/modules/assets.ts +++ b/packages/contentstack-export/src/export/modules/assets.ts @@ -108,6 +108,8 @@ export default class ExportAssets extends BaseClass { chunkFileSizeMb: csAssetsModuleConfig?.chunkFileSizeMb, apiConcurrency: csAssetsModuleConfig?.apiConcurrency, downloadAssetsConcurrency: csAssetsModuleConfig?.downloadAssetsConcurrency, + pageSize: csAssetsModuleConfig?.pageSize, + fetchConcurrency: csAssetsModuleConfig?.fetchConcurrency, }); exporter.setParentProgressManager(progress); await exporter.start(); diff --git a/packages/contentstack-export/src/types/default-config.ts b/packages/contentstack-export/src/types/default-config.ts index ba03e91ee..0c684b702 100644 --- a/packages/contentstack-export/src/types/default-config.ts +++ b/packages/contentstack-export/src/types/default-config.ts @@ -110,6 +110,10 @@ export default interface DefaultConfig { apiConcurrency: number; /** Parallel downloads per AM workspace export. */ downloadAssetsConcurrency: number; + /** Items per page for paginated GET requests (assets, folders, spaces). */ + pageSize: number; + /** Parallel page fetches for paginated GET requests. */ + fetchConcurrency: number; dependencies?: Modules[]; }; content_types: { diff --git a/packages/contentstack-export/test/unit/export/modules/assets.test.ts b/packages/contentstack-export/test/unit/export/modules/assets.test.ts index 6358c744b..520820df7 100644 --- a/packages/contentstack-export/test/unit/export/modules/assets.test.ts +++ b/packages/contentstack-export/test/unit/export/modules/assets.test.ts @@ -146,6 +146,8 @@ describe('ExportAssets', () => { chunkFileSizeMb: 1, apiConcurrency: 5, downloadAssetsConcurrency: 5, + pageSize: 100, + fetchConcurrency: 5, }, content_types: { dirName: 'content_types', diff --git a/packages/contentstack-export/test/unit/export/modules/base-class.test.ts b/packages/contentstack-export/test/unit/export/modules/base-class.test.ts index 5ff9a9dc7..c9d7961da 100644 --- a/packages/contentstack-export/test/unit/export/modules/base-class.test.ts +++ b/packages/contentstack-export/test/unit/export/modules/base-class.test.ts @@ -163,6 +163,8 @@ describe('BaseClass', () => { chunkFileSizeMb: 1, apiConcurrency: 5, downloadAssetsConcurrency: 5, + pageSize: 100, + fetchConcurrency: 5, }, content_types: { dirName: 'content_types', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0529b5982..2bd785f3a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -25,13 +25,13 @@ importers: dependencies: '@apollo/client': specifier: ^3.14.1 - version: 3.14.1(graphql@16.14.1) + version: 3.14.1(graphql@16.14.2) '@contentstack/cli-command': specifier: ~2.0.0-beta.8 version: 2.0.0-beta.8(@types/node@20.19.42) '@contentstack/cli-launch': specifier: ^1.10.0 - version: 1.10.0(@types/node@20.19.42)(tslib@2.8.1)(typescript@5.9.3) + version: 1.10.1(@types/node@20.19.42)(tslib@2.8.1)(typescript@5.9.3) '@contentstack/cli-utilities': specifier: ~2.0.0-beta.9 version: 2.0.0-beta.9(@types/node@20.19.42) @@ -129,10 +129,16 @@ importers: '@contentstack/cli-utilities': specifier: ~2.0.0-beta.9 version: 2.0.0-beta.9(@types/node@20.19.42) + lodash: + specifier: 4.18.1 + version: 4.18.1 devDependencies: '@types/chai': specifier: ^4.3.11 version: 4.3.20 + '@types/lodash': + specifier: ^4.17.0 + version: 4.17.24 '@types/mocha': specifier: ^10.0.6 version: 10.0.10 @@ -442,7 +448,7 @@ importers: version: 10.1.8(eslint@10.4.1) eslint-plugin-prettier: specifier: ^5.5.5 - version: 5.5.6(eslint-config-prettier@10.1.8(eslint@10.4.1))(eslint@10.4.1)(prettier@3.8.3) + version: 5.5.6(eslint-config-prettier@10.1.8(eslint@10.4.1))(eslint@10.4.1)(prettier@3.8.4) husky: specifier: ^9.1.7 version: 9.1.7 @@ -460,7 +466,7 @@ importers: version: 4.23.12(@types/node@20.19.42) prettier: specifier: ^3.8.3 - version: 3.8.3 + version: 3.8.4 shx: specifier: ^0.4.0 version: 0.4.0 @@ -578,7 +584,7 @@ importers: version: 2.0.0-beta.9(@types/node@22.19.20) '@contentstack/types-generator': specifier: ^3.10.0 - version: 3.10.1(graphql@16.14.1) + version: 3.10.1(graphql@16.14.2) devDependencies: '@oclif/plugin-help': specifier: ^6.2.49 @@ -2376,8 +2382,8 @@ packages: '@contentstack/cli-dev-dependencies@2.0.0-beta.0': resolution: {integrity: sha512-tLP05taIeepvp5Xte2LKDTKeYtDjCxOLlNWzwMFhMFYU1Z7oOgiCu8RVHNz+EkAm5xScKORx1OyEgyNLFoTLBw==} - '@contentstack/cli-launch@1.10.0': - resolution: {integrity: sha512-NDeXmdaFS3ZlNYcmyJCaJwwF8ZLCEDciuwe8RZn6HmIFTOFsrqDYTZdcCmE8B6pC+y8n2pD4Pgb9HtzA4W1Alg==} + '@contentstack/cli-launch@1.10.1': + resolution: {integrity: sha512-FaEvN35aOlPgEJDwUqmh7YMJakctL08dn5cU9wM9z6e3J8DXih2D1VnrJGAS76pDMTefD8RYbz6cS5NqDkea3Q==} engines: {node: '>=22.0.0'} hasBin: true @@ -3224,128 +3230,128 @@ packages: rollup: optional: true - '@rollup/rollup-android-arm-eabi@4.61.0': - resolution: {integrity: sha512-dnxczajOqt0gesZlN5pGQ1s1imQVrsmCw5G2Ci4oM+0WvNz3pyRnlWrT7McoZIb8VlFwCawdmbWRmxRn7HI+VQ==} + '@rollup/rollup-android-arm-eabi@4.61.1': + resolution: {integrity: sha512-JnBB8MdXj45cajvTuO5FmPlvFVJRQgvrz1uSEl3NwqFnReAPGwb8EanbGi4z2nRaqLzjJSv5/JmycoTKlRZxHA==} cpu: [arm] os: [android] - '@rollup/rollup-android-arm64@4.61.0': - resolution: {integrity: sha512-Bp3JpGP00Vu3f238ivRrjf7z3xSzVPXqCmaJYA9t2c+c8vKYvOzmXF7LkkeUalTEGd6cZcSWe+PFIP3Vy48fRg==} + '@rollup/rollup-android-arm64@4.61.1': + resolution: {integrity: sha512-Jx2g7iSjw4AOT0HDPHM9RV3GNjRXwybWtSFZiZAYUTjUwjVrYIwq3kBf+LnhqJlzXFAqTAh2F7IGI+O568exPw==} cpu: [arm64] os: [android] - '@rollup/rollup-darwin-arm64@4.61.0': - resolution: {integrity: sha512-zaYIpr670mUmmZ1tVzUFplbQbG7h3Gugx3L5FoqhsC2m/YnLlR1a7zVLmXNPy+iY1tFPEbNG+HHBXZGyId0G5w==} + '@rollup/rollup-darwin-arm64@4.61.1': + resolution: {integrity: sha512-0F1L/Z3Eqv8mT2n3dCpeO8GcTvHvVqkP5/t6DMsn0KzhYVcg+s7Ncl5DS8qjKYEeio6Az0Gt6nyBORay5qIlCA==} cpu: [arm64] os: [darwin] - '@rollup/rollup-darwin-x64@4.61.0': - resolution: {integrity: sha512-+P49fvkv2dSoeevUW+lgZ/I2JHSsJCK1Lyjj7Cu6E4UHG4tS9XIefzIjo5qhgELjAclnen1rLzK2PMKJdo+Dyg==} + '@rollup/rollup-darwin-x64@4.61.1': + resolution: {integrity: sha512-qLttcH871ujY4YcVfUSShhOw+CsoTatYz8gRbHO7Bb92QH059/P0y5do1KMs41fY0BpD2x4AJH/gID0zFiqVKQ==} cpu: [x64] os: [darwin] - '@rollup/rollup-freebsd-arm64@4.61.0': - resolution: {integrity: sha512-l3FAAOyKJXH2ea6KNFN+MMgC/rnE94YGLXs2ehYqDcCoHt1DpvgWX75BhUJxN38XojP7Ul+4H8PRn7EdyqSDrw==} + '@rollup/rollup-freebsd-arm64@4.61.1': + resolution: {integrity: sha512-fUI4RapGE0Oh3mb8mgfvC1O2nU1RpDZUKnDQm3xB1Ipg7C2wTs5Kstz7G2uWK99a8S2yTMq8/P4uycwNa0nJyw==} cpu: [arm64] os: [freebsd] - '@rollup/rollup-freebsd-x64@4.61.0': - resolution: {integrity: sha512-VokPN3TSctKj65cyCNPaUh4vMFA8awxOot/0sp+4J7ZlNRKQEhXhawqPwajoi8H5ZFt61i0ugZJuTKXBjGJ17Q==} + '@rollup/rollup-freebsd-x64@4.61.1': + resolution: {integrity: sha512-H5YrdvJaDtI/U9/emrD4b++xkvp3y/JvOe4rizHbxvkyMfRS/CiRYdji+Pl8D0brEaNFWUh1drQxgAGIl6Xudw==} cpu: [x64] os: [freebsd] - '@rollup/rollup-linux-arm-gnueabihf@4.61.0': - resolution: {integrity: sha512-DxH0P3wxm+Yzs/p3zrk9dw1rURu8p0Nv5+MRK/L7OtnLNg5rLZraSBFZ8iUXOd9f2BlhJyEpIZUH/emjq4UJ4g==} + '@rollup/rollup-linux-arm-gnueabihf@4.61.1': + resolution: {integrity: sha512-Q8CBCCQtDFrYtXoeUXSrnFXKOnyUhx6bz+SkL6A0E7V8kAiCJ5pamq1WtbfpVGhR5TSpXY6ak3avmDc5fHTyJA==} cpu: [arm] os: [linux] - '@rollup/rollup-linux-arm-musleabihf@4.61.0': - resolution: {integrity: sha512-T6ZvMNe84kAz6TBWHC7hGAoEtzP1LWYw/AqayGWEF6uISt3Abk/st06LqRD9THd7Xz3NxzurUpzAuEAUbZf+nw==} + '@rollup/rollup-linux-arm-musleabihf@4.61.1': + resolution: {integrity: sha512-nwnhk1581l0FBVellGcVCAT0Oi06onEA3WB53sf01VO3I0UPBkMH9sXONYME2K0ovXcNayJfNtHfm6mpJElatQ==} cpu: [arm] os: [linux] - '@rollup/rollup-linux-arm64-gnu@4.61.0': - resolution: {integrity: sha512-q/4hzvQkDs8b4jIBab1pnLiiM0ayTZsN2amBFPDzuyZxjEd4wDwx0UJFYM3cOZzSf5Kw8fnWSprJzIBMkcR44Q==} + '@rollup/rollup-linux-arm64-gnu@4.61.1': + resolution: {integrity: sha512-x5Xr49hwt3hdW75UOZm3395YwwzPyauktslv29KpWL/T+vVAzoT3azLcTWv0eMciBNrx+DYjH4paehHoLpPvpg==} cpu: [arm64] os: [linux] - '@rollup/rollup-linux-arm64-musl@4.61.0': - resolution: {integrity: sha512-vvYWX3akdEAY6km+9wAqFDnk6pQsbJKVnj7xawcvs/+fdlYBGp+U+Qq/lLfpIxYIZvZLHMAKD9HLdacSx/r3dw==} + '@rollup/rollup-linux-arm64-musl@4.61.1': + resolution: {integrity: sha512-unMS3H73DpaoPyyEVPjGKleM/s0mkmsauTENpw4INQY8y4+IuLNjkueQ5QCtC0D3N38Y38yhAU8OoZ20S2Tm6w==} cpu: [arm64] os: [linux] - '@rollup/rollup-linux-loong64-gnu@4.61.0': - resolution: {integrity: sha512-DePa5cqOxDP/Zp0VOXpeWaGew5iIv5DXp9NYbzkX5PFQyWVX9184WCTh3hvr/7lhXo8ZVlbFLkz8+o/q1dU6gA==} + '@rollup/rollup-linux-loong64-gnu@4.61.1': + resolution: {integrity: sha512-zNZzGRnAhwjFEYmvphJRV5XaQGjs62cCmeYYHUT//NbvEnHauw+I85nGG+SiVg5ld4GX8D1IbKIX+ozITQnhMQ==} cpu: [loong64] os: [linux] - '@rollup/rollup-linux-loong64-musl@4.61.0': - resolution: {integrity: sha512-LV8aWMB8UChglMCEzs7RkN0GsH29RJaLLqwm9fCIjlqwxQTiWAqNcc7wjBkH31hV0PU/yVxGYvrYsgfea2qw6g==} + '@rollup/rollup-linux-loong64-musl@4.61.1': + resolution: {integrity: sha512-LdpWGL8X209B2SIvWjqlc8VZgM6PKfontSerGepuldQmHYrAOtnMCXeJkxXGbC+PPZVOuu5czJo7fNV6aeW8rQ==} cpu: [loong64] os: [linux] - '@rollup/rollup-linux-ppc64-gnu@4.61.0': - resolution: {integrity: sha512-QoNSnwQtaeNu5grdBbsL0tt1uyl5EnS8DA8Mr3nluMXbhdQNyhN+G4tBax7VCdxLKj8YJ0/4OO9Ho84jMnJtKA==} + '@rollup/rollup-linux-ppc64-gnu@4.61.1': + resolution: {integrity: sha512-EC5kTtNaNGOmbMGqar8dvJy6y/hg99GAwjfBz++pxZhQATXGcRjd6c5en5wcbru0vkRmiMGsQKdMJOOf6sza4g==} cpu: [ppc64] os: [linux] - '@rollup/rollup-linux-ppc64-musl@4.61.0': - resolution: {integrity: sha512-/zZp5MKapIIApE8trN8qLGNSiRN9TUoaUZ1cmVu4XnVdd5LQLOXTtyi+vtfUbNnT3iyjzpPqYeKXmvJ+gJGYWw==} + '@rollup/rollup-linux-ppc64-musl@4.61.1': + resolution: {integrity: sha512-8hiwp6D4acEcNK78I4rP0/XtS1sknWIAMJBPdR4l6zUtyTm5KiTDr5bXmWt4foY7nAN7AThDHgkLIEZOWKbzWw==} cpu: [ppc64] os: [linux] - '@rollup/rollup-linux-riscv64-gnu@4.61.0': - resolution: {integrity: sha512-RbrzcD3aJ1k3UbtMRRBNwojdVVyXjuVAFTfn/xPa6EEl6GE9Sm/akPgFTb9aAC9pMKGJ6CtWxaGrqWcabH+ySg==} + '@rollup/rollup-linux-riscv64-gnu@4.61.1': + resolution: {integrity: sha512-10dh/h/BqA7DuMPWSxkR8uks18FRwnwOEqr5zOTEl+NOwP/OMzKX8OFR/Of9xxDA7D5qef1Nzar5WDD2kCCr1g==} cpu: [riscv64] os: [linux] - '@rollup/rollup-linux-riscv64-musl@4.61.0': - resolution: {integrity: sha512-ZF+onDsBso8PJf1XaG9lB+O9RnBpKGnY6OrzC4CSHrtC1jb6jWLTKK4bRqdoCXHd22gyr2hiYmEAm8Wns/BOCw==} + '@rollup/rollup-linux-riscv64-musl@4.61.1': + resolution: {integrity: sha512-YKJ5lg35DP17gcAOggnihe+APw9HLyj1Xn7gsmGumBJAUDa6NGXNixJzmkWLhcK9TOuuyQjdamzvJefkO7qHZQ==} cpu: [riscv64] os: [linux] - '@rollup/rollup-linux-s390x-gnu@4.61.0': - resolution: {integrity: sha512-Atk0aSIk5Zx2Wuh9dgRQgLP0Koc8hOeYpbWryMXyk8G8/HmPkwPPkMqIIDhrXHHYqfUzSJA/I7IWSBv8xSmRBA==} + '@rollup/rollup-linux-s390x-gnu@4.61.1': + resolution: {integrity: sha512-Mlil5G2Jj6a7B3LWGctg+XPL9vdXYuzCtNXfxOQ0nPjc2m6ueUktocPGH9bnAM0bNRKb/bAWTujUU7IJQdQA+g==} cpu: [s390x] os: [linux] - '@rollup/rollup-linux-x64-gnu@4.61.0': - resolution: {integrity: sha512-0uMOcf3eZ5K+K4cYHkdxShFMPlPXCOdfDFEFn9dNYAEEd2cVvmOfH7zFgRVoDgmtQ1m9k5q7qfrHzyMAubKYUA==} + '@rollup/rollup-linux-x64-gnu@4.61.1': + resolution: {integrity: sha512-bVWIOIk6pV01p4CdUbPP7CJ/434z+OooYjDuFcR+44N35YvKUC66G8MGnvcWx5mWKW3g61J+t74l3Kj15Kwn2Q==} cpu: [x64] os: [linux] - '@rollup/rollup-linux-x64-musl@4.61.0': - resolution: {integrity: sha512-mvFtE4A/t/7hRJ7X8Ozmu8FsIkAUat2nzl12pgU337BRmq87AQUJztwHz2Zv5/tjo9/C95E66CK03SI/ToEDJw==} + '@rollup/rollup-linux-x64-musl@4.61.1': + resolution: {integrity: sha512-qy5pBvZbqNFheBz61R1rzsezjm0J7O2oNGoWtGoY89SZYLUfxAJTBAqDChqAIdB4rCiIbi9nF7yZ83GnNiLwSw==} cpu: [x64] os: [linux] - '@rollup/rollup-openbsd-x64@4.61.0': - resolution: {integrity: sha512-z9b9+aTxvt8n2rNltMPvyaUfB8NJ+CVyOrGK/MdIKHx7B+lXmZpm/XbRsU7Rpf3fRqJ2uS6mBJiJveCtq8LHDg==} + '@rollup/rollup-openbsd-x64@4.61.1': + resolution: {integrity: sha512-E83TXjI4zm0+5f2qO+UOudaCYIhYwpJ5jq6YCZNIZ+6CbfhKrkAGezeiASBL9ElxAxFsRS9ZhESv8mfnj6TKeg==} cpu: [x64] os: [openbsd] - '@rollup/rollup-openharmony-arm64@4.61.0': - resolution: {integrity: sha512-jXaXFqKMehsOc+g8R6oo33RRC6w07G9jDBxAE5eAKX7mOcCbZloYIPNhfG9Wl+P9O9IWHFO4OJgPi1Ml2qkt7w==} + '@rollup/rollup-openharmony-arm64@4.61.1': + resolution: {integrity: sha512-fbWnKqVkjrJN38vNe3ahkbk6iejS/3b0Nt7EEtPpE6RBacZcGXNKbzfHN3GUUlXOPghUg0j6XUGrtjX9z1sIvA==} cpu: [arm64] os: [openharmony] - '@rollup/rollup-win32-arm64-msvc@4.61.0': - resolution: {integrity: sha512-OXNWVFocS2IA4+QplhTZZ2a+8hPZR7T8KuozsNmJKK8y7cp83StHvGksfHzPG3wczWTczyWHVQuqeiTUbjiyBg==} + '@rollup/rollup-win32-arm64-msvc@4.61.1': + resolution: {integrity: sha512-ArMl38iVAbk0New1ogihQNY6iphLi4ZaRsa037gUzv5yeKPY8TD3Dmy4x2RNC1VztU/uqm+G+/RwFrSka3Oy2g==} cpu: [arm64] os: [win32] - '@rollup/rollup-win32-ia32-msvc@4.61.0': - resolution: {integrity: sha512-AlAbNtBO637LxSldqV43z0FfXoGfl2TW1DgAg/bs7aQswFbDewz2SJm3BUhiGfbOVtW571xbc9p+REdxhyN/Eg==} + '@rollup/rollup-win32-ia32-msvc@4.61.1': + resolution: {integrity: sha512-0mYtjHS9ucAbcATycCNK9IGBk/cCe/ma7EmSLGZdsxnOA8cjRIyU04wDpVAD9NiOfLUR9KTxdiO53uOkherqjQ==} cpu: [ia32] os: [win32] - '@rollup/rollup-win32-x64-gnu@4.61.0': - resolution: {integrity: sha512-QRSrQXyJ1M4tjNXdR0/G/IgV6lzfQQJYBjlWIEYkY2Xs86DRl/iEpQ4blMDjJxSl7n19eDKKXMg0AmuBVYy8pQ==} + '@rollup/rollup-win32-x64-gnu@4.61.1': + resolution: {integrity: sha512-gK1iCEPfpoSG9wfBihXxvBMi8ZfcWffYkEsC/Eih+iFENTaewvNcrEQ69lIOWYO5pePHKLHHO7nq5AILGO/HQQ==} cpu: [x64] os: [win32] - '@rollup/rollup-win32-x64-msvc@4.61.0': - resolution: {integrity: sha512-tkuFxhvKO/HlGd0VsINF6vHSYH8AF8W0TcNxKDK6JZmrehngFj78pToc8iemtnvwilDjs2G/qSzYFhe9U8q+fw==} + '@rollup/rollup-win32-x64-msvc@4.61.1': + resolution: {integrity: sha512-X+zaP2x+j4RXGfbp/seSoRHWnPxzApilDszisZxbYH5C/jTxFhCtDNdPGZb9lJyYPs24wGxruPF7Y+sIXt9Gzw==} cpu: [x64] os: [win32] @@ -5998,8 +6004,8 @@ packages: peerDependencies: graphql: ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 - graphql@16.14.1: - resolution: {integrity: sha512-cQOsSMS/IrDz82PVyRDvf/Q1F/bRbBVjJlh+xYOkI1qw2bWRvWGiWc+m2O0d6l4Bt1fyY+8kzJ8JFWGJqNeDBg==} + graphql@16.14.2: + resolution: {integrity: sha512-Chq1s4CY7jmh8gO2qvLIJyfCDIN+EHLFW/9iShnp1z8FjBQMoodWP1kDC36VAMXXIvAjj4ARa7ntfAV2BrjsbA==} engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} handlebars@4.7.9: @@ -6473,8 +6479,8 @@ packages: resolution: {integrity: sha512-NkwHbo3E00oybX6NGJi6ar0B29vxyvNwoC7eJ4G4Yq28UfY758Hgn/heV8VRFhevPED4LXfFz0DQ8z/0kw9zMg==} engines: {node: '>=8'} - istanbul-lib-processinfo@3.0.0: - resolution: {integrity: sha512-P7nLXRRlo7Sqinty6lNa7+4o9jBUYGpqtejqCOZKfgXlRoxY/QArflcB86YO500Ahj4pDJEG34JjMRbQgePLnQ==} + istanbul-lib-processinfo@3.0.1: + resolution: {integrity: sha512-s3mX05h5wGZeScG6XnOanygPh4SJu5ujMc9YbvpnLGXWy1cRiGbp0NdVcjHxgoZt3WfQppfBsa0y+gWdYJ2pGQ==} engines: {node: 20 || >=22} istanbul-lib-report@3.0.1: @@ -7748,8 +7754,8 @@ packages: resolution: {integrity: sha512-SxToR7P8Y2lWmv/kTzVLC1t/GDI2WGjMwNhLLE9qtH8Q13C+aEmuRlzDst4Up4s0Wc8sF2M+J57iB3cMLqftfg==} engines: {node: '>=6.0.0'} - prettier@3.8.3: - resolution: {integrity: sha512-7igPTM53cGHMW8xWuVTydi2KO233VFiTNyF5hLJqpilHfmn8C8gPf+PS7dUT64YcXFbiMGZxS9pCSxL/Dxm/Jw==} + prettier@3.8.4: + resolution: {integrity: sha512-N2MylSdi48+5N/6S5j+maeHbUSIzzZ5uOcX5Hm4QpV8Dkb1HFjfAKTKX6yNPJQD9AhcT3ifHNB66tWTTJDi11Q==} engines: {node: '>=14'} hasBin: true @@ -8131,8 +8137,8 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} hasBin: true - rollup@4.61.0: - resolution: {integrity: sha512-T9mWdbWfQtp0B5lv/HX+wrhYsmXRlcWnXXmJbXqKJhlRaoS6KMhq0gpyzW4UJfclcxrEdLnTgjT2NjruLONu0g==} + rollup@4.61.1: + resolution: {integrity: sha512-I4KW6iuRpuu2uHBLraZ1wNZe0DP7lnRha+VJ9tNaYVaVgKhW0aI3h4RYnoRPeql0flHm/Co55b7snEDcOfOJrA==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true @@ -9355,14 +9361,14 @@ packages: snapshots: - '@apollo/client@3.14.1(graphql@16.14.1)': + '@apollo/client@3.14.1(graphql@16.14.2)': dependencies: - '@graphql-typed-document-node/core': 3.2.0(graphql@16.14.1) + '@graphql-typed-document-node/core': 3.2.0(graphql@16.14.2) '@wry/caches': 1.0.1 '@wry/equality': 0.5.7 '@wry/trie': 0.5.0 - graphql: 16.14.1 - graphql-tag: 2.12.6(graphql@16.14.1) + graphql: 16.14.2 + graphql-tag: 2.12.6(graphql@16.14.2) hoist-non-react-statics: 3.3.2 optimism: 0.18.1 prop-types: 15.8.1 @@ -10489,17 +10495,17 @@ snapshots: transitivePeerDependencies: - supports-color - '@contentstack/cli-launch@1.10.0(@types/node@20.19.42)(tslib@2.8.1)(typescript@5.9.3)': + '@contentstack/cli-launch@1.10.1(@types/node@20.19.42)(tslib@2.8.1)(typescript@5.9.3)': dependencies: - '@apollo/client': 3.14.1(graphql@16.14.1) + '@apollo/client': 3.14.1(graphql@16.14.2) '@contentstack/cli-command': 1.8.3(@types/node@20.19.42) '@contentstack/cli-utilities': 1.18.4(@types/node@20.19.42) '@oclif/core': 4.11.4 '@oclif/plugin-help': 6.2.50 - '@rollup/plugin-commonjs': 28.0.9(rollup@4.61.0) - '@rollup/plugin-json': 6.1.0(rollup@4.61.0) - '@rollup/plugin-node-resolve': 16.0.3(rollup@4.61.0) - '@rollup/plugin-typescript': 12.3.0(rollup@4.61.0)(tslib@2.8.1)(typescript@5.9.3) + '@rollup/plugin-commonjs': 28.0.9(rollup@4.61.1) + '@rollup/plugin-json': 6.1.0(rollup@4.61.1) + '@rollup/plugin-node-resolve': 16.0.3(rollup@4.61.1) + '@rollup/plugin-typescript': 12.3.0(rollup@4.61.1)(tslib@2.8.1)(typescript@5.9.3) '@types/express': 4.17.25 '@types/express-serve-static-core': 4.19.8 adm-zip: 0.5.17 @@ -10508,11 +10514,11 @@ snapshots: dotenv: 16.6.1 express: 4.22.2 form-data: 4.0.4 - graphql: 16.14.1 + graphql: 16.14.2 ini: 3.0.1 lodash: 4.18.1 open: 8.4.2 - rollup: 4.61.0 + rollup: 4.61.1 winston: 3.19.0 transitivePeerDependencies: - '@types/node' @@ -10809,14 +10815,14 @@ snapshots: - debug - supports-color - '@contentstack/types-generator@3.10.1(graphql@16.14.1)': + '@contentstack/types-generator@3.10.1(graphql@16.14.2)': dependencies: '@contentstack/delivery-sdk': 5.2.1 - '@gql2ts/from-schema': 2.0.0-4(graphql@16.14.1) + '@gql2ts/from-schema': 2.0.0-4(graphql@16.14.2) async: 3.2.6 axios: 1.16.1 lodash: 4.18.1 - prettier: 3.8.3 + prettier: 3.8.4 transitivePeerDependencies: - debug - graphql @@ -11050,27 +11056,27 @@ snapshots: lodash.isundefined: 3.0.1 lodash.uniq: 4.5.0 - '@gql2ts/from-schema@2.0.0-4(graphql@16.14.1)': + '@gql2ts/from-schema@2.0.0-4(graphql@16.14.2)': dependencies: - '@gql2ts/language-typescript': 2.0.0-0(graphql@16.14.1) - '@gql2ts/util': 2.0.0-0(graphql@16.14.1) + '@gql2ts/language-typescript': 2.0.0-0(graphql@16.14.2) + '@gql2ts/util': 2.0.0-0(graphql@16.14.2) dedent: 0.7.0 - graphql: 16.14.1 + graphql: 16.14.2 - '@gql2ts/language-typescript@2.0.0-0(graphql@16.14.1)': + '@gql2ts/language-typescript@2.0.0-0(graphql@16.14.2)': dependencies: - '@gql2ts/util': 2.0.0-0(graphql@16.14.1) + '@gql2ts/util': 2.0.0-0(graphql@16.14.2) humps: 2.0.1 transitivePeerDependencies: - graphql - '@gql2ts/util@2.0.0-0(graphql@16.14.1)': + '@gql2ts/util@2.0.0-0(graphql@16.14.2)': dependencies: - graphql: 16.14.1 + graphql: 16.14.2 - '@graphql-typed-document-node/core@3.2.0(graphql@16.14.1)': + '@graphql-typed-document-node/core@3.2.0(graphql@16.14.2)': dependencies: - graphql: 16.14.1 + graphql: 16.14.2 '@humanfs/core@0.19.2': dependencies: @@ -12306,9 +12312,9 @@ snapshots: '@rolldown/pluginutils@1.0.1': {} - '@rollup/plugin-commonjs@28.0.9(rollup@4.61.0)': + '@rollup/plugin-commonjs@28.0.9(rollup@4.61.1)': dependencies: - '@rollup/pluginutils': 5.4.0(rollup@4.61.0) + '@rollup/pluginutils': 5.4.0(rollup@4.61.1) commondir: 1.0.1 estree-walker: 2.0.2 fdir: 6.5.0(picomatch@4.0.4) @@ -12316,114 +12322,114 @@ snapshots: magic-string: 0.30.21 picomatch: 4.0.4 optionalDependencies: - rollup: 4.61.0 + rollup: 4.61.1 - '@rollup/plugin-json@6.1.0(rollup@4.61.0)': + '@rollup/plugin-json@6.1.0(rollup@4.61.1)': dependencies: - '@rollup/pluginutils': 5.4.0(rollup@4.61.0) + '@rollup/pluginutils': 5.4.0(rollup@4.61.1) optionalDependencies: - rollup: 4.61.0 + rollup: 4.61.1 - '@rollup/plugin-node-resolve@16.0.3(rollup@4.61.0)': + '@rollup/plugin-node-resolve@16.0.3(rollup@4.61.1)': dependencies: - '@rollup/pluginutils': 5.4.0(rollup@4.61.0) + '@rollup/pluginutils': 5.4.0(rollup@4.61.1) '@types/resolve': 1.20.2 deepmerge: 4.3.1 is-module: 1.0.0 resolve: 1.22.12 optionalDependencies: - rollup: 4.61.0 + rollup: 4.61.1 - '@rollup/plugin-typescript@12.3.0(rollup@4.61.0)(tslib@2.8.1)(typescript@5.9.3)': + '@rollup/plugin-typescript@12.3.0(rollup@4.61.1)(tslib@2.8.1)(typescript@5.9.3)': dependencies: - '@rollup/pluginutils': 5.4.0(rollup@4.61.0) + '@rollup/pluginutils': 5.4.0(rollup@4.61.1) resolve: 1.22.12 typescript: 5.9.3 optionalDependencies: - rollup: 4.61.0 + rollup: 4.61.1 tslib: 2.8.1 - '@rollup/pluginutils@5.4.0(rollup@4.61.0)': + '@rollup/pluginutils@5.4.0(rollup@4.61.1)': dependencies: '@types/estree': 1.0.9 estree-walker: 2.0.2 picomatch: 4.0.4 optionalDependencies: - rollup: 4.61.0 + rollup: 4.61.1 - '@rollup/rollup-android-arm-eabi@4.61.0': + '@rollup/rollup-android-arm-eabi@4.61.1': optional: true - '@rollup/rollup-android-arm64@4.61.0': + '@rollup/rollup-android-arm64@4.61.1': optional: true - '@rollup/rollup-darwin-arm64@4.61.0': + '@rollup/rollup-darwin-arm64@4.61.1': optional: true - '@rollup/rollup-darwin-x64@4.61.0': + '@rollup/rollup-darwin-x64@4.61.1': optional: true - '@rollup/rollup-freebsd-arm64@4.61.0': + '@rollup/rollup-freebsd-arm64@4.61.1': optional: true - '@rollup/rollup-freebsd-x64@4.61.0': + '@rollup/rollup-freebsd-x64@4.61.1': optional: true - '@rollup/rollup-linux-arm-gnueabihf@4.61.0': + '@rollup/rollup-linux-arm-gnueabihf@4.61.1': optional: true - '@rollup/rollup-linux-arm-musleabihf@4.61.0': + '@rollup/rollup-linux-arm-musleabihf@4.61.1': optional: true - '@rollup/rollup-linux-arm64-gnu@4.61.0': + '@rollup/rollup-linux-arm64-gnu@4.61.1': optional: true - '@rollup/rollup-linux-arm64-musl@4.61.0': + '@rollup/rollup-linux-arm64-musl@4.61.1': optional: true - '@rollup/rollup-linux-loong64-gnu@4.61.0': + '@rollup/rollup-linux-loong64-gnu@4.61.1': optional: true - '@rollup/rollup-linux-loong64-musl@4.61.0': + '@rollup/rollup-linux-loong64-musl@4.61.1': optional: true - '@rollup/rollup-linux-ppc64-gnu@4.61.0': + '@rollup/rollup-linux-ppc64-gnu@4.61.1': optional: true - '@rollup/rollup-linux-ppc64-musl@4.61.0': + '@rollup/rollup-linux-ppc64-musl@4.61.1': optional: true - '@rollup/rollup-linux-riscv64-gnu@4.61.0': + '@rollup/rollup-linux-riscv64-gnu@4.61.1': optional: true - '@rollup/rollup-linux-riscv64-musl@4.61.0': + '@rollup/rollup-linux-riscv64-musl@4.61.1': optional: true - '@rollup/rollup-linux-s390x-gnu@4.61.0': + '@rollup/rollup-linux-s390x-gnu@4.61.1': optional: true - '@rollup/rollup-linux-x64-gnu@4.61.0': + '@rollup/rollup-linux-x64-gnu@4.61.1': optional: true - '@rollup/rollup-linux-x64-musl@4.61.0': + '@rollup/rollup-linux-x64-musl@4.61.1': optional: true - '@rollup/rollup-openbsd-x64@4.61.0': + '@rollup/rollup-openbsd-x64@4.61.1': optional: true - '@rollup/rollup-openharmony-arm64@4.61.0': + '@rollup/rollup-openharmony-arm64@4.61.1': optional: true - '@rollup/rollup-win32-arm64-msvc@4.61.0': + '@rollup/rollup-win32-arm64-msvc@4.61.1': optional: true - '@rollup/rollup-win32-ia32-msvc@4.61.0': + '@rollup/rollup-win32-ia32-msvc@4.61.1': optional: true - '@rollup/rollup-win32-x64-gnu@4.61.0': + '@rollup/rollup-win32-x64-gnu@4.61.1': optional: true - '@rollup/rollup-win32-x64-msvc@4.61.0': + '@rollup/rollup-win32-x64-msvc@4.61.1': optional: true '@rtsao/scc@1.1.0': {} @@ -16248,10 +16254,10 @@ snapshots: - supports-color - typescript - eslint-plugin-prettier@5.5.6(eslint-config-prettier@10.1.8(eslint@10.4.1))(eslint@10.4.1)(prettier@3.8.3): + eslint-plugin-prettier@5.5.6(eslint-config-prettier@10.1.8(eslint@10.4.1))(eslint@10.4.1)(prettier@3.8.4): dependencies: eslint: 10.4.1 - prettier: 3.8.3 + prettier: 3.8.4 prettier-linter-helpers: 1.0.1 synckit: 0.11.13 optionalDependencies: @@ -16693,7 +16699,7 @@ snapshots: '@types/chai': 4.3.20 '@types/lodash': 4.17.24 '@types/node': 20.19.42 - '@types/sinon': 21.0.1 + '@types/sinon': 17.0.4 lodash: 4.18.1 mock-stdin: 1.0.0 nock: 13.5.6 @@ -17095,12 +17101,12 @@ snapshots: graphemer@1.4.0: {} - graphql-tag@2.12.6(graphql@16.14.1): + graphql-tag@2.12.6(graphql@16.14.2): dependencies: - graphql: 16.14.1 + graphql: 16.14.2 tslib: 2.8.1 - graphql@16.14.1: {} + graphql@16.14.2: {} handlebars@4.7.9: dependencies: @@ -17632,14 +17638,13 @@ snapshots: rimraf: 3.0.2 uuid: 14.0.0 - istanbul-lib-processinfo@3.0.0: + istanbul-lib-processinfo@3.0.1: dependencies: archy: 1.0.0 cross-spawn: 7.0.6 istanbul-lib-coverage: 3.2.2 p-map: 3.0.0 rimraf: 6.1.3 - uuid: 14.0.0 istanbul-lib-report@3.0.1: dependencies: @@ -19181,7 +19186,7 @@ snapshots: istanbul-lib-coverage: 3.2.2 istanbul-lib-hook: 3.0.0 istanbul-lib-instrument: 6.0.3 - istanbul-lib-processinfo: 3.0.0 + istanbul-lib-processinfo: 3.0.1 istanbul-lib-report: 3.0.1 istanbul-lib-source-maps: 4.0.1 istanbul-reports: 3.2.0 @@ -19606,7 +19611,7 @@ snapshots: dependencies: fast-diff: 1.3.0 - prettier@3.8.3: {} + prettier@3.8.4: {} pretty-format@26.6.2: dependencies: @@ -19752,7 +19757,7 @@ snapshots: read@1.0.7: dependencies: - mute-stream: 0.0.8 + mute-stream: 0.0.7 readable-stream@2.3.8: dependencies: @@ -20004,35 +20009,35 @@ snapshots: '@rolldown/binding-win32-arm64-msvc': 1.0.3 '@rolldown/binding-win32-x64-msvc': 1.0.3 - rollup@4.61.0: + rollup@4.61.1: dependencies: '@types/estree': 1.0.9 optionalDependencies: - '@rollup/rollup-android-arm-eabi': 4.61.0 - '@rollup/rollup-android-arm64': 4.61.0 - '@rollup/rollup-darwin-arm64': 4.61.0 - '@rollup/rollup-darwin-x64': 4.61.0 - '@rollup/rollup-freebsd-arm64': 4.61.0 - '@rollup/rollup-freebsd-x64': 4.61.0 - '@rollup/rollup-linux-arm-gnueabihf': 4.61.0 - '@rollup/rollup-linux-arm-musleabihf': 4.61.0 - '@rollup/rollup-linux-arm64-gnu': 4.61.0 - '@rollup/rollup-linux-arm64-musl': 4.61.0 - '@rollup/rollup-linux-loong64-gnu': 4.61.0 - '@rollup/rollup-linux-loong64-musl': 4.61.0 - '@rollup/rollup-linux-ppc64-gnu': 4.61.0 - '@rollup/rollup-linux-ppc64-musl': 4.61.0 - '@rollup/rollup-linux-riscv64-gnu': 4.61.0 - '@rollup/rollup-linux-riscv64-musl': 4.61.0 - '@rollup/rollup-linux-s390x-gnu': 4.61.0 - '@rollup/rollup-linux-x64-gnu': 4.61.0 - '@rollup/rollup-linux-x64-musl': 4.61.0 - '@rollup/rollup-openbsd-x64': 4.61.0 - '@rollup/rollup-openharmony-arm64': 4.61.0 - '@rollup/rollup-win32-arm64-msvc': 4.61.0 - '@rollup/rollup-win32-ia32-msvc': 4.61.0 - '@rollup/rollup-win32-x64-gnu': 4.61.0 - '@rollup/rollup-win32-x64-msvc': 4.61.0 + '@rollup/rollup-android-arm-eabi': 4.61.1 + '@rollup/rollup-android-arm64': 4.61.1 + '@rollup/rollup-darwin-arm64': 4.61.1 + '@rollup/rollup-darwin-x64': 4.61.1 + '@rollup/rollup-freebsd-arm64': 4.61.1 + '@rollup/rollup-freebsd-x64': 4.61.1 + '@rollup/rollup-linux-arm-gnueabihf': 4.61.1 + '@rollup/rollup-linux-arm-musleabihf': 4.61.1 + '@rollup/rollup-linux-arm64-gnu': 4.61.1 + '@rollup/rollup-linux-arm64-musl': 4.61.1 + '@rollup/rollup-linux-loong64-gnu': 4.61.1 + '@rollup/rollup-linux-loong64-musl': 4.61.1 + '@rollup/rollup-linux-ppc64-gnu': 4.61.1 + '@rollup/rollup-linux-ppc64-musl': 4.61.1 + '@rollup/rollup-linux-riscv64-gnu': 4.61.1 + '@rollup/rollup-linux-riscv64-musl': 4.61.1 + '@rollup/rollup-linux-s390x-gnu': 4.61.1 + '@rollup/rollup-linux-x64-gnu': 4.61.1 + '@rollup/rollup-linux-x64-musl': 4.61.1 + '@rollup/rollup-openbsd-x64': 4.61.1 + '@rollup/rollup-openharmony-arm64': 4.61.1 + '@rollup/rollup-win32-arm64-msvc': 4.61.1 + '@rollup/rollup-win32-ia32-msvc': 4.61.1 + '@rollup/rollup-win32-x64-gnu': 4.61.1 + '@rollup/rollup-win32-x64-msvc': 4.61.1 fsevents: 2.3.3 rrweb-cssom@0.6.0: {}