From c98fafeedf50d5800d69f6955423b1998f07bc7c Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Mon, 2 Mar 2026 17:51:54 +0530 Subject: [PATCH 01/22] feat: add asset management package for AM2.0 support --- .talismanrc | 23 +- .../contentstack-asset-management/.eslintrc | 54 ++++ .../contentstack-asset-management/.gitignore | 3 + .../contentstack-asset-management/README.md | 49 ++++ .../package.json | 58 ++++ .../src/constants/index.ts | 47 ++++ .../src/export/asset-types.ts | 28 ++ .../src/export/assets.ts | 100 +++++++ .../src/export/base.ts | 101 +++++++ .../src/export/fields.ts | 26 ++ .../src/export/index.ts | 7 + .../src/export/spaces.ts | 131 +++++++++ .../src/export/workspaces.ts | 37 +++ .../src/index.ts | 4 + .../src/types/asset-management-api.ts | 140 ++++++++++ .../src/types/export-types.ts | 16 ++ .../src/types/index.ts | 2 + .../src/utils/asset-management-api-adapter.ts | 149 +++++++++++ .../src/utils/export-helpers.ts | 41 +++ .../src/utils/index.ts | 9 + .../test/unit/export/asset-types.test.ts | 88 ++++++ .../test/unit/export/assets.test.ts | 252 ++++++++++++++++++ .../test/unit/export/base.test.ts | 235 ++++++++++++++++ .../test/unit/export/fields.test.ts | 88 ++++++ .../test/unit/export/spaces.test.ts | 142 ++++++++++ .../test/unit/export/workspaces.test.ts | 117 ++++++++ .../asset-management-api-adapter.test.ts | 219 +++++++++++++++ .../test/unit/utils/export-helpers.test.ts | 125 +++++++++ .../tsconfig.json | 31 +++ packages/contentstack-export/package.json | 3 +- .../src/export/modules/assets.ts | 76 ++++-- .../src/export/modules/stack.ts | 51 ++-- .../src/types/export-config.ts | 1 + .../contentstack-export/src/types/index.ts | 1 + .../src/utils/constants.ts | 16 ++ .../src/utils/get-linked-workspaces.ts | 33 +++ .../contentstack-export/src/utils/index.ts | 1 + .../src/utils/progress-strategy-registry.ts | 18 ++ .../test/unit/export/modules/stack.test.ts | 12 +- 39 files changed, 2464 insertions(+), 70 deletions(-) create mode 100644 packages/contentstack-asset-management/.eslintrc create mode 100644 packages/contentstack-asset-management/.gitignore create mode 100644 packages/contentstack-asset-management/README.md create mode 100644 packages/contentstack-asset-management/package.json create mode 100644 packages/contentstack-asset-management/src/constants/index.ts create mode 100644 packages/contentstack-asset-management/src/export/asset-types.ts create mode 100644 packages/contentstack-asset-management/src/export/assets.ts create mode 100644 packages/contentstack-asset-management/src/export/base.ts create mode 100644 packages/contentstack-asset-management/src/export/fields.ts create mode 100644 packages/contentstack-asset-management/src/export/index.ts create mode 100644 packages/contentstack-asset-management/src/export/spaces.ts create mode 100644 packages/contentstack-asset-management/src/export/workspaces.ts create mode 100644 packages/contentstack-asset-management/src/index.ts create mode 100644 packages/contentstack-asset-management/src/types/asset-management-api.ts create mode 100644 packages/contentstack-asset-management/src/types/export-types.ts create mode 100644 packages/contentstack-asset-management/src/types/index.ts create mode 100644 packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts create mode 100644 packages/contentstack-asset-management/src/utils/export-helpers.ts create mode 100644 packages/contentstack-asset-management/src/utils/index.ts create mode 100644 packages/contentstack-asset-management/test/unit/export/asset-types.test.ts create mode 100644 packages/contentstack-asset-management/test/unit/export/assets.test.ts create mode 100644 packages/contentstack-asset-management/test/unit/export/base.test.ts create mode 100644 packages/contentstack-asset-management/test/unit/export/fields.test.ts create mode 100644 packages/contentstack-asset-management/test/unit/export/spaces.test.ts create mode 100644 packages/contentstack-asset-management/test/unit/export/workspaces.test.ts create mode 100644 packages/contentstack-asset-management/test/unit/utils/asset-management-api-adapter.test.ts create mode 100644 packages/contentstack-asset-management/test/unit/utils/export-helpers.test.ts create mode 100644 packages/contentstack-asset-management/tsconfig.json create mode 100644 packages/contentstack-export/src/utils/get-linked-workspaces.ts diff --git a/.talismanrc b/.talismanrc index 97313279b..29a879ed5 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,8 +1,19 @@ fileignoreconfig: - - filename: package-lock.json - checksum: 45100667793fc7dfaae3e24787871257e7f29e06df69ba10ec05b358d59ff15d - - filename: pnpm-lock.yaml - checksum: 87d001c32b1d7f9df30a289c277e0ea13cfd8a0e2e5fa5118956ff4183683e5c - - filename: .husky/pre-commit - checksum: 7a12030ddfea18d6f85edc25f1721fb2009df00fdd42bab66b05de25ab3e32b2 +- filename: packages/contentstack-asset-management/.eslintrc + checksum: 136f03481c8c59575d2eafd4c78d105119f85fb10fe88e02af8cffaf3eb7c090 +- filename: packages/contentstack-asset-management/src/export/base.ts + checksum: fcae2679bdeb93a6786cb290b60ba98f222a9c682552c6474370d17bf59ae1b4 +- filename: packages/contentstack-export/src/export/modules/stack.ts + checksum: 82f7df78993942debb79e690c8c27d0998157428ef506d0b07ea31d5a1f71aba +- filename: packages/contentstack-asset-management/src/types/export-types.ts + checksum: d00ca608006d864f516e21b76d552c0ecf52ff89b3dcb361ed11ac600abed989 +- filename: packages/contentstack-asset-management/src/utils/export-helpers.ts + checksum: 1a533a4e4d56a952f61ced63aa6f1bc8fbb3855fd7acecdd9fd40dd71e5fab6d +- filename: packages/contentstack-asset-management/test/unit/utils/export-helpers.test.ts + checksum: 0e8751163491fc45e7ae3999282d336ae1ab8a9f88e601cbb85b4f44e8db96b8 +- filename: packages/contentstack-asset-management/test/unit/utils/asset-management-api-adapter.test.ts + checksum: ff688f37f40de3f7cbef378ec682ca1167720d902d8d84370464af7feb36c124 +- filename: packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts + checksum: 6f5e11d3685b6093d6c4def7fc4199f673d9a56e5fbc2858ed72f69d764f1260 +version: "1.0" version: '1.0' diff --git a/packages/contentstack-asset-management/.eslintrc b/packages/contentstack-asset-management/.eslintrc new file mode 100644 index 000000000..4d2aef653 --- /dev/null +++ b/packages/contentstack-asset-management/.eslintrc @@ -0,0 +1,54 @@ +{ + "env": { + "node": true + }, + "parser": "@typescript-eslint/parser", + "parserOptions": { + "project": "tsconfig.json", + "sourceType": "module" + }, + "extends": [ + "oclif-typescript", + "plugin:@typescript-eslint/recommended" + ], + "rules": { + "@typescript-eslint/no-unused-vars": [ + "error", + { + "args": "none" + } + ], + "@typescript-eslint/prefer-namespace-keyword": "error", + "@typescript-eslint/quotes": [ + "error", + "single", + { + "avoidEscape": true, + "allowTemplateLiterals": true + } + ], + "semi": "off", + "@typescript-eslint/type-annotation-spacing": "error", + "@typescript-eslint/no-redeclare": "off", + "eqeqeq": [ + "error", + "smart" + ], + "id-match": "error", + "no-eval": "error", + "no-var": "error", + "quotes": "off", + "indent": "off", + "camelcase": "off", + "comma-dangle": "off", + "arrow-parens": "off", + "operator-linebreak": "off", + "object-curly-spacing": "off", + "node/no-missing-import": "off", + "padding-line-between-statements": "off", + "@typescript-eslint/ban-ts-ignore": "off", + "unicorn/no-abusive-eslint-disable": "off", + "unicorn/consistent-function-scoping": "off", + "@typescript-eslint/no-use-before-define": "off" + } +} diff --git a/packages/contentstack-asset-management/.gitignore b/packages/contentstack-asset-management/.gitignore new file mode 100644 index 000000000..6cf8928fa --- /dev/null +++ b/packages/contentstack-asset-management/.gitignore @@ -0,0 +1,3 @@ +lib/ +node_modules/ +*.tsbuildinfo diff --git a/packages/contentstack-asset-management/README.md b/packages/contentstack-asset-management/README.md new file mode 100644 index 000000000..87867c247 --- /dev/null +++ b/packages/contentstack-asset-management/README.md @@ -0,0 +1,49 @@ +# @contentstack/cli-asset-management + +Asset Management 2.0 API adapter for Contentstack CLI export and import. Used by the export and import plugins when Asset Management (AM 2.0) is enabled. To learn how to export and import content in Contentstack, refer to the [Migration guide](https://www.contentstack.com/docs/developers/cli/migration/). + +[![License](https://img.shields.io/npm/l/@contentstack/cli)](https://github.com/contentstack/cli/blob/main/LICENSE) + + +* [@contentstack/cli-asset-management](#contentstackcli-asset-management) +* [Overview](#overview) +* [Usage](#usage) +* [Exports](#exports) + + +# Overview + +This package provides: + +- **AssetManagementAdapter** – HTTP client for the Asset Management API (spaces, assets, folders, fields, asset types). +- **exportSpaceStructure** – Exports space metadata and full workspace structure (metadata, folders, assets, fields, asset types) for linked workspaces. +- **Types** – `AssetManagementExportOptions`, `LinkedWorkspace`, `IAssetManagementAdapter`, and related types for export/import integration. + +# Usage + +This package is consumed by the export and import plugins. When using the export CLI with the `--asset-management` flag (or when the host app enables AM 2.0), the export plugin calls `exportSpaceStructure` with linked workspaces and options: + +```ts +import { exportSpaceStructure } from '@contentstack/cli-asset-management'; + +await exportSpaceStructure({ + linkedWorkspaces, + exportDir, + branchName: 'main', + assetManagementUrl, + org_uid, + context, + progressManager, + progressProcessName, + updateStatus, + downloadAsset, // optional +}); +``` + +# Exports + +| Export | Description | +|--------|-------------| +| `exportSpaceStructure` | Async function to export space structure for given linked workspaces. | +| `AssetManagementAdapter` | Class to call the Asset Management API (getSpace, getWorkspaceFields, getWorkspaceAssets, etc.). | +| Types from `./types` | `AssetManagementExportOptions`, `ExportSpaceOptions`, `ChunkedJsonWriteOptions`, `LinkedWorkspace`, `SpaceResponse`, `FieldsResponse`, `AssetTypesResponse`, and related API types. | diff --git a/packages/contentstack-asset-management/package.json b/packages/contentstack-asset-management/package.json new file mode 100644 index 000000000..2ac90a3dd --- /dev/null +++ b/packages/contentstack-asset-management/package.json @@ -0,0 +1,58 @@ +{ + "name": "@contentstack/cli-asset-management", + "version": "1.0.0", + "description": "Asset Management 2.0 API adapter for export and import", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "files": [ + "lib", + "oclif.manifest.json" + ], + "scripts": { + "build": "pnpm compile", + "clean": "rm -rf ./lib ./node_modules tsconfig.build.tsbuildinfo", + "compile": "tsc -b tsconfig.json", + "postpack": "rm -f oclif.manifest.json", + "prepack": "pnpm compile && oclif manifest && oclif readme", + "version": "oclif readme && git add README.md", + "lint": "eslint src/**/*.ts", + "format": "eslint src/**/*.ts --fix", + "test": "nyc --extension .ts mocha --require ts-node/register --forbid-only \"test/**/*.test.ts\"", + "posttest": "npm run lint", + "test:unit": "mocha --require ts-node/register --forbid-only \"test/unit/**/*.test.ts\"", + "test:unit:report": "nyc --extension .ts mocha --require ts-node/register --forbid-only \"test/unit/**/*.test.ts\"" + }, + "keywords": [ + "contentstack", + "asset-management", + "cli" + ], + "license": "MIT", + "dependencies": { + "@contentstack/cli-utilities": "~2.0.0-beta" + }, + "oclif": { + "commands": "./lib/commands", + "bin": "csdx", + "devPlugins": [ + "@oclif/plugin-help" + ], + "repositoryPrefix": "<%- repo %>/blob/main/packages/contentstack-asset-management/<%- commandPath %>" + }, + "devDependencies": { + "@types/chai": "^4.3.11", + "@types/mocha": "^10.0.6", + "@types/node": "^20.17.50", + "@types/sinon": "^17.0.2", + "chai": "^4.4.1", + "eslint": "^8.57.1", + "eslint-config-oclif": "^6.0.68", + "mocha": "^10.8.2", + "nyc": "^15.1.0", + "oclif": "^4.17.46", + "sinon": "^17.0.1", + "source-map-support": "^0.5.21", + "ts-node": "^10.9.2", + "typescript": "^5.8.3" + } +} \ No newline at end of file diff --git a/packages/contentstack-asset-management/src/constants/index.ts b/packages/contentstack-asset-management/src/constants/index.ts new file mode 100644 index 000000000..0a0b469a7 --- /dev/null +++ b/packages/contentstack-asset-management/src/constants/index.ts @@ -0,0 +1,47 @@ +/** + * Main process name for Asset Management 2.0 export (single progress bar). + * Use this when adding/starting the process and for all ticks. + */ +export const AM_MAIN_PROCESS_NAME = 'Asset Management 2.0'; + +/** + * Process names for Asset Management 2.0 export progress (for tick labels). + */ +export const PROCESS_NAMES = { + AM_SPACE_METADATA: 'Space metadata', + AM_FOLDERS: 'Folders', + AM_ASSETS: 'Assets', + AM_FIELDS: 'Fields', + AM_ASSET_TYPES: 'Asset types', + AM_DOWNLOADS: 'Asset downloads', +} as const; + +/** + * Status messages for each process (exporting, fetching, failed). + */ +export const PROCESS_STATUS = { + [PROCESS_NAMES.AM_SPACE_METADATA]: { + EXPORTING: 'Exporting space metadata...', + FAILED: 'Failed to export space metadata.', + }, + [PROCESS_NAMES.AM_FOLDERS]: { + FETCHING: 'Fetching folders...', + FAILED: 'Failed to fetch folders.', + }, + [PROCESS_NAMES.AM_ASSETS]: { + FETCHING: 'Fetching assets...', + FAILED: 'Failed to fetch assets.', + }, + [PROCESS_NAMES.AM_FIELDS]: { + FETCHING: 'Fetching fields...', + FAILED: 'Failed to fetch fields.', + }, + [PROCESS_NAMES.AM_ASSET_TYPES]: { + FETCHING: 'Fetching asset types...', + FAILED: 'Failed to fetch asset types.', + }, + [PROCESS_NAMES.AM_DOWNLOADS]: { + DOWNLOADING: 'Downloading asset files...', + FAILED: 'Failed to download assets.', + }, +} as const; diff --git a/packages/contentstack-asset-management/src/export/asset-types.ts b/packages/contentstack-asset-management/src/export/asset-types.ts new file mode 100644 index 000000000..2a88e9d19 --- /dev/null +++ b/packages/contentstack-asset-management/src/export/asset-types.ts @@ -0,0 +1,28 @@ +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig } from '../types/asset-management-api'; +import type { ExportContext } from '../types/export-types'; +import { AssetManagementExportAdapter } from './base'; +import { getArrayFromResponse } from '../utils/export-helpers'; +import { PROCESS_NAMES } from '../constants/index'; + +export default class ExportAssetTypes extends AssetManagementExportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) { + super(apiConfig, exportContext); + } + + async start(spaceUid: string): Promise { + await this.init(); + const assetTypesData = await this.getWorkspaceAssetTypes(spaceUid); + const items = getArrayFromResponse(assetTypesData, 'asset_types'); + const dir = this.getAssetTypesDir(); + log.debug( + items.length === 0 + ? 'No asset types, wrote empty asset-types' + : `Writing ${items.length} shared asset types`, + this.exportContext.context, + ); + await this.writeItemsToChunkedJson(dir, 'asset-types.json', 'asset_types', ['uid', 'title', 'category', 'file_extension'], items); + this.tick(true, PROCESS_NAMES.AM_ASSET_TYPES, null); + } +} diff --git a/packages/contentstack-asset-management/src/export/assets.ts b/packages/contentstack-asset-management/src/export/assets.ts new file mode 100644 index 000000000..140b56645 --- /dev/null +++ b/packages/contentstack-asset-management/src/export/assets.ts @@ -0,0 +1,100 @@ +import { resolve as pResolve } from 'node:path'; +import { Readable } from 'node:stream'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { configHandler, log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, LinkedWorkspace } from '../types/asset-management-api'; +import type { ExportContext } from '../types/export-types'; +import { AssetManagementExportAdapter } from './base'; +import { getAssetItems, writeStreamToFile } from '../utils/export-helpers'; +import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; + +export default class ExportAssets extends AssetManagementExportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) { + super(apiConfig, exportContext); + } + + async start(workspace: LinkedWorkspace, spaceDir: string): Promise { + await this.init(); + const assetsDir = pResolve(spaceDir, 'assets'); + await mkdir(assetsDir, { recursive: true }); + log.debug(`Fetching folders and assets for space ${workspace.space_uid}`, this.exportContext.context); + + const [folders, assetsData] = await Promise.all([ + this.getWorkspaceFolders(workspace.space_uid), + this.getWorkspaceAssets(workspace.space_uid), + ]); + + await writeFile(pResolve(assetsDir, 'folders.json'), JSON.stringify(folders, null, 2)); + this.tick(true, `folders: ${workspace.space_uid}`, null); + log.debug(`Wrote folders.json for space ${workspace.space_uid}`, this.exportContext.context); + + const assetItems = getAssetItems(assetsData); + log.debug( + assetItems.length === 0 + ? `No assets for space ${workspace.space_uid}, wrote empty assets.json` + : `Writing ${assetItems.length} assets metadata for space ${workspace.space_uid}`, + this.exportContext.context, + ); + await this.writeItemsToChunkedJson( + assetsDir, + 'assets.json', + 'assets', + ['uid', 'url', 'filename', 'file_name', 'parent_uid'], + assetItems, + ); + this.tick(true, `assets: ${workspace.space_uid} (${assetItems.length})`, null); + + await this.downloadWorkspaceAssets(assetsData, assetsDir, workspace.space_uid); + } + + private async downloadWorkspaceAssets( + assetsData: unknown, + assetsDir: string, + spaceUid: string, + ): Promise { + const items = getAssetItems(assetsData); + if (items.length === 0) { + log.debug('No assets to download', this.exportContext.context); + return; + } + + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].DOWNLOADING); + log.debug(`Downloading ${items.length} asset file(s) for space ${spaceUid}...`, this.exportContext.context); + const filesDir = pResolve(assetsDir, 'files'); + await mkdir(filesDir, { recursive: true }); + + const securedAssets = this.exportContext.securedAssets ?? false; + const authtoken = securedAssets ? configHandler.get('authtoken') : null; + let lastError: string | null = null; + let allSuccess = true; + + for (const asset of items) { + const uid = asset.uid ?? asset._uid; + const url = asset.url; + const filename = asset.filename ?? asset.file_name ?? 'asset'; + if (!url || !uid) continue; + try { + const separator = url.includes('?') ? '&' : '?'; + const downloadUrl = securedAssets && authtoken ? `${url}${separator}authtoken=${authtoken}` : url; + const response = await fetch(downloadUrl); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const body = response.body; + if (!body) throw new Error('No response body'); + const nodeStream = Readable.fromWeb(body as Parameters[0]); + const assetFolderPath = pResolve(filesDir, uid); + await mkdir(assetFolderPath, { recursive: true }); + const filePath = pResolve(assetFolderPath, filename); + await writeStreamToFile(nodeStream, filePath); + log.debug(`Downloaded asset ${uid}`, this.exportContext.context); + } catch (e) { + allSuccess = false; + lastError = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED; + log.debug(`Failed to download asset ${uid}: ${e}`, this.exportContext.context); + } + } + + this.tick(allSuccess, `downloads: ${spaceUid}`, lastError); + log.debug('Asset downloads completed', this.exportContext.context); + } +} diff --git a/packages/contentstack-asset-management/src/export/base.ts b/packages/contentstack-asset-management/src/export/base.ts new file mode 100644 index 000000000..8a55ab29d --- /dev/null +++ b/packages/contentstack-asset-management/src/export/base.ts @@ -0,0 +1,101 @@ +import { resolve as pResolve } from 'node:path'; +import { writeFile } from 'node:fs/promises'; +import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig } from '../types/asset-management-api'; +import type { ExportContext } from '../types/export-types'; +import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; +import { AM_MAIN_PROCESS_NAME } from '../constants/index'; +import { BATCH_SIZE, CHUNK_FILE_SIZE_MB } from '../utils/export-helpers'; + +export type { ExportContext }; + +/** + * Base class for export modules. Extends the API adapter and adds export context, + * internal progress management, and shared write helpers. + */ +export class AssetManagementExportAdapter extends AssetManagementAdapter { + protected readonly apiConfig: AssetManagementAPIConfig; + protected readonly exportContext: ExportContext; + protected progressManager: CLIProgressManager | null = null; + protected parentProgressManager: CLIProgressManager | null = null; + protected readonly processName: string = AM_MAIN_PROCESS_NAME; + + constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) { + super(apiConfig); + this.apiConfig = apiConfig; + this.exportContext = exportContext; + } + + public setParentProgressManager(parent: CLIProgressManager): void { + this.parentProgressManager = parent; + } + + protected get progressOrParent(): CLIProgressManager | null { + return this.parentProgressManager ?? this.progressManager; + } + + protected createNestedProgress(moduleName: string): CLIProgressManager { + if (this.parentProgressManager) { + this.progressManager = this.parentProgressManager; + return this.parentProgressManager; + } + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; + this.progressManager = CLIProgressManager.createNested(moduleName, showConsoleLogs); + return this.progressManager; + } + + protected tick(success: boolean, itemName: string, error: string | null, processName?: string): void { + this.progressOrParent?.tick?.(success, itemName, error, processName ?? this.processName); + } + + protected updateStatus(message: string, processName?: string): void { + this.progressOrParent?.updateStatus?.(message, processName ?? this.processName); + } + + protected completeProcess(processName: string, success: boolean): void { + if (!this.parentProgressManager) { + this.progressManager?.completeProcess?.(processName, success); + } + } + + protected get spacesRootPath(): string { + return this.exportContext.spacesRootPath; + } + + protected getAssetTypesDir(): string { + return pResolve(this.exportContext.spacesRootPath, 'asset_types'); + } + + protected getFieldsDir(): string { + return pResolve(this.exportContext.spacesRootPath, 'fields'); + } + + protected async writeItemsToChunkedJson( + dir: string, + indexFileName: string, + moduleName: string, + metaPickKeys: string[], + items: unknown[], + ): Promise { + if (items.length === 0) { + await writeFile(pResolve(dir, indexFileName), '{}'); + return; + } + const fs = new FsUtility({ + basePath: dir, + indexFileName, + chunkFileSize: CHUNK_FILE_SIZE_MB, + moduleName, + fileExt: 'json', + metaPickKeys, + keepMetadata: true, + }); + for (let i = 0; i < items.length; i += BATCH_SIZE) { + const batch = items.slice(i, i + BATCH_SIZE); + fs.writeIntoFile(batch as Record[], { mapKeyVal: true }); + } + fs.completeFile(true); + } +} diff --git a/packages/contentstack-asset-management/src/export/fields.ts b/packages/contentstack-asset-management/src/export/fields.ts new file mode 100644 index 000000000..2960c024c --- /dev/null +++ b/packages/contentstack-asset-management/src/export/fields.ts @@ -0,0 +1,26 @@ +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig } from '../types/asset-management-api'; +import type { ExportContext } from '../types/export-types'; +import { AssetManagementExportAdapter } from './base'; +import { getArrayFromResponse } from '../utils/export-helpers'; +import { PROCESS_NAMES } from '../constants/index'; + +export default class ExportFields extends AssetManagementExportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) { + super(apiConfig, exportContext); + } + + async start(spaceUid: string): Promise { + await this.init(); + const fieldsData = await this.getWorkspaceFields(spaceUid); + const items = getArrayFromResponse(fieldsData, 'fields'); + const dir = this.getFieldsDir(); + log.debug( + items.length === 0 ? 'No field items, wrote empty fields' : `Writing ${items.length} shared fields`, + this.exportContext.context, + ); + await this.writeItemsToChunkedJson(dir, 'fields.json', 'fields', ['uid', 'title', 'display_type'], items); + this.tick(true, PROCESS_NAMES.AM_FIELDS, null); + } +} diff --git a/packages/contentstack-asset-management/src/export/index.ts b/packages/contentstack-asset-management/src/export/index.ts new file mode 100644 index 000000000..7d71e361e --- /dev/null +++ b/packages/contentstack-asset-management/src/export/index.ts @@ -0,0 +1,7 @@ +export { ExportSpaces, exportSpaceStructure } from './spaces'; +export { default as ExportAssetTypes } from './asset-types'; +export { default as ExportFields } from './fields'; +export { default as ExportAssets } from './assets'; +export { default as ExportWorkspace } from './workspaces'; +export { AssetManagementExportAdapter } from './base'; +export type { ExportContext } from './base'; diff --git a/packages/contentstack-asset-management/src/export/spaces.ts b/packages/contentstack-asset-management/src/export/spaces.ts new file mode 100644 index 000000000..cf3ff2c30 --- /dev/null +++ b/packages/contentstack-asset-management/src/export/spaces.ts @@ -0,0 +1,131 @@ +import { resolve as pResolve } from 'node:path'; +import { mkdir } from 'node:fs/promises'; +import { log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import type { AssetManagementExportOptions, AssetManagementAPIConfig } from '../types/asset-management-api'; +import type { ExportContext } from '../types/export-types'; +import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; +import { AM_MAIN_PROCESS_NAME, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +import ExportAssetTypes from './asset-types'; +import ExportFields from './fields'; +import ExportWorkspace from './workspaces'; + +/** + * Orchestrates the full Asset Management 2.0 export: shared asset types and fields, + * then per-workspace metadata and assets (including internal download). + * Progress and download are fully owned by this package. + */ +export class ExportSpaces { + private readonly options: AssetManagementExportOptions; + private parentProgressManager: CLIProgressManager | null = null; + private progressManager: CLIProgressManager | null = null; + + constructor(options: AssetManagementExportOptions) { + this.options = options; + } + + public setParentProgressManager(parent: CLIProgressManager): void { + this.parentProgressManager = parent; + } + + async start(): Promise { + const { + linkedWorkspaces, + exportDir, + branchName, + assetManagementUrl, + org_uid, + context, + securedAssets, + } = this.options; + + if (!linkedWorkspaces.length) { + log.debug('No linked workspaces to export', context); + return; + } + + log.debug(`Exporting Asset Management 2.0 (${linkedWorkspaces.length} space(s))`, context); + log.debug(`Spaces: ${linkedWorkspaces.map((ws) => ws.space_uid).join(', ')}`, context); + + const spacesRootPath = pResolve(exportDir, branchName || 'main', 'spaces'); + await mkdir(spacesRootPath, { recursive: true }); + log.debug(`Spaces root path: ${spacesRootPath}`, context); + + const totalSteps = 2 + linkedWorkspaces.length * 4; + const progress = this.createProgress(); + progress.addProcess(AM_MAIN_PROCESS_NAME, totalSteps); + progress.startProcess(AM_MAIN_PROCESS_NAME).updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_FIELDS].FETCHING, AM_MAIN_PROCESS_NAME); + + const apiConfig: AssetManagementAPIConfig = { + baseURL: assetManagementUrl, + headers: { organization_uid: org_uid }, + context, + }; + const exportContext: ExportContext = { + spacesRootPath, + context, + securedAssets, + }; + + const sharedFieldsDir = pResolve(spacesRootPath, 'fields'); + const sharedAssetTypesDir = pResolve(spacesRootPath, 'asset_types'); + await mkdir(sharedFieldsDir, { recursive: true }); + await mkdir(sharedAssetTypesDir, { recursive: true }); + + const firstSpaceUid = linkedWorkspaces[0].space_uid; + try { + const exportAssetTypes = new ExportAssetTypes(apiConfig, exportContext); + exportAssetTypes.setParentProgressManager(progress); + await exportAssetTypes.start(firstSpaceUid); + + const exportFields = new ExportFields(apiConfig, exportContext); + exportFields.setParentProgressManager(progress); + await exportFields.start(firstSpaceUid); + + for (const ws of linkedWorkspaces) { + progress.updateStatus(`Exporting space: ${ws.space_uid}...`, AM_MAIN_PROCESS_NAME); + log.debug(`Exporting space: ${ws.space_uid}`, context); + const spaceDir = pResolve(spacesRootPath, ws.space_uid); + try { + const exportWorkspace = new ExportWorkspace(apiConfig, exportContext); + exportWorkspace.setParentProgressManager(progress); + await exportWorkspace.start(ws, spaceDir, branchName || 'main'); + log.debug(`Exported workspace structure for space ${ws.space_uid}`, context); + } catch (err) { + log.debug(`Failed to export workspace for space ${ws.space_uid}: ${err}`, context); + progress.tick( + false, + `space: ${ws.space_uid}`, + (err as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_SPACE_METADATA].FAILED, + AM_MAIN_PROCESS_NAME, + ); + throw err; + } + } + + progress.completeProcess(AM_MAIN_PROCESS_NAME, true); + log.debug('Asset Management 2.0 export completed', context); + } catch (err) { + progress.completeProcess(AM_MAIN_PROCESS_NAME, false); + throw err; + } + } + + private createProgress(): CLIProgressManager { + if (this.parentProgressManager) { + this.progressManager = this.parentProgressManager; + return this.parentProgressManager; + } + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; + this.progressManager = CLIProgressManager.createNested(AM_MAIN_PROCESS_NAME, showConsoleLogs); + return this.progressManager; + } +} + +/** + * Entry point for callers that prefer a function. Delegates to ExportSpaces. + */ +export async function exportSpaceStructure(options: AssetManagementExportOptions): Promise { + await new ExportSpaces(options).start(); +} diff --git a/packages/contentstack-asset-management/src/export/workspaces.ts b/packages/contentstack-asset-management/src/export/workspaces.ts new file mode 100644 index 000000000..88cfd3976 --- /dev/null +++ b/packages/contentstack-asset-management/src/export/workspaces.ts @@ -0,0 +1,37 @@ +import { resolve as pResolve } from 'node:path'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, LinkedWorkspace } from '../types/asset-management-api'; +import type { ExportContext } from '../types/export-types'; +import { AssetManagementExportAdapter } from './base'; +import ExportAssets from './assets'; +import { PROCESS_NAMES } from '../constants/index'; + +export default class ExportWorkspace extends AssetManagementExportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) { + super(apiConfig, exportContext); + } + + async start(workspace: LinkedWorkspace, spaceDir: string, branchName: string): Promise { + await this.init(); + const spaceResponse = await this.getSpace(workspace.space_uid); + const space = spaceResponse.space; + await mkdir(spaceDir, { recursive: true }); + + const metadata = { + ...space, + workspace_uid: workspace.uid, + is_default: workspace.is_default, + branch: branchName || 'main', + }; + await writeFile(pResolve(spaceDir, 'metadata.json'), JSON.stringify(metadata, null, 2)); + this.tick(true, `space: ${workspace.space_uid}`, null); + log.debug(`Space metadata written for ${workspace.space_uid}`, this.exportContext.context); + + const assetsExporter = new ExportAssets(this.apiConfig, this.exportContext); + if (this.progressOrParent) assetsExporter.setParentProgressManager(this.progressOrParent); + await assetsExporter.start(workspace, spaceDir); + log.debug(`Exported workspace structure for space ${workspace.space_uid}`, this.exportContext.context); + } +} diff --git a/packages/contentstack-asset-management/src/index.ts b/packages/contentstack-asset-management/src/index.ts new file mode 100644 index 000000000..f0ff59bdd --- /dev/null +++ b/packages/contentstack-asset-management/src/index.ts @@ -0,0 +1,4 @@ +export * from './constants/index'; +export * from './types'; +export * from './utils'; +export * from './export'; diff --git a/packages/contentstack-asset-management/src/types/asset-management-api.ts b/packages/contentstack-asset-management/src/types/asset-management-api.ts new file mode 100644 index 000000000..733ecada7 --- /dev/null +++ b/packages/contentstack-asset-management/src/types/asset-management-api.ts @@ -0,0 +1,140 @@ +/** + * Linked workspace from CMA branch settings (am_v2.linked_workspaces). + * Consumed by export/import after fetching branch with include_settings: true. + */ +export type LinkedWorkspace = { + uid: string; + space_uid: string; + is_default: boolean; +}; + +/** + * Space details from GET /api/spaces/{space_uid}. + */ +export type Space = { + uid: string; + title?: string; + description?: string; + org_uid?: string; + owner_uid?: string; + default_locale?: string; + default_workspace?: string; + tags?: string[]; + settings?: Record; + created_by?: string; + updated_by?: string; + created_at?: string; + updated_at?: string; + meta_info?: { + assets_count?: number; + folders_count?: number; + storage?: number; + last_modified_at?: string; + }; +}; + +/** Response shape of GET /api/spaces/{space_uid}. */ +export type SpaceResponse = { space: Space }; + +/** + * Field structure from GET /api/fields (org-level). + */ +export type FieldStruct = { + uid: string; + title?: string; + description?: string | null; + display_type?: string; + is_system?: boolean; + is_multiple?: boolean; + is_mandatory?: boolean; + asset_types_count?: number; + created_at?: string; + created_by?: string; + updated_at?: string; + updated_by?: string; +}; + +/** Response shape of GET /api/fields. */ +export type FieldsResponse = { + count: number; + relation: string; + fields: FieldStruct[]; +}; + +/** + * Options object for asset type (from GET /api/asset_types). + */ +export type AssetTypeOptions = { + title?: string; + publishable?: boolean; + is_page?: boolean; + singleton?: boolean; + sub_title?: string[]; + url_pattern?: string; + url_prefix?: string; +}; + +/** + * Asset type structure from GET /api/asset_types (org-level). + */ +export type AssetTypeStruct = { + uid: string; + title?: string; + is_system?: boolean; + fields?: string[]; + options?: AssetTypeOptions; + description?: string; + content_type?: string; + file_extension?: string; + created_by?: string; + updated_by?: string; + created_at?: string; + updated_at?: string; + category?: string; + preview_image_url?: string; + category_detail?: string; +}; + +/** Response shape of GET /api/asset_types. */ +export type AssetTypesResponse = { + count: number; + relation: string; + asset_types: AssetTypeStruct[]; +}; + +/** + * Configuration for AssetManagementAdapter constructor. + */ +export type AssetManagementAPIConfig = { + baseURL: string; + headers?: Record; + /** Optional context for logging (e.g. exportConfig.context) */ + context?: Record; +}; + +/** + * Adapter interface for Asset Management API calls. + * Used by export and (future) import. + */ +export interface IAssetManagementAdapter { + init(): Promise; + getSpace(spaceUid: string): Promise; + getWorkspaceFields(spaceUid: string): Promise; + getWorkspaceAssets(spaceUid: string): Promise; + getWorkspaceFolders(spaceUid: string): Promise; + getWorkspaceAssetTypes(spaceUid: string): Promise; +} + +/** + * Options for exporting space structure (used by export app after fetching linked workspaces). + */ +export type AssetManagementExportOptions = { + linkedWorkspaces: LinkedWorkspace[]; + exportDir: string; + branchName: string; + assetManagementUrl: string; + org_uid: string; + context?: Record; + /** When true, the AM package will add authtoken to asset download URLs. */ + securedAssets?: boolean; +}; diff --git a/packages/contentstack-asset-management/src/types/export-types.ts b/packages/contentstack-asset-management/src/types/export-types.ts new file mode 100644 index 000000000..25b8dcace --- /dev/null +++ b/packages/contentstack-asset-management/src/types/export-types.ts @@ -0,0 +1,16 @@ +export type ExportContext = { + spacesRootPath: string; + context?: Record; + securedAssets?: boolean; +}; + +/** + * Options for writing a list of items to chunked JSON files via FsUtility. + */ +export type ChunkedJsonWriteOptions = { + dir: string; + indexFileName: string; + moduleName: string; + metaPickKeys: string[]; + items: unknown[]; +}; diff --git a/packages/contentstack-asset-management/src/types/index.ts b/packages/contentstack-asset-management/src/types/index.ts new file mode 100644 index 000000000..c673e1893 --- /dev/null +++ b/packages/contentstack-asset-management/src/types/index.ts @@ -0,0 +1,2 @@ +export * from './asset-management-api'; +export * from './export-types'; diff --git a/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts b/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts new file mode 100644 index 000000000..b159cc330 --- /dev/null +++ b/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts @@ -0,0 +1,149 @@ +import { HttpClient, log, authenticationHandler } from '@contentstack/cli-utilities'; + +import type { + AssetManagementAPIConfig, + AssetTypesResponse, + FieldsResponse, + IAssetManagementAdapter, + SpaceResponse, +} from '../types/asset-management-api'; + +export class AssetManagementAdapter implements IAssetManagementAdapter { + private readonly config: AssetManagementAPIConfig; + private readonly apiClient: HttpClient; + + constructor(config: AssetManagementAPIConfig) { + this.config = config; + this.apiClient = new HttpClient(); + const baseURL = config.baseURL?.replace(/\/$/, '') ?? ''; + this.apiClient.baseUrl(baseURL); + const defaultHeaders = { Accept: 'application/json', 'x-cs-api-version': '4' }; + this.apiClient.headers(config.headers ? { ...defaultHeaders, ...config.headers } : defaultHeaders); + log.debug('AssetManagementAdapter initialized', config.context); + } + + /** + * Build query string from params. Supports string and string[] values. + * Returns empty string when params are empty so we never append "?" with no keys. + */ + private buildQueryString(params: Record): string { + const entries = Object.entries(params).filter( + ([, v]) => v !== undefined && v !== null && (typeof v === 'string' || Array.isArray(v)), + ); + if (entries.length === 0) return ''; + const parts: string[] = []; + for (const [key, value] of entries) { + if (Array.isArray(value)) { + for (const v of value) { + parts.push(`${encodeURIComponent(key)}=${encodeURIComponent(String(v))}`); + } + } else { + parts.push(`${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`); + } + } + return '?' + parts.join('&'); + } + + /** + * GET a space-level endpoint (e.g. /api/spaces/{uid}). Builds path + query string and performs the request. + */ + private async getSpaceLevel( + _spaceUid: string, + path: string, + queryParams: Record = {}, + ): Promise { + await this.init(); + const safeParams: Record = {}; + for (const [k, v] of Object.entries(queryParams)) { + let value: string | string[] | undefined; + if (typeof v === 'string') value = v; + else if (Array.isArray(v) && v.every((x) => typeof x === 'string')) value = v; + else value = undefined; + if (value !== undefined) safeParams[k] = value; + } + const queryString = this.buildQueryString(safeParams); + const fullPath = path + queryString; + log.debug(`GET ${fullPath}`, this.config.context); + const response = await this.apiClient.get(fullPath); + if (response.status < 200 || response.status >= 300) { + throw new Error(`Asset Management API error: status ${response.status}, path ${path}`); + } + return response.data as T; + } + + async init(): Promise { + try { + log.debug('Initializing Asset Management adapter...', this.config.context); + await authenticationHandler.getAuthDetails(); + const token = authenticationHandler.accessToken; + log.debug( + `Authentication type: ${authenticationHandler.isOauthEnabled ? 'OAuth' : 'Token'}`, + this.config.context, + ); + const authHeader = authenticationHandler.isOauthEnabled ? { authorization: token } : { access_token: token }; + this.apiClient.headers(this.config.headers ? { ...authHeader, ...this.config.headers } : authHeader); + log.debug('Asset Management adapter initialization completed', this.config.context); + } catch (error: unknown) { + log.debug(`Asset Management adapter initialization failed: ${error}`, this.config.context); + throw error; + } + } + + async getSpace(spaceUid: string): Promise { + log.debug(`Fetching space: ${spaceUid}`, this.config.context); + const path = `/api/spaces/${spaceUid}`; + const queryParams: Record = { + addl_fields: ['meta_info', 'users'], + }; + const result = await this.getSpaceLevel(spaceUid, path, queryParams); + log.debug(`Fetched space: ${spaceUid}`, this.config.context); + return result; + } + + async getWorkspaceFields(spaceUid: string): Promise { + log.debug(`Fetching fields for space: ${spaceUid}`, this.config.context); + const result = await this.getSpaceLevel(spaceUid, '/api/fields', {}); + log.debug(`Fetched fields (count: ${result?.count ?? '?'})`, this.config.context); + return result; + } + + /** + * GET a workspace collection (assets or folders), log count, and return result. + */ + private async getWorkspaceCollection( + spaceUid: string, + path: string, + logLabel: string, + ): Promise { + log.debug(`Fetching ${logLabel} for space: ${spaceUid}`, this.config.context); + const result = await this.getSpaceLevel(spaceUid, path, {}); + const count = (result as { count?: number })?.count ?? (Array.isArray(result) ? result.length : '?'); + log.debug(`Fetched ${logLabel} (count: ${count})`, this.config.context); + return result; + } + + async getWorkspaceAssets(spaceUid: string): Promise { + return this.getWorkspaceCollection( + spaceUid, + `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, + 'assets', + ); + } + + async getWorkspaceFolders(spaceUid: string): Promise { + return this.getWorkspaceCollection( + spaceUid, + `/api/spaces/${encodeURIComponent(spaceUid)}/folders`, + 'folders', + ); + } + + async getWorkspaceAssetTypes(spaceUid: string): Promise { + log.debug(`Fetching asset types for space: ${spaceUid}`, this.config.context); + const result = await this.getSpaceLevel(spaceUid, '/api/asset_types', { + include_fields: 'true', + }); + log.debug(`Fetched asset types (count: ${result?.count ?? '?'})`, this.config.context); + return result; + } +} diff --git a/packages/contentstack-asset-management/src/utils/export-helpers.ts b/packages/contentstack-asset-management/src/utils/export-helpers.ts new file mode 100644 index 000000000..2083951ec --- /dev/null +++ b/packages/contentstack-asset-management/src/utils/export-helpers.ts @@ -0,0 +1,41 @@ +import { createWriteStream } from 'node:fs'; + +export const BATCH_SIZE = 50; +export const CHUNK_FILE_SIZE_MB = 1; + +export function getArrayFromResponse(data: unknown, arrayKey: string): unknown[] { + if (Array.isArray(data)) return data; + if (data != null && typeof data === 'object' && arrayKey in data) { + const arr = (data as Record)[arrayKey]; + return Array.isArray(arr) ? arr : []; + } + return []; +} + +export function getAssetItems( + assetsData: unknown, +): Array<{ uid?: string; _uid?: string; url?: string; filename?: string; file_name?: string }> { + if (Array.isArray(assetsData)) return assetsData; + const data = assetsData as Record; + const items = data?.items ?? data?.assets; + return Array.isArray(items) ? items : []; +} + +export function getReadableStreamFromDownloadResponse( + response: { data?: NodeJS.ReadableStream } | NodeJS.ReadableStream | null, +): NodeJS.ReadableStream | null { + if (!response) return null; + const withData = response as { data?: NodeJS.ReadableStream }; + if (withData?.data != null) return withData.data; + const stream = response as NodeJS.ReadableStream; + return typeof stream?.pipe === 'function' ? stream : null; +} + +export function writeStreamToFile(stream: NodeJS.ReadableStream, filePath: string): Promise { + const writer = createWriteStream(filePath); + stream.pipe(writer); + return new Promise((resolve, reject) => { + writer.on('finish', () => resolve()); + writer.on('error', reject); + }); +} diff --git a/packages/contentstack-asset-management/src/utils/index.ts b/packages/contentstack-asset-management/src/utils/index.ts new file mode 100644 index 000000000..fdbfd6133 --- /dev/null +++ b/packages/contentstack-asset-management/src/utils/index.ts @@ -0,0 +1,9 @@ +export { AssetManagementAdapter } from './asset-management-api-adapter'; +export { + BATCH_SIZE, + CHUNK_FILE_SIZE_MB, + getArrayFromResponse, + getAssetItems, + getReadableStreamFromDownloadResponse, + writeStreamToFile, +} from './export-helpers'; diff --git a/packages/contentstack-asset-management/test/unit/export/asset-types.test.ts b/packages/contentstack-asset-management/test/unit/export/asset-types.test.ts new file mode 100644 index 000000000..af052e2db --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/export/asset-types.test.ts @@ -0,0 +1,88 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; + +import ExportAssetTypes from '../../../src/export/asset-types'; +import { AssetManagementExportAdapter } from '../../../src/export/base'; +import { PROCESS_NAMES } from '../../../src/constants/index'; + +import type { AssetManagementAPIConfig } from '../../../src/types/asset-management-api'; +import type { ExportContext } from '../../../src/types/export-types'; + +describe('ExportAssetTypes', () => { + const apiConfig: AssetManagementAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + + const exportContext: ExportContext = { + spacesRootPath: '/tmp/export/spaces', + }; + + const spaceUid = 'space-uid-1'; + const assetTypesDir = '/tmp/export/spaces/asset_types'; + + const assetTypesResponse = { + count: 2, + relation: 'organization', + asset_types: [ + { uid: 'at1', title: 'Image', category: 'image', file_extension: 'png' }, + { uid: 'at2', title: 'Document', category: 'document', file_extension: 'pdf' }, + ], + }; + + beforeEach(() => { + sinon.stub(AssetManagementExportAdapter.prototype, 'init' as any).resolves(); + sinon.stub(AssetManagementExportAdapter.prototype, 'writeItemsToChunkedJson' as any).resolves(); + sinon.stub(AssetManagementExportAdapter.prototype, 'tick' as any); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe('start method', () => { + it('should call getWorkspaceAssetTypes with the correct spaceUid', async () => { + const getStub = sinon.stub(ExportAssetTypes.prototype, 'getWorkspaceAssetTypes').resolves(assetTypesResponse); + const exporter = new ExportAssetTypes(apiConfig, exportContext); + await exporter.start(spaceUid); + + expect(getStub.firstCall.args[0]).to.equal(spaceUid); + }); + + it('should write asset types with correct chunked JSON args', async () => { + sinon.stub(ExportAssetTypes.prototype, 'getWorkspaceAssetTypes').resolves(assetTypesResponse); + const exporter = new ExportAssetTypes(apiConfig, exportContext); + await exporter.start(spaceUid); + + const writeStub = (AssetManagementExportAdapter.prototype as any).writeItemsToChunkedJson as sinon.SinonStub; + const args = writeStub.firstCall.args; + expect(args[0]).to.equal(assetTypesDir); + expect(args[1]).to.equal('asset-types.json'); + expect(args[2]).to.equal('asset_types'); + expect(args[3]).to.deep.equal(['uid', 'title', 'category', 'file_extension']); + expect(args[4]).to.deep.equal(assetTypesResponse.asset_types); + }); + + it('should write empty items when no asset types returned', async () => { + sinon.stub(ExportAssetTypes.prototype, 'getWorkspaceAssetTypes').resolves({ + count: 0, + relation: 'organization', + asset_types: [], + }); + const exporter = new ExportAssetTypes(apiConfig, exportContext); + await exporter.start(spaceUid); + + const writeStub = (AssetManagementExportAdapter.prototype as any).writeItemsToChunkedJson as sinon.SinonStub; + expect(writeStub.firstCall.args[4]).to.deep.equal([]); + }); + + it('should tick with success=true, the asset types process name, and null error', async () => { + sinon.stub(ExportAssetTypes.prototype, 'getWorkspaceAssetTypes').resolves(assetTypesResponse); + const exporter = new ExportAssetTypes(apiConfig, exportContext); + await exporter.start(spaceUid); + + const tickStub = (AssetManagementExportAdapter.prototype as any).tick as sinon.SinonStub; + expect(tickStub.firstCall.args).to.deep.equal([true, PROCESS_NAMES.AM_ASSET_TYPES, null]); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/export/assets.test.ts b/packages/contentstack-asset-management/test/unit/export/assets.test.ts new file mode 100644 index 000000000..763285ca2 --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/export/assets.test.ts @@ -0,0 +1,252 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { configHandler } from '@contentstack/cli-utilities'; + +import ExportAssets from '../../../src/export/assets'; +import { AssetManagementExportAdapter } from '../../../src/export/base'; + +import type { AssetManagementAPIConfig, LinkedWorkspace } from '../../../src/types/asset-management-api'; +import type { ExportContext } from '../../../src/types/export-types'; + +const foldersData = [{ uid: 'folder-1', name: 'Images' }]; +const assetsResponseWithItems = { + items: [ + { uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'image.png' }, + { uid: 'a2', url: 'https://cdn.example.com/a2.pdf', file_name: 'doc.pdf' }, + ], +}; +const emptyAssetsResponse = { items: [] as any[] }; + +describe('ExportAssets', () => { + const apiConfig: AssetManagementAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + + const exportContext: ExportContext = { + spacesRootPath: '/tmp/export/spaces', + }; + + const workspace: LinkedWorkspace = { + uid: 'ws-1', + space_uid: 'space-uid-1', + is_default: true, + }; + + const spaceDir = '/tmp/export/spaces/space-uid-1'; + + let fetchStub: sinon.SinonStub; + + const makeFetchResponse = () => { + const webStream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('file-content')); + controller.close(); + }, + }); + return { ok: true, status: 200, body: webStream }; + }; + + beforeEach(() => { + sinon.stub(AssetManagementExportAdapter.prototype, 'init' as any).resolves(); + sinon.stub(AssetManagementExportAdapter.prototype, 'writeItemsToChunkedJson' as any).resolves(); + sinon.stub(AssetManagementExportAdapter.prototype, 'tick' as any); + sinon.stub(AssetManagementExportAdapter.prototype, 'updateStatus' as any); + fetchStub = sinon.stub(globalThis, 'fetch'); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe('start method', () => { + it('should fetch folders and assets using the workspace space_uid', async () => { + const foldersStub = sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); + const assetsStub = sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(emptyAssetsResponse); + + const exporter = new ExportAssets(apiConfig, exportContext); + await exporter.start(workspace, spaceDir); + + expect(foldersStub.firstCall.args[0]).to.equal(workspace.space_uid); + expect(assetsStub.firstCall.args[0]).to.equal(workspace.space_uid); + }); + + it('should write chunked assets metadata with correct args', async () => { + sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); + sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(assetsResponseWithItems); + fetchStub.callsFake(async () => makeFetchResponse() as any); + + const exporter = new ExportAssets(apiConfig, exportContext); + await exporter.start(workspace, spaceDir); + + const writeStub = (AssetManagementExportAdapter.prototype as any).writeItemsToChunkedJson as sinon.SinonStub; + const args = writeStub.firstCall.args; + expect(args[1]).to.equal('assets.json'); + expect(args[2]).to.equal('assets'); + expect(args[3]).to.deep.equal(['uid', 'url', 'filename', 'file_name', 'parent_uid']); + expect(args[4]).to.have.length(2); + }); + + it('should not attempt any downloads when the asset list is empty', async () => { + sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); + sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(emptyAssetsResponse); + + const exporter = new ExportAssets(apiConfig, exportContext); + await exporter.start(workspace, spaceDir); + + expect(fetchStub.callCount).to.equal(0); + const tickStub = (AssetManagementExportAdapter.prototype as any).tick as sinon.SinonStub; + const downloadTick = tickStub.getCalls().find((c) => String(c.args[1]).startsWith('downloads:')); + expect(downloadTick).to.be.undefined; + }); + + it('should tick with success=false and the error message on download failure', async () => { + sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); + sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(assetsResponseWithItems); + fetchStub.rejects(new Error('network failure')); + + const exporter = new ExportAssets(apiConfig, exportContext); + await exporter.start(workspace, spaceDir); + + const tickStub = (AssetManagementExportAdapter.prototype as any).tick as sinon.SinonStub; + const downloadTick = tickStub.getCalls().find((c) => String(c.args[1]).startsWith('downloads:')); + expect(downloadTick!.args[0]).to.be.false; + expect(downloadTick!.args[2]).to.equal('network failure'); + }); + + it('should tick with success=true and null error on successful downloads', async () => { + sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); + sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(assetsResponseWithItems); + fetchStub.callsFake(async () => makeFetchResponse() as any); + + const exporter = new ExportAssets(apiConfig, exportContext); + await exporter.start(workspace, spaceDir); + + const tickStub = (AssetManagementExportAdapter.prototype as any).tick as sinon.SinonStub; + const downloadTick = tickStub.getCalls().find((c) => String(c.args[1]).startsWith('downloads:')); + expect(downloadTick!.args[0]).to.be.true; + expect(downloadTick!.args[2]).to.be.null; + }); + + it('should skip assets that have neither a url nor a uid', async () => { + const incompleteAssets = { + items: [ + { uid: 'a1', url: null as any }, + { url: 'https://cdn.example.com/a2.png', filename: 'img.png' }, + { uid: null as any, url: null as any }, + ], + }; + sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); + sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(incompleteAssets); + + const exporter = new ExportAssets(apiConfig, exportContext); + await exporter.start(workspace, spaceDir); + + expect(fetchStub.callCount).to.equal(0); + }); + + it('should process assets that have _uid instead of uid without skipping them', async () => { + const assetsWithUnderscoreUid = { + items: [{ _uid: 'a-uid', url: 'https://cdn.example.com/a.png', filename: 'a.png' }], + }; + sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); + sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(assetsWithUnderscoreUid); + fetchStub.callsFake(async () => makeFetchResponse() as any); + + const exporter = new ExportAssets(apiConfig, exportContext); + await exporter.start(workspace, spaceDir); + + expect(fetchStub.firstCall.args[0]).to.equal('https://cdn.example.com/a.png'); + const tickStub = (AssetManagementExportAdapter.prototype as any).tick as sinon.SinonStub; + const downloadTick = tickStub.getCalls().find((c) => String(c.args[1]).startsWith('downloads:')); + expect(downloadTick!.args[0]).to.be.true; + expect(downloadTick!.args[2]).to.be.null; + }); + + it('should download assets that use file_name, and fall back to "asset" when both names are absent', async () => { + const assetsNoFilename = { + items: [ + { uid: 'a1', url: 'https://cdn.example.com/a1.pdf', file_name: 'named.pdf' }, + { uid: 'a2', url: 'https://cdn.example.com/a2.bin' }, + ], + }; + sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); + sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(assetsNoFilename); + fetchStub.callsFake(async () => makeFetchResponse() as any); + + const exporter = new ExportAssets(apiConfig, exportContext); + await exporter.start(workspace, spaceDir); + + expect(fetchStub.callCount).to.equal(2); + expect(fetchStub.firstCall.args[0]).to.equal('https://cdn.example.com/a1.pdf'); + expect(fetchStub.secondCall.args[0]).to.equal('https://cdn.example.com/a2.bin'); + const tickStub = (AssetManagementExportAdapter.prototype as any).tick as sinon.SinonStub; + const downloadTick = tickStub.getCalls().find((c) => String(c.args[1]).startsWith('downloads:')); + expect(downloadTick!.args[0]).to.be.true; + }); + + it('should append authtoken to URL when securedAssets is true', async () => { + sinon.stub(configHandler, 'get').returns('my-auth-token'); + sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); + sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves({ + items: [{ uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'img.png' }], + }); + fetchStub.callsFake(async () => makeFetchResponse() as any); + + const securedContext: typeof exportContext = { ...exportContext, securedAssets: true }; + const exporter = new ExportAssets(apiConfig, securedContext); + await exporter.start(workspace, spaceDir); + + const downloadUrl = fetchStub.firstCall.args[0] as string; + expect(downloadUrl).to.include('authtoken=my-auth-token'); + }); + + it('should use "&" separator when URL already contains "?"', async () => { + sinon.stub(configHandler, 'get').returns('my-token'); + sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); + sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves({ + items: [{ uid: 'a1', url: 'https://cdn.example.com/a1?v=1', filename: 'img.png' }], + }); + fetchStub.callsFake(async () => makeFetchResponse() as any); + + const securedContext: typeof exportContext = { ...exportContext, securedAssets: true }; + const exporter = new ExportAssets(apiConfig, securedContext); + await exporter.start(workspace, spaceDir); + + const downloadUrl = fetchStub.firstCall.args[0] as string; + expect(downloadUrl).to.include('?v=1&authtoken='); + }); + + it('should tick with success=false and the HTTP status code on non-ok response', async () => { + sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); + sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves({ + items: [{ uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'img.png' }], + }); + fetchStub.resolves({ ok: false, status: 403, body: null } as any); + + const exporter = new ExportAssets(apiConfig, exportContext); + await exporter.start(workspace, spaceDir); + + const tickStub = (AssetManagementExportAdapter.prototype as any).tick as sinon.SinonStub; + const downloadTick = tickStub.getCalls().find((c) => String(c.args[1]).startsWith('downloads:')); + expect(downloadTick!.args[0]).to.be.false; + expect(downloadTick!.args[2]).to.include('403'); + }); + + it('should tick with success=false and "No response body" when body is null', async () => { + sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); + sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves({ + items: [{ uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'img.png' }], + }); + fetchStub.resolves({ ok: true, status: 200, body: null } as any); + + const exporter = new ExportAssets(apiConfig, exportContext); + await exporter.start(workspace, spaceDir); + + const tickStub = (AssetManagementExportAdapter.prototype as any).tick as sinon.SinonStub; + const downloadTick = tickStub.getCalls().find((c) => String(c.args[1]).startsWith('downloads:')); + expect(downloadTick!.args[0]).to.be.false; + expect(downloadTick!.args[2]).to.equal('No response body'); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/export/base.test.ts b/packages/contentstack-asset-management/test/unit/export/base.test.ts new file mode 100644 index 000000000..84264fcc8 --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/export/base.test.ts @@ -0,0 +1,235 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { FsUtility, CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import { AssetManagementExportAdapter } from '../../../src/export/base'; + +import type { AssetManagementAPIConfig } from '../../../src/types/asset-management-api'; +import type { ExportContext } from '../../../src/types/export-types'; + +class TestAdapter extends AssetManagementExportAdapter { + public callCreateNestedProgress(name: string) { + return this.createNestedProgress(name); + } + public callTick(success: boolean, name: string, error: string | null) { + return this.tick(success, name, error); + } + public callUpdateStatus(msg: string) { + return this.updateStatus(msg); + } + public callCompleteProcess(name: string, success: boolean) { + return this.completeProcess(name, success); + } + public callWriteItemsToChunkedJson(...args: Parameters) { + return this.writeItemsToChunkedJson(...args); + } + public getProgressOrParent() { + return this.progressOrParent; + } + public getAssetTypesDirPublic() { + return this.getAssetTypesDir(); + } + public getFieldsDirPublic() { + return this.getFieldsDir(); + } + public get spacesRootPathPublic() { + return this.spacesRootPath; + } +} + +describe('AssetManagementExportAdapter (base)', () => { + const apiConfig: AssetManagementAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + + const exportContext: ExportContext = { + spacesRootPath: '/tmp/export/spaces', + }; + + beforeEach(() => { + sinon.stub(AssetManagementExportAdapter.prototype, 'init' as any).resolves(); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe('constructor + path helpers', () => { + it('should expose spacesRootPath from exportContext', () => { + const adapter = new TestAdapter(apiConfig, exportContext); + expect(adapter.spacesRootPathPublic).to.equal('/tmp/export/spaces'); + }); + + it('should build getAssetTypesDir as /asset_types', () => { + const adapter = new TestAdapter(apiConfig, exportContext); + const expected = require('node:path').join('/tmp/export/spaces', 'asset_types'); + expect(adapter.getAssetTypesDirPublic()).to.equal(expected); + }); + + it('should build getFieldsDir as /fields', () => { + const adapter = new TestAdapter(apiConfig, exportContext); + const expected = require('node:path').join('/tmp/export/spaces', 'fields'); + expect(adapter.getFieldsDirPublic()).to.equal(expected); + }); + }); + + describe('setParentProgressManager / progressOrParent getter', () => { + it('should return null when no progress manager is set', () => { + const adapter = new TestAdapter(apiConfig, exportContext); + expect(adapter.getProgressOrParent()).to.be.null; + }); + + it('should return parentProgressManager when set', () => { + const fakeParent = { tick: sinon.stub() } as any; + const adapter = new TestAdapter(apiConfig, exportContext); + adapter.setParentProgressManager(fakeParent); + expect(adapter.getProgressOrParent()).to.equal(fakeParent); + }); + + it('should return progressManager when parentProgressManager is not set', () => { + sinon.stub(configHandler, 'get').returns({}); + const fakeProgress = { tick: sinon.stub() } as any; + sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress); + + const adapter = new TestAdapter(apiConfig, exportContext); + adapter.callCreateNestedProgress('test'); + expect(adapter.getProgressOrParent()).to.equal(fakeProgress); + }); + }); + + describe('createNestedProgress', () => { + it('should create a new CLIProgressManager with the given name and showConsoleLogs flag', () => { + sinon.stub(configHandler, 'get').returns({ showConsoleLogs: true }); + const fakeProgress = { tick: sinon.stub() } as any; + const createNestedStub = sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress); + + const adapter = new TestAdapter(apiConfig, exportContext); + const result = adapter.callCreateNestedProgress('my-module'); + + expect(createNestedStub.firstCall.args[0]).to.equal('my-module'); + expect(result).to.equal(fakeProgress); + }); + + it('should return parentProgressManager directly when parent is set', () => { + const fakeParent = { tick: sinon.stub() } as any; + const adapter = new TestAdapter(apiConfig, exportContext); + adapter.setParentProgressManager(fakeParent); + + const result = adapter.callCreateNestedProgress('ignored'); + expect(result).to.equal(fakeParent); + }); + + it('should default showConsoleLogs to false when log config is missing', () => { + sinon.stub(configHandler, 'get').returns(null); + const fakeProgress = { tick: sinon.stub() } as any; + const createNestedStub = sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress); + + const adapter = new TestAdapter(apiConfig, exportContext); + adapter.callCreateNestedProgress('test'); + + expect(createNestedStub.firstCall.args[1]).to.be.false; + }); + }); + + describe('tick', () => { + it('should forward success, item name, and error to the progress manager tick', () => { + const fakeParent = { tick: sinon.stub(), updateStatus: sinon.stub() } as any; + const adapter = new TestAdapter(apiConfig, exportContext); + adapter.setParentProgressManager(fakeParent); + + adapter.callTick(true, 'my-item', null); + + expect(fakeParent.tick.firstCall.args[0]).to.equal(true); + expect(fakeParent.tick.firstCall.args[1]).to.equal('my-item'); + expect(fakeParent.tick.firstCall.args[2]).to.be.null; + }); + + it('should not throw when progressOrParent is null', () => { + const adapter = new TestAdapter(apiConfig, exportContext); + expect(() => adapter.callTick(true, 'item', null)).to.not.throw(); + }); + }); + + describe('updateStatus', () => { + it('should forward the status message to the progress manager', () => { + const fakeParent = { tick: sinon.stub(), updateStatus: sinon.stub() } as any; + const adapter = new TestAdapter(apiConfig, exportContext); + adapter.setParentProgressManager(fakeParent); + + adapter.callUpdateStatus('Fetching...'); + + expect(fakeParent.updateStatus.firstCall.args[0]).to.equal('Fetching...'); + }); + + it('should not throw when progressOrParent is null', () => { + const adapter = new TestAdapter(apiConfig, exportContext); + expect(() => adapter.callUpdateStatus('msg')).to.not.throw(); + }); + }); + + describe('completeProcess', () => { + it('should call completeProcess on progressManager with the given name and success flag', () => { + sinon.stub(configHandler, 'get').returns({}); + const fakeProgress = { tick: sinon.stub(), completeProcess: sinon.stub() } as any; + sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress); + + const adapter = new TestAdapter(apiConfig, exportContext); + adapter.callCreateNestedProgress('test'); + adapter.callCompleteProcess('test', true); + + expect(fakeProgress.completeProcess.firstCall.args).to.deep.equal(['test', true]); + }); + + it('should NOT call completeProcess when parentProgressManager is set', () => { + const fakeParent = { tick: sinon.stub(), completeProcess: sinon.stub() } as any; + const adapter = new TestAdapter(apiConfig, exportContext); + adapter.setParentProgressManager(fakeParent); + + adapter.callCompleteProcess('test', true); + + expect(fakeParent.completeProcess.callCount).to.equal(0); + }); + }); + + describe('writeItemsToChunkedJson', () => { + it('should write {} to an empty file when items array is empty', async () => { + const os = require('node:os'); + const path = require('node:path'); + const fsReal = require('node:fs'); + const tmpDir = os.tmpdir(); + const adapter = new TestAdapter(apiConfig, exportContext); + await adapter.callWriteItemsToChunkedJson(tmpDir, 'test-empty.json', 'items', ['uid'], []); + + const written = fsReal.readFileSync(path.join(tmpDir, 'test-empty.json'), 'utf-8'); + expect(written).to.equal('{}'); + fsReal.unlinkSync(path.join(tmpDir, 'test-empty.json')); + }); + + it('should write all items in a single batch and complete the file when count is below BATCH_SIZE', async () => { + const writeIntoFileStub = sinon.stub(FsUtility.prototype, 'writeIntoFile'); + const completeFileStub = sinon.stub(FsUtility.prototype, 'completeFile'); + + const items = Array.from({ length: 3 }, (_, i) => ({ uid: `item-${i}` })); + const adapter = new TestAdapter(apiConfig, exportContext); + await adapter.callWriteItemsToChunkedJson('/tmp/dir', 'items.json', 'items', ['uid'], items); + + expect(writeIntoFileStub.firstCall.args[0]).to.have.length(3); + expect(completeFileStub.firstCall.args[0]).to.be.true; + }); + + it('should write items in batches of BATCH_SIZE (50)', async () => { + const writeIntoFileStub = sinon.stub(FsUtility.prototype, 'writeIntoFile'); + sinon.stub(FsUtility.prototype, 'completeFile'); + + const items = Array.from({ length: 120 }, (_, i) => ({ uid: `item-${i}` })); + const adapter = new TestAdapter(apiConfig, exportContext); + await adapter.callWriteItemsToChunkedJson('/tmp/dir', 'items.json', 'items', ['uid'], items); + + expect(writeIntoFileStub.callCount).to.equal(3); + expect(writeIntoFileStub.firstCall.args[0]).to.have.length(50); + expect(writeIntoFileStub.secondCall.args[0]).to.have.length(50); + expect(writeIntoFileStub.thirdCall.args[0]).to.have.length(20); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/export/fields.test.ts b/packages/contentstack-asset-management/test/unit/export/fields.test.ts new file mode 100644 index 000000000..a039dcb75 --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/export/fields.test.ts @@ -0,0 +1,88 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; + +import ExportFields from '../../../src/export/fields'; +import { AssetManagementExportAdapter } from '../../../src/export/base'; +import { PROCESS_NAMES } from '../../../src/constants/index'; + +import type { AssetManagementAPIConfig } from '../../../src/types/asset-management-api'; +import type { ExportContext } from '../../../src/types/export-types'; + +describe('ExportFields', () => { + const apiConfig: AssetManagementAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + + const exportContext: ExportContext = { + spacesRootPath: '/tmp/export/spaces', + }; + + const spaceUid = 'space-uid-1'; + const fieldsDir = '/tmp/export/spaces/fields'; + + const fieldsResponse = { + count: 2, + relation: 'organization', + fields: [ + { uid: 'f1', title: 'Tags', display_type: 'text' }, + { uid: 'f2', title: 'Description', display_type: 'textarea' }, + ], + }; + + beforeEach(() => { + sinon.stub(AssetManagementExportAdapter.prototype, 'init' as any).resolves(); + sinon.stub(AssetManagementExportAdapter.prototype, 'writeItemsToChunkedJson' as any).resolves(); + sinon.stub(AssetManagementExportAdapter.prototype, 'tick' as any); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe('start method', () => { + it('should call getWorkspaceFields with the correct spaceUid', async () => { + const getFieldsStub = sinon.stub(ExportFields.prototype, 'getWorkspaceFields').resolves(fieldsResponse); + const exporter = new ExportFields(apiConfig, exportContext); + await exporter.start(spaceUid); + + expect(getFieldsStub.firstCall.args[0]).to.equal(spaceUid); + }); + + it('should write fields with correct chunked JSON args', async () => { + sinon.stub(ExportFields.prototype, 'getWorkspaceFields').resolves(fieldsResponse); + const exporter = new ExportFields(apiConfig, exportContext); + await exporter.start(spaceUid); + + const writeStub = (AssetManagementExportAdapter.prototype as any).writeItemsToChunkedJson as sinon.SinonStub; + const args = writeStub.firstCall.args; + expect(args[0]).to.equal(fieldsDir); + expect(args[1]).to.equal('fields.json'); + expect(args[2]).to.equal('fields'); + expect(args[3]).to.deep.equal(['uid', 'title', 'display_type']); + expect(args[4]).to.deep.equal(fieldsResponse.fields); + }); + + it('should write empty items when no fields returned', async () => { + sinon.stub(ExportFields.prototype, 'getWorkspaceFields').resolves({ + count: 0, + relation: 'organization', + fields: [], + }); + const exporter = new ExportFields(apiConfig, exportContext); + await exporter.start(spaceUid); + + const writeStub = (AssetManagementExportAdapter.prototype as any).writeItemsToChunkedJson as sinon.SinonStub; + expect(writeStub.firstCall.args[4]).to.deep.equal([]); + }); + + it('should tick with success=true, the fields process name, and null error', async () => { + sinon.stub(ExportFields.prototype, 'getWorkspaceFields').resolves(fieldsResponse); + const exporter = new ExportFields(apiConfig, exportContext); + await exporter.start(spaceUid); + + const tickStub = (AssetManagementExportAdapter.prototype as any).tick as sinon.SinonStub; + expect(tickStub.firstCall.args).to.deep.equal([true, PROCESS_NAMES.AM_FIELDS, null]); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/export/spaces.test.ts b/packages/contentstack-asset-management/test/unit/export/spaces.test.ts new file mode 100644 index 000000000..935b7d262 --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/export/spaces.test.ts @@ -0,0 +1,142 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import { ExportSpaces, exportSpaceStructure } from '../../../src/export/spaces'; +import ExportAssetTypes from '../../../src/export/asset-types'; +import ExportFields from '../../../src/export/fields'; +import ExportWorkspace from '../../../src/export/workspaces'; +import { AssetManagementExportAdapter } from '../../../src/export/base'; +import { AM_MAIN_PROCESS_NAME } from '../../../src/constants/index'; + +import type { AssetManagementExportOptions, LinkedWorkspace } from '../../../src/types/asset-management-api'; + +describe('ExportSpaces', () => { + const baseOptions: AssetManagementExportOptions = { + linkedWorkspaces: [ + { uid: 'ws-1', space_uid: 'space-1', is_default: true }, + { uid: 'ws-2', space_uid: 'space-2', is_default: false }, + ], + exportDir: '/tmp/export', + branchName: 'main', + assetManagementUrl: 'https://am.example.com', + org_uid: 'org-1', + }; + + const fakeProgress = { + addProcess: sinon.stub().returnsThis(), + startProcess: sinon.stub().returnsThis(), + updateStatus: sinon.stub().returnsThis(), + tick: sinon.stub(), + completeProcess: sinon.stub(), + }; + + beforeEach(() => { + sinon.stub(AssetManagementExportAdapter.prototype, 'init' as any).resolves(); + sinon.stub(configHandler, 'get').returns({ showConsoleLogs: false }); + sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress as any); + sinon.stub(ExportAssetTypes.prototype, 'start').resolves(); + sinon.stub(ExportAssetTypes.prototype, 'setParentProgressManager'); + sinon.stub(ExportFields.prototype, 'start').resolves(); + sinon.stub(ExportFields.prototype, 'setParentProgressManager'); + sinon.stub(ExportWorkspace.prototype, 'start').resolves(); + sinon.stub(ExportWorkspace.prototype, 'setParentProgressManager'); + + fakeProgress.addProcess.returnsThis(); + fakeProgress.startProcess.returnsThis(); + fakeProgress.updateStatus.returnsThis(); + fakeProgress.tick.reset(); + fakeProgress.completeProcess.reset(); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe('start method', () => { + it('should return early without starting any export when linkedWorkspaces is empty', async () => { + const exporter = new ExportSpaces({ ...baseOptions, linkedWorkspaces: [] }); + await exporter.start(); + + expect((CLIProgressManager.createNested as sinon.SinonStub).callCount).to.equal(0); + expect((ExportAssetTypes.prototype.start as sinon.SinonStub).callCount).to.equal(0); + expect((ExportFields.prototype.start as sinon.SinonStub).callCount).to.equal(0); + expect((ExportWorkspace.prototype.start as sinon.SinonStub).callCount).to.equal(0); + }); + + it('should export shared asset types and fields from the first workspace space_uid', async () => { + const exporter = new ExportSpaces(baseOptions); + await exporter.start(); + + const atStub = ExportAssetTypes.prototype.start as sinon.SinonStub; + expect(atStub.firstCall.args[0]).to.equal('space-1'); + + const fieldsStub = ExportFields.prototype.start as sinon.SinonStub; + expect(fieldsStub.firstCall.args[0]).to.equal('space-1'); + }); + + it('should iterate over all workspaces in order', async () => { + const exporter = new ExportSpaces(baseOptions); + await exporter.start(); + + const wsStub = ExportWorkspace.prototype.start as sinon.SinonStub; + expect(wsStub.callCount).to.equal(2); + expect(wsStub.firstCall.args[0]).to.deep.include({ uid: 'ws-1', space_uid: 'space-1' }); + expect(wsStub.secondCall.args[0]).to.deep.include({ uid: 'ws-2', space_uid: 'space-2' }); + }); + + it('should register and complete the progress process with success', async () => { + const totalSteps = 2 + baseOptions.linkedWorkspaces.length * 4; // 10 + const exporter = new ExportSpaces(baseOptions); + await exporter.start(); + + expect(fakeProgress.addProcess.firstCall.args).to.deep.equal([AM_MAIN_PROCESS_NAME, totalSteps]); + expect(fakeProgress.startProcess.firstCall.args[0]).to.equal(AM_MAIN_PROCESS_NAME); + expect(fakeProgress.completeProcess.firstCall.args).to.deep.equal([AM_MAIN_PROCESS_NAME, true]); + }); + + it('should mark progress as failed and re-throw when a workspace export errors', async () => { + (ExportWorkspace.prototype.start as sinon.SinonStub).rejects(new Error('workspace-error')); + + const exporter = new ExportSpaces(baseOptions); + try { + await exporter.start(); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.equal('workspace-error'); + } + + expect(fakeProgress.completeProcess.firstCall.args).to.deep.equal([AM_MAIN_PROCESS_NAME, false]); + }); + + it('should use the provided parentProgressManager instead of creating a new one', async () => { + const fakeParent = { + addProcess: sinon.stub().returnsThis(), + startProcess: sinon.stub().returnsThis(), + updateStatus: sinon.stub().returnsThis(), + tick: sinon.stub(), + completeProcess: sinon.stub(), + }; + const totalSteps = 2 + baseOptions.linkedWorkspaces.length * 4; + + const exporter = new ExportSpaces(baseOptions); + exporter.setParentProgressManager(fakeParent as any); + await exporter.start(); + + expect((CLIProgressManager.createNested as sinon.SinonStub).callCount).to.equal(0); + expect(fakeParent.addProcess.firstCall.args).to.deep.equal([AM_MAIN_PROCESS_NAME, totalSteps]); + expect(fakeParent.startProcess.firstCall.args[0]).to.equal(AM_MAIN_PROCESS_NAME); + expect(fakeParent.completeProcess.firstCall.args).to.deep.equal([AM_MAIN_PROCESS_NAME, true]); + }); + }); + + describe('exportSpaceStructure', () => { + it('should be a thin wrapper that delegates to ExportSpaces.start', async () => { + const startSpy = sinon.stub(ExportSpaces.prototype, 'start').resolves(); + const options: AssetManagementExportOptions = { ...baseOptions, linkedWorkspaces: [] as LinkedWorkspace[] }; + await exportSpaceStructure(options); + + expect(startSpy.callCount).to.equal(1); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/export/workspaces.test.ts b/packages/contentstack-asset-management/test/unit/export/workspaces.test.ts new file mode 100644 index 000000000..0a4503b04 --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/export/workspaces.test.ts @@ -0,0 +1,117 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; + +import ExportWorkspace from '../../../src/export/workspaces'; +import ExportAssets from '../../../src/export/assets'; +import { AssetManagementExportAdapter } from '../../../src/export/base'; + +import type { AssetManagementAPIConfig, LinkedWorkspace, SpaceResponse } from '../../../src/types/asset-management-api'; +import type { ExportContext } from '../../../src/types/export-types'; + +describe('ExportWorkspace', () => { + const apiConfig: AssetManagementAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + + const exportContext: ExportContext = { + spacesRootPath: '/tmp/export/spaces', + }; + + const workspace: LinkedWorkspace = { + uid: 'ws-1', + space_uid: 'space-uid-1', + is_default: true, + }; + + const spaceDir = '/tmp/export/spaces/space-uid-1'; + const branchName = 'develop'; + + const spaceResponse: SpaceResponse = { + space: { + uid: 'space-uid-1', + title: 'My Space', + org_uid: 'org-1', + }, + }; + + beforeEach(() => { + sinon.stub(AssetManagementExportAdapter.prototype, 'init' as any).resolves(); + sinon.stub(AssetManagementExportAdapter.prototype, 'tick' as any); + sinon.stub(ExportAssets.prototype, 'start').resolves(); + sinon.stub(ExportAssets.prototype, 'setParentProgressManager'); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe('start method', () => { + it('should call getSpace with the workspace space_uid', async () => { + const getSpaceStub = sinon.stub(ExportWorkspace.prototype, 'getSpace').resolves(spaceResponse); + const exporter = new ExportWorkspace(apiConfig, exportContext); + await exporter.start(workspace, spaceDir, branchName); + + expect(getSpaceStub.firstCall.args[0]).to.equal(workspace.space_uid); + }); + + it('should tick success after writing metadata', async () => { + sinon.stub(ExportWorkspace.prototype, 'getSpace').resolves(spaceResponse); + const exporter = new ExportWorkspace(apiConfig, exportContext); + await exporter.start(workspace, spaceDir, branchName); + + const tickStub = (AssetManagementExportAdapter.prototype as any).tick as sinon.SinonStub; + expect(tickStub.firstCall.args).to.deep.equal([true, `space: ${workspace.space_uid}`, null]); + }); + + it('should delegate to ExportAssets.start with workspace and spaceDir', async () => { + sinon.stub(ExportWorkspace.prototype, 'getSpace').resolves(spaceResponse); + const exporter = new ExportWorkspace(apiConfig, exportContext); + await exporter.start(workspace, spaceDir, branchName); + + const startStub = ExportAssets.prototype.start as sinon.SinonStub; + expect(startStub.firstCall.args[0]).to.deep.equal(workspace); + expect(startStub.firstCall.args[1]).to.equal(spaceDir); + }); + + it('should write "main" as branch in metadata when branchName is empty', async () => { + const os = require('node:os'); + const path = require('node:path'); + const fs = require('node:fs'); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-ws-')); + + sinon.stub(ExportWorkspace.prototype, 'getSpace').resolves(spaceResponse); + const exporter = new ExportWorkspace(apiConfig, exportContext); + await exporter.start(workspace, tmpDir, ''); + + const metadata = JSON.parse(fs.readFileSync(path.join(tmpDir, 'metadata.json'), 'utf-8')); + expect(metadata.branch).to.equal('main'); + expect(metadata.workspace_uid).to.equal(workspace.uid); + expect(metadata.is_default).to.equal(workspace.is_default); + + fs.rmSync(tmpDir, { recursive: true }); + }); + + it('should NOT call setParentProgressManager on assets exporter when progressOrParent is null', async () => { + sinon.stub(ExportWorkspace.prototype, 'getSpace').resolves(spaceResponse); + const setParentStub = ExportAssets.prototype.setParentProgressManager as sinon.SinonStub; + + const exporter = new ExportWorkspace(apiConfig, exportContext); + await exporter.start(workspace, spaceDir, branchName); + + expect(setParentStub.callCount).to.equal(0); + }); + + it('should call setParentProgressManager on assets exporter when a progress manager is set', async () => { + sinon.stub(ExportWorkspace.prototype, 'getSpace').resolves(spaceResponse); + const fakeProgress = { tick: sinon.stub(), updateStatus: sinon.stub() } as any; + const setParentStub = ExportAssets.prototype.setParentProgressManager as sinon.SinonStub; + + const exporter = new ExportWorkspace(apiConfig, exportContext); + exporter.setParentProgressManager(fakeProgress); + await exporter.start(workspace, spaceDir, branchName); + + expect(setParentStub.firstCall.args[0]).to.equal(fakeProgress); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/utils/asset-management-api-adapter.test.ts b/packages/contentstack-asset-management/test/unit/utils/asset-management-api-adapter.test.ts new file mode 100644 index 000000000..f7e774912 --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/utils/asset-management-api-adapter.test.ts @@ -0,0 +1,219 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { HttpClient, authenticationHandler } from '@contentstack/cli-utilities'; + +import { AssetManagementAdapter } from '../../../src/utils/asset-management-api-adapter'; + +import type { AssetManagementAPIConfig } from '../../../src/types/asset-management-api'; + +describe('AssetManagementAdapter', () => { + const baseConfig: AssetManagementAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + + let headersStub: sinon.SinonStub; + let baseUrlStub: sinon.SinonStub; + let getStub: sinon.SinonStub; + + beforeEach(() => { + headersStub = sinon.stub(HttpClient.prototype, 'headers').returnsThis(); + baseUrlStub = sinon.stub(HttpClient.prototype, 'baseUrl').returnsThis(); + getStub = sinon.stub(HttpClient.prototype, 'get'); + sinon.stub(authenticationHandler, 'getAuthDetails').resolves(); + sinon.stub(authenticationHandler, 'isOauthEnabled').get(() => false); + sinon.stub(authenticationHandler, 'accessToken').get(() => 'test-token-123'); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe('constructor', () => { + it('should set the baseURL with trailing slash stripped', () => { + new AssetManagementAdapter({ baseURL: 'https://am.example.com/' }); + expect(baseUrlStub.firstCall.args[0]).to.equal('https://am.example.com'); + }); + + it('should set default headers with x-cs-api-version when no extra headers provided', () => { + new AssetManagementAdapter({ baseURL: 'https://am.example.com' }); + const allHeaderArgs = headersStub.getCalls().map((c) => c.args[0]); + const apiVersionCall = allHeaderArgs.find((h) => 'x-cs-api-version' in h); + expect(apiVersionCall).to.exist; + expect(apiVersionCall['x-cs-api-version']).to.equal('4'); + expect(apiVersionCall['Accept']).to.equal('application/json'); + }); + + it('should merge extra headers with default headers', () => { + new AssetManagementAdapter(baseConfig); + const allHeaderArgs = headersStub.getCalls().map((c) => c.args[0]); + const apiVersionCall = allHeaderArgs.find((h) => 'x-cs-api-version' in h); + expect(apiVersionCall).to.exist; + expect(apiVersionCall['x-cs-api-version']).to.equal('4'); + expect(apiVersionCall['organization_uid']).to.equal('org-1'); + }); + + it('should handle empty baseURL gracefully', () => { + new AssetManagementAdapter({ baseURL: '' }); + expect(baseUrlStub.firstCall.args[0]).to.equal(''); + }); + }); + + describe('init', () => { + it('should set access_token header when OAuth is disabled', async () => { + const adapter = new AssetManagementAdapter(baseConfig); + await adapter.init(); + + const authCallArgs = headersStub.getCalls().map((c) => c.args[0]); + const authCall = authCallArgs.find((a) => 'access_token' in a); + expect(authCall).to.exist; + expect(authCall.access_token).to.equal('test-token-123'); + }); + + describe('when OAuth is enabled', () => { + beforeEach(() => { + sinon.restore(); + sinon.stub(HttpClient.prototype, 'headers').returnsThis(); + sinon.stub(HttpClient.prototype, 'baseUrl').returnsThis(); + sinon.stub(HttpClient.prototype, 'get'); + sinon.stub(authenticationHandler, 'getAuthDetails').resolves(); + sinon.stub(authenticationHandler, 'isOauthEnabled').get(() => true); + sinon.stub(authenticationHandler, 'accessToken').get(() => 'oauth-bearer-token'); + }); + + it('should set authorization header', async () => { + const capturedHeaders = HttpClient.prototype.headers as sinon.SinonStub; + const adapter = new AssetManagementAdapter(baseConfig); + await adapter.init(); + + const authCallArgs = capturedHeaders.getCalls().map((c) => c.args[0]); + const authCall = authCallArgs.find((a: any) => 'authorization' in a); + expect(authCall).to.exist; + expect(authCall.authorization).to.equal('oauth-bearer-token'); + }); + }); + + it('should re-throw errors from getAuthDetails', async () => { + (authenticationHandler.getAuthDetails as sinon.SinonStub).rejects(new Error('auth-failed')); + const adapter = new AssetManagementAdapter(baseConfig); + + try { + await adapter.init(); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.equal('auth-failed'); + } + }); + + it('should merge config headers with auth header when config.headers is present', async () => { + const adapter = new AssetManagementAdapter(baseConfig); + await adapter.init(); + + const capturedHeaders = headersStub.getCalls().map((c) => c.args[0]); + const authCall = capturedHeaders.find((a) => 'access_token' in a); + expect(authCall).to.include({ organization_uid: 'org-1' }); + }); + }); + + describe('getSpace', () => { + it('should GET /api/spaces/{spaceUid}?addl_fields=... and return the space', async () => { + getStub.resolves({ status: 200, data: { space: { uid: 'sp-1' } } }); + const adapter = new AssetManagementAdapter(baseConfig); + const result = await adapter.getSpace('sp-1'); + + const path = getStub.firstCall.args[0] as string; + expect(path).to.include('/api/spaces/sp-1'); + expect(path).to.include('addl_fields'); + expect(result).to.deep.equal({ space: { uid: 'sp-1' } }); + }); + + it('should throw when response status is non-2xx', async () => { + getStub.resolves({ status: 404, data: null }); + const adapter = new AssetManagementAdapter(baseConfig); + + try { + await adapter.getSpace('missing-space'); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('404'); + } + }); + }); + + describe('getWorkspaceFields', () => { + it('should GET /api/fields and return the response data', async () => { + const fieldsResponse = { count: 1, relation: 'org', fields: [{ uid: 'f1' }] }; + getStub.resolves({ status: 200, data: fieldsResponse }); + const adapter = new AssetManagementAdapter(baseConfig); + const result = await adapter.getWorkspaceFields('sp-1'); + + expect(getStub.firstCall.args[0]).to.equal('/api/fields'); + expect(result).to.deep.equal(fieldsResponse); + }); + }); + + describe('getWorkspaceAssets', () => { + it('should GET /api/spaces/{spaceUid}/assets', async () => { + getStub.resolves({ status: 200, data: { items: [] } }); + const adapter = new AssetManagementAdapter(baseConfig); + await adapter.getWorkspaceAssets('sp-1'); + + expect(getStub.firstCall.args[0]).to.include('/api/spaces/sp-1/assets'); + }); + + it('should URL-encode the spaceUid in the path', async () => { + getStub.resolves({ status: 200, data: { items: [] } }); + const adapter = new AssetManagementAdapter(baseConfig); + await adapter.getWorkspaceAssets('sp uid/special'); + + const path = getStub.firstCall.args[0] as string; + expect(path).to.include('sp%20uid%2Fspecial'); + }); + }); + + describe('getWorkspaceFolders', () => { + it('should GET /api/spaces/{spaceUid}/folders', async () => { + getStub.resolves({ status: 200, data: [] }); + const adapter = new AssetManagementAdapter(baseConfig); + await adapter.getWorkspaceFolders('sp-1'); + + expect(getStub.firstCall.args[0]).to.include('/api/spaces/sp-1/folders'); + }); + }); + + describe('getWorkspaceAssetTypes', () => { + it('should GET /api/asset_types?include_fields=true and return the response data', async () => { + const atResponse = { count: 1, relation: 'org', asset_types: [{ uid: 'at1' }] }; + getStub.resolves({ status: 200, data: atResponse }); + const adapter = new AssetManagementAdapter(baseConfig); + const result = await adapter.getWorkspaceAssetTypes('sp-1'); + + const path = getStub.firstCall.args[0] as string; + expect(path).to.include('/api/asset_types'); + expect(path).to.include('include_fields=true'); + expect(result).to.deep.equal(atResponse); + }); + }); + + describe('buildQueryString (via public methods)', () => { + it('should encode array values as repeated key=value pairs', async () => { + getStub.resolves({ status: 200, data: { space: { uid: 'sp-1' } } }); + const adapter = new AssetManagementAdapter(baseConfig); + await adapter.getSpace('sp-1'); + + const path = getStub.firstCall.args[0] as string; + expect(path).to.include('addl_fields=meta_info'); + expect(path).to.include('addl_fields=users'); + }); + + it('should return empty string and no "?" when params are empty', async () => { + getStub.resolves({ status: 200, data: { count: 0, relation: '', fields: [] } }); + const adapter = new AssetManagementAdapter(baseConfig); + await adapter.getWorkspaceFields('sp-1'); + + const path = getStub.firstCall.args[0] as string; + expect(path).to.equal('/api/fields'); + expect(path).to.not.include('?'); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/utils/export-helpers.test.ts b/packages/contentstack-asset-management/test/unit/utils/export-helpers.test.ts new file mode 100644 index 000000000..adcd20896 --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/utils/export-helpers.test.ts @@ -0,0 +1,125 @@ +import { expect } from 'chai'; +import { PassThrough } from 'node:stream'; + +import { + getArrayFromResponse, + getAssetItems, + getReadableStreamFromDownloadResponse, + writeStreamToFile, +} from '../../../src/utils/export-helpers'; + +describe('export-helpers', () => { + describe('getArrayFromResponse', () => { + it('should return the input when it is already an array', () => { + const arr = [1, 2, 3]; + expect(getArrayFromResponse(arr, 'items')).to.equal(arr); + }); + + it('should extract nested array by key', () => { + const data = { fields: [{ uid: 'f1' }, { uid: 'f2' }] }; + const result = getArrayFromResponse(data, 'fields'); + expect(result).to.deep.equal([{ uid: 'f1' }, { uid: 'f2' }]); + }); + + it('should return [] when key exists but value is not an array', () => { + const data = { fields: 'not-an-array' }; + expect(getArrayFromResponse(data, 'fields')).to.deep.equal([]); + }); + + it('should return [] when key is missing', () => { + const data = { other: [1] }; + expect(getArrayFromResponse(data, 'fields')).to.deep.equal([]); + }); + + it('should return [] for null input', () => { + expect(getArrayFromResponse(null, 'key')).to.deep.equal([]); + }); + + it('should return [] for undefined input', () => { + expect(getArrayFromResponse(undefined, 'key')).to.deep.equal([]); + }); + + it('should return [] for non-object input (number)', () => { + expect(getArrayFromResponse(42, 'key')).to.deep.equal([]); + }); + }); + + describe('getAssetItems', () => { + it('should return the input when it is already an array', () => { + const arr = [{ uid: 'a1' }]; + expect(getAssetItems(arr)).to.equal(arr); + }); + + it('should extract from data.items', () => { + const data = { items: [{ uid: 'a1', url: 'http://example.com/a1' }] }; + expect(getAssetItems(data)).to.deep.equal(data.items); + }); + + it('should extract from data.assets', () => { + const data = { assets: [{ uid: 'a2', filename: 'img.png' }] }; + expect(getAssetItems(data)).to.deep.equal(data.assets); + }); + + it('should prefer data.items over data.assets', () => { + const data = { items: [{ uid: 'from-items' }], assets: [{ uid: 'from-assets' }] }; + expect(getAssetItems(data)).to.deep.equal([{ uid: 'from-items' }]); + }); + + it('should return [] when neither key exists', () => { + expect(getAssetItems({ other: 'value' })).to.deep.equal([]); + }); + + it('should return [] for null input', () => { + expect(getAssetItems(null)).to.deep.equal([]); + }); + }); + + describe('getReadableStreamFromDownloadResponse', () => { + it('should return null for null input', () => { + expect(getReadableStreamFromDownloadResponse(null)).to.be.null; + }); + + it('should extract response.data when present', () => { + const inner = new PassThrough(); + const response = { data: inner }; + expect(getReadableStreamFromDownloadResponse(response)).to.equal(inner); + }); + + it('should return the stream itself if it has .pipe', () => { + const stream = new PassThrough(); + expect(getReadableStreamFromDownloadResponse(stream as any)).to.equal(stream); + }); + + it('should return null for non-stream objects without data', () => { + const obj = { something: 'else' } as any; + expect(getReadableStreamFromDownloadResponse(obj)).to.be.null; + }); + }); + + describe('writeStreamToFile', () => { + it('should resolve when stream finishes writing', async () => { + const source = new PassThrough(); + const tmpPath = require('node:path').join(require('node:os').tmpdir(), `test-write-${Date.now()}.txt`); + + const promise = writeStreamToFile(source, tmpPath); + source.end('hello world'); + await promise; + + const content = require('node:fs').readFileSync(tmpPath, 'utf-8'); + expect(content).to.equal('hello world'); + require('node:fs').unlinkSync(tmpPath); + }); + + it('should reject when the write stream errors', async () => { + const source = new PassThrough(); + const badPath = '/nonexistent-dir-xyz/file.txt'; + + try { + await writeStreamToFile(source, badPath); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.code).to.equal('ENOENT'); + } + }); + }); +}); diff --git a/packages/contentstack-asset-management/tsconfig.json b/packages/contentstack-asset-management/tsconfig.json new file mode 100644 index 000000000..513662339 --- /dev/null +++ b/packages/contentstack-asset-management/tsconfig.json @@ -0,0 +1,31 @@ +{ + "compilerOptions": { + "baseUrl": ".", + "declaration": true, + "importHelpers": true, + "module": "commonjs", + "outDir": "lib", + "rootDir": "src", + "strict": false, + "target": "es2017", + "allowJs": true, + "skipLibCheck": true, + "sourceMap": false, + "esModuleInterop": true, + "noImplicitAny": true, + "lib": [ + "ES2019", + "es2020.promise" + ], + "strictPropertyInitialization": false, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "src/**/*", + "types/*" + ], + "exclude": [ + "node_modules", + "lib" + ] +} \ No newline at end of file diff --git a/packages/contentstack-export/package.json b/packages/contentstack-export/package.json index 7453b36f4..5fdfaab83 100644 --- a/packages/contentstack-export/package.json +++ b/packages/contentstack-export/package.json @@ -8,6 +8,7 @@ "@contentstack/cli-command": "~2.0.0-beta", "@contentstack/cli-utilities": "~2.0.0-beta.1", "@contentstack/cli-variants": "~2.0.0-beta.7", + "@contentstack/cli-asset-management": "1.0.0", "@oclif/core": "^4.8.0", "async": "^3.2.6", "big-json": "^3.2.0", @@ -94,4 +95,4 @@ } }, "repository": "https://github.com/contentstack/cli" -} +} \ No newline at end of file diff --git a/packages/contentstack-export/src/export/modules/assets.ts b/packages/contentstack-export/src/export/modules/assets.ts index 0d6233f0e..1812ba0ab 100644 --- a/packages/contentstack-export/src/export/modules/assets.ts +++ b/packages/contentstack-export/src/export/modules/assets.ts @@ -25,7 +25,8 @@ import { PATH_CONSTANTS } from '../../constants'; import config from '../../config'; import { ModuleClassParams } from '../../types'; import BaseClass, { CustomPromiseHandler, CustomPromiseHandlerInput } from './base-class'; -import { PROCESS_NAMES, MODULE_CONTEXTS, PROCESS_STATUS, MODULE_NAMES } from '../../utils'; +import { ExportSpaces } from '@contentstack/cli-asset-management'; +import { PROCESS_NAMES, MODULE_CONTEXTS, PROCESS_STATUS, MODULE_NAMES, getOrgUid } from '../../utils'; export default class ExportAssets extends BaseClass { private assetsRootPath: string; @@ -48,7 +49,48 @@ export default class ExportAssets extends BaseClass { } async start(): Promise { - this.assetsRootPath = pResolve( + const linkedWorkspaces = this.exportConfig.linkedWorkspaces ?? []; + + if (linkedWorkspaces.length > 0) { + const assetManagementUrl = this.exportConfig.region?.assetManagementUrl; + if (!assetManagementUrl) { + this.completeProgress( + false, + 'Asset Management URL is required for AM 2.0 export. Ensure your region is configured with assetManagementUrl.', + ); + throw new Error( + 'Asset Management URL is required for AM 2.0 export. Ensure your region is configured with assetManagementUrl.', + ); + } + log.debug( + `Exporting with AM 2.0: ${assetManagementUrl} (linked_workspaces from exportConfig)`, + this.exportConfig.context, + ); + this.exportConfig.org_uid = this.exportConfig.org_uid || (await getOrgUid(this.exportConfig)); + const progress = this.createNestedProgress(this.currentModuleName); + try { + const exporter = new ExportSpaces({ + linkedWorkspaces, + exportDir: this.exportConfig.exportDir, + branchName: this.exportConfig.branchName || 'main', + assetManagementUrl, + org_uid: this.exportConfig.org_uid ?? '', + context: this.exportConfig.context as unknown as Record, + securedAssets: this.exportConfig.securedAssets, + }); + exporter.setParentProgressManager(progress); + await exporter.start(); + this.completeProgressWithMessage(); + } catch (error) { + this.completeProgress(false, (error as Error)?.message ?? 'Asset Management export failed'); + throw error; + } + return; + } + + log.debug('Using legacy asset export (no linked_workspaces in exportConfig)', this.exportConfig.context); + + this.assetsRootPath = pResolve( this.exportConfig.exportDir, this.exportConfig.branchName || '', this.assetConfig.dirName, @@ -78,10 +120,7 @@ export default class ExportAssets extends BaseClass { if (typeof assetsFolderCount === 'number' && assetsFolderCount > 0) { progress .startProcess(PROCESS_NAMES.ASSET_FOLDERS) - .updateStatus( - PROCESS_STATUS[PROCESS_NAMES.ASSET_FOLDERS].FETCHING, - PROCESS_NAMES.ASSET_FOLDERS, - ); + .updateStatus(PROCESS_STATUS[PROCESS_NAMES.ASSET_FOLDERS].FETCHING, PROCESS_NAMES.ASSET_FOLDERS); await this.getAssetsFolders(assetsFolderCount); progress.completeProcess(PROCESS_NAMES.ASSET_FOLDERS, true); } @@ -90,10 +129,7 @@ export default class ExportAssets extends BaseClass { if (typeof assetsCount === 'number' && assetsCount > 0) { progress .startProcess(PROCESS_NAMES.ASSET_METADATA) - .updateStatus( - PROCESS_STATUS[PROCESS_NAMES.ASSET_METADATA].FETCHING, - PROCESS_NAMES.ASSET_METADATA, - ); + .updateStatus(PROCESS_STATUS[PROCESS_NAMES.ASSET_METADATA].FETCHING, PROCESS_NAMES.ASSET_METADATA); await this.getAssets(assetsCount); progress.completeProcess(PROCESS_NAMES.ASSET_METADATA, true); } @@ -112,17 +148,13 @@ export default class ExportAssets extends BaseClass { if (typeof assetsCount === 'number' && assetsCount > 0) { progress .startProcess(PROCESS_NAMES.ASSET_DOWNLOADS) - .updateStatus( - PROCESS_STATUS[PROCESS_NAMES.ASSET_DOWNLOADS].DOWNLOADING, - PROCESS_NAMES.ASSET_DOWNLOADS, - ); + .updateStatus(PROCESS_STATUS[PROCESS_NAMES.ASSET_DOWNLOADS].DOWNLOADING, PROCESS_NAMES.ASSET_DOWNLOADS); log.debug('Starting download of all assets...', this.exportConfig.context); await this.downloadAssets(); progress.completeProcess(PROCESS_NAMES.ASSET_DOWNLOADS, true); } this.completeProgressWithMessage(); - } catch (error) { this.completeProgress(false, error?.message || 'Asset export failed'); } @@ -148,12 +180,7 @@ export default class ExportAssets extends BaseClass { if (!isEmpty(items)) { this.assetsFolder.push(...items); items.forEach((folder: any) => { - this.progressManager?.tick( - true, - `folder: ${folder.name || folder.uid}`, - null, - PROCESS_NAMES.ASSET_FOLDERS, - ); + this.progressManager?.tick(true, `folder: ${folder.name || folder.uid}`, null, PROCESS_NAMES.ASSET_FOLDERS); }); } }; @@ -254,12 +281,7 @@ export default class ExportAssets extends BaseClass { fs?.writeIntoFile(items, { mapKeyVal: true }); // Track progress for each asset with process name items.forEach((asset: any) => { - this.progressManager?.tick( - true, - `asset: ${asset.filename || asset.uid}`, - null, - PROCESS_NAMES.ASSET_METADATA, - ); + this.progressManager?.tick(true, `asset: ${asset.filename || asset.uid}`, null, PROCESS_NAMES.ASSET_METADATA); }); } }; diff --git a/packages/contentstack-export/src/export/modules/stack.ts b/packages/contentstack-export/src/export/modules/stack.ts index 5007235da..e4898815d 100644 --- a/packages/contentstack-export/src/export/modules/stack.ts +++ b/packages/contentstack-export/src/export/modules/stack.ts @@ -1,11 +1,6 @@ import find from 'lodash/find'; import { resolve as pResolve } from 'node:path'; -import { - handleAndLogError, - isAuthenticated, - managementSDKClient, - log, -} from '@contentstack/cli-utilities'; +import { handleAndLogError, isAuthenticated, managementSDKClient, log } from '@contentstack/cli-utilities'; import { PATH_CONSTANTS } from '../../constants'; import BaseClass from './base-class'; @@ -15,6 +10,7 @@ import { MODULE_CONTEXTS, PROCESS_STATUS, MODULE_NAMES, + getLinkedWorkspacesForBranch, } from '../../utils'; import { StackConfig, ModuleClassParams } from '../../types'; @@ -79,10 +75,7 @@ export default class ExportStack extends BaseClass { if (!this.exportConfig.management_token) { progress .startProcess(PROCESS_NAMES.STACK_SETTINGS) - .updateStatus( - PROCESS_STATUS[PROCESS_NAMES.STACK_SETTINGS].EXPORTING, - PROCESS_NAMES.STACK_SETTINGS, - ); + .updateStatus(PROCESS_STATUS[PROCESS_NAMES.STACK_SETTINGS].EXPORTING, PROCESS_NAMES.STACK_SETTINGS); await this.exportStackSettings(); progress.completeProcess(PROCESS_NAMES.STACK_SETTINGS, true); } else { @@ -95,10 +88,7 @@ export default class ExportStack extends BaseClass { if (!this.exportConfig.preserveStackVersion && !this.exportConfig.hasOwnProperty('master_locale')) { progress .startProcess(PROCESS_NAMES.STACK_LOCALE) - .updateStatus( - PROCESS_STATUS[PROCESS_NAMES.STACK_LOCALE].FETCHING, - PROCESS_NAMES.STACK_LOCALE, - ); + .updateStatus(PROCESS_STATUS[PROCESS_NAMES.STACK_LOCALE].FETCHING, PROCESS_NAMES.STACK_LOCALE); const masterLocale = await this.getLocales(); progress.completeProcess(PROCESS_NAMES.STACK_LOCALE, true); @@ -112,10 +102,7 @@ export default class ExportStack extends BaseClass { } else if (this.exportConfig.preserveStackVersion) { progress .startProcess(PROCESS_NAMES.STACK_DETAILS) - .updateStatus( - PROCESS_STATUS[PROCESS_NAMES.STACK_DETAILS].EXPORTING, - PROCESS_NAMES.STACK_DETAILS, - ); + .updateStatus(PROCESS_STATUS[PROCESS_NAMES.STACK_DETAILS].EXPORTING, PROCESS_NAMES.STACK_DETAILS); const stackResult = await this.exportStack(); progress.completeProcess(PROCESS_NAMES.STACK_DETAILS, true); @@ -126,7 +113,6 @@ export default class ExportStack extends BaseClass { } this.completeProgressWithMessage(); - } catch (error) { log.debug('Error occurred during stack export', this.exportConfig.context); handleAndLogError(error, { ...this.exportConfig.context }); @@ -233,18 +219,13 @@ export default class ExportStack extends BaseClass { return this.stack .fetch() - .then((resp: any) => { + .then(async (resp: any) => { const stackFilePath = pResolve(this.stackFolderPath, this.stackConfig.fileName); log.debug(`Writing stack data to: '${stackFilePath}'`, this.exportConfig.context); fsUtil.writeFile(stackFilePath, resp); // Track progress for stack export completion - this.progressManager?.tick( - true, - `stack: ${this.exportConfig.apiKey}`, - null, - PROCESS_NAMES.STACK_DETAILS, - ); + this.progressManager?.tick(true, `stack: ${this.exportConfig.apiKey}`, null, PROCESS_NAMES.STACK_DETAILS); log.success( `Stack details exported successfully for stack ${this.exportConfig.apiKey}`, @@ -270,14 +251,26 @@ export default class ExportStack extends BaseClass { await fsUtil.makeDirectory(this.stackFolderPath); return this.stack .settings() - .then((resp: any) => { - fsUtil.writeFile(pResolve(this.stackFolderPath, PATH_CONSTANTS.FILES.SETTINGS), resp); + .then(async (resp: any) => { + const linked = await getLinkedWorkspacesForBranch( + this.stack, + this.exportConfig.branchName || 'main', + this.exportConfig.context as unknown as Record, + ); + const settings = { + ...resp, + am_v2: { ...(resp.am_v2 ?? {}), linked_workspaces: linked }, + }; + fsUtil.writeFile(pResolve(this.stackFolderPath, PATH_CONSTANTS.FILES.SETTINGS), settings); + + this.exportConfig.linkedWorkspaces = linked; // Track progress for stack settings completion this.progressManager?.tick(true, 'stack settings', null, PROCESS_NAMES.STACK_SETTINGS); + log.debug(`Included ${linked.length} linked workspace(s) in settings`, this.exportConfig.context); log.success('Exported stack settings successfully!', this.exportConfig.context); - return resp; + return settings; }) .catch((error: any) => { this.progressManager?.tick( diff --git a/packages/contentstack-export/src/types/export-config.ts b/packages/contentstack-export/src/types/export-config.ts index 8b0e1b37b..978a6c04a 100644 --- a/packages/contentstack-export/src/types/export-config.ts +++ b/packages/contentstack-export/src/types/export-config.ts @@ -36,6 +36,7 @@ export default interface ExportConfig extends DefaultConfig { skipStackSettings?: boolean; skipDependencies?: boolean; authenticationMethod?: string; + linkedWorkspaces?: Array<{ uid: string; space_uid: string; is_default: boolean }>; } type branch = { diff --git a/packages/contentstack-export/src/types/index.ts b/packages/contentstack-export/src/types/index.ts index 63baf41e6..60d044a1a 100644 --- a/packages/contentstack-export/src/types/index.ts +++ b/packages/contentstack-export/src/types/index.ts @@ -32,6 +32,7 @@ export interface Region { cma: string; cda: string; uiHost: string; + assetManagementUrl?: string; } export type Modules = diff --git a/packages/contentstack-export/src/utils/constants.ts b/packages/contentstack-export/src/utils/constants.ts index fc2f7dd28..323e3ab89 100644 --- a/packages/contentstack-export/src/utils/constants.ts +++ b/packages/contentstack-export/src/utils/constants.ts @@ -3,6 +3,11 @@ export const PROCESS_NAMES = { ASSET_FOLDERS: 'Folders', ASSET_METADATA: 'Metadata', ASSET_DOWNLOADS: 'Downloads', + /** Used when Assets module runs Asset Management 2.0 path (spaces, metadata, folders, assets, downloads). */ + ASSET_MANAGEMENT_SPACES: 'Spaces & assets', + + // Asset Management 2.0 module + ASSET_MANAGEMENT_EXPORT: 'Asset Management 2.0', // Custom Roles module FETCH_ROLES: 'Fetch Roles', @@ -37,6 +42,7 @@ export const PROCESS_NAMES = { export const MODULE_CONTEXTS = { ASSETS: 'assets', + ASSET_MANAGEMENT: 'asset-management', CONTENT_TYPES: 'content-types', CUSTOM_ROLES: 'custom-roles', ENTRIES: 'entries', @@ -56,6 +62,7 @@ export const MODULE_CONTEXTS = { // Display names for modules to avoid scattering user-facing strings export const MODULE_NAMES = { [MODULE_CONTEXTS.ASSETS]: 'Assets', + [MODULE_CONTEXTS.ASSET_MANAGEMENT]: 'Asset Management 2.0', [MODULE_CONTEXTS.CONTENT_TYPES]: 'Content Types', [MODULE_CONTEXTS.CUSTOM_ROLES]: 'Custom Roles', [MODULE_CONTEXTS.ENTRIES]: 'Entries', @@ -87,6 +94,15 @@ export const PROCESS_STATUS = { FAILED: 'Failed to download asset:', }, // Custom Roles + [PROCESS_NAMES.ASSET_MANAGEMENT_SPACES]: { + EXPORTING: 'Exporting spaces & assets...', + FAILED: 'Failed to export spaces & assets.', + }, + // Asset Management 2.0 + [PROCESS_NAMES.ASSET_MANAGEMENT_EXPORT]: { + EXPORTING: 'Exporting...', + FAILED: 'Asset Management export failed.', + }, [PROCESS_NAMES.FETCH_ROLES]: { FETCHING: 'Fetching custom roles...', FAILED: 'Failed to fetch custom roles.', diff --git a/packages/contentstack-export/src/utils/get-linked-workspaces.ts b/packages/contentstack-export/src/utils/get-linked-workspaces.ts new file mode 100644 index 000000000..c9f17338e --- /dev/null +++ b/packages/contentstack-export/src/utils/get-linked-workspaces.ts @@ -0,0 +1,33 @@ +import { log, handleAndLogError } from '@contentstack/cli-utilities'; +import type { LinkedWorkspace } from '@contentstack/cli-asset-management'; + +/** Stack client with branch().fetch() for CMA branch details */ +type StackWithBranch = { branch: (name: string) => { fetch: (params?: Record) => Promise } }; + +/** + * Fetch branch details with include_settings: true and return linked workspaces (am_v2). + * Reused by stack export (included in settings.json) and asset-management module. + */ +export async function getLinkedWorkspacesForBranch( + stack: StackWithBranch, + branchName: string, + context?: Record, +): Promise { + log.debug(`Fetching branch details for: ${branchName}`, context); + try { + const branch = await stack.branch(branchName).fetch({ include_settings: true } as Record); + const linked = (branch as any)?.settings?.am_v2?.linked_workspaces; + if (!Array.isArray(linked)) { + log.debug('No linked_workspaces in branch settings', context); + return []; + } + log.info( + `Found ${linked.length} linked workspace(s) for branch ${branchName}`, + context, + ); + return linked as LinkedWorkspace[]; + } catch (error) { + handleAndLogError(error as Error, context as any, 'Failed to fetch branch settings'); + return []; + } +} diff --git a/packages/contentstack-export/src/utils/index.ts b/packages/contentstack-export/src/utils/index.ts index 9cbd32cac..2b5dd9eea 100644 --- a/packages/contentstack-export/src/utils/index.ts +++ b/packages/contentstack-export/src/utils/index.ts @@ -8,4 +8,5 @@ export { log, unlinkFileLogger } from './logger'; export { default as login } from './basic-login'; export * from './common-helper'; export * from './marketplace-app-helper'; +export { getLinkedWorkspacesForBranch } from './get-linked-workspaces'; export { MODULE_CONTEXTS, MODULE_NAMES, PROCESS_NAMES, PROCESS_STATUS } from './constants'; diff --git a/packages/contentstack-export/src/utils/progress-strategy-registry.ts b/packages/contentstack-export/src/utils/progress-strategy-registry.ts index ed6823138..b50c1e86b 100644 --- a/packages/contentstack-export/src/utils/progress-strategy-registry.ts +++ b/packages/contentstack-export/src/utils/progress-strategy-registry.ts @@ -1,3 +1,4 @@ +import { AM_MAIN_PROCESS_NAME } from '@contentstack/cli-asset-management'; import { MODULE_CONTEXTS, MODULE_NAMES, PROCESS_NAMES } from './constants'; /** * Progress Strategy Registrations for Export Modules @@ -30,6 +31,23 @@ try { failures: downloadsProcess.failureCount, }; } + // Asset Management 2.0 path (process name owned by AM package) + const amProcess = processes.get(AM_MAIN_PROCESS_NAME); + if (amProcess) { + return { + total: amProcess.total, + success: amProcess.successCount, + failures: amProcess.failureCount, + }; + } + const spacesProcess = processes.get(PROCESS_NAMES.ASSET_MANAGEMENT_SPACES); + if (spacesProcess) { + return { + total: spacesProcess.total, + success: spacesProcess.successCount, + failures: spacesProcess.failureCount, + }; + } // Fallback to metadata process if downloads don't exist const metadataProcess = processes.get(PROCESS_NAMES.ASSET_METADATA); diff --git a/packages/contentstack-export/test/unit/export/modules/stack.test.ts b/packages/contentstack-export/test/unit/export/modules/stack.test.ts index 52645c028..5b7ed1d9b 100644 --- a/packages/contentstack-export/test/unit/export/modules/stack.test.ts +++ b/packages/contentstack-export/test/unit/export/modules/stack.test.ts @@ -482,18 +482,16 @@ describe('ExportStack', () => { description: 'Settings description', settings: { global: { example: 'value' } }, }; + const expectedSettings = { ...settingsData, am_v2: { linked_workspaces: [] as any[] } }; mockStackClient.settings = sinon.stub().resolves(settingsData); const result = await exportStack.exportStackSettings(); - expect(writeFileStub.called).to.be.true; - expect(makeDirectoryStub.called).to.be.true; - // Should return the settings data - expect(result).to.deep.equal(settingsData); - // Verify file was written with correct path - const writeCall = writeFileStub.getCall(0); + expect(result).to.deep.equal(expectedSettings); + const writeCall = writeFileStub.firstCall; expect(writeCall.args[0]).to.include('settings.json'); - expect(writeCall.args[1]).to.deep.equal(settingsData); + expect(writeCall.args[1]).to.deep.equal(expectedSettings); + expect(makeDirectoryStub.firstCall).to.not.be.null; }); it('should handle errors when exporting settings without throwing', async () => { From 4a815c093f1f07653fffbe487d83df0494ee3ea4 Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Wed, 4 Mar 2026 12:08:26 +0530 Subject: [PATCH 02/22] chore: moved constants to constants file --- packages/contentstack-asset-management/src/constants/index.ts | 3 +++ packages/contentstack-asset-management/src/export/base.ts | 2 +- .../contentstack-asset-management/src/utils/export-helpers.ts | 3 --- packages/contentstack-asset-management/src/utils/index.ts | 3 +-- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/contentstack-asset-management/src/constants/index.ts b/packages/contentstack-asset-management/src/constants/index.ts index 0a0b469a7..96ff84906 100644 --- a/packages/contentstack-asset-management/src/constants/index.ts +++ b/packages/contentstack-asset-management/src/constants/index.ts @@ -1,3 +1,6 @@ +export const BATCH_SIZE = 50; +export const CHUNK_FILE_SIZE_MB = 1; + /** * Main process name for Asset Management 2.0 export (single progress bar). * Use this when adding/starting the process and for all ticks. diff --git a/packages/contentstack-asset-management/src/export/base.ts b/packages/contentstack-asset-management/src/export/base.ts index 8a55ab29d..7521eae53 100644 --- a/packages/contentstack-asset-management/src/export/base.ts +++ b/packages/contentstack-asset-management/src/export/base.ts @@ -6,7 +6,7 @@ import type { AssetManagementAPIConfig } from '../types/asset-management-api'; import type { ExportContext } from '../types/export-types'; import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; import { AM_MAIN_PROCESS_NAME } from '../constants/index'; -import { BATCH_SIZE, CHUNK_FILE_SIZE_MB } from '../utils/export-helpers'; +import { BATCH_SIZE, CHUNK_FILE_SIZE_MB } from '../constants/index'; export type { ExportContext }; diff --git a/packages/contentstack-asset-management/src/utils/export-helpers.ts b/packages/contentstack-asset-management/src/utils/export-helpers.ts index 2083951ec..9bc772c0f 100644 --- a/packages/contentstack-asset-management/src/utils/export-helpers.ts +++ b/packages/contentstack-asset-management/src/utils/export-helpers.ts @@ -1,8 +1,5 @@ import { createWriteStream } from 'node:fs'; -export const BATCH_SIZE = 50; -export const CHUNK_FILE_SIZE_MB = 1; - export function getArrayFromResponse(data: unknown, arrayKey: string): unknown[] { if (Array.isArray(data)) return data; if (data != null && typeof data === 'object' && arrayKey in data) { diff --git a/packages/contentstack-asset-management/src/utils/index.ts b/packages/contentstack-asset-management/src/utils/index.ts index fdbfd6133..0837e33fd 100644 --- a/packages/contentstack-asset-management/src/utils/index.ts +++ b/packages/contentstack-asset-management/src/utils/index.ts @@ -1,7 +1,6 @@ export { AssetManagementAdapter } from './asset-management-api-adapter'; +export { BATCH_SIZE, CHUNK_FILE_SIZE_MB } from '../constants'; export { - BATCH_SIZE, - CHUNK_FILE_SIZE_MB, getArrayFromResponse, getAssetItems, getReadableStreamFromDownloadResponse, From bd80e5abe1ee47ef0681de8ceb44c32cbbb5d341 Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Wed, 4 Mar 2026 12:14:31 +0530 Subject: [PATCH 03/22] chore: update lock file --- pnpm-lock.yaml | 151 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 117 insertions(+), 34 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d371a2179..b9fe282ae 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,6 +15,55 @@ importers: specifier: ^10.28.0 version: 10.30.3 + packages/contentstack-asset-management: + dependencies: + '@contentstack/cli-utilities': + specifier: ~2.0.0-beta + version: 2.0.0-beta.1(@types/node@20.19.35) + devDependencies: + '@types/chai': + specifier: ^4.3.11 + version: 4.3.20 + '@types/mocha': + specifier: ^10.0.6 + version: 10.0.10 + '@types/node': + specifier: ^20.17.50 + version: 20.19.35 + '@types/sinon': + specifier: ^17.0.2 + version: 17.0.4 + chai: + specifier: ^4.4.1 + version: 4.5.0 + eslint: + specifier: ^8.57.1 + version: 8.57.1 + eslint-config-oclif: + specifier: ^6.0.68 + version: 6.0.146(eslint@8.57.1)(typescript@5.9.3) + mocha: + specifier: ^10.8.2 + version: 10.8.2 + nyc: + specifier: ^15.1.0 + version: 15.1.0 + oclif: + specifier: ^4.17.46 + version: 4.22.81(@types/node@20.19.35) + sinon: + specifier: ^17.0.1 + version: 17.0.2 + source-map-support: + specifier: ^0.5.21 + version: 0.5.21 + ts-node: + specifier: ^10.9.2 + version: 10.9.2(@types/node@20.19.35)(typescript@5.9.3) + typescript: + specifier: ^5.8.3 + version: 5.9.3 + packages/contentstack-audit: dependencies: '@contentstack/cli-command': @@ -333,6 +382,9 @@ importers: packages/contentstack-export: dependencies: + '@contentstack/cli-asset-management': + specifier: 1.0.0 + version: link:../contentstack-asset-management '@contentstack/cli-command': specifier: ~2.0.0-beta version: 2.0.0-beta.1(@types/node@22.19.13) @@ -8106,7 +8158,7 @@ snapshots: '@jest/console@29.7.0': dependencies: '@jest/types': 29.6.3 - '@types/node': 14.18.63 + '@types/node': 20.19.35 chalk: 4.1.2 jest-message-util: 29.7.0 jest-util: 29.7.0 @@ -8119,14 +8171,14 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 14.18.63 + '@types/node': 20.19.35 ansi-escapes: 4.3.2 chalk: 4.1.2 ci-info: 3.9.0 exit: 0.1.2 graceful-fs: 4.2.11 jest-changed-files: 29.7.0 - jest-config: 29.7.0(@types/node@14.18.63)(ts-node@8.10.2(typescript@4.9.5)) + jest-config: 29.7.0(@types/node@20.19.35)(ts-node@8.10.2(typescript@4.9.5)) jest-haste-map: 29.7.0 jest-message-util: 29.7.0 jest-regex-util: 29.6.3 @@ -8151,7 +8203,7 @@ snapshots: dependencies: '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 14.18.63 + '@types/node': 20.19.35 jest-mock: 29.7.0 '@jest/expect-utils@29.7.0': @@ -8169,7 +8221,7 @@ snapshots: dependencies: '@jest/types': 29.6.3 '@sinonjs/fake-timers': 10.3.0 - '@types/node': 14.18.63 + '@types/node': 20.19.35 jest-message-util: 29.7.0 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -8191,7 +8243,7 @@ snapshots: '@jest/transform': 29.7.0 '@jest/types': 29.6.3 '@jridgewell/trace-mapping': 0.3.31 - '@types/node': 14.18.63 + '@types/node': 20.19.35 chalk: 4.1.2 collect-v8-coverage: 1.0.3 exit: 0.1.2 @@ -8260,7 +8312,7 @@ snapshots: dependencies: '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 14.18.63 + '@types/node': 20.19.35 '@types/yargs': 15.0.20 chalk: 4.1.2 @@ -8269,7 +8321,7 @@ snapshots: '@jest/schemas': 29.6.3 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 14.18.63 + '@types/node': 20.19.35 '@types/yargs': 17.0.35 chalk: 4.1.2 @@ -8899,7 +8951,7 @@ snapshots: '@types/graceful-fs@4.1.9': dependencies: - '@types/node': 14.18.63 + '@types/node': 20.19.35 '@types/http-cache-semantics@4.2.0': {} @@ -8944,7 +8996,7 @@ snapshots: '@types/mkdirp@1.0.2': dependencies: - '@types/node': 14.18.63 + '@types/node': 20.19.35 '@types/mocha@10.0.10': {} @@ -8988,12 +9040,12 @@ snapshots: '@types/tar@6.1.13': dependencies: - '@types/node': 14.18.63 + '@types/node': 20.19.35 minipass: 4.2.8 '@types/through@0.0.33': dependencies: - '@types/node': 14.18.63 + '@types/node': 20.19.35 '@types/tmp@0.2.6': {} @@ -10596,7 +10648,7 @@ snapshots: '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1)(typescript@5.9.3) '@typescript-eslint/parser': 6.21.0(eslint@8.57.1)(typescript@5.9.3) eslint-config-xo-space: 0.35.0(eslint@8.57.1) - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@8.57.1) eslint-plugin-import: 2.32.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1)(eslint@8.57.1) eslint-plugin-mocha: 10.5.0(eslint@8.57.1) eslint-plugin-n: 15.7.0(eslint@8.57.1) @@ -10658,8 +10710,8 @@ snapshots: eslint-config-oclif: 5.2.2(eslint@8.57.1) eslint-config-xo: 0.49.0(eslint@8.57.1) eslint-config-xo-space: 0.35.0(eslint@8.57.1) - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1) - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1)(eslint@8.57.1) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) eslint-plugin-jsdoc: 50.8.0(eslint@8.57.1) eslint-plugin-mocha: 10.5.0(eslint@8.57.1) eslint-plugin-n: 17.24.0(eslint@8.57.1)(typescript@5.9.3) @@ -10700,7 +10752,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1): + eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1): dependencies: '@nolyfill/is-core-module': 1.0.39 debug: 4.4.3(supports-color@8.1.1) @@ -10711,7 +10763,7 @@ snapshots: tinyglobby: 0.2.15 unrs-resolver: 1.11.1 optionalDependencies: - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1)(eslint@8.57.1) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) transitivePeerDependencies: - supports-color @@ -10726,7 +10778,7 @@ snapshots: tinyglobby: 0.2.15 unrs-resolver: 1.11.1 optionalDependencies: - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@4.9.5))(eslint-import-resolver-typescript@3.10.1)(eslint@8.57.1) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1)(eslint@8.57.1) transitivePeerDependencies: - supports-color @@ -10741,14 +10793,14 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.1(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1): + eslint-module-utils@2.12.1(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1)(eslint@8.57.1): dependencies: debug: 3.2.7 optionalDependencies: '@typescript-eslint/parser': 6.21.0(eslint@8.57.1)(typescript@5.9.3) eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@8.57.1) transitivePeerDependencies: - supports-color @@ -10763,14 +10815,14 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.1(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1): + eslint-module-utils@2.12.1(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1): dependencies: debug: 3.2.7 optionalDependencies: '@typescript-eslint/parser': 8.56.1(eslint@8.57.1)(typescript@5.9.3) eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1) transitivePeerDependencies: - supports-color @@ -10827,7 +10879,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.1(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) + eslint-module-utils: 2.12.1(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1)(eslint@8.57.1) hasown: 2.0.2 is-core-module: 2.16.1 is-glob: 4.0.3 @@ -10874,7 +10926,7 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1)(eslint@8.57.1): + eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1): dependencies: '@rtsao/scc': 1.1.0 array-includes: 3.1.9 @@ -10885,7 +10937,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) + eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.56.1(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) hasown: 2.0.2 is-core-module: 2.16.1 is-glob: 4.0.3 @@ -12061,7 +12113,7 @@ snapshots: '@jest/expect': 29.7.0 '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 14.18.63 + '@types/node': 20.19.35 chalk: 4.1.2 co: 4.6.0 dedent: 1.7.2 @@ -12131,6 +12183,37 @@ snapshots: - babel-plugin-macros - supports-color + jest-config@29.7.0(@types/node@20.19.35)(ts-node@8.10.2(typescript@4.9.5)): + dependencies: + '@babel/core': 7.29.0 + '@jest/test-sequencer': 29.7.0 + '@jest/types': 29.6.3 + babel-jest: 29.7.0(@babel/core@7.29.0) + chalk: 4.1.2 + ci-info: 3.9.0 + deepmerge: 4.3.1 + glob: 7.2.3 + graceful-fs: 4.2.11 + jest-circus: 29.7.0 + jest-environment-node: 29.7.0 + jest-get-type: 29.6.3 + jest-regex-util: 29.6.3 + jest-resolve: 29.7.0 + jest-runner: 29.7.0 + jest-util: 29.7.0 + jest-validate: 29.7.0 + micromatch: 4.0.8 + parse-json: 5.2.0 + pretty-format: 29.7.0 + slash: 3.0.0 + strip-json-comments: 3.1.1 + optionalDependencies: + '@types/node': 20.19.35 + ts-node: 8.10.2(typescript@4.9.5) + transitivePeerDependencies: + - babel-plugin-macros + - supports-color + jest-diff@26.6.2: dependencies: chalk: 4.1.2 @@ -12162,7 +12245,7 @@ snapshots: '@jest/environment': 29.7.0 '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 14.18.63 + '@types/node': 20.19.35 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -12174,7 +12257,7 @@ snapshots: dependencies: '@jest/types': 29.6.3 '@types/graceful-fs': 4.1.9 - '@types/node': 14.18.63 + '@types/node': 20.19.35 anymatch: 3.1.3 fb-watchman: 2.0.2 graceful-fs: 4.2.11 @@ -12213,7 +12296,7 @@ snapshots: jest-mock@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 14.18.63 + '@types/node': 20.19.35 jest-util: 29.7.0 jest-pnp-resolver@1.2.3(jest-resolve@29.7.0): @@ -12248,7 +12331,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 14.18.63 + '@types/node': 20.19.35 chalk: 4.1.2 emittery: 0.13.1 graceful-fs: 4.2.11 @@ -12276,7 +12359,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 14.18.63 + '@types/node': 20.19.35 chalk: 4.1.2 cjs-module-lexer: 1.4.3 collect-v8-coverage: 1.0.3 @@ -12322,7 +12405,7 @@ snapshots: jest-util@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 14.18.63 + '@types/node': 20.19.35 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -12341,7 +12424,7 @@ snapshots: dependencies: '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 14.18.63 + '@types/node': 20.19.35 ansi-escapes: 4.3.2 chalk: 4.1.2 emittery: 0.13.1 @@ -12350,7 +12433,7 @@ snapshots: jest-worker@29.7.0: dependencies: - '@types/node': 14.18.63 + '@types/node': 20.19.35 jest-util: 29.7.0 merge-stream: 2.0.0 supports-color: 8.1.1 From 64c6e36daf734b36683d18e38818d89d0e427374 Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Fri, 20 Mar 2026 19:50:21 +0530 Subject: [PATCH 04/22] feat: add support for AM in import module --- .talismanrc | 46 ++-- .../src/constants/index.ts | 23 +- .../src/export/assets.ts | 4 +- .../src/export/spaces.ts | 15 +- .../src/import/asset-types.ts | 88 +++++++ .../src/import/assets.ts | 227 ++++++++++++++++++ .../src/import/base.ts | 96 ++++++++ .../src/import/fields.ts | 88 +++++++ .../src/import/index.ts | 7 + .../src/import/spaces.ts | 165 +++++++++++++ .../src/import/workspaces.ts | 82 +++++++ .../src/index.ts | 1 + .../src/types/asset-management-api.ts | 123 +++++++++- .../src/utils/asset-management-api-adapter.ts | 139 ++++++++++- .../src/export/modules/assets.ts | 1 + packages/contentstack-import/package.json | 3 +- .../contentstack-import/src/config/index.ts | 9 + .../src/constants/index.ts | 1 + .../src/import/modules/assets.ts | 116 +++++++-- .../src/import/modules/stack.ts | 13 +- .../src/types/default-config.ts | 9 + .../src/types/import-config.ts | 2 + .../contentstack-import/src/types/index.ts | 1 + .../src/utils/import-config-handler.ts | 36 ++- 24 files changed, 1230 insertions(+), 65 deletions(-) create mode 100644 packages/contentstack-asset-management/src/import/asset-types.ts create mode 100644 packages/contentstack-asset-management/src/import/assets.ts create mode 100644 packages/contentstack-asset-management/src/import/base.ts create mode 100644 packages/contentstack-asset-management/src/import/fields.ts create mode 100644 packages/contentstack-asset-management/src/import/index.ts create mode 100644 packages/contentstack-asset-management/src/import/spaces.ts create mode 100644 packages/contentstack-asset-management/src/import/workspaces.ts diff --git a/.talismanrc b/.talismanrc index 86b85bda6..20cce845c 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,26 +1,24 @@ fileignoreconfig: - - filename: packages/contentstack-audit/test/unit/mock/contents/environments/environments.json - checksum: d983bc17ab56937c66a6d25f449ecbe285d00c807923ed56deacb3a571aa3448 - - filename: package-lock.json - checksum: 8fd93d7b01c7d064fa797fba81d09da612b5639468c02003e11b2f3b599e49a2 - - filename: pnpm-lock.yaml - checksum: 458cb8d135905583080023055430cf8344f46e92dd1c9e90cdb6f78bbd02eecb - - filename: packages/contentstack-seed/src/commands/cm/stacks/seed.ts - checksum: 5c59296f3d5ba078f16bca23a47c920dc2180cff3b8250a341176185a4dabc39 - - filename: packages/contentstack-seed/README.md - checksum: d2a017a8206aae1058d4a91d445a7f7d50e919a0d2dd69605a66529c4f4ebe2e - - filename: packages/contentstack-seed/src/seed/github/client.ts - checksum: 44d491ab5253ebb6c24bb0ac5c8d985320bdc4cc3711de77adfe79f7fb1874a1 - - filename: packages/contentstack-seed/test/commands/cm/stacks/seed.test.ts - checksum: 999b1afe970452691318c76d5e9abd8852384fcf2d826cecda19f156de75fb59 - - filename: packages/contentstack-seed/src/seed/index.ts - checksum: 2bd73b2562618a37e02247459141432515471277ee5b85f5053f169891a54eb5 - - filename: packages/contentstack-export-to-csv/src/utils/interactive.ts - checksum: 8aa3870a6694e404f4f8df3ed884dd69521099fe66851c4f8c9860276254da9d - - filename: packages/contentstack-import/test/unit/import/modules/marketplace-apps.test.ts - checksum: e927e670f70374bfa09a4866faf2af0a65476709412882122ea2811717e528aa - - filename: packages/contentstack-export/test/unit/export/modules/marketplace-apps.test.ts - checksum: 6cb665ee2ea09372b3f80c3d2c38d1c3c3889ce0a1ba7d8488614a73aedf17b7 - - filename: packages/contentstack-import/test/unit/import/module-importer.test.ts - checksum: baf0ffc77d2afe9084da2d1fecac4a3a6ef875739677ef6186cd7de278886406 + - filename: packages/contentstack-export/src/export/modules/assets.ts + checksum: 1eacc8e86cb50fe283febe6688965854f420e02cf1b49555a15661fa0c3e3c7a + - filename: packages/contentstack-import/src/utils/import-config-handler.ts + checksum: f831cef1b7c3bd97bdbc170cff452350cee0f448d97df02e25aa41d6c4d64ad3 + - filename: packages/contentstack-asset-management/src/import/asset-types.ts + checksum: a39caa373b2a736d1e57063326cfb2073ae78376efa931b27d2c7110997708a5 + - filename: packages/contentstack-asset-management/src/export/spaces.ts + checksum: 3ec11c8f710b60ae495c69344025587df2e6195c872a0c82feaf04ac044ecefa + - filename: packages/contentstack-import/src/import/modules/assets.ts + checksum: d9f4a29a29e8b8a2a36e498f2380d39e1c5c0ec13ff894ef450abd817f2a646e + - filename: packages/contentstack-asset-management/src/import/fields.ts + checksum: bbae69c28ec69bf67c2c7b4df3620380ef3fca488b3288e137b65a60ee738b9e + - filename: packages/contentstack-asset-management/src/import/spaces.ts + checksum: 79cf2f1b55523d28c218d970155f887255a00dc095a941556b709d1f19c6a8a0 + - filename: packages/contentstack-asset-management/src/types/asset-management-api.ts + checksum: 716df03dcba70b2cc0f77b1f6338524553ba740080d7087a8699147c3ce8f0ba + - filename: packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts + checksum: 92bcad2feabc1954ead89b370d284b7af5f38ec1dca60a41752371977ef106ff + - filename: packages/contentstack-asset-management/src/import/base.ts + checksum: 513b55cf7e92bdbe8b815141822ba10579b6a728bec4424fc664794246ad33bb + - filename: packages/contentstack-asset-management/src/import/assets.ts + checksum: 2d34fa57f5ab269f6c535dff3242cc1135dbe1decd84fa0bc8997d0410d520b2 version: '1.0' diff --git a/packages/contentstack-asset-management/src/constants/index.ts b/packages/contentstack-asset-management/src/constants/index.ts index 96ff84906..6763629d0 100644 --- a/packages/contentstack-asset-management/src/constants/index.ts +++ b/packages/contentstack-asset-management/src/constants/index.ts @@ -17,10 +17,15 @@ export const PROCESS_NAMES = { AM_FIELDS: 'Fields', AM_ASSET_TYPES: 'Asset types', AM_DOWNLOADS: 'Asset downloads', + // Import process names + AM_IMPORT_FIELDS: 'Import fields', + AM_IMPORT_ASSET_TYPES: 'Import asset types', + AM_IMPORT_FOLDERS: 'Import folders', + AM_IMPORT_ASSETS: 'Import assets', } as const; /** - * Status messages for each process (exporting, fetching, failed). + * Status messages for each process (exporting, fetching, importing, failed). */ export const PROCESS_STATUS = { [PROCESS_NAMES.AM_SPACE_METADATA]: { @@ -47,4 +52,20 @@ export const PROCESS_STATUS = { DOWNLOADING: 'Downloading asset files...', FAILED: 'Failed to download assets.', }, + [PROCESS_NAMES.AM_IMPORT_FIELDS]: { + IMPORTING: 'Importing shared fields...', + FAILED: 'Failed to import fields.', + }, + [PROCESS_NAMES.AM_IMPORT_ASSET_TYPES]: { + IMPORTING: 'Importing shared asset types...', + FAILED: 'Failed to import asset types.', + }, + [PROCESS_NAMES.AM_IMPORT_FOLDERS]: { + IMPORTING: 'Importing folders...', + FAILED: 'Failed to import folders.', + }, + [PROCESS_NAMES.AM_IMPORT_ASSETS]: { + IMPORTING: 'Importing assets...', + FAILED: 'Failed to import assets.', + }, } as const; diff --git a/packages/contentstack-asset-management/src/export/assets.ts b/packages/contentstack-asset-management/src/export/assets.ts index 140b56645..280166914 100644 --- a/packages/contentstack-asset-management/src/export/assets.ts +++ b/packages/contentstack-asset-management/src/export/assets.ts @@ -21,8 +21,8 @@ export default class ExportAssets extends AssetManagementExportAdapter { log.debug(`Fetching folders and assets for space ${workspace.space_uid}`, this.exportContext.context); const [folders, assetsData] = await Promise.all([ - this.getWorkspaceFolders(workspace.space_uid), - this.getWorkspaceAssets(workspace.space_uid), + this.getWorkspaceFolders(workspace.space_uid, workspace.uid), + this.getWorkspaceAssets(workspace.space_uid, workspace.uid), ]); await writeFile(pResolve(assetsDir, 'folders.json'), JSON.stringify(folders, null, 2)); diff --git a/packages/contentstack-asset-management/src/export/spaces.ts b/packages/contentstack-asset-management/src/export/spaces.ts index cf3ff2c30..a5abd54a7 100644 --- a/packages/contentstack-asset-management/src/export/spaces.ts +++ b/packages/contentstack-asset-management/src/export/spaces.ts @@ -29,15 +29,8 @@ export class ExportSpaces { } async start(): Promise { - const { - linkedWorkspaces, - exportDir, - branchName, - assetManagementUrl, - org_uid, - context, - securedAssets, - } = this.options; + const { linkedWorkspaces, exportDir, branchName, assetManagementUrl, org_uid, apiKey, context, securedAssets } = + this.options; if (!linkedWorkspaces.length) { log.debug('No linked workspaces to export', context); @@ -54,7 +47,9 @@ export class ExportSpaces { const totalSteps = 2 + linkedWorkspaces.length * 4; const progress = this.createProgress(); progress.addProcess(AM_MAIN_PROCESS_NAME, totalSteps); - progress.startProcess(AM_MAIN_PROCESS_NAME).updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_FIELDS].FETCHING, AM_MAIN_PROCESS_NAME); + progress + .startProcess(AM_MAIN_PROCESS_NAME) + .updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_FIELDS].FETCHING, AM_MAIN_PROCESS_NAME); const apiConfig: AssetManagementAPIConfig = { baseURL: assetManagementUrl, diff --git a/packages/contentstack-asset-management/src/import/asset-types.ts b/packages/contentstack-asset-management/src/import/asset-types.ts new file mode 100644 index 000000000..9305657c5 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/asset-types.ts @@ -0,0 +1,88 @@ +import omit from 'lodash/omit'; +import isEqual from 'lodash/isEqual'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; + +const STRIP_KEYS = ['created_at', 'created_by', 'updated_at', 'updated_by', 'is_system', 'category', 'preview_image_url', 'category_detail']; + +/** + * Reads shared asset types from `spaces/asset_types/asset-types.json` and POSTs + * each to the target org-level AM endpoint (`POST /api/asset_types`). + * + * Strategy: Fetch → Diff → Create only missing, warn on conflict + * 1. Fetch asset types that already exist in the target org. + * 2. Skip entries where is_system=true (platform-owned, cannot be created via API). + * 3. If uid already exists and definition differs → warn and skip. + * 4. If uid already exists and definition matches → silently skip. + * 5. Strip read-only/computed keys from the POST body before creating new asset types. + */ +export default class ImportAssetTypes extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + async start(): Promise { + await this.init(); + + const dir = this.getAssetTypesDir(); + const items = await this.readAllChunkedJson>(dir, 'asset-types.json'); + + if (items.length === 0) { + log.debug('No shared asset types to import', this.importContext.context); + return; + } + + // Fetch existing asset types from the target org keyed by uid for diff comparison. + // Asset types are org-level; the spaceUid param in getWorkspaceAssetTypes is unused in the path. + const existingByUid = new Map>(); + try { + const existing = await this.getWorkspaceAssetTypes(''); + for (const at of existing.asset_types ?? []) { + existingByUid.set(at.uid, at as Record); + } + log.debug(`Target org has ${existingByUid.size} existing asset type(s)`, this.importContext.context); + } catch (e) { + log.debug(`Could not fetch existing asset types, will attempt to create all: ${e}`, this.importContext.context); + } + + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + + for (const assetType of items) { + const uid = assetType.uid as string; + + if (assetType.is_system) { + log.debug(`Skipping system asset type: ${uid}`, this.importContext.context); + continue; + } + + const existing = existingByUid.get(uid); + if (existing) { + const exportedClean = omit(assetType, STRIP_KEYS); + const existingClean = omit(existing, STRIP_KEYS); + if (!isEqual(exportedClean, existingClean)) { + log.warn( + `Asset type "${uid}" already exists in the target org with a different definition. Skipping — to apply the exported definition, delete the asset type from the target org first.`, + this.importContext.context, + ); + } else { + log.debug(`Asset type "${uid}" already exists with matching definition, skipping`, this.importContext.context); + } + this.tick(true, `asset-type: ${uid} (skipped, already exists)`, null, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + continue; + } + + const payload = omit(assetType, STRIP_KEYS); + try { + await this.createAssetType(payload as any); + this.tick(true, `asset-type: ${uid}`, null, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + log.debug(`Imported asset type: ${uid}`, this.importContext.context); + } catch (e) { + this.tick(false, `asset-type: ${uid}`, (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].FAILED, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + log.debug(`Failed to import asset type ${uid}: ${e}`, this.importContext.context); + } + } + } +} diff --git a/packages/contentstack-asset-management/src/import/assets.ts b/packages/contentstack-asset-management/src/import/assets.ts new file mode 100644 index 000000000..ab7986db1 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/assets.ts @@ -0,0 +1,227 @@ +import { resolve as pResolve, join } from 'node:path'; +import { existsSync, readFileSync } from 'node:fs'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import { getArrayFromResponse } from '../utils/export-helpers'; +import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; + +type FolderRecord = { + uid: string; + title: string; + description?: string; + parent_uid?: string; +}; + +type AssetRecord = { + uid: string; + url: string; + filename?: string; + file_name?: string; + parent_uid?: string; + title?: string; + description?: string; +}; + +/** + * Imports folders and assets for a single AM space. + * - Reads `spaces/{spaceUid}/assets/folders.json` → creates folders, builds folderUidMap + * - Reads chunked `assets.json` → uploads each file from `files/{oldUid}/{filename}` + * - Builds UID and URL mapper entries for entries.ts consumption + * Mirrors ExportAssets. + */ +export default class ImportAssets extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + /** + * Loads chunked `assets.json` when present; shared by reuse (identity maps) and upload path. + */ + private async loadExportedAssetItems(spaceDir: string): Promise { + const assetsDir = pResolve(spaceDir, 'assets'); + if (!existsSync(join(assetsDir, 'assets.json'))) { + return null; + } + return this.readAllChunkedJson(assetsDir, 'assets.json'); + } + + /** + * Build identity uid/url mappers from export JSON only (reuse path — no upload). + * Keys and values are equal so lookupAssets contract is satisfied without remapping. + */ + async buildIdentityMappersFromExport( + spaceDir: string, + ): Promise<{ uidMap: Record; urlMap: Record }> { + const uidMap: Record = {}; + const urlMap: Record = {}; + + const assetItems = await this.loadExportedAssetItems(spaceDir); + if (!assetItems) { + log.debug( + `No assets.json index in ${pResolve(spaceDir, 'assets')}, identity mappers empty`, + this.importContext.context, + ); + return { uidMap, urlMap }; + } + log.debug( + `Building identity mappers for ${assetItems.length} exported asset(s) (reuse path)`, + this.importContext.context, + ); + + for (const asset of assetItems) { + if (asset.uid) { + uidMap[asset.uid] = asset.uid; + } + if (asset.url) { + urlMap[asset.url] = asset.url; + } + } + + return { uidMap, urlMap }; + } + + async start( + newSpaceUid: string, + spaceDir: string, + ): Promise<{ uidMap: Record; urlMap: Record }> { + const assetsDir = pResolve(spaceDir, 'assets'); + const uidMap: Record = {}; + const urlMap: Record = {}; + + // ----------------------------------------------------------------------- + // 1. Import folders + // ----------------------------------------------------------------------- + const folderUidMap: Record = {}; + const foldersFilePath = join(assetsDir, 'folders.json'); + + if (existsSync(foldersFilePath)) { + let foldersData: unknown; + try { + foldersData = JSON.parse(readFileSync(foldersFilePath, 'utf8')); + } catch (e) { + log.debug(`Could not read folders.json: ${e}`, this.importContext.context); + } + + if (foldersData) { + const folders = getArrayFromResponse(foldersData, 'folders') as FolderRecord[]; + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FOLDERS].IMPORTING, PROCESS_NAMES.AM_IMPORT_FOLDERS); + log.debug(`Importing ${folders.length} folder(s) for space ${newSpaceUid}`, this.importContext.context); + await this.importFolders(newSpaceUid, folders, folderUidMap); + } + } + + // ----------------------------------------------------------------------- + // 2. Import assets (chunked) + // ----------------------------------------------------------------------- + const assetItems = await this.loadExportedAssetItems(spaceDir); + if (!assetItems) { + log.debug(`No assets.json index found in ${assetsDir}, skipping asset upload`, this.importContext.context); + return { uidMap, urlMap }; + } + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSETS].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSETS); + log.debug(`Uploading ${assetItems.length} asset(s) for space ${newSpaceUid}`, this.importContext.context); + + for (const asset of assetItems) { + const oldUid = asset.uid; + const filename = asset.filename ?? asset.file_name ?? 'asset'; + const filePath = pResolve(assetsDir, 'files', oldUid, filename); + + if (!existsSync(filePath)) { + log.debug(`Asset file not found: ${filePath}, skipping`, this.importContext.context); + this.tick(false, `asset: ${oldUid}`, 'File not found on disk', PROCESS_NAMES.AM_IMPORT_ASSETS); + continue; + } + + const assetParent = asset.parent_uid && asset.parent_uid !== 'root' ? asset.parent_uid : undefined; + const mappedParentUid = assetParent ? folderUidMap[assetParent] ?? undefined : undefined; + + try { + const { asset: created } = await this.uploadAsset(newSpaceUid, filePath, { + title: asset.title ?? filename, + description: asset.description, + parent_uid: mappedParentUid, + }); + + uidMap[oldUid] = created.uid; + + // Map old AM direct URL → new AM direct URL. + if (asset.url && created.url) { + urlMap[asset.url] = created.url; + } + + this.tick(true, `asset: ${oldUid}`, null, PROCESS_NAMES.AM_IMPORT_ASSETS); + log.debug(`Uploaded asset ${oldUid} → ${created.uid}`, this.importContext.context); + } catch (e) { + this.tick( + false, + `asset: ${oldUid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSETS].FAILED, + PROCESS_NAMES.AM_IMPORT_ASSETS, + ); + log.debug(`Failed to upload asset ${oldUid}: ${e}`, this.importContext.context); + } + } + + return { uidMap, urlMap }; + } + + /** + * Creates folders respecting hierarchy: parents before children. + * Uses multiple passes to handle arbitrary depth without requiring sorted input. + */ + private async importFolders( + newSpaceUid: string, + folders: FolderRecord[], + folderUidMap: Record, + ): Promise { + let remaining = [...folders]; + let prevLength = -1; + + while (remaining.length > 0 && remaining.length !== prevLength) { + prevLength = remaining.length; + const nextPass: FolderRecord[] = []; + + for (const folder of remaining) { + const { parent_uid: parentUid } = folder; + // "root" is the AM API sentinel for a top-level folder + const isRootParent = !parentUid || parentUid === 'root'; + const parentMapped = isRootParent || folderUidMap[parentUid] !== undefined; + + if (!parentMapped) { + nextPass.push(folder); + continue; + } + + try { + const { folder: created } = await this.createFolder(newSpaceUid, { + title: folder.title, + description: folder.description, + parent_uid: isRootParent ? undefined : folderUidMap[parentUid], + }); + folderUidMap[folder.uid] = created.uid; + this.tick(true, `folder: ${folder.uid}`, null, PROCESS_NAMES.AM_IMPORT_FOLDERS); + log.debug(`Created folder ${folder.uid} → ${created.uid}`, this.importContext.context); + } catch (e) { + this.tick( + false, + `folder: ${folder.uid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FOLDERS].FAILED, + PROCESS_NAMES.AM_IMPORT_FOLDERS, + ); + log.debug(`Failed to create folder ${folder.uid}: ${e}`, this.importContext.context); + } + } + + remaining = nextPass; + } + + if (remaining.length > 0) { + log.debug( + `${remaining.length} folder(s) could not be imported (unresolved parent UIDs)`, + this.importContext.context, + ); + } + } +} diff --git a/packages/contentstack-asset-management/src/import/base.ts b/packages/contentstack-asset-management/src/import/base.ts new file mode 100644 index 000000000..db82f1462 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/base.ts @@ -0,0 +1,96 @@ +import { resolve as pResolve } from 'node:path'; +import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; +import { AM_MAIN_PROCESS_NAME } from '../constants/index'; + +export type { ImportContext }; + +/** + * Base class for all AM 2.0 import modules. Mirrors AssetManagementExportAdapter + * but carries ImportContext (spacesRootPath, apiKey, host, etc.) instead of ExportContext. + */ +export class AssetManagementImportAdapter extends AssetManagementAdapter { + protected readonly apiConfig: AssetManagementAPIConfig; + protected readonly importContext: ImportContext; + protected progressManager: CLIProgressManager | null = null; + protected parentProgressManager: CLIProgressManager | null = null; + protected readonly processName: string = AM_MAIN_PROCESS_NAME; + + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig); + this.apiConfig = apiConfig; + this.importContext = importContext; + } + + public setParentProgressManager(parent: CLIProgressManager): void { + this.parentProgressManager = parent; + } + + protected get progressOrParent(): CLIProgressManager | null { + return this.parentProgressManager ?? this.progressManager; + } + + protected createNestedProgress(moduleName: string): CLIProgressManager { + if (this.parentProgressManager) { + this.progressManager = this.parentProgressManager; + return this.parentProgressManager; + } + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; + this.progressManager = CLIProgressManager.createNested(moduleName, showConsoleLogs); + return this.progressManager; + } + + protected tick(success: boolean, itemName: string, error: string | null, processName?: string): void { + this.progressOrParent?.tick?.(success, itemName, error, processName ?? this.processName); + } + + protected updateStatus(message: string, processName?: string): void { + this.progressOrParent?.updateStatus?.(message, processName ?? this.processName); + } + + protected completeProcess(processName: string, success: boolean): void { + if (!this.parentProgressManager) { + this.progressManager?.completeProcess?.(processName, success); + } + } + + protected get spacesRootPath(): string { + return this.importContext.spacesRootPath; + } + + protected getAssetTypesDir(): string { + return pResolve(this.importContext.spacesRootPath, 'asset_types'); + } + + protected getFieldsDir(): string { + return pResolve(this.importContext.spacesRootPath, 'fields'); + } + + /** + * Reads all items from a FsUtility chunked JSON store (index file + chunk files). + * Returns a flat array of all items across all chunks. + */ + protected async readAllChunkedJson>(dir: string, indexFileName: string): Promise { + try { + const fs = new FsUtility({ basePath: dir, indexFileName }); + const indexer = fs.indexFileContent; + const items: T[] = []; + for (const _ in indexer) { + const chunk = await fs.readChunkFiles.next().catch((err: unknown): null => { + log.debug(`Error reading chunk: ${err}`, this.importContext.context); + return null; + }); + if (chunk) { + items.push(...(Object.values(chunk as Record))); + } + } + return items; + } catch (err) { + log.debug(`readAllChunkedJson failed for ${dir}/${indexFileName}: ${err}`, this.importContext.context); + return []; + } + } +} diff --git a/packages/contentstack-asset-management/src/import/fields.ts b/packages/contentstack-asset-management/src/import/fields.ts new file mode 100644 index 000000000..86f45086a --- /dev/null +++ b/packages/contentstack-asset-management/src/import/fields.ts @@ -0,0 +1,88 @@ +import omit from 'lodash/omit'; +import isEqual from 'lodash/isEqual'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; + +const STRIP_KEYS = ['created_at', 'created_by', 'updated_at', 'updated_by', 'is_system', 'asset_types_count']; + +/** + * Reads shared fields from `spaces/fields/fields.json` and POSTs each to the + * target org-level AM fields endpoint (`POST /api/fields`). + * + * Strategy: Fetch → Diff → Create only missing, warn on conflict + * 1. Fetch fields that already exist in the target org. + * 2. Skip entries where is_system=true (platform-owned, cannot be created via API). + * 3. If uid already exists and definition differs → warn and skip. + * 4. If uid already exists and definition matches → silently skip. + * 5. Strip read-only/computed keys from the POST body before creating new fields. + */ +export default class ImportFields extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + async start(): Promise { + await this.init(); + + const dir = this.getFieldsDir(); + const items = await this.readAllChunkedJson>(dir, 'fields.json'); + + if (items.length === 0) { + log.debug('No shared fields to import', this.importContext.context); + return; + } + + // Fetch existing fields from the target org keyed by uid for diff comparison. + // Fields are org-level; the spaceUid param in getWorkspaceFields is unused in the path. + const existingByUid = new Map>(); + try { + const existing = await this.getWorkspaceFields(''); + for (const f of existing.fields ?? []) { + existingByUid.set(f.uid, f as Record); + } + log.debug(`Target org has ${existingByUid.size} existing field(s)`, this.importContext.context); + } catch (e) { + log.debug(`Could not fetch existing fields, will attempt to create all: ${e}`, this.importContext.context); + } + + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FIELDS].IMPORTING, PROCESS_NAMES.AM_IMPORT_FIELDS); + + for (const field of items) { + const uid = field.uid as string; + + if (field.is_system) { + log.debug(`Skipping system field: ${uid}`, this.importContext.context); + continue; + } + + const existing = existingByUid.get(uid); + if (existing) { + const exportedClean = omit(field, STRIP_KEYS); + const existingClean = omit(existing, STRIP_KEYS); + if (!isEqual(exportedClean, existingClean)) { + log.warn( + `Field "${uid}" already exists in the target org with a different definition. Skipping — to apply the exported definition, delete the field from the target org first.`, + this.importContext.context, + ); + } else { + log.debug(`Field "${uid}" already exists with matching definition, skipping`, this.importContext.context); + } + this.tick(true, `field: ${uid} (skipped, already exists)`, null, PROCESS_NAMES.AM_IMPORT_FIELDS); + continue; + } + + const payload = omit(field, STRIP_KEYS); + try { + await this.createField(payload as any); + this.tick(true, `field: ${uid}`, null, PROCESS_NAMES.AM_IMPORT_FIELDS); + log.debug(`Imported field: ${uid}`, this.importContext.context); + } catch (e) { + this.tick(false, `field: ${uid}`, (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FIELDS].FAILED, PROCESS_NAMES.AM_IMPORT_FIELDS); + log.debug(`Failed to import field ${uid}: ${e}`, this.importContext.context); + } + } + } +} diff --git a/packages/contentstack-asset-management/src/import/index.ts b/packages/contentstack-asset-management/src/import/index.ts new file mode 100644 index 000000000..61d8a457e --- /dev/null +++ b/packages/contentstack-asset-management/src/import/index.ts @@ -0,0 +1,7 @@ +export { ImportSpaces } from './spaces'; +export { default as ImportWorkspace } from './workspaces'; +export { default as ImportAssets } from './assets'; +export { default as ImportFields } from './fields'; +export { default as ImportAssetTypes } from './asset-types'; +export { AssetManagementImportAdapter } from './base'; +export type { ImportContext } from './base'; diff --git a/packages/contentstack-asset-management/src/import/spaces.ts b/packages/contentstack-asset-management/src/import/spaces.ts new file mode 100644 index 000000000..f43885154 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/spaces.ts @@ -0,0 +1,165 @@ +import { resolve as pResolve, join } from 'node:path'; +import { readdirSync, statSync } from 'node:fs'; +import { log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import type { + AssetManagementAPIConfig, + AssetManagementImportOptions, + ImportContext, + ImportResult, + SpaceMapping, +} from '../types/asset-management-api'; +import { AM_MAIN_PROCESS_NAME } from '../constants/index'; +import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; +import ImportAssetTypes from './asset-types'; +import ImportFields from './fields'; +import ImportWorkspace from './workspaces'; + +/** + * Top-level orchestrator for AM 2.0 import. + * Mirrors ExportSpaces: creates shared fields + asset types, then imports each space. + * Returns combined uidMap, urlMap, and spaceMappings for the bridge module. + */ +export class ImportSpaces { + private readonly options: AssetManagementImportOptions; + private parentProgressManager: CLIProgressManager | null = null; + private progressManager: CLIProgressManager | null = null; + + constructor(options: AssetManagementImportOptions) { + this.options = options; + } + + public setParentProgressManager(parent: CLIProgressManager): void { + this.parentProgressManager = parent; + } + + async start(): Promise { + const { contentDir, assetManagementUrl, org_uid, apiKey, host, sourceApiKey, context } = this.options; + + const spacesRootPath = pResolve(contentDir, 'spaces'); + + const importContext: ImportContext = { + spacesRootPath, + sourceApiKey, + apiKey, + host, + org_uid, + context, + }; + + const apiConfig: AssetManagementAPIConfig = { + baseURL: assetManagementUrl, + headers: { organization_uid: org_uid }, + context, + }; + + // Discover space directories + let spaceDirs: string[] = []; + try { + spaceDirs = readdirSync(spacesRootPath).filter((entry) => { + try { + return statSync(join(spacesRootPath, entry)).isDirectory() && entry.startsWith('am'); + } catch { + return false; + } + }); + } catch (e) { + log.debug(`Could not read spaces root path ${spacesRootPath}: ${e}`, context); + } + + const totalSteps = 2 + spaceDirs.length * 2; + const progress = this.createProgress(); + progress.addProcess(AM_MAIN_PROCESS_NAME, totalSteps); + progress.startProcess(AM_MAIN_PROCESS_NAME); + + const allUidMap: Record = {}; + const allUrlMap: Record = {}; + const allSpaceUidMap: Record = {}; + const spaceMappings: SpaceMapping[] = []; + let hasFailures = false; + + // Space UIDs already present in the target org — reuse when export dir name matches a uid here. + const existingSpaceUids = new Set(); + try { + const adapterForList = new AssetManagementAdapter(apiConfig); + await adapterForList.init(); + const { spaces } = await adapterForList.listSpaces(); + for (const s of spaces) { + if (s.uid) existingSpaceUids.add(s.uid); + } + log.debug(`Found ${existingSpaceUids.size} existing space uid(s) in target org`, context); + } catch (e) { + log.debug(`Could not fetch existing spaces — reuse-by-uid disabled: ${e}`, context); + } + + try { + // 1. Import shared fields + progress.updateStatus(`Importing shared fields...`, AM_MAIN_PROCESS_NAME); + const fieldsImporter = new ImportFields(apiConfig, importContext); + fieldsImporter.setParentProgressManager(progress); + await fieldsImporter.start(); + + // 2. Import shared asset types + progress.updateStatus('Importing shared asset types...', AM_MAIN_PROCESS_NAME); + const assetTypesImporter = new ImportAssetTypes(apiConfig, importContext); + assetTypesImporter.setParentProgressManager(progress); + await assetTypesImporter.start(); + + // 3. Import each space — continue on failure so partially-imported data is never lost + for (const spaceUid of spaceDirs) { + const spaceDir = join(spacesRootPath, spaceUid); + progress.updateStatus(`Importing space: ${spaceUid}...`, AM_MAIN_PROCESS_NAME); + log.debug(`Importing space: ${spaceUid}`, context); + + try { + const workspaceImporter = new ImportWorkspace(apiConfig, importContext); + workspaceImporter.setParentProgressManager(progress); + const result = await workspaceImporter.start(spaceUid, spaceDir, existingSpaceUids); + + // Newly created spaces get a new uid — add so later iterations in this run see it. + existingSpaceUids.add(result.newSpaceUid); + + Object.assign(allUidMap, result.uidMap); + Object.assign(allUrlMap, result.urlMap); + allSpaceUidMap[result.oldSpaceUid] = result.newSpaceUid; + spaceMappings.push({ + oldSpaceUid: result.oldSpaceUid, + newSpaceUid: result.newSpaceUid, + workspaceUid: result.workspaceUid, + isDefault: result.isDefault, + }); + + log.debug(`Imported space ${spaceUid} → ${result.newSpaceUid}`, context); + } catch (err) { + hasFailures = true; + progress.tick( + false, + `space: ${spaceUid}`, + (err as Error)?.message ?? 'Failed to import space', + AM_MAIN_PROCESS_NAME, + ); + log.debug(`Failed to import space ${spaceUid}: ${err}`, context); + } + } + + progress.completeProcess(AM_MAIN_PROCESS_NAME, !hasFailures); + log.debug('Asset Management 2.0 import completed', context); + } catch (err) { + progress.completeProcess(AM_MAIN_PROCESS_NAME, false); + throw err; + } + + return { uidMap: allUidMap, urlMap: allUrlMap, spaceMappings, spaceUidMap: allSpaceUidMap }; + } + + private createProgress(): CLIProgressManager { + if (this.parentProgressManager) { + this.progressManager = this.parentProgressManager; + return this.parentProgressManager; + } + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; + this.progressManager = CLIProgressManager.createNested(AM_MAIN_PROCESS_NAME, showConsoleLogs); + return this.progressManager; + } +} diff --git a/packages/contentstack-asset-management/src/import/workspaces.ts b/packages/contentstack-asset-management/src/import/workspaces.ts new file mode 100644 index 000000000..5d13450de --- /dev/null +++ b/packages/contentstack-asset-management/src/import/workspaces.ts @@ -0,0 +1,82 @@ +import { join } from 'node:path'; +import { readFileSync } from 'node:fs'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext, SpaceMapping } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import ImportAssets from './assets'; +import { PROCESS_NAMES } from '../constants/index'; + +type WorkspaceResult = SpaceMapping & { + uidMap: Record; + urlMap: Record; +}; + +/** + * Handles import for a single AM 2.0 space directory. + * Reads `metadata.json`, creates the space in the target org when its uid is not + * already present, or reuses the existing space and emits identity mappers only. + * Returns the SpaceMapping plus UID/URL maps for the mapper files. + */ +export default class ImportWorkspace extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + async start( + oldSpaceUid: string, + spaceDir: string, + existingSpaceUids: Set = new Set(), + ): Promise { + await this.init(); + + // Read exported metadata + const metadataPath = join(spaceDir, 'metadata.json'); + let metadata: Record = {}; + try { + metadata = JSON.parse(readFileSync(metadataPath, 'utf8')) as Record; + } catch (e) { + log.debug(`Could not read metadata.json for space ${oldSpaceUid}: ${e}`, this.importContext.context); + } + + const exportedTitle = (metadata.title as string) ?? oldSpaceUid; + const description = metadata.description as string | undefined; + const isDefault = (metadata.is_default as boolean) ?? false; + const workspaceUid = 'main'; + + const assetsImporter = new ImportAssets(this.apiConfig, this.importContext); + if (this.progressOrParent) assetsImporter.setParentProgressManager(this.progressOrParent); + + // Reuse: target org already has a space with the same uid as the export directory. + if (existingSpaceUids.has(oldSpaceUid)) { + log.info( + `Reusing existing AM space "${oldSpaceUid}" (uid matches export directory); skipping create and upload.`, + this.importContext.context, + ); + const newSpaceUid = oldSpaceUid; + const { uidMap, urlMap } = await assetsImporter.buildIdentityMappersFromExport(spaceDir); + this.tick(true, `space: ${oldSpaceUid} → ${newSpaceUid} (reused)`, null, PROCESS_NAMES.AM_SPACE_METADATA); + return { + oldSpaceUid, + newSpaceUid, + workspaceUid, + isDefault, + uidMap, + urlMap, + }; + } + + // Create new space with exact exported title + log.debug(`Creating space "${exportedTitle}" (old uid: ${oldSpaceUid})`, this.importContext.context); + + const { space } = await this.createSpace({ title: exportedTitle, description }); + const newSpaceUid = space.uid; + + log.debug(`Created space ${newSpaceUid} (old: ${oldSpaceUid})`, this.importContext.context); + this.tick(true, `space: ${oldSpaceUid} → ${newSpaceUid}`, null, PROCESS_NAMES.AM_SPACE_METADATA); + + const { uidMap, urlMap } = await assetsImporter.start(newSpaceUid, spaceDir); + + return { oldSpaceUid, newSpaceUid, workspaceUid, isDefault, uidMap, urlMap }; + } +} diff --git a/packages/contentstack-asset-management/src/index.ts b/packages/contentstack-asset-management/src/index.ts index f0ff59bdd..c66c638d0 100644 --- a/packages/contentstack-asset-management/src/index.ts +++ b/packages/contentstack-asset-management/src/index.ts @@ -2,3 +2,4 @@ export * from './constants/index'; export * from './types'; export * from './utils'; export * from './export'; +export * from './import'; diff --git a/packages/contentstack-asset-management/src/types/asset-management-api.ts b/packages/contentstack-asset-management/src/types/asset-management-api.ts index 733ecada7..3d3b01d1b 100644 --- a/packages/contentstack-asset-management/src/types/asset-management-api.ts +++ b/packages/contentstack-asset-management/src/types/asset-management-api.ts @@ -36,6 +36,9 @@ export type Space = { /** Response shape of GET /api/spaces/{space_uid}. */ export type SpaceResponse = { space: Space }; +/** Response shape of GET /api/spaces (list all spaces in the org). */ +export type SpacesListResponse = { spaces: Space[]; count?: number }; + /** * Field structure from GET /api/fields (org-level). */ @@ -118,10 +121,11 @@ export type AssetManagementAPIConfig = { */ export interface IAssetManagementAdapter { init(): Promise; + listSpaces(): Promise; getSpace(spaceUid: string): Promise; getWorkspaceFields(spaceUid: string): Promise; - getWorkspaceAssets(spaceUid: string): Promise; - getWorkspaceFolders(spaceUid: string): Promise; + getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise; + getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise; getWorkspaceAssetTypes(spaceUid: string): Promise; } @@ -137,4 +141,119 @@ export type AssetManagementExportOptions = { context?: Record; /** When true, the AM package will add authtoken to asset download URLs. */ securedAssets?: boolean; + /** + * API key of the stack being exported. + * Saved to `spaces/export-metadata.json` so that during import the URL mapper + * can reconstruct old CMA proxy URLs (format: /v3/assets/{apiKey}/{amUid}/...). + */ + apiKey?: string; +}; + +// --------------------------------------------------------------------------- +// Import types +// --------------------------------------------------------------------------- + +/** + * Context passed down to every import adapter class. + * Mirrors ExportContext but carries the import-specific fields needed for + * URL mapper reconstruction and API calls. + */ +export type ImportContext = { + /** Absolute path to the root `spaces/` directory inside the backup/content dir. */ + spacesRootPath: string; + /** Source stack API key — used to reconstruct old CMA proxy URLs. */ + sourceApiKey?: string; + /** Target stack API key — used to build new CMA proxy URLs. */ + apiKey: string; + /** Target CMA host (may include /v3), e.g. "https://api.contentstack.io/v3". */ + host: string; + /** Target org UID — required as `x-organization-uid` header when creating spaces. */ + org_uid: string; + /** Optional logging context (same shape as ExportConfig.context). */ + context?: Record; +}; + +/** + * Options accepted by the top-level `ImportSpaces` class. + */ +export type AssetManagementImportOptions = { + /** Absolute path to the root content / backup directory. */ + contentDir: string; + /** AM 2.0 base URL (e.g. "https://am.contentstack.io"). */ + assetManagementUrl: string; + /** Target organisation UID. */ + org_uid: string; + /** Target stack API key. */ + apiKey: string; + /** Target CMA host. */ + host: string; + /** Source stack API key — used for old CMA proxy URL reconstruction. */ + sourceApiKey?: string; + /** Optional logging context. */ + context?: Record; +}; + +/** + * Maps an old source-org space UID to the newly created target-org space UID. + */ +export type SpaceMapping = { + oldSpaceUid: string; + newSpaceUid: string; + /** Workspace identifier inside the space (typically "main"). */ + workspaceUid: string; + isDefault: boolean; +}; + +/** + * The value returned by `ImportSpaces.start()`. + * Written to `mapper/assets/uid-mapping.json` and `mapper/assets/url-mapping.json` + * by the bridge module so `entries.ts` can resolve asset references. + */ +export type ImportResult = { + uidMap: Record; + urlMap: Record; + spaceMappings: SpaceMapping[]; + /** old space UID → new space UID, written to mapper/assets/space-uid-mapping.json */ + spaceUidMap: Record; +}; + +// --------------------------------------------------------------------------- +// Import payload types (confirmed from Postman collection) +// --------------------------------------------------------------------------- + +export type CreateSpacePayload = { + title: string; + description?: string; +}; + +export type CreateFolderPayload = { + title: string; + description?: string; + parent_uid?: string; +}; + +export type CreateAssetMetadata = { + title?: string; + description?: string; + parent_uid?: string; +}; + +export type CreateFieldPayload = { + uid: string; + title: string; + display_type?: string; + child?: unknown[]; + is_mandatory?: boolean; + is_multiple?: boolean; + [key: string]: unknown; +}; + +export type CreateAssetTypePayload = { + uid: string; + title: string; + description?: string; + content_type?: string; + file_extension?: string | string[]; + fields?: string[]; + [key: string]: unknown; }; diff --git a/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts b/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts index b159cc330..b26b2664a 100644 --- a/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts +++ b/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts @@ -1,11 +1,20 @@ +import { readFileSync } from 'node:fs'; +import { basename } from 'node:path'; import { HttpClient, log, authenticationHandler } from '@contentstack/cli-utilities'; import type { AssetManagementAPIConfig, AssetTypesResponse, + CreateAssetMetadata, + CreateAssetTypePayload, + CreateFieldPayload, + CreateFolderPayload, + CreateSpacePayload, FieldsResponse, IAssetManagementAdapter, + Space, SpaceResponse, + SpacesListResponse, } from '../types/asset-management-api'; export class AssetManagementAdapter implements IAssetManagementAdapter { @@ -89,6 +98,13 @@ export class AssetManagementAdapter implements IAssetManagementAdapter { } } + async listSpaces(): Promise { + log.debug('Fetching all spaces in org', this.config.context); + const result = await this.getSpaceLevel('', '/api/spaces', {}); + log.debug(`Fetched ${result?.count ?? result?.spaces?.length ?? '?'} space(s)`, this.config.context); + return result; + } + async getSpace(spaceUid: string): Promise { log.debug(`Fetching space: ${spaceUid}`, this.config.context); const path = `/api/spaces/${spaceUid}`; @@ -114,27 +130,30 @@ export class AssetManagementAdapter implements IAssetManagementAdapter { spaceUid: string, path: string, logLabel: string, + queryParams: Record = {}, ): Promise { log.debug(`Fetching ${logLabel} for space: ${spaceUid}`, this.config.context); - const result = await this.getSpaceLevel(spaceUid, path, {}); + const result = await this.getSpaceLevel(spaceUid, path, queryParams); const count = (result as { count?: number })?.count ?? (Array.isArray(result) ? result.length : '?'); log.debug(`Fetched ${logLabel} (count: ${count})`, this.config.context); return result; } - async getWorkspaceAssets(spaceUid: string): Promise { + async getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise { return this.getWorkspaceCollection( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, 'assets', + workspaceUid ? { workspace: workspaceUid } : {}, ); } - async getWorkspaceFolders(spaceUid: string): Promise { + async getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise { return this.getWorkspaceCollection( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/folders`, 'folders', + workspaceUid ? { workspace: workspaceUid } : {}, ); } @@ -146,4 +165,118 @@ export class AssetManagementAdapter implements IAssetManagementAdapter { log.debug(`Fetched asset types (count: ${result?.count ?? '?'})`, this.config.context); return result; } + + // --------------------------------------------------------------------------- + // POST helpers + // --------------------------------------------------------------------------- + + /** + * Build headers for outgoing POST requests. + */ + private async getPostHeaders(extraHeaders: Record = {}): Promise> { + await authenticationHandler.getAuthDetails(); + const token = authenticationHandler.accessToken; + const authHeader: Record = authenticationHandler.isOauthEnabled + ? { authorization: token } + : { access_token: token }; + return { + Accept: 'application/json', + 'x-cs-api-version': '4', + ...(this.config.headers ?? {}), + ...authHeader, + ...extraHeaders, + }; + } + + private async postJson(path: string, body: unknown, extraHeaders: Record = {}): Promise { + const baseUrl = this.config.baseURL?.replace(/\/$/, '') ?? ''; + const headers = await this.getPostHeaders({ 'Content-Type': 'application/json', ...extraHeaders }); + log.debug(`POST ${path}`, this.config.context); + const response = await fetch(`${baseUrl}${path}`, { + method: 'POST', + headers, + body: JSON.stringify(body), + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`AM API POST error: status ${response.status}, path ${path}, body: ${text}`); + } + return response.json() as Promise; + } + + private async postMultipart(path: string, form: FormData, extraHeaders: Record = {}): Promise { + const baseUrl = this.config.baseURL?.replace(/\/$/, '') ?? ''; + const headers = await this.getPostHeaders(extraHeaders); + log.debug(`POST (multipart) ${path}`, this.config.context); + const response = await fetch(`${baseUrl}${path}`, { + method: 'POST', + headers, + body: form, + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`AM API multipart POST error: status ${response.status}, path ${path}, body: ${text}`); + } + return response.json() as Promise; + } + + // --------------------------------------------------------------------------- + // Import API methods + // --------------------------------------------------------------------------- + + /** + * POST /api/spaces — creates a new space in the target org. + */ + async createSpace(payload: CreateSpacePayload): Promise<{ space: Space }> { + const orgUid = (this.config.headers as Record | undefined)?.organization_uid ?? ''; + return this.postJson<{ space: Space }>('/api/spaces', payload, { + 'x-organization-uid': orgUid, + }); + } + + /** + * POST /api/spaces/{spaceUid}/folders — creates a folder inside a space. + */ + async createFolder(spaceUid: string, payload: CreateFolderPayload): Promise<{ folder: { uid: string } }> { + return this.postJson<{ folder: { uid: string } }>(`/api/spaces/${encodeURIComponent(spaceUid)}/folders`, payload, { + space_key: spaceUid, + }); + } + + /** + * POST /api/spaces/{spaceUid}/assets — uploads an asset file as multipart form-data. + */ + async uploadAsset( + spaceUid: string, + filePath: string, + metadata: CreateAssetMetadata, + ): Promise<{ asset: { uid: string; url: string } }> { + const filename = basename(filePath); + const fileBuffer = readFileSync(filePath); + const blob = new Blob([fileBuffer]); + const form = new FormData(); + form.append('file', blob, filename); + if (metadata.title) form.append('title', metadata.title); + if (metadata.description) form.append('description', metadata.description); + if (metadata.parent_uid) form.append('parent_uid', metadata.parent_uid); + return this.postMultipart<{ asset: { uid: string; url: string } }>( + `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, + form, + { space_key: spaceUid }, + ); + } + + /** + * POST /api/fields — creates a shared field. + */ + async createField(payload: CreateFieldPayload): Promise<{ field: { uid: string } }> { + return this.postJson<{ field: { uid: string } }>('/api/fields', payload); + } + + /** + * POST /api/asset_types — creates a shared asset type. + */ + async createAssetType(payload: CreateAssetTypePayload): Promise<{ asset_type: { uid: string } }> { + return this.postJson<{ asset_type: { uid: string } }>('/api/asset_types', payload); + } } diff --git a/packages/contentstack-export/src/export/modules/assets.ts b/packages/contentstack-export/src/export/modules/assets.ts index 1812ba0ab..d896af4e3 100644 --- a/packages/contentstack-export/src/export/modules/assets.ts +++ b/packages/contentstack-export/src/export/modules/assets.ts @@ -75,6 +75,7 @@ export default class ExportAssets extends BaseClass { branchName: this.exportConfig.branchName || 'main', assetManagementUrl, org_uid: this.exportConfig.org_uid ?? '', + apiKey: this.exportConfig.apiKey, context: this.exportConfig.context as unknown as Record, securedAssets: this.exportConfig.securedAssets, }); diff --git a/packages/contentstack-import/package.json b/packages/contentstack-import/package.json index a1cddafbd..b64247075 100644 --- a/packages/contentstack-import/package.json +++ b/packages/contentstack-import/package.json @@ -5,6 +5,7 @@ "author": "Contentstack", "bugs": "https://github.com/contentstack/cli/issues", "dependencies": { + "@contentstack/cli-asset-management": "1.0.0", "@contentstack/cli-audit": "~2.0.0-beta.6", "@contentstack/cli-command": "~2.0.0-beta.2", "@contentstack/cli-utilities": "~2.0.0-beta.2", @@ -90,4 +91,4 @@ } }, "repository": "https://github.com/contentstack/cli" -} +} \ No newline at end of file diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index a18d7d077..fd0a5d1e4 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -101,6 +101,15 @@ const config: DefaultConfig = { folderValidKeys: ['name', 'parent_uid'], validKeys: ['title', 'parent_uid', 'description', 'tags'], }, + 'asset-management': { + dirName: 'spaces', + fieldsDir: 'fields', + assetTypesDir: 'asset_types', + foldersFileName: 'folders.json', + assetsFileName: 'assets.json', + uploadAssetsConcurrency: 2, + importFoldersConcurrency: 1, + }, 'assets-old': { dirName: 'assets', fileName: 'assets.json', diff --git a/packages/contentstack-import/src/constants/index.ts b/packages/contentstack-import/src/constants/index.ts index 5872ec29a..e3455787a 100644 --- a/packages/contentstack-import/src/constants/index.ts +++ b/packages/contentstack-import/src/constants/index.ts @@ -20,6 +20,7 @@ export const PATH_CONSTANTS = { INDEX: 'index.json', FOLDER_MAPPING: 'folder-mapping.json', VERSIONED_ASSETS: 'versioned-assets.json', + SPACE_UID_MAPPING: 'space-uid-mapping.json', }, /** Module subdirectory names within mapper */ diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index 007f21875..215d2775a 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -5,20 +5,17 @@ import unionBy from 'lodash/unionBy'; import orderBy from 'lodash/orderBy'; import isEmpty from 'lodash/isEmpty'; import uniq from 'lodash/uniq'; -import { existsSync } from 'node:fs'; +import { existsSync, mkdirSync } from 'node:fs'; import includes from 'lodash/includes'; import { v4 as uuid } from 'uuid'; import { resolve as pResolve, join } from 'node:path'; -import { - FsUtility, - log, - handleAndLogError, -} from '@contentstack/cli-utilities'; +import { FsUtility, log, handleAndLogError } from '@contentstack/cli-utilities'; +import { ImportSpaces } from '@contentstack/cli-asset-management'; import { PATH_CONSTANTS } from '../../constants'; import config from '../../config'; import { ModuleClassParams } from '../../types'; -import { formatDate, PROCESS_NAMES, MODULE_CONTEXTS, MODULE_NAMES, PROCESS_STATUS } from '../../utils'; +import { formatDate, fsUtil, PROCESS_NAMES, MODULE_CONTEXTS, MODULE_NAMES, PROCESS_STATUS } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; export default class ImportAssets extends BaseClass { @@ -42,22 +39,14 @@ export default class ImportAssets extends BaseClass { this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.ASSETS]; this.assetsPath = join(this.importConfig.backupDir, PATH_CONSTANTS.CONTENT_DIRS.ASSETS); - this.mapperDirPath = join( - this.importConfig.backupDir, - PATH_CONSTANTS.MAPPER, - PATH_CONSTANTS.MAPPER_MODULES.ASSETS, - ); + this.mapperDirPath = join(this.importConfig.backupDir, PATH_CONSTANTS.MAPPER, PATH_CONSTANTS.MAPPER_MODULES.ASSETS); this.assetUidMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.UID_MAPPING); this.assetUrlMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.URL_MAPPING); this.assetFolderUidMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.FOLDER_MAPPING); this.assetsRootPath = join(this.importConfig.backupDir, this.assetConfig.dirName); this.fs = new FsUtility({ basePath: this.mapperDirPath }); this.environments = this.fs.readFile( - join( - this.importConfig.backupDir, - PATH_CONSTANTS.CONTENT_DIRS.ENVIRONMENTS, - PATH_CONSTANTS.FILES.ENVIRONMENTS, - ), + join(this.importConfig.backupDir, PATH_CONSTANTS.CONTENT_DIRS.ENVIRONMENTS, PATH_CONSTANTS.FILES.ENVIRONMENTS), true, ) as Record; } @@ -70,6 +59,95 @@ export default class ImportAssets extends BaseClass { try { log.debug('Starting assets import process...', this.importConfig.context); + // AM 2.0: assetManagementEnabled is set in the config handler when spaces/ + am_v2 are detected. + if (this.importConfig.assetManagementEnabled) { + const assetManagementUrl = this.importConfig.assetManagementUrl; + if (!assetManagementUrl) { + log.info( + 'AM 2.0 export detected but assetManagementUrl is not configured in the region settings. Skipping AM 2.0 asset import.', + this.importConfig.context, + ); + return; + } + + const progress = this.createNestedProgress(this.currentModuleName); + try { + const importer = new ImportSpaces({ + contentDir: this.importConfig.contentDir, + assetManagementUrl, + org_uid: this.importConfig.org_uid ?? '', + apiKey: this.importConfig.apiKey, + host: this.importConfig.region?.cma ?? this.importConfig.host ?? '', + sourceApiKey: this.importConfig.source_stack, + context: this.importConfig.context as unknown as Record, + }); + importer.setParentProgressManager(progress); + + const { uidMap, urlMap, spaceMappings, spaceUidMap } = await importer.start(); + + const mapperDirPath = join( + this.importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.ASSETS, + ); + mkdirSync(mapperDirPath, { recursive: true }); + await fsUtil.writeFile(join(mapperDirPath, PATH_CONSTANTS.FILES.UID_MAPPING), uidMap); + await fsUtil.writeFile(join(mapperDirPath, PATH_CONSTANTS.FILES.URL_MAPPING), urlMap); + await fsUtil.writeFile(join(mapperDirPath, PATH_CONSTANTS.FILES.SPACE_UID_MAPPING), spaceUidMap); + log.debug('Wrote AM 2.0 mapper files (uid, url, space-uid)', this.importConfig.context); + + // Link newly-created spaces to the target stack via branch settings POST. + if (spaceMappings.length > 0) { + try { + const branchUid = this.importConfig.branchName ?? 'main'; + + // Fetch the current branch settings to get already-linked workspaces. + const branchData = (await this.stack.branch(branchUid).fetch({ include_settings: true })) as Record< + string, + any + >; + const currentLinked = (branchData?.settings?.am_v2?.linked_workspaces ?? []) as Array<{ + uid: string; + space_uid: string; + is_default: boolean; + operation?: string; + }>; + + const newWorkspaces = spaceMappings.map(({ newSpaceUid, workspaceUid }) => ({ + uid: workspaceUid, + space_uid: newSpaceUid, + is_default: false, + operation: 'LINK', + })); + + const combinedWorkspaces = [...currentLinked, ...newWorkspaces]; + + await this.stack.branch(branchUid).updateSettings({ + branch: { settings: { am_v2: { linked_workspaces: combinedWorkspaces } } }, + }); + log.success( + `Linked ${newWorkspaces.length} space(s) to branch "${branchUid}"`, + this.importConfig.context, + ); + } catch (linkErr) { + log.warn( + `AM 2.0 spaces were imported but could not be linked to the target stack: ${ + (linkErr as Error)?.message + }. Re-run the import or link manually.`, + this.importConfig.context, + ); + } + } + + this.completeProgressWithMessage(); + } catch (error) { + this.completeProgress(false, (error as Error)?.message ?? 'AM 2.0 asset import failed'); + throw error; + } + return; + } + // Legacy flow continues below + // Step 1: Analyze import data const [foldersCount, assetsCount, versionedAssetsCount, publishableAssetsCount] = await this.withLoadingSpinner( 'ASSETS: Analyzing import data...', @@ -221,9 +299,7 @@ export default class ImportAssets extends BaseClass { */ async importAssets(isVersion = false): Promise { const processName = isVersion ? 'import versioned assets' : 'import assets'; - const indexFileName = isVersion - ? PATH_CONSTANTS.FILES.VERSIONED_ASSETS - : this.assetConfig.fileName; + const indexFileName = isVersion ? PATH_CONSTANTS.FILES.VERSIONED_ASSETS : this.assetConfig.fileName; const basePath = isVersion ? join(this.assetsPath, 'versions') : this.assetsPath; const progressProcessName = isVersion ? PROCESS_NAMES.ASSET_VERSIONS : PROCESS_NAMES.ASSET_UPLOAD; diff --git a/packages/contentstack-import/src/import/modules/stack.ts b/packages/contentstack-import/src/import/modules/stack.ts index d6d920b0c..969ad4c94 100644 --- a/packages/contentstack-import/src/import/modules/stack.ts +++ b/packages/contentstack-import/src/import/modules/stack.ts @@ -1,4 +1,5 @@ import { join } from 'node:path'; +import { existsSync } from 'node:fs'; import { log, handleAndLogError } from '@contentstack/cli-utilities'; import { PATH_CONSTANTS } from '../../constants'; @@ -59,7 +60,6 @@ export default class ImportStack extends BaseClass { await this.importStackSettings(); this.completeProgressWithMessage(); - } catch (error) { this.completeProgress(false, 'Stack settings import failed'); handleAndLogError(error, { ...this.importConfig.context }); @@ -69,6 +69,17 @@ export default class ImportStack extends BaseClass { private async importStackSettings(): Promise { log.debug('Processing stack settings for import', this.importConfig.context); + // Old source-org space UIDs must not be written to the target stack — + // the asset-management module will apply the correct am_v2.linked_workspaces. + if (existsSync(join(this.importConfig.contentDir, 'spaces'))) { + const { am_v2, ...settingsWithoutAm } = this.stackSettings as any; + this.stackSettings = settingsWithoutAm; + log.debug( + 'Stripped am_v2 from stack settings; asset-management module will apply it after space creation', + this.importConfig.context, + ); + } + // Update environment UID mapping if live preview is configured if (this.stackSettings?.live_preview && this.stackSettings?.live_preview['default-env'] !== undefined) { const oldEnvUid = this.stackSettings.live_preview['default-env']; diff --git a/packages/contentstack-import/src/types/default-config.ts b/packages/contentstack-import/src/types/default-config.ts index 2b7c3bd95..dc7ce3153 100644 --- a/packages/contentstack-import/src/types/default-config.ts +++ b/packages/contentstack-import/src/types/default-config.ts @@ -72,6 +72,15 @@ export default interface DefaultConfig { uploadAssetsConcurrency: number; importFoldersConcurrency: number; }; + 'asset-management': { + dirName: string; + fieldsDir: string; + assetTypesDir: string; + foldersFileName: string; + assetsFileName: string; + uploadAssetsConcurrency: number; + importFoldersConcurrency: number; + }; content_types: { dirName: string; fileName: string; diff --git a/packages/contentstack-import/src/types/import-config.ts b/packages/contentstack-import/src/types/import-config.ts index 86db5668d..5025c82a3 100644 --- a/packages/contentstack-import/src/types/import-config.ts +++ b/packages/contentstack-import/src/types/import-config.ts @@ -58,6 +58,8 @@ export default interface ImportConfig extends DefaultConfig, ExternalConfig { personalizeProjectName?: string; 'exclude-global-modules': false; context: Context; + assetManagementUrl?: string; + assetManagementEnabled?: boolean; } type branch = { diff --git a/packages/contentstack-import/src/types/index.ts b/packages/contentstack-import/src/types/index.ts index 70bf9110a..a73584b36 100644 --- a/packages/contentstack-import/src/types/index.ts +++ b/packages/contentstack-import/src/types/index.ts @@ -19,6 +19,7 @@ export interface Region { cma: string; cda: string; uiHost: string; + assetManagementUrl?: string; } export interface InquirePayload { diff --git a/packages/contentstack-import/src/utils/import-config-handler.ts b/packages/contentstack-import/src/utils/import-config-handler.ts index 0eb0ee297..927e34483 100644 --- a/packages/contentstack-import/src/utils/import-config-handler.ts +++ b/packages/contentstack-import/src/utils/import-config-handler.ts @@ -1,5 +1,6 @@ import merge from 'merge'; import * as path from 'path'; +import { existsSync, readFileSync } from 'node:fs'; import { omit, filter, includes, isArray } from 'lodash'; import { configHandler, isAuthenticated, cliux, sanitizePath, log } from '@contentstack/cli-utilities'; import defaultConfig from '../config'; @@ -21,7 +22,6 @@ const setupConfig = async (importCmdFlags: any): Promise => { if (importCmdFlags['config']) { let externalConfig = await readFile(importCmdFlags['config']); - if (isArray(externalConfig['modules'])) { config.modules.types = filter(config.modules.types, (module) => includes(externalConfig['modules'], module)); externalConfig = omit(externalConfig, ['modules']); @@ -126,6 +126,40 @@ const setupConfig = async (importCmdFlags: any): Promise => { config['exclude-global-modules'] = importCmdFlags['exclude-global-modules']; } + const spacesDir = path.join(config.contentDir, 'spaces'); + const stackSettingsPath = path.join(config.contentDir, 'stack', 'settings.json'); + + if (existsSync(spacesDir) && existsSync(stackSettingsPath)) { + try { + const stackSettings = JSON.parse(readFileSync(stackSettingsPath, 'utf8')); + if (stackSettings?.am_v2) { + config.assetManagementEnabled = true; + config.assetManagementUrl = configHandler.get('region')?.assetManagementUrl; + + const branchesJsonCandidates = [ + path.join(config.contentDir, 'branches.json'), + path.join(config.contentDir, '..', 'branches.json'), + ]; + for (const branchesJsonPath of branchesJsonCandidates) { + if (existsSync(branchesJsonPath)) { + try { + const branches = JSON.parse(readFileSync(branchesJsonPath, 'utf8')); + const apiKey = branches?.[0]?.stackHeaders?.api_key; + if (apiKey) { + config.source_stack = apiKey; + } + } catch { + // branches.json unreadable — URL mapping will be skipped + } + break; + } + } + } + } catch { + // stack settings unreadable — not an AM 2.0 export we can process + } + } + // Add authentication details to config for context tracking config.authenticationMethod = authenticationMethod; log.debug('Import configuration setup completed.', { ...config }); From a01a6fc818593d3660b6280af2f6cf9b1a177982 Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Tue, 24 Mar 2026 16:14:42 +0530 Subject: [PATCH 05/22] chore: update implementation for AM import --- .talismanrc | 22 ++---------- .../src/constants/index.ts | 14 ++++++++ .../src/import/asset-types.ts | 18 ++++++++-- .../src/import/assets.ts | 30 ++++++++++++---- .../src/import/base.ts | 7 +++- .../src/import/fields.ts | 18 ++++++++-- .../src/import/spaces.ts | 34 +++++++++++++++++-- .../src/types/asset-management-api.ts | 15 ++++++-- .../src/utils/concurrent-batch.ts | 34 +++++++++++++++++++ .../src/utils/index.ts | 1 + .../src/import/modules/assets.ts | 27 ++++----------- 11 files changed, 161 insertions(+), 59 deletions(-) create mode 100644 packages/contentstack-asset-management/src/utils/concurrent-batch.ts diff --git a/.talismanrc b/.talismanrc index 20cce845c..01e7de833 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,24 +1,6 @@ fileignoreconfig: - - filename: packages/contentstack-export/src/export/modules/assets.ts - checksum: 1eacc8e86cb50fe283febe6688965854f420e02cf1b49555a15661fa0c3e3c7a - - filename: packages/contentstack-import/src/utils/import-config-handler.ts - checksum: f831cef1b7c3bd97bdbc170cff452350cee0f448d97df02e25aa41d6c4d64ad3 - filename: packages/contentstack-asset-management/src/import/asset-types.ts - checksum: a39caa373b2a736d1e57063326cfb2073ae78376efa931b27d2c7110997708a5 - - filename: packages/contentstack-asset-management/src/export/spaces.ts - checksum: 3ec11c8f710b60ae495c69344025587df2e6195c872a0c82feaf04ac044ecefa - - filename: packages/contentstack-import/src/import/modules/assets.ts - checksum: d9f4a29a29e8b8a2a36e498f2380d39e1c5c0ec13ff894ef450abd817f2a646e + checksum: 78f9334477ccb4b59a2b14b09214377f2eaf271e2dc68d1976321910d3b4e91c - filename: packages/contentstack-asset-management/src/import/fields.ts - checksum: bbae69c28ec69bf67c2c7b4df3620380ef3fca488b3288e137b65a60ee738b9e - - filename: packages/contentstack-asset-management/src/import/spaces.ts - checksum: 79cf2f1b55523d28c218d970155f887255a00dc095a941556b709d1f19c6a8a0 - - filename: packages/contentstack-asset-management/src/types/asset-management-api.ts - checksum: 716df03dcba70b2cc0f77b1f6338524553ba740080d7087a8699147c3ce8f0ba - - filename: packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts - checksum: 92bcad2feabc1954ead89b370d284b7af5f38ec1dca60a41752371977ef106ff - - filename: packages/contentstack-asset-management/src/import/base.ts - checksum: 513b55cf7e92bdbe8b815141822ba10579b6a728bec4424fc664794246ad33bb - - filename: packages/contentstack-asset-management/src/import/assets.ts - checksum: 2d34fa57f5ab269f6c535dff3242cc1135dbe1decd84fa0bc8997d0410d520b2 + checksum: 2a079cb7b58c11cf69068a9373f4b9a871487bc7ad03d2bc1efb45a3c1e333be version: '1.0' diff --git a/packages/contentstack-asset-management/src/constants/index.ts b/packages/contentstack-asset-management/src/constants/index.ts index 6763629d0..f2cad6efb 100644 --- a/packages/contentstack-asset-management/src/constants/index.ts +++ b/packages/contentstack-asset-management/src/constants/index.ts @@ -1,6 +1,20 @@ export const BATCH_SIZE = 50; export const CHUNK_FILE_SIZE_MB = 1; +/** Default parallel AM API calls when import caller does not set apiConcurrency. */ +export const DEFAULT_AM_API_CONCURRENCY = 5; + +/** + * Mapper output paths — must stay aligned with contentstack-import `PATH_CONSTANTS` + * (`mapper` / `assets` / uid, url, space-uid file names). + */ +export const IMPORT_ASSETS_MAPPER_DIR_SEGMENTS = ['mapper', 'assets'] as const; +export const IMPORT_ASSETS_MAPPER_FILES = { + UID_MAPPING: 'uid-mapping.json', + URL_MAPPING: 'url-mapping.json', + SPACE_UID_MAPPING: 'space-uid-mapping.json', +} as const; + /** * Main process name for Asset Management 2.0 export (single progress bar). * Use this when adding/starting the process and for all ticks. diff --git a/packages/contentstack-asset-management/src/import/asset-types.ts b/packages/contentstack-asset-management/src/import/asset-types.ts index 9305657c5..1a4a01c96 100644 --- a/packages/contentstack-asset-management/src/import/asset-types.ts +++ b/packages/contentstack-asset-management/src/import/asset-types.ts @@ -5,6 +5,7 @@ import { log } from '@contentstack/cli-utilities'; import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; import { AssetManagementImportAdapter } from './base'; import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +import { runInBatches } from '../utils/concurrent-batch'; const STRIP_KEYS = ['created_at', 'created_by', 'updated_at', 'updated_by', 'is_system', 'category', 'preview_image_url', 'category_detail']; @@ -50,6 +51,9 @@ export default class ImportAssetTypes extends AssetManagementImportAdapter { this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + type ToCreate = { uid: string; payload: Record }; + const toCreate: ToCreate[] = []; + for (const assetType of items) { const uid = assetType.uid as string; @@ -74,15 +78,23 @@ export default class ImportAssetTypes extends AssetManagementImportAdapter { continue; } - const payload = omit(assetType, STRIP_KEYS); + toCreate.push({ uid, payload: omit(assetType, STRIP_KEYS) as Record }); + } + + await runInBatches(toCreate, this.apiConcurrency, async ({ uid, payload }) => { try { await this.createAssetType(payload as any); this.tick(true, `asset-type: ${uid}`, null, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); log.debug(`Imported asset type: ${uid}`, this.importContext.context); } catch (e) { - this.tick(false, `asset-type: ${uid}`, (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].FAILED, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + this.tick( + false, + `asset-type: ${uid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].FAILED, + PROCESS_NAMES.AM_IMPORT_ASSET_TYPES, + ); log.debug(`Failed to import asset type ${uid}: ${e}`, this.importContext.context); } - } + }); } } diff --git a/packages/contentstack-asset-management/src/import/assets.ts b/packages/contentstack-asset-management/src/import/assets.ts index ab7986db1..92384b1a9 100644 --- a/packages/contentstack-asset-management/src/import/assets.ts +++ b/packages/contentstack-asset-management/src/import/assets.ts @@ -5,6 +5,7 @@ import { log } from '@contentstack/cli-utilities'; import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; import { AssetManagementImportAdapter } from './base'; import { getArrayFromResponse } from '../utils/export-helpers'; +import { runInBatches } from '../utils/concurrent-batch'; import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; type FolderRecord = { @@ -123,6 +124,14 @@ export default class ImportAssets extends AssetManagementImportAdapter { this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSETS].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSETS); log.debug(`Uploading ${assetItems.length} asset(s) for space ${newSpaceUid}`, this.importContext.context); + type UploadJob = { + asset: AssetRecord; + filePath: string; + mappedParentUid: string | undefined; + oldUid: string; + }; + const uploadJobs: UploadJob[] = []; + for (const asset of assetItems) { const oldUid = asset.uid; const filename = asset.filename ?? asset.file_name ?? 'asset'; @@ -137,6 +146,11 @@ export default class ImportAssets extends AssetManagementImportAdapter { const assetParent = asset.parent_uid && asset.parent_uid !== 'root' ? asset.parent_uid : undefined; const mappedParentUid = assetParent ? folderUidMap[assetParent] ?? undefined : undefined; + uploadJobs.push({ asset, filePath, mappedParentUid, oldUid }); + } + + await runInBatches(uploadJobs, this.apiConcurrency, async ({ asset, filePath, mappedParentUid, oldUid }) => { + const filename = asset.filename ?? asset.file_name ?? 'asset'; try { const { asset: created } = await this.uploadAsset(newSpaceUid, filePath, { title: asset.title ?? filename, @@ -146,7 +160,6 @@ export default class ImportAssets extends AssetManagementImportAdapter { uidMap[oldUid] = created.uid; - // Map old AM direct URL → new AM direct URL. if (asset.url && created.url) { urlMap[asset.url] = created.url; } @@ -162,7 +175,7 @@ export default class ImportAssets extends AssetManagementImportAdapter { ); log.debug(`Failed to upload asset ${oldUid}: ${e}`, this.importContext.context); } - } + }); return { uidMap, urlMap }; } @@ -181,24 +194,29 @@ export default class ImportAssets extends AssetManagementImportAdapter { while (remaining.length > 0 && remaining.length !== prevLength) { prevLength = remaining.length; + const ready: FolderRecord[] = []; const nextPass: FolderRecord[] = []; for (const folder of remaining) { const { parent_uid: parentUid } = folder; - // "root" is the AM API sentinel for a top-level folder const isRootParent = !parentUid || parentUid === 'root'; const parentMapped = isRootParent || folderUidMap[parentUid] !== undefined; if (!parentMapped) { nextPass.push(folder); - continue; + } else { + ready.push(folder); } + } + await runInBatches(ready, this.apiConcurrency, async (folder) => { + const { parent_uid: parentUid } = folder; + const isRootParent = !parentUid || parentUid === 'root'; try { const { folder: created } = await this.createFolder(newSpaceUid, { title: folder.title, description: folder.description, - parent_uid: isRootParent ? undefined : folderUidMap[parentUid], + parent_uid: isRootParent ? undefined : folderUidMap[parentUid!], }); folderUidMap[folder.uid] = created.uid; this.tick(true, `folder: ${folder.uid}`, null, PROCESS_NAMES.AM_IMPORT_FOLDERS); @@ -212,7 +230,7 @@ export default class ImportAssets extends AssetManagementImportAdapter { ); log.debug(`Failed to create folder ${folder.uid}: ${e}`, this.importContext.context); } - } + }); remaining = nextPass; } diff --git a/packages/contentstack-asset-management/src/import/base.ts b/packages/contentstack-asset-management/src/import/base.ts index db82f1462..145aa0e69 100644 --- a/packages/contentstack-asset-management/src/import/base.ts +++ b/packages/contentstack-asset-management/src/import/base.ts @@ -3,7 +3,7 @@ import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; -import { AM_MAIN_PROCESS_NAME } from '../constants/index'; +import { AM_MAIN_PROCESS_NAME, DEFAULT_AM_API_CONCURRENCY } from '../constants/index'; export type { ImportContext }; @@ -61,6 +61,11 @@ export class AssetManagementImportAdapter extends AssetManagementAdapter { return this.importContext.spacesRootPath; } + /** Parallel AM API limit for import batches. */ + protected get apiConcurrency(): number { + return this.importContext.apiConcurrency ?? DEFAULT_AM_API_CONCURRENCY; + } + protected getAssetTypesDir(): string { return pResolve(this.importContext.spacesRootPath, 'asset_types'); } diff --git a/packages/contentstack-asset-management/src/import/fields.ts b/packages/contentstack-asset-management/src/import/fields.ts index 86f45086a..b2cf4e5c0 100644 --- a/packages/contentstack-asset-management/src/import/fields.ts +++ b/packages/contentstack-asset-management/src/import/fields.ts @@ -5,6 +5,7 @@ import { log } from '@contentstack/cli-utilities'; import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; import { AssetManagementImportAdapter } from './base'; import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +import { runInBatches } from '../utils/concurrent-batch'; const STRIP_KEYS = ['created_at', 'created_by', 'updated_at', 'updated_by', 'is_system', 'asset_types_count']; @@ -50,6 +51,9 @@ export default class ImportFields extends AssetManagementImportAdapter { this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FIELDS].IMPORTING, PROCESS_NAMES.AM_IMPORT_FIELDS); + type ToCreate = { uid: string; payload: Record }; + const toCreate: ToCreate[] = []; + for (const field of items) { const uid = field.uid as string; @@ -74,15 +78,23 @@ export default class ImportFields extends AssetManagementImportAdapter { continue; } - const payload = omit(field, STRIP_KEYS); + toCreate.push({ uid, payload: omit(field, STRIP_KEYS) as Record }); + } + + await runInBatches(toCreate, this.apiConcurrency, async ({ uid, payload }) => { try { await this.createField(payload as any); this.tick(true, `field: ${uid}`, null, PROCESS_NAMES.AM_IMPORT_FIELDS); log.debug(`Imported field: ${uid}`, this.importContext.context); } catch (e) { - this.tick(false, `field: ${uid}`, (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FIELDS].FAILED, PROCESS_NAMES.AM_IMPORT_FIELDS); + this.tick( + false, + `field: ${uid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FIELDS].FAILED, + PROCESS_NAMES.AM_IMPORT_FIELDS, + ); log.debug(`Failed to import field ${uid}: ${e}`, this.importContext.context); } - } + }); } } diff --git a/packages/contentstack-asset-management/src/import/spaces.ts b/packages/contentstack-asset-management/src/import/spaces.ts index f43885154..ac3e88f92 100644 --- a/packages/contentstack-asset-management/src/import/spaces.ts +++ b/packages/contentstack-asset-management/src/import/spaces.ts @@ -1,5 +1,6 @@ import { resolve as pResolve, join } from 'node:path'; -import { readdirSync, statSync } from 'node:fs'; +import { mkdirSync, readdirSync, statSync } from 'node:fs'; +import { writeFile } from 'node:fs/promises'; import { log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; import type { @@ -9,7 +10,11 @@ import type { ImportResult, SpaceMapping, } from '../types/asset-management-api'; -import { AM_MAIN_PROCESS_NAME } from '../constants/index'; +import { + AM_MAIN_PROCESS_NAME, + IMPORT_ASSETS_MAPPER_DIR_SEGMENTS, + IMPORT_ASSETS_MAPPER_FILES, +} from '../constants/index'; import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; import ImportAssetTypes from './asset-types'; import ImportFields from './fields'; @@ -34,7 +39,16 @@ export class ImportSpaces { } async start(): Promise { - const { contentDir, assetManagementUrl, org_uid, apiKey, host, sourceApiKey, context } = this.options; + const { + contentDir, + assetManagementUrl, + org_uid, + apiKey, + host, + sourceApiKey, + context, + apiConcurrency, + } = this.options; const spacesRootPath = pResolve(contentDir, 'spaces'); @@ -45,6 +59,7 @@ export class ImportSpaces { host, org_uid, context, + apiConcurrency, }; const apiConfig: AssetManagementAPIConfig = { @@ -142,6 +157,19 @@ export class ImportSpaces { } } + if (this.options.backupDir) { + const mapperDir = join(this.options.backupDir, ...IMPORT_ASSETS_MAPPER_DIR_SEGMENTS); + mkdirSync(mapperDir, { recursive: true }); + await writeFile(join(mapperDir, IMPORT_ASSETS_MAPPER_FILES.UID_MAPPING), JSON.stringify(allUidMap), 'utf8'); + await writeFile(join(mapperDir, IMPORT_ASSETS_MAPPER_FILES.URL_MAPPING), JSON.stringify(allUrlMap), 'utf8'); + await writeFile( + join(mapperDir, IMPORT_ASSETS_MAPPER_FILES.SPACE_UID_MAPPING), + JSON.stringify(allSpaceUidMap), + 'utf8', + ); + log.debug('Wrote AM 2.0 mapper files (uid, url, space-uid)', context); + } + progress.completeProcess(AM_MAIN_PROCESS_NAME, !hasFailures); log.debug('Asset Management 2.0 import completed', context); } catch (err) { diff --git a/packages/contentstack-asset-management/src/types/asset-management-api.ts b/packages/contentstack-asset-management/src/types/asset-management-api.ts index 3d3b01d1b..7b979ad66 100644 --- a/packages/contentstack-asset-management/src/types/asset-management-api.ts +++ b/packages/contentstack-asset-management/src/types/asset-management-api.ts @@ -171,6 +171,11 @@ export type ImportContext = { org_uid: string; /** Optional logging context (same shape as ExportConfig.context). */ context?: Record; + /** + * Max parallel AM API calls for import (fields, asset types, folders batch, uploads). + * Set from `AssetManagementImportOptions.apiConcurrency`. + */ + apiConcurrency?: number; }; /** @@ -191,6 +196,12 @@ export type AssetManagementImportOptions = { sourceApiKey?: string; /** Optional logging context. */ context?: Record; + /** + * When set, mapper files are written under `{backupDir}/mapper/assets/` after import. + */ + backupDir?: string; + /** Parallel AM API limit; defaults to package constant when omitted. */ + apiConcurrency?: number; }; /** @@ -206,8 +217,8 @@ export type SpaceMapping = { /** * The value returned by `ImportSpaces.start()`. - * Written to `mapper/assets/uid-mapping.json` and `mapper/assets/url-mapping.json` - * by the bridge module so `entries.ts` can resolve asset references. + * When `backupDir` is set on options, the AM package also writes these maps under + * `mapper/assets/` for `entries.ts` to resolve asset references. */ export type ImportResult = { uidMap: Record; diff --git a/packages/contentstack-asset-management/src/utils/concurrent-batch.ts b/packages/contentstack-asset-management/src/utils/concurrent-batch.ts new file mode 100644 index 000000000..dcd916c4b --- /dev/null +++ b/packages/contentstack-asset-management/src/utils/concurrent-batch.ts @@ -0,0 +1,34 @@ +/** + * Split an array into chunks of at most `size` elements. + */ +export function chunkArray(items: T[], size: number): T[][] { + if (size <= 0) { + return items.length ? [items] : []; + } + const out: T[][] = []; + for (let i = 0; i < items.length; i += size) { + out.push(items.slice(i, i + size)); + } + return out; +} + +/** + * Run async work in batches of at most `concurrency` parallel tasks at a time. + * Uses Promise.allSettled per batch so one failure does not abort the batch. + */ +export async function runInBatches( + items: T[], + concurrency: number, + fn: (item: T, index: number) => Promise, +): Promise { + if (items.length === 0) { + return; + } + const limit = Math.max(1, concurrency); + const batches = chunkArray(items, limit); + let offset = 0; + for (const batch of batches) { + await Promise.allSettled(batch.map((item, j) => fn(item, offset + j))); + offset += batch.length; + } +} diff --git a/packages/contentstack-asset-management/src/utils/index.ts b/packages/contentstack-asset-management/src/utils/index.ts index 0837e33fd..9c2ffd3cc 100644 --- a/packages/contentstack-asset-management/src/utils/index.ts +++ b/packages/contentstack-asset-management/src/utils/index.ts @@ -6,3 +6,4 @@ export { getReadableStreamFromDownloadResponse, writeStreamToFile, } from './export-helpers'; +export { chunkArray, runInBatches } from './concurrent-batch'; diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index 215d2775a..57f77045f 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -5,7 +5,7 @@ import unionBy from 'lodash/unionBy'; import orderBy from 'lodash/orderBy'; import isEmpty from 'lodash/isEmpty'; import uniq from 'lodash/uniq'; -import { existsSync, mkdirSync } from 'node:fs'; +import { existsSync } from 'node:fs'; import includes from 'lodash/includes'; import { v4 as uuid } from 'uuid'; import { resolve as pResolve, join } from 'node:path'; @@ -80,28 +80,18 @@ export default class ImportAssets extends BaseClass { host: this.importConfig.region?.cma ?? this.importConfig.host ?? '', sourceApiKey: this.importConfig.source_stack, context: this.importConfig.context as unknown as Record, + backupDir: this.importConfig.backupDir, + apiConcurrency: this.importConfig.modules?.apiConcurrency, }); importer.setParentProgressManager(progress); - const { uidMap, urlMap, spaceMappings, spaceUidMap } = await importer.start(); + const { spaceMappings } = await importer.start(); - const mapperDirPath = join( - this.importConfig.backupDir, - PATH_CONSTANTS.MAPPER, - PATH_CONSTANTS.MAPPER_MODULES.ASSETS, - ); - mkdirSync(mapperDirPath, { recursive: true }); - await fsUtil.writeFile(join(mapperDirPath, PATH_CONSTANTS.FILES.UID_MAPPING), uidMap); - await fsUtil.writeFile(join(mapperDirPath, PATH_CONSTANTS.FILES.URL_MAPPING), urlMap); - await fsUtil.writeFile(join(mapperDirPath, PATH_CONSTANTS.FILES.SPACE_UID_MAPPING), spaceUidMap); - log.debug('Wrote AM 2.0 mapper files (uid, url, space-uid)', this.importConfig.context); - - // Link newly-created spaces to the target stack via branch settings POST. + // Link imported AM spaces to the target stack via CMA branch settings. if (spaceMappings.length > 0) { try { const branchUid = this.importConfig.branchName ?? 'main'; - // Fetch the current branch settings to get already-linked workspaces. const branchData = (await this.stack.branch(branchUid).fetch({ include_settings: true })) as Record< string, any @@ -130,12 +120,7 @@ export default class ImportAssets extends BaseClass { this.importConfig.context, ); } catch (linkErr) { - log.warn( - `AM 2.0 spaces were imported but could not be linked to the target stack: ${ - (linkErr as Error)?.message - }. Re-run the import or link manually.`, - this.importConfig.context, - ); + handleAndLogError(linkErr, { ...this.importConfig.context }); } } From cbd6d7fd0925e0f55784ab33cd7c109060d57913 Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Fri, 27 Mar 2026 13:07:37 +0530 Subject: [PATCH 06/22] chore: made implementation consistent with other modules --- .talismanrc | 12 ++++- .../src/constants/index.ts | 45 +++++++++++----- .../src/export/base.ts | 15 ++++-- .../src/export/spaces.ts | 16 +++++- .../src/import/asset-types.ts | 15 +++--- .../src/import/assets.ts | 11 ++-- .../src/import/base.ts | 33 +++--------- .../src/import/fields.ts | 15 +++--- .../src/import/spaces.ts | 53 ++++++++++++++----- .../src/types/asset-management-api.ts | 38 +++++++++++++ .../src/types/export-types.ts | 3 ++ .../src/utils/chunked-json-read.ts | 30 +++++++++++ .../src/utils/index.ts | 8 ++- packages/contentstack-export/package.json | 1 + .../contentstack-export/src/config/index.ts | 4 ++ .../src/export/modules/assets.ts | 3 ++ .../src/types/default-config.ts | 7 +++ .../contentstack-import/src/config/index.ts | 25 +++++++++ .../src/import/modules/assets.ts | 17 ++++++ .../src/types/default-config.ts | 9 ++++ 20 files changed, 279 insertions(+), 81 deletions(-) create mode 100644 packages/contentstack-asset-management/src/utils/chunked-json-read.ts diff --git a/.talismanrc b/.talismanrc index dbe7be688..848d28eab 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,4 +1,12 @@ fileignoreconfig: - - filename: pnpm-lock.yaml - checksum: 9ad01ca900e007d8ea4f5ffcef9f166665aba852b3b9a0a5f4506c04fd362138 + - filename: packages/contentstack-asset-management/src/import/fields.ts + checksum: e46c3eb94bba78ae06af0139a1b0fd4113c3dcfe879ed40cfe77659781526d5c + - filename: packages/contentstack-asset-management/src/import/asset-types.ts + checksum: 3525703fd2ac0f7ab3e963966c0abfa53d87c09f73dabab0889c7f41a5f1b003 + - filename: packages/contentstack-import/src/types/default-config.ts + checksum: 1c09acba953cfd7058a3e0d63f0a9bfbb8f28e903538eaa015fdc611402bbd4f + - filename: packages/contentstack-asset-management/src/types/asset-management-api.ts + checksum: 9394e67f4d0dea9ad27a8592adf99441f40731b788335cd7699c78205bdaad58 + - filename: packages/contentstack-import/src/import/modules/assets.ts + checksum: 98cea49283eb5d975168dc51e27b0e2fb541a9ed9b4368206f99b434e104f096 version: '1.0' diff --git a/packages/contentstack-asset-management/src/constants/index.ts b/packages/contentstack-asset-management/src/constants/index.ts index f2cad6efb..43c1b6d15 100644 --- a/packages/contentstack-asset-management/src/constants/index.ts +++ b/packages/contentstack-asset-management/src/constants/index.ts @@ -1,19 +1,36 @@ -export const BATCH_SIZE = 50; -export const CHUNK_FILE_SIZE_MB = 1; +/** Fallback when export/import do not pass `chunkWriteBatchSize`. */ +export const FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE = 50; +/** Fallback when export/import do not pass `chunkFileSizeMb`. */ +export const FALLBACK_AM_CHUNK_FILE_SIZE_MB = 1; +/** Fallback when import does not pass `apiConcurrency`. */ +export const FALLBACK_AM_API_CONCURRENCY = 5; +/** @deprecated Use FALLBACK_AM_API_CONCURRENCY */ +export const DEFAULT_AM_API_CONCURRENCY = FALLBACK_AM_API_CONCURRENCY; -/** Default parallel AM API calls when import caller does not set apiConcurrency. */ -export const DEFAULT_AM_API_CONCURRENCY = 5; +/** Fallback strip lists when import options omit `fieldsImportInvalidKeys` / `assetTypesImportInvalidKeys`. */ +export const FALLBACK_FIELDS_IMPORT_INVALID_KEYS = [ + 'created_at', + 'created_by', + 'updated_at', + 'updated_by', + 'is_system', + 'asset_types_count', +] as const; +export const FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS = [ + 'created_at', + 'created_by', + 'updated_at', + 'updated_by', + 'is_system', + 'category', + 'preview_image_url', + 'category_detail', +] as const; -/** - * Mapper output paths — must stay aligned with contentstack-import `PATH_CONSTANTS` - * (`mapper` / `assets` / uid, url, space-uid file names). - */ -export const IMPORT_ASSETS_MAPPER_DIR_SEGMENTS = ['mapper', 'assets'] as const; -export const IMPORT_ASSETS_MAPPER_FILES = { - UID_MAPPING: 'uid-mapping.json', - URL_MAPPING: 'url-mapping.json', - SPACE_UID_MAPPING: 'space-uid-mapping.json', -} as const; +/** @deprecated Use FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE */ +export const BATCH_SIZE = FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE; +/** @deprecated Use FALLBACK_AM_CHUNK_FILE_SIZE_MB */ +export const CHUNK_FILE_SIZE_MB = FALLBACK_AM_CHUNK_FILE_SIZE_MB; /** * Main process name for Asset Management 2.0 export (single progress bar). diff --git a/packages/contentstack-asset-management/src/export/base.ts b/packages/contentstack-asset-management/src/export/base.ts index 7521eae53..9990cf21a 100644 --- a/packages/contentstack-asset-management/src/export/base.ts +++ b/packages/contentstack-asset-management/src/export/base.ts @@ -5,8 +5,11 @@ import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack import type { AssetManagementAPIConfig } from '../types/asset-management-api'; import type { ExportContext } from '../types/export-types'; import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; -import { AM_MAIN_PROCESS_NAME } from '../constants/index'; -import { BATCH_SIZE, CHUNK_FILE_SIZE_MB } from '../constants/index'; +import { + AM_MAIN_PROCESS_NAME, + FALLBACK_AM_CHUNK_FILE_SIZE_MB, + FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE, +} from '../constants/index'; export type { ExportContext }; @@ -83,17 +86,19 @@ export class AssetManagementExportAdapter extends AssetManagementAdapter { await writeFile(pResolve(dir, indexFileName), '{}'); return; } + const chunkMb = this.exportContext.chunkFileSizeMb ?? FALLBACK_AM_CHUNK_FILE_SIZE_MB; + const batchSize = this.exportContext.chunkWriteBatchSize ?? FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE; const fs = new FsUtility({ basePath: dir, indexFileName, - chunkFileSize: CHUNK_FILE_SIZE_MB, + chunkFileSize: chunkMb, moduleName, fileExt: 'json', metaPickKeys, keepMetadata: true, }); - for (let i = 0; i < items.length; i += BATCH_SIZE) { - const batch = items.slice(i, i + BATCH_SIZE); + for (let i = 0; i < items.length; i += batchSize) { + const batch = items.slice(i, i + batchSize); fs.writeIntoFile(batch as Record[], { mapKeyVal: true }); } fs.completeFile(true); diff --git a/packages/contentstack-asset-management/src/export/spaces.ts b/packages/contentstack-asset-management/src/export/spaces.ts index a5abd54a7..cdc23c932 100644 --- a/packages/contentstack-asset-management/src/export/spaces.ts +++ b/packages/contentstack-asset-management/src/export/spaces.ts @@ -29,8 +29,18 @@ export class ExportSpaces { } async start(): Promise { - const { linkedWorkspaces, exportDir, branchName, assetManagementUrl, org_uid, apiKey, context, securedAssets } = - this.options; + const { + linkedWorkspaces, + exportDir, + branchName, + assetManagementUrl, + org_uid, + apiKey, + context, + securedAssets, + chunkWriteBatchSize, + chunkFileSizeMb, + } = this.options; if (!linkedWorkspaces.length) { log.debug('No linked workspaces to export', context); @@ -60,6 +70,8 @@ export class ExportSpaces { spacesRootPath, context, securedAssets, + chunkWriteBatchSize, + chunkFileSizeMb, }; const sharedFieldsDir = pResolve(spacesRootPath, 'fields'); diff --git a/packages/contentstack-asset-management/src/import/asset-types.ts b/packages/contentstack-asset-management/src/import/asset-types.ts index 1a4a01c96..cb23b617a 100644 --- a/packages/contentstack-asset-management/src/import/asset-types.ts +++ b/packages/contentstack-asset-management/src/import/asset-types.ts @@ -4,10 +4,9 @@ import { log } from '@contentstack/cli-utilities'; import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; import { AssetManagementImportAdapter } from './base'; -import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +import { FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; import { runInBatches } from '../utils/concurrent-batch'; - -const STRIP_KEYS = ['created_at', 'created_by', 'updated_at', 'updated_by', 'is_system', 'category', 'preview_image_url', 'category_detail']; +import { readChunkedJsonItems } from '../utils/chunked-json-read'; /** * Reads shared asset types from `spaces/asset_types/asset-types.json` and POSTs @@ -28,8 +27,10 @@ export default class ImportAssetTypes extends AssetManagementImportAdapter { async start(): Promise { await this.init(); + const stripKeys = this.importContext.assetTypesImportInvalidKeys ?? [...FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS]; const dir = this.getAssetTypesDir(); - const items = await this.readAllChunkedJson>(dir, 'asset-types.json'); + const indexName = this.importContext.assetTypesFileName ?? 'asset-types.json'; + const items = await readChunkedJsonItems>(dir, indexName, this.importContext.context); if (items.length === 0) { log.debug('No shared asset types to import', this.importContext.context); @@ -64,8 +65,8 @@ export default class ImportAssetTypes extends AssetManagementImportAdapter { const existing = existingByUid.get(uid); if (existing) { - const exportedClean = omit(assetType, STRIP_KEYS); - const existingClean = omit(existing, STRIP_KEYS); + const exportedClean = omit(assetType, stripKeys); + const existingClean = omit(existing, stripKeys); if (!isEqual(exportedClean, existingClean)) { log.warn( `Asset type "${uid}" already exists in the target org with a different definition. Skipping — to apply the exported definition, delete the asset type from the target org first.`, @@ -78,7 +79,7 @@ export default class ImportAssetTypes extends AssetManagementImportAdapter { continue; } - toCreate.push({ uid, payload: omit(assetType, STRIP_KEYS) as Record }); + toCreate.push({ uid, payload: omit(assetType, stripKeys) as Record }); } await runInBatches(toCreate, this.apiConcurrency, async ({ uid, payload }) => { diff --git a/packages/contentstack-asset-management/src/import/assets.ts b/packages/contentstack-asset-management/src/import/assets.ts index 92384b1a9..4b942a186 100644 --- a/packages/contentstack-asset-management/src/import/assets.ts +++ b/packages/contentstack-asset-management/src/import/assets.ts @@ -5,6 +5,7 @@ import { log } from '@contentstack/cli-utilities'; import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; import { AssetManagementImportAdapter } from './base'; import { getArrayFromResponse } from '../utils/export-helpers'; +import { readChunkedJsonItems } from '../utils/chunked-json-read'; import { runInBatches } from '../utils/concurrent-batch'; import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; @@ -42,10 +43,11 @@ export default class ImportAssets extends AssetManagementImportAdapter { */ private async loadExportedAssetItems(spaceDir: string): Promise { const assetsDir = pResolve(spaceDir, 'assets'); - if (!existsSync(join(assetsDir, 'assets.json'))) { + const assetsIndex = this.importContext.assetsFileName ?? 'assets.json'; + if (!existsSync(join(assetsDir, assetsIndex))) { return null; } - return this.readAllChunkedJson(assetsDir, 'assets.json'); + return readChunkedJsonItems(assetsDir, assetsIndex, this.importContext.context); } /** @@ -95,14 +97,15 @@ export default class ImportAssets extends AssetManagementImportAdapter { // 1. Import folders // ----------------------------------------------------------------------- const folderUidMap: Record = {}; - const foldersFilePath = join(assetsDir, 'folders.json'); + const foldersFileName = this.importContext.foldersFileName ?? 'folders.json'; + const foldersFilePath = join(assetsDir, foldersFileName); if (existsSync(foldersFilePath)) { let foldersData: unknown; try { foldersData = JSON.parse(readFileSync(foldersFilePath, 'utf8')); } catch (e) { - log.debug(`Could not read folders.json: ${e}`, this.importContext.context); + log.debug(`Could not read ${foldersFileName}: ${e}`, this.importContext.context); } if (foldersData) { diff --git a/packages/contentstack-asset-management/src/import/base.ts b/packages/contentstack-asset-management/src/import/base.ts index 145aa0e69..298f7f613 100644 --- a/packages/contentstack-asset-management/src/import/base.ts +++ b/packages/contentstack-asset-management/src/import/base.ts @@ -1,9 +1,10 @@ import { resolve as pResolve } from 'node:path'; -import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; +import { CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; -import { AM_MAIN_PROCESS_NAME, DEFAULT_AM_API_CONCURRENCY } from '../constants/index'; +import { AM_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY } from '../constants/index'; +import { readChunkedJsonItems } from '../utils/chunked-json-read'; export type { ImportContext }; @@ -63,39 +64,21 @@ export class AssetManagementImportAdapter extends AssetManagementAdapter { /** Parallel AM API limit for import batches. */ protected get apiConcurrency(): number { - return this.importContext.apiConcurrency ?? DEFAULT_AM_API_CONCURRENCY; + return this.importContext.apiConcurrency ?? FALLBACK_AM_API_CONCURRENCY; } protected getAssetTypesDir(): string { - return pResolve(this.importContext.spacesRootPath, 'asset_types'); + return pResolve(this.importContext.spacesRootPath, this.importContext.assetTypesDir ?? 'asset_types'); } protected getFieldsDir(): string { - return pResolve(this.importContext.spacesRootPath, 'fields'); + return pResolve(this.importContext.spacesRootPath, this.importContext.fieldsDir ?? 'fields'); } /** - * Reads all items from a FsUtility chunked JSON store (index file + chunk files). - * Returns a flat array of all items across all chunks. + * Reads all items from a chunked JSON store via {@link readChunkedJsonItems} (FsUtility). */ protected async readAllChunkedJson>(dir: string, indexFileName: string): Promise { - try { - const fs = new FsUtility({ basePath: dir, indexFileName }); - const indexer = fs.indexFileContent; - const items: T[] = []; - for (const _ in indexer) { - const chunk = await fs.readChunkFiles.next().catch((err: unknown): null => { - log.debug(`Error reading chunk: ${err}`, this.importContext.context); - return null; - }); - if (chunk) { - items.push(...(Object.values(chunk as Record))); - } - } - return items; - } catch (err) { - log.debug(`readAllChunkedJson failed for ${dir}/${indexFileName}: ${err}`, this.importContext.context); - return []; - } + return readChunkedJsonItems(dir, indexFileName, this.importContext.context); } } diff --git a/packages/contentstack-asset-management/src/import/fields.ts b/packages/contentstack-asset-management/src/import/fields.ts index b2cf4e5c0..a153eb31d 100644 --- a/packages/contentstack-asset-management/src/import/fields.ts +++ b/packages/contentstack-asset-management/src/import/fields.ts @@ -4,10 +4,9 @@ import { log } from '@contentstack/cli-utilities'; import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; import { AssetManagementImportAdapter } from './base'; -import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +import { FALLBACK_FIELDS_IMPORT_INVALID_KEYS, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; import { runInBatches } from '../utils/concurrent-batch'; - -const STRIP_KEYS = ['created_at', 'created_by', 'updated_at', 'updated_by', 'is_system', 'asset_types_count']; +import { readChunkedJsonItems } from '../utils/chunked-json-read'; /** * Reads shared fields from `spaces/fields/fields.json` and POSTs each to the @@ -28,8 +27,10 @@ export default class ImportFields extends AssetManagementImportAdapter { async start(): Promise { await this.init(); + const stripKeys = this.importContext.fieldsImportInvalidKeys ?? [...FALLBACK_FIELDS_IMPORT_INVALID_KEYS]; const dir = this.getFieldsDir(); - const items = await this.readAllChunkedJson>(dir, 'fields.json'); + const indexName = this.importContext.fieldsFileName ?? 'fields.json'; + const items = await readChunkedJsonItems>(dir, indexName, this.importContext.context); if (items.length === 0) { log.debug('No shared fields to import', this.importContext.context); @@ -64,8 +65,8 @@ export default class ImportFields extends AssetManagementImportAdapter { const existing = existingByUid.get(uid); if (existing) { - const exportedClean = omit(field, STRIP_KEYS); - const existingClean = omit(existing, STRIP_KEYS); + const exportedClean = omit(field, stripKeys); + const existingClean = omit(existing, stripKeys); if (!isEqual(exportedClean, existingClean)) { log.warn( `Field "${uid}" already exists in the target org with a different definition. Skipping — to apply the exported definition, delete the field from the target org first.`, @@ -78,7 +79,7 @@ export default class ImportFields extends AssetManagementImportAdapter { continue; } - toCreate.push({ uid, payload: omit(field, STRIP_KEYS) as Record }); + toCreate.push({ uid, payload: omit(field, stripKeys) as Record }); } await runInBatches(toCreate, this.apiConcurrency, async ({ uid, payload }) => { diff --git a/packages/contentstack-asset-management/src/import/spaces.ts b/packages/contentstack-asset-management/src/import/spaces.ts index ac3e88f92..d44fc795b 100644 --- a/packages/contentstack-asset-management/src/import/spaces.ts +++ b/packages/contentstack-asset-management/src/import/spaces.ts @@ -10,11 +10,7 @@ import type { ImportResult, SpaceMapping, } from '../types/asset-management-api'; -import { - AM_MAIN_PROCESS_NAME, - IMPORT_ASSETS_MAPPER_DIR_SEGMENTS, - IMPORT_ASSETS_MAPPER_FILES, -} from '../constants/index'; +import { AM_MAIN_PROCESS_NAME } from '../constants/index'; import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; import ImportAssetTypes from './asset-types'; import ImportFields from './fields'; @@ -48,9 +44,23 @@ export class ImportSpaces { sourceApiKey, context, apiConcurrency, + spacesDirName, + fieldsDir, + assetTypesDir, + fieldsFileName, + assetTypesFileName, + foldersFileName, + assetsFileName, + fieldsImportInvalidKeys, + assetTypesImportInvalidKeys, + mapperRootDir, + mapperAssetsModuleDir, + mapperUidFileName, + mapperUrlFileName, + mapperSpaceUidFileName, } = this.options; - const spacesRootPath = pResolve(contentDir, 'spaces'); + const spacesRootPath = pResolve(contentDir, spacesDirName ?? 'spaces'); const importContext: ImportContext = { spacesRootPath, @@ -60,6 +70,20 @@ export class ImportSpaces { org_uid, context, apiConcurrency, + spacesDirName, + fieldsDir, + assetTypesDir, + fieldsFileName, + assetTypesFileName, + foldersFileName, + assetsFileName, + fieldsImportInvalidKeys, + assetTypesImportInvalidKeys, + mapperRootDir, + mapperAssetsModuleDir, + mapperUidFileName, + mapperUrlFileName, + mapperSpaceUidFileName, }; const apiConfig: AssetManagementAPIConfig = { @@ -158,15 +182,16 @@ export class ImportSpaces { } if (this.options.backupDir) { - const mapperDir = join(this.options.backupDir, ...IMPORT_ASSETS_MAPPER_DIR_SEGMENTS); + const mapperRoot = this.options.mapperRootDir ?? 'mapper'; + const mapperAssetsMod = this.options.mapperAssetsModuleDir ?? 'assets'; + const mapperDir = join(this.options.backupDir, mapperRoot, mapperAssetsMod); mkdirSync(mapperDir, { recursive: true }); - await writeFile(join(mapperDir, IMPORT_ASSETS_MAPPER_FILES.UID_MAPPING), JSON.stringify(allUidMap), 'utf8'); - await writeFile(join(mapperDir, IMPORT_ASSETS_MAPPER_FILES.URL_MAPPING), JSON.stringify(allUrlMap), 'utf8'); - await writeFile( - join(mapperDir, IMPORT_ASSETS_MAPPER_FILES.SPACE_UID_MAPPING), - JSON.stringify(allSpaceUidMap), - 'utf8', - ); + const uidFile = this.options.mapperUidFileName ?? 'uid-mapping.json'; + const urlFile = this.options.mapperUrlFileName ?? 'url-mapping.json'; + const spaceUidFile = this.options.mapperSpaceUidFileName ?? 'space-uid-mapping.json'; + await writeFile(join(mapperDir, uidFile), JSON.stringify(allUidMap), 'utf8'); + await writeFile(join(mapperDir, urlFile), JSON.stringify(allUrlMap), 'utf8'); + await writeFile(join(mapperDir, spaceUidFile), JSON.stringify(allSpaceUidMap), 'utf8'); log.debug('Wrote AM 2.0 mapper files (uid, url, space-uid)', context); } diff --git a/packages/contentstack-asset-management/src/types/asset-management-api.ts b/packages/contentstack-asset-management/src/types/asset-management-api.ts index 7b979ad66..8ae90f6ad 100644 --- a/packages/contentstack-asset-management/src/types/asset-management-api.ts +++ b/packages/contentstack-asset-management/src/types/asset-management-api.ts @@ -147,6 +147,14 @@ export type AssetManagementExportOptions = { * can reconstruct old CMA proxy URLs (format: /v3/assets/{apiKey}/{amUid}/...). */ apiKey?: string; + /** + * Chunked JSON write batch size (items per FsUtility write). From export `modules['asset-management']`. + */ + chunkWriteBatchSize?: number; + /** + * FsUtility `chunkFileSize` in MB for AM export chunked writes. + */ + chunkFileSizeMb?: number; }; // --------------------------------------------------------------------------- @@ -176,6 +184,22 @@ export type ImportContext = { * Set from `AssetManagementImportOptions.apiConcurrency`. */ apiConcurrency?: number; + /** Relative dir under content dir for AM export root (e.g. `spaces`). */ + spacesDirName?: string; + fieldsDir?: string; + assetTypesDir?: string; + fieldsFileName?: string; + assetTypesFileName?: string; + foldersFileName?: string; + assetsFileName?: string; + fieldsImportInvalidKeys?: string[]; + assetTypesImportInvalidKeys?: string[]; + /** `{backupDir}/{mapperRootDir}/{mapperAssetsModuleDir}/` for AM mapper JSON. */ + mapperRootDir?: string; + mapperAssetsModuleDir?: string; + mapperUidFileName?: string; + mapperUrlFileName?: string; + mapperSpaceUidFileName?: string; }; /** @@ -202,6 +226,20 @@ export type AssetManagementImportOptions = { backupDir?: string; /** Parallel AM API limit; defaults to package constant when omitted. */ apiConcurrency?: number; + spacesDirName?: string; + fieldsDir?: string; + assetTypesDir?: string; + fieldsFileName?: string; + assetTypesFileName?: string; + foldersFileName?: string; + assetsFileName?: string; + fieldsImportInvalidKeys?: string[]; + assetTypesImportInvalidKeys?: string[]; + mapperRootDir?: string; + mapperAssetsModuleDir?: string; + mapperUidFileName?: string; + mapperUrlFileName?: string; + mapperSpaceUidFileName?: string; }; /** diff --git a/packages/contentstack-asset-management/src/types/export-types.ts b/packages/contentstack-asset-management/src/types/export-types.ts index 25b8dcace..d1480adb2 100644 --- a/packages/contentstack-asset-management/src/types/export-types.ts +++ b/packages/contentstack-asset-management/src/types/export-types.ts @@ -2,6 +2,9 @@ export type ExportContext = { spacesRootPath: string; context?: Record; securedAssets?: boolean; + /** From export config; AM falls back to package constants when unset. */ + chunkWriteBatchSize?: number; + chunkFileSizeMb?: number; }; /** diff --git a/packages/contentstack-asset-management/src/utils/chunked-json-read.ts b/packages/contentstack-asset-management/src/utils/chunked-json-read.ts new file mode 100644 index 000000000..4e09616c0 --- /dev/null +++ b/packages/contentstack-asset-management/src/utils/chunked-json-read.ts @@ -0,0 +1,30 @@ +import { FsUtility, log } from '@contentstack/cli-utilities'; + +/** + * Read all items from a chunked JSON store (index + chunk files) using FsUtility, + * matching the pattern used in contentstack-import entry modules. + */ +export async function readChunkedJsonItems>( + basePath: string, + indexFileName: string, + context?: Record, +): Promise { + try { + const fs = new FsUtility({ basePath, indexFileName }); + const indexer = fs.indexFileContent; + const items: T[] = []; + for (const _ in indexer) { + const chunk = await fs.readChunkFiles.next().catch((err: unknown): null => { + log.debug(`Error reading chunk: ${err}`, context); + return null; + }); + if (chunk) { + items.push(...(Object.values(chunk as Record))); + } + } + return items; + } catch (err) { + log.debug(`readChunkedJsonItems failed for ${basePath}/${indexFileName}: ${err}`, context); + return []; + } +} diff --git a/packages/contentstack-asset-management/src/utils/index.ts b/packages/contentstack-asset-management/src/utils/index.ts index 9c2ffd3cc..b6925a876 100644 --- a/packages/contentstack-asset-management/src/utils/index.ts +++ b/packages/contentstack-asset-management/src/utils/index.ts @@ -1,5 +1,11 @@ export { AssetManagementAdapter } from './asset-management-api-adapter'; -export { BATCH_SIZE, CHUNK_FILE_SIZE_MB } from '../constants'; +export { + BATCH_SIZE, + CHUNK_FILE_SIZE_MB, + FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE, + FALLBACK_AM_CHUNK_FILE_SIZE_MB, +} from '../constants'; +export { readChunkedJsonItems } from './chunked-json-read'; export { getArrayFromResponse, getAssetItems, diff --git a/packages/contentstack-export/package.json b/packages/contentstack-export/package.json index 5f3e14c46..ef7459409 100644 --- a/packages/contentstack-export/package.json +++ b/packages/contentstack-export/package.json @@ -8,6 +8,7 @@ "@contentstack/cli-command": "~2.0.0-beta.4", "@contentstack/cli-utilities": "~2.0.0-beta.4", "@contentstack/cli-variants": "~2.0.0-beta.10", + "@contentstack/cli-asset-management": "~1.0.0", "@oclif/core": "^4.8.0", "async": "^3.2.6", "big-json": "^3.2.0", diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index 14fe590b3..788e76042 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -112,6 +112,10 @@ const config: DefaultConfig = { enableDownloadStatus: false, includeVersionedAssets: false, }, + 'asset-management': { + chunkWriteBatchSize: 50, + chunkFileSizeMb: 1, + }, content_types: { dirName: 'content_types', fileName: 'content_types.json', diff --git a/packages/contentstack-export/src/export/modules/assets.ts b/packages/contentstack-export/src/export/modules/assets.ts index 7d85f7bca..bf8ff1937 100644 --- a/packages/contentstack-export/src/export/modules/assets.ts +++ b/packages/contentstack-export/src/export/modules/assets.ts @@ -69,6 +69,7 @@ export default class ExportAssets extends BaseClass { this.exportConfig.org_uid = this.exportConfig.org_uid || (await getOrgUid(this.exportConfig)); const progress = this.createNestedProgress(this.currentModuleName); try { + const assetManagementModuleConfig = this.exportConfig.modules['asset-management']; const exporter = new ExportSpaces({ linkedWorkspaces, exportDir: this.exportConfig.exportDir, @@ -78,6 +79,8 @@ export default class ExportAssets extends BaseClass { apiKey: this.exportConfig.apiKey, context: this.exportConfig.context as unknown as Record, securedAssets: this.exportConfig.securedAssets, + chunkWriteBatchSize: assetManagementModuleConfig?.chunkWriteBatchSize, + chunkFileSizeMb: assetManagementModuleConfig?.chunkFileSizeMb, }); exporter.setParentProgressManager(progress); await exporter.start(); diff --git a/packages/contentstack-export/src/types/default-config.ts b/packages/contentstack-export/src/types/default-config.ts index b08239927..30c58e06f 100644 --- a/packages/contentstack-export/src/types/default-config.ts +++ b/packages/contentstack-export/src/types/default-config.ts @@ -96,6 +96,13 @@ export default interface DefaultConfig { includeVersionedAssets: boolean; dependencies?: Modules[]; }; + 'asset-management': { + /** Batch size for AM 2.0 chunked JSON writes (shared fields / asset types / etc.). */ + chunkWriteBatchSize: number; + /** Passed to FsUtility chunkFileSize (MB) when writing chunked export JSON. */ + chunkFileSizeMb: number; + dependencies?: Modules[]; + }; content_types: { dirName: string; fileName: string; diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index fd0a5d1e4..a15332baa 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -105,8 +105,33 @@ const config: DefaultConfig = { dirName: 'spaces', fieldsDir: 'fields', assetTypesDir: 'asset_types', + fieldsFileName: 'fields.json', + assetTypesFileName: 'asset-types.json', foldersFileName: 'folders.json', assetsFileName: 'assets.json', + fieldsImportInvalidKeys: [ + 'created_at', + 'created_by', + 'updated_at', + 'updated_by', + 'is_system', + 'asset_types_count', + ], + assetTypesImportInvalidKeys: [ + 'created_at', + 'created_by', + 'updated_at', + 'updated_by', + 'is_system', + 'category', + 'preview_image_url', + 'category_detail', + ], + mapperRootDir: 'mapper', + mapperAssetsModuleDir: 'assets', + mapperUidFileName: 'uid-mapping.json', + mapperUrlFileName: 'url-mapping.json', + mapperSpaceUidFileName: 'space-uid-mapping.json', uploadAssetsConcurrency: 2, importFoldersConcurrency: 1, }, diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index 57f77045f..46e2100c7 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -72,6 +72,7 @@ export default class ImportAssets extends BaseClass { const progress = this.createNestedProgress(this.currentModuleName); try { + const assetManagementModuleConfig = this.importConfig.modules['asset-management']; const importer = new ImportSpaces({ contentDir: this.importConfig.contentDir, assetManagementUrl, @@ -82,6 +83,22 @@ export default class ImportAssets extends BaseClass { context: this.importConfig.context as unknown as Record, backupDir: this.importConfig.backupDir, apiConcurrency: this.importConfig.modules?.apiConcurrency, + spacesDirName: assetManagementModuleConfig?.dirName, + fieldsDir: assetManagementModuleConfig?.fieldsDir, + assetTypesDir: assetManagementModuleConfig?.assetTypesDir, + fieldsFileName: assetManagementModuleConfig?.fieldsFileName, + assetTypesFileName: assetManagementModuleConfig?.assetTypesFileName, + foldersFileName: assetManagementModuleConfig?.foldersFileName, + assetsFileName: assetManagementModuleConfig?.assetsFileName, + fieldsImportInvalidKeys: assetManagementModuleConfig?.fieldsImportInvalidKeys, + assetTypesImportInvalidKeys: assetManagementModuleConfig?.assetTypesImportInvalidKeys, + mapperRootDir: assetManagementModuleConfig?.mapperRootDir ?? PATH_CONSTANTS.MAPPER, + mapperAssetsModuleDir: + assetManagementModuleConfig?.mapperAssetsModuleDir ?? PATH_CONSTANTS.MAPPER_MODULES.ASSETS, + mapperUidFileName: assetManagementModuleConfig?.mapperUidFileName ?? PATH_CONSTANTS.FILES.UID_MAPPING, + mapperUrlFileName: assetManagementModuleConfig?.mapperUrlFileName ?? PATH_CONSTANTS.FILES.URL_MAPPING, + mapperSpaceUidFileName: + assetManagementModuleConfig?.mapperSpaceUidFileName ?? PATH_CONSTANTS.FILES.SPACE_UID_MAPPING, }); importer.setParentProgressManager(progress); diff --git a/packages/contentstack-import/src/types/default-config.ts b/packages/contentstack-import/src/types/default-config.ts index dc7ce3153..29e859806 100644 --- a/packages/contentstack-import/src/types/default-config.ts +++ b/packages/contentstack-import/src/types/default-config.ts @@ -76,8 +76,17 @@ export default interface DefaultConfig { dirName: string; fieldsDir: string; assetTypesDir: string; + fieldsFileName: string; + assetTypesFileName: string; foldersFileName: string; assetsFileName: string; + fieldsImportInvalidKeys: string[]; + assetTypesImportInvalidKeys: string[]; + mapperRootDir: string; + mapperAssetsModuleDir: string; + mapperUidFileName: string; + mapperUrlFileName: string; + mapperSpaceUidFileName: string; uploadAssetsConcurrency: number; importFoldersConcurrency: number; }; From 2ef8e3a8e5baccfd9cf01ff08bcee187a164f2ef Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Tue, 31 Mar 2026 18:32:41 +0530 Subject: [PATCH 07/22] chore: updated implementation for AM import --- .talismanrc | 26 +-- .../package.json | 2 +- .../src/constants/index.ts | 4 - .../src/export/base.ts | 12 +- .../src/export/spaces.ts | 2 - .../src/import/asset-types.ts | 54 +++++- .../src/import/assets.ts | 174 +++++++++++------- .../src/import/base.ts | 18 +- .../src/import/fields.ts | 52 +++++- .../src/import/spaces.ts | 4 + .../src/types/asset-management-api.ts | 12 +- .../src/types/export-types.ts | 2 - .../src/utils/chunked-json-read.ts | 30 --- .../src/utils/chunked-json-reader.ts | 66 +++++++ .../src/utils/index.ts | 9 +- .../test/unit/export/base.test.ts | 11 +- .../contentstack-export/src/config/index.ts | 1 - .../src/export/modules/assets.ts | 1 - .../src/types/default-config.ts | 2 - .../src/import/modules/assets.ts | 39 +--- .../utils/asset-management-import-options.ts | 43 +++++ .../contentstack-import/src/utils/index.ts | 1 + 22 files changed, 350 insertions(+), 215 deletions(-) delete mode 100644 packages/contentstack-asset-management/src/utils/chunked-json-read.ts create mode 100644 packages/contentstack-asset-management/src/utils/chunked-json-reader.ts create mode 100644 packages/contentstack-import/src/utils/asset-management-import-options.ts diff --git a/.talismanrc b/.talismanrc index ca749ad2b..09ab9653d 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,18 +1,12 @@ fileignoreconfig: - - filename: pnpm-lock.yaml - checksum: 9c19eb613068c193fac35e72327198fbc86e759968391f07cc876c56b2b1a63d - - filename: packages/contentstack-export/test/unit/export/modules/base-class.test.ts - checksum: bd2b28305fff90ca26bce56b2c5c61751a62225d310a2553874e9ec009ed78e8 - - filename: packages/contentstack-export/test/unit/export/modules/assets.test.ts - checksum: 73ff01e2d19c8d1384dca2ee7087f8c19e0b1fac6b29c75a02ca523a36b7cb92 - - filename: packages/contentstack-export/src/types/default-config.ts - checksum: a204b00fc47046fd638f952c1326b6d88615dd96e37dc83e4c7e9404dc0435bb - - filename: packages/contentstack-export/src/types/index.ts - checksum: fa36c236abac338b03bf307102a99f25dddac9afe75b6b34fb82e318e7759799 - - filename: packages/contentstack-export/src/config/index.ts - checksum: 3998e30abaf9838f86025c3422cab085441bc38e958ed9e63084f928dbb7995c - - filename: packages/contentstack-export/test/unit/export/modules/stack.test.ts - checksum: 79876b8f635037a2d8ba38dac055e7625bf85db6a3cf5729434e6a97e44857d6 - - filename: packages/contentstack-export/src/export/modules/stack.ts - checksum: 375c0c5f58d43430b355050d122d3283083ca91891abe8105a4b4fd9433ece97 + - filename: packages/contentstack-asset-management/src/export/base.ts + checksum: 01cb2158fea9cbe05449c04efb42d8d416d24868d411bd7300d8fa99c9a4ab01 + - filename: packages/contentstack-asset-management/test/unit/export/base.test.ts + checksum: 164fc2e5a4337a2739903499b66eecc66a85bb9b50aa2e71079bdd046a195a94 + - filename: packages/contentstack-asset-management/src/import/fields.ts + checksum: 1cd52254ddbfd186d8ade2c73fc799dd1caa0f10bdd3c6b151621c27207ee173 + - filename: packages/contentstack-import/src/utils/asset-management-import-options.ts + checksum: 96b61b683109be2cc2dbab5231e1ded19282fbf176cd8492c75cc4861f2efea0 + - filename: packages/contentstack-asset-management/src/import/asset-types.ts + checksum: fa2aeea704fd259628f9c86eacd3cf9f6f12543b06b387bd65db7df6b6f5fc49 version: '1.0' diff --git a/packages/contentstack-asset-management/package.json b/packages/contentstack-asset-management/package.json index 2ac90a3dd..f41f31f1d 100644 --- a/packages/contentstack-asset-management/package.json +++ b/packages/contentstack-asset-management/package.json @@ -29,7 +29,7 @@ ], "license": "MIT", "dependencies": { - "@contentstack/cli-utilities": "~2.0.0-beta" + "@contentstack/cli-utilities": "~2.0.0-beta.5" }, "oclif": { "commands": "./lib/commands", diff --git a/packages/contentstack-asset-management/src/constants/index.ts b/packages/contentstack-asset-management/src/constants/index.ts index 43c1b6d15..9d6bca636 100644 --- a/packages/contentstack-asset-management/src/constants/index.ts +++ b/packages/contentstack-asset-management/src/constants/index.ts @@ -1,5 +1,3 @@ -/** Fallback when export/import do not pass `chunkWriteBatchSize`. */ -export const FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE = 50; /** Fallback when export/import do not pass `chunkFileSizeMb`. */ export const FALLBACK_AM_CHUNK_FILE_SIZE_MB = 1; /** Fallback when import does not pass `apiConcurrency`. */ @@ -27,8 +25,6 @@ export const FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS = [ 'category_detail', ] as const; -/** @deprecated Use FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE */ -export const BATCH_SIZE = FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE; /** @deprecated Use FALLBACK_AM_CHUNK_FILE_SIZE_MB */ export const CHUNK_FILE_SIZE_MB = FALLBACK_AM_CHUNK_FILE_SIZE_MB; diff --git a/packages/contentstack-asset-management/src/export/base.ts b/packages/contentstack-asset-management/src/export/base.ts index 9990cf21a..6fff78b43 100644 --- a/packages/contentstack-asset-management/src/export/base.ts +++ b/packages/contentstack-asset-management/src/export/base.ts @@ -5,11 +5,7 @@ import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack import type { AssetManagementAPIConfig } from '../types/asset-management-api'; import type { ExportContext } from '../types/export-types'; import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; -import { - AM_MAIN_PROCESS_NAME, - FALLBACK_AM_CHUNK_FILE_SIZE_MB, - FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE, -} from '../constants/index'; +import { AM_MAIN_PROCESS_NAME, FALLBACK_AM_CHUNK_FILE_SIZE_MB } from '../constants/index'; export type { ExportContext }; @@ -87,7 +83,6 @@ export class AssetManagementExportAdapter extends AssetManagementAdapter { return; } const chunkMb = this.exportContext.chunkFileSizeMb ?? FALLBACK_AM_CHUNK_FILE_SIZE_MB; - const batchSize = this.exportContext.chunkWriteBatchSize ?? FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE; const fs = new FsUtility({ basePath: dir, indexFileName, @@ -97,10 +92,7 @@ export class AssetManagementExportAdapter extends AssetManagementAdapter { metaPickKeys, keepMetadata: true, }); - for (let i = 0; i < items.length; i += batchSize) { - const batch = items.slice(i, i + batchSize); - fs.writeIntoFile(batch as Record[], { mapKeyVal: true }); - } + fs.writeIntoFile(items as Record[], { mapKeyVal: true }); fs.completeFile(true); } } diff --git a/packages/contentstack-asset-management/src/export/spaces.ts b/packages/contentstack-asset-management/src/export/spaces.ts index cdc23c932..5bb6c7472 100644 --- a/packages/contentstack-asset-management/src/export/spaces.ts +++ b/packages/contentstack-asset-management/src/export/spaces.ts @@ -38,7 +38,6 @@ export class ExportSpaces { apiKey, context, securedAssets, - chunkWriteBatchSize, chunkFileSizeMb, } = this.options; @@ -70,7 +69,6 @@ export class ExportSpaces { spacesRootPath, context, securedAssets, - chunkWriteBatchSize, chunkFileSizeMb, }; diff --git a/packages/contentstack-asset-management/src/import/asset-types.ts b/packages/contentstack-asset-management/src/import/asset-types.ts index cb23b617a..e5cc4f155 100644 --- a/packages/contentstack-asset-management/src/import/asset-types.ts +++ b/packages/contentstack-asset-management/src/import/asset-types.ts @@ -1,3 +1,5 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; import omit from 'lodash/omit'; import isEqual from 'lodash/isEqual'; import { log } from '@contentstack/cli-utilities'; @@ -6,7 +8,9 @@ import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-man import { AssetManagementImportAdapter } from './base'; import { FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; import { runInBatches } from '../utils/concurrent-batch'; -import { readChunkedJsonItems } from '../utils/chunked-json-read'; +import { forEachChunkedJsonStore } from '../utils/chunked-json-reader'; + +type AssetTypeToCreate = { uid: string; payload: Record }; /** * Reads shared asset types from `spaces/asset_types/asset-types.json` and POSTs @@ -30,15 +34,37 @@ export default class ImportAssetTypes extends AssetManagementImportAdapter { const stripKeys = this.importContext.assetTypesImportInvalidKeys ?? [...FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS]; const dir = this.getAssetTypesDir(); const indexName = this.importContext.assetTypesFileName ?? 'asset-types.json'; - const items = await readChunkedJsonItems>(dir, indexName, this.importContext.context); + const indexPath = join(dir, indexName); - if (items.length === 0) { - log.debug('No shared asset types to import', this.importContext.context); + if (!existsSync(indexPath)) { + log.debug('No shared asset types to import (index missing)', this.importContext.context); return; } - // Fetch existing asset types from the target org keyed by uid for diff comparison. - // Asset types are org-level; the spaceUid param in getWorkspaceAssetTypes is unused in the path. + const existingByUid = await this.loadExistingAssetTypesMap(); + + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + + await forEachChunkedJsonStore>( + dir, + indexName, + { + context: this.importContext.context, + chunkReadLogLabel: 'asset-types', + onOpenError: (e) => + log.debug(`Could not open chunked asset-types index: ${e}`, this.importContext.context), + onEmptyIndexer: () => + log.debug('No shared asset types to import (empty indexer)', this.importContext.context), + }, + async (records) => { + const toCreate = this.buildAssetTypesToCreate(records, existingByUid, stripKeys); + await this.importAssetTypesCreates(toCreate); + }, + ); + } + + /** Org-level asset types keyed by uid for diff; empty map if list API fails. */ + private async loadExistingAssetTypesMap(): Promise>> { const existingByUid = new Map>(); try { const existing = await this.getWorkspaceAssetTypes(''); @@ -49,11 +75,15 @@ export default class ImportAssetTypes extends AssetManagementImportAdapter { } catch (e) { log.debug(`Could not fetch existing asset types, will attempt to create all: ${e}`, this.importContext.context); } + return existingByUid; + } - this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); - - type ToCreate = { uid: string; payload: Record }; - const toCreate: ToCreate[] = []; + private buildAssetTypesToCreate( + items: Record[], + existingByUid: Map>, + stripKeys: string[], + ): AssetTypeToCreate[] { + const toCreate: AssetTypeToCreate[] = []; for (const assetType of items) { const uid = assetType.uid as string; @@ -82,6 +112,10 @@ export default class ImportAssetTypes extends AssetManagementImportAdapter { toCreate.push({ uid, payload: omit(assetType, stripKeys) as Record }); } + return toCreate; + } + + private async importAssetTypesCreates(toCreate: AssetTypeToCreate[]): Promise { await runInBatches(toCreate, this.apiConcurrency, async ({ uid, payload }) => { try { await this.createAssetType(payload as any); diff --git a/packages/contentstack-asset-management/src/import/assets.ts b/packages/contentstack-asset-management/src/import/assets.ts index 4b942a186..14c5baed6 100644 --- a/packages/contentstack-asset-management/src/import/assets.ts +++ b/packages/contentstack-asset-management/src/import/assets.ts @@ -1,12 +1,12 @@ import { resolve as pResolve, join } from 'node:path'; import { existsSync, readFileSync } from 'node:fs'; -import { log } from '@contentstack/cli-utilities'; +import { FsUtility, log } from '@contentstack/cli-utilities'; import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; import { AssetManagementImportAdapter } from './base'; import { getArrayFromResponse } from '../utils/export-helpers'; -import { readChunkedJsonItems } from '../utils/chunked-json-read'; import { runInBatches } from '../utils/concurrent-batch'; +import { forEachChunkRecordsFromFs } from '../utils/chunked-json-reader'; import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; type FolderRecord = { @@ -26,6 +26,13 @@ type AssetRecord = { description?: string; }; +type UploadJob = { + asset: AssetRecord; + filePath: string; + mappedParentUid: string | undefined; + oldUid: string; +}; + /** * Imports folders and assets for a single AM space. * - Reads `spaces/{spaceUid}/assets/folders.json` → creates folders, builds folderUidMap @@ -38,16 +45,13 @@ export default class ImportAssets extends AssetManagementImportAdapter { super(apiConfig, importContext); } - /** - * Loads chunked `assets.json` when present; shared by reuse (identity maps) and upload path. - */ - private async loadExportedAssetItems(spaceDir: string): Promise { + private resolveAssetsChunkedLocation(spaceDir: string): { assetsDir: string; indexName: string } | null { const assetsDir = pResolve(spaceDir, 'assets'); - const assetsIndex = this.importContext.assetsFileName ?? 'assets.json'; - if (!existsSync(join(assetsDir, assetsIndex))) { + const indexName = this.importContext.assetsFileName ?? 'assets.json'; + if (!existsSync(join(assetsDir, indexName))) { return null; } - return readChunkedJsonItems(assetsDir, assetsIndex, this.importContext.context); + return { assetsDir, indexName }; } /** @@ -60,28 +64,39 @@ export default class ImportAssets extends AssetManagementImportAdapter { const uidMap: Record = {}; const urlMap: Record = {}; - const assetItems = await this.loadExportedAssetItems(spaceDir); - if (!assetItems) { + const loc = this.resolveAssetsChunkedLocation(spaceDir); + if (!loc) { log.debug( `No assets.json index in ${pResolve(spaceDir, 'assets')}, identity mappers empty`, this.importContext.context, ); return { uidMap, urlMap }; } + + const fs = new FsUtility({ basePath: loc.assetsDir, indexFileName: loc.indexName }); + let totalRows = 0; + + await forEachChunkRecordsFromFs( + fs, + { context: this.importContext.context, chunkReadLogLabel: 'assets' }, + async (records) => { + totalRows += records.length; + for (const asset of records) { + if (asset.uid) { + uidMap[asset.uid] = asset.uid; + } + if (asset.url) { + urlMap[asset.url] = asset.url; + } + } + }, + ); + log.debug( - `Building identity mappers for ${assetItems.length} exported asset(s) (reuse path)`, + `Built identity mappers for ${totalRows} exported asset row(s) (reuse path, chunked read)`, this.importContext.context, ); - for (const asset of assetItems) { - if (asset.uid) { - uidMap[asset.uid] = asset.uid; - } - if (asset.url) { - urlMap[asset.url] = asset.url; - } - } - return { uidMap, urlMap }; } @@ -117,68 +132,85 @@ export default class ImportAssets extends AssetManagementImportAdapter { } // ----------------------------------------------------------------------- - // 2. Import assets (chunked) + // 2. Import assets (chunked on disk — process one chunk file at a time) // ----------------------------------------------------------------------- - const assetItems = await this.loadExportedAssetItems(spaceDir); - if (!assetItems) { + const loc = this.resolveAssetsChunkedLocation(spaceDir); + if (!loc) { log.debug(`No assets.json index found in ${assetsDir}, skipping asset upload`, this.importContext.context); return { uidMap, urlMap }; } + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSETS].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSETS); - log.debug(`Uploading ${assetItems.length} asset(s) for space ${newSpaceUid}`, this.importContext.context); - - type UploadJob = { - asset: AssetRecord; - filePath: string; - mappedParentUid: string | undefined; - oldUid: string; - }; - const uploadJobs: UploadJob[] = []; - - for (const asset of assetItems) { - const oldUid = asset.uid; - const filename = asset.filename ?? asset.file_name ?? 'asset'; - const filePath = pResolve(assetsDir, 'files', oldUid, filename); - - if (!existsSync(filePath)) { - log.debug(`Asset file not found: ${filePath}, skipping`, this.importContext.context); - this.tick(false, `asset: ${oldUid}`, 'File not found on disk', PROCESS_NAMES.AM_IMPORT_ASSETS); - continue; - } + log.debug( + `Uploading assets for space ${newSpaceUid} from chunked export (incremental chunks)`, + this.importContext.context, + ); - const assetParent = asset.parent_uid && asset.parent_uid !== 'root' ? asset.parent_uid : undefined; - const mappedParentUid = assetParent ? folderUidMap[assetParent] ?? undefined : undefined; + const assetFs = new FsUtility({ basePath: loc.assetsDir, indexFileName: loc.indexName }); + let exportRowCount = 0; - uploadJobs.push({ asset, filePath, mappedParentUid, oldUid }); - } + await forEachChunkRecordsFromFs( + assetFs, + { context: this.importContext.context, chunkReadLogLabel: 'assets' }, + async (assetChunk) => { + exportRowCount += assetChunk.length; + const uploadJobs: UploadJob[] = []; - await runInBatches(uploadJobs, this.apiConcurrency, async ({ asset, filePath, mappedParentUid, oldUid }) => { - const filename = asset.filename ?? asset.file_name ?? 'asset'; - try { - const { asset: created } = await this.uploadAsset(newSpaceUid, filePath, { - title: asset.title ?? filename, - description: asset.description, - parent_uid: mappedParentUid, - }); + for (const asset of assetChunk) { + const oldUid = asset.uid; + const filename = asset.filename ?? asset.file_name ?? 'asset'; + const filePath = pResolve(assetsDir, 'files', oldUid, filename); - uidMap[oldUid] = created.uid; + if (!existsSync(filePath)) { + log.debug(`Asset file not found: ${filePath}, skipping`, this.importContext.context); + this.tick(false, `asset: ${oldUid}`, 'File not found on disk', PROCESS_NAMES.AM_IMPORT_ASSETS); + continue; + } - if (asset.url && created.url) { - urlMap[asset.url] = created.url; + const assetParent = asset.parent_uid && asset.parent_uid !== 'root' ? asset.parent_uid : undefined; + const mappedParentUid = assetParent ? folderUidMap[assetParent] ?? undefined : undefined; + + uploadJobs.push({ asset, filePath, mappedParentUid, oldUid }); } - this.tick(true, `asset: ${oldUid}`, null, PROCESS_NAMES.AM_IMPORT_ASSETS); - log.debug(`Uploaded asset ${oldUid} → ${created.uid}`, this.importContext.context); - } catch (e) { - this.tick( - false, - `asset: ${oldUid}`, - (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSETS].FAILED, - PROCESS_NAMES.AM_IMPORT_ASSETS, + await runInBatches( + uploadJobs, + this.uploadAssetsBatchConcurrency, + async ({ asset, filePath, mappedParentUid, oldUid }) => { + const filename = asset.filename ?? asset.file_name ?? 'asset'; + try { + const { asset: created } = await this.uploadAsset(newSpaceUid, filePath, { + title: asset.title ?? filename, + description: asset.description, + parent_uid: mappedParentUid, + }); + + uidMap[oldUid] = created.uid; + + if (asset.url && created.url) { + urlMap[asset.url] = created.url; + } + + this.tick(true, `asset: ${oldUid}`, null, PROCESS_NAMES.AM_IMPORT_ASSETS); + log.debug(`Uploaded asset ${oldUid} → ${created.uid}`, this.importContext.context); + } catch (e) { + this.tick( + false, + `asset: ${oldUid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSETS].FAILED, + PROCESS_NAMES.AM_IMPORT_ASSETS, + ); + log.debug(`Failed to upload asset ${oldUid}: ${e}`, this.importContext.context); + } + }, ); - log.debug(`Failed to upload asset ${oldUid}: ${e}`, this.importContext.context); - } - }); + }, + ); + + log.debug( + `Finished asset uploads for space ${newSpaceUid} (${exportRowCount} row(s) read from export chunks)`, + this.importContext.context, + ); return { uidMap, urlMap }; } @@ -212,7 +244,7 @@ export default class ImportAssets extends AssetManagementImportAdapter { } } - await runInBatches(ready, this.apiConcurrency, async (folder) => { + await runInBatches(ready, this.importFoldersBatchConcurrency, async (folder) => { const { parent_uid: parentUid } = folder; const isRootParent = !parentUid || parentUid === 'root'; try { diff --git a/packages/contentstack-asset-management/src/import/base.ts b/packages/contentstack-asset-management/src/import/base.ts index 298f7f613..ef1d4c0f5 100644 --- a/packages/contentstack-asset-management/src/import/base.ts +++ b/packages/contentstack-asset-management/src/import/base.ts @@ -4,7 +4,6 @@ import { CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; import { AM_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY } from '../constants/index'; -import { readChunkedJsonItems } from '../utils/chunked-json-read'; export type { ImportContext }; @@ -67,6 +66,16 @@ export class AssetManagementImportAdapter extends AssetManagementAdapter { return this.importContext.apiConcurrency ?? FALLBACK_AM_API_CONCURRENCY; } + /** Upload batch size; falls back to {@link apiConcurrency}. */ + protected get uploadAssetsBatchConcurrency(): number { + return this.importContext.uploadAssetsConcurrency ?? this.apiConcurrency; + } + + /** Folder creation batch size; falls back to {@link apiConcurrency}. */ + protected get importFoldersBatchConcurrency(): number { + return this.importContext.importFoldersConcurrency ?? this.apiConcurrency; + } + protected getAssetTypesDir(): string { return pResolve(this.importContext.spacesRootPath, this.importContext.assetTypesDir ?? 'asset_types'); } @@ -74,11 +83,4 @@ export class AssetManagementImportAdapter extends AssetManagementAdapter { protected getFieldsDir(): string { return pResolve(this.importContext.spacesRootPath, this.importContext.fieldsDir ?? 'fields'); } - - /** - * Reads all items from a chunked JSON store via {@link readChunkedJsonItems} (FsUtility). - */ - protected async readAllChunkedJson>(dir: string, indexFileName: string): Promise { - return readChunkedJsonItems(dir, indexFileName, this.importContext.context); - } } diff --git a/packages/contentstack-asset-management/src/import/fields.ts b/packages/contentstack-asset-management/src/import/fields.ts index a153eb31d..1e431b14f 100644 --- a/packages/contentstack-asset-management/src/import/fields.ts +++ b/packages/contentstack-asset-management/src/import/fields.ts @@ -1,3 +1,5 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; import omit from 'lodash/omit'; import isEqual from 'lodash/isEqual'; import { log } from '@contentstack/cli-utilities'; @@ -6,7 +8,9 @@ import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-man import { AssetManagementImportAdapter } from './base'; import { FALLBACK_FIELDS_IMPORT_INVALID_KEYS, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; import { runInBatches } from '../utils/concurrent-batch'; -import { readChunkedJsonItems } from '../utils/chunked-json-read'; +import { forEachChunkedJsonStore } from '../utils/chunked-json-reader'; + +type FieldToCreate = { uid: string; payload: Record }; /** * Reads shared fields from `spaces/fields/fields.json` and POSTs each to the @@ -30,15 +34,35 @@ export default class ImportFields extends AssetManagementImportAdapter { const stripKeys = this.importContext.fieldsImportInvalidKeys ?? [...FALLBACK_FIELDS_IMPORT_INVALID_KEYS]; const dir = this.getFieldsDir(); const indexName = this.importContext.fieldsFileName ?? 'fields.json'; - const items = await readChunkedJsonItems>(dir, indexName, this.importContext.context); + const indexPath = join(dir, indexName); - if (items.length === 0) { - log.debug('No shared fields to import', this.importContext.context); + if (!existsSync(indexPath)) { + log.debug('No shared fields to import (index missing)', this.importContext.context); return; } - // Fetch existing fields from the target org keyed by uid for diff comparison. - // Fields are org-level; the spaceUid param in getWorkspaceFields is unused in the path. + const existingByUid = await this.loadExistingFieldsMap(); + + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FIELDS].IMPORTING, PROCESS_NAMES.AM_IMPORT_FIELDS); + + await forEachChunkedJsonStore>( + dir, + indexName, + { + context: this.importContext.context, + chunkReadLogLabel: 'fields', + onOpenError: (e) => log.debug(`Could not open chunked fields index: ${e}`, this.importContext.context), + onEmptyIndexer: () => log.debug('No shared fields to import (empty indexer)', this.importContext.context), + }, + async (records) => { + const toCreate = this.buildFieldsToCreate(records, existingByUid, stripKeys); + await this.importFieldsCreates(toCreate); + }, + ); + } + + /** Org-level fields keyed by uid for diff; empty map if list API fails. */ + private async loadExistingFieldsMap(): Promise>> { const existingByUid = new Map>(); try { const existing = await this.getWorkspaceFields(''); @@ -49,11 +73,15 @@ export default class ImportFields extends AssetManagementImportAdapter { } catch (e) { log.debug(`Could not fetch existing fields, will attempt to create all: ${e}`, this.importContext.context); } + return existingByUid; + } - this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FIELDS].IMPORTING, PROCESS_NAMES.AM_IMPORT_FIELDS); - - type ToCreate = { uid: string; payload: Record }; - const toCreate: ToCreate[] = []; + private buildFieldsToCreate( + items: Record[], + existingByUid: Map>, + stripKeys: string[], + ): FieldToCreate[] { + const toCreate: FieldToCreate[] = []; for (const field of items) { const uid = field.uid as string; @@ -82,6 +110,10 @@ export default class ImportFields extends AssetManagementImportAdapter { toCreate.push({ uid, payload: omit(field, stripKeys) as Record }); } + return toCreate; + } + + private async importFieldsCreates(toCreate: FieldToCreate[]): Promise { await runInBatches(toCreate, this.apiConcurrency, async ({ uid, payload }) => { try { await this.createField(payload as any); diff --git a/packages/contentstack-asset-management/src/import/spaces.ts b/packages/contentstack-asset-management/src/import/spaces.ts index d44fc795b..c2642baa8 100644 --- a/packages/contentstack-asset-management/src/import/spaces.ts +++ b/packages/contentstack-asset-management/src/import/spaces.ts @@ -44,6 +44,8 @@ export class ImportSpaces { sourceApiKey, context, apiConcurrency, + uploadAssetsConcurrency, + importFoldersConcurrency, spacesDirName, fieldsDir, assetTypesDir, @@ -70,6 +72,8 @@ export class ImportSpaces { org_uid, context, apiConcurrency, + uploadAssetsConcurrency, + importFoldersConcurrency, spacesDirName, fieldsDir, assetTypesDir, diff --git a/packages/contentstack-asset-management/src/types/asset-management-api.ts b/packages/contentstack-asset-management/src/types/asset-management-api.ts index 8ae90f6ad..65ecfb3d5 100644 --- a/packages/contentstack-asset-management/src/types/asset-management-api.ts +++ b/packages/contentstack-asset-management/src/types/asset-management-api.ts @@ -147,10 +147,6 @@ export type AssetManagementExportOptions = { * can reconstruct old CMA proxy URLs (format: /v3/assets/{apiKey}/{amUid}/...). */ apiKey?: string; - /** - * Chunked JSON write batch size (items per FsUtility write). From export `modules['asset-management']`. - */ - chunkWriteBatchSize?: number; /** * FsUtility `chunkFileSize` in MB for AM export chunked writes. */ @@ -180,10 +176,14 @@ export type ImportContext = { /** Optional logging context (same shape as ExportConfig.context). */ context?: Record; /** - * Max parallel AM API calls for import (fields, asset types, folders batch, uploads). + * Max parallel AM API calls for import (fields, asset types, and default for folders/uploads). * Set from `AssetManagementImportOptions.apiConcurrency`. */ apiConcurrency?: number; + /** Overrides parallel limit for asset uploads when set (import `modules['asset-management'].uploadAssetsConcurrency`). */ + uploadAssetsConcurrency?: number; + /** Overrides parallel limit for folder creation batches when set (import `modules['asset-management'].importFoldersConcurrency`). */ + importFoldersConcurrency?: number; /** Relative dir under content dir for AM export root (e.g. `spaces`). */ spacesDirName?: string; fieldsDir?: string; @@ -226,6 +226,8 @@ export type AssetManagementImportOptions = { backupDir?: string; /** Parallel AM API limit; defaults to package constant when omitted. */ apiConcurrency?: number; + uploadAssetsConcurrency?: number; + importFoldersConcurrency?: number; spacesDirName?: string; fieldsDir?: string; assetTypesDir?: string; diff --git a/packages/contentstack-asset-management/src/types/export-types.ts b/packages/contentstack-asset-management/src/types/export-types.ts index d1480adb2..7cefa319b 100644 --- a/packages/contentstack-asset-management/src/types/export-types.ts +++ b/packages/contentstack-asset-management/src/types/export-types.ts @@ -2,8 +2,6 @@ export type ExportContext = { spacesRootPath: string; context?: Record; securedAssets?: boolean; - /** From export config; AM falls back to package constants when unset. */ - chunkWriteBatchSize?: number; chunkFileSizeMb?: number; }; diff --git a/packages/contentstack-asset-management/src/utils/chunked-json-read.ts b/packages/contentstack-asset-management/src/utils/chunked-json-read.ts deleted file mode 100644 index 4e09616c0..000000000 --- a/packages/contentstack-asset-management/src/utils/chunked-json-read.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { FsUtility, log } from '@contentstack/cli-utilities'; - -/** - * Read all items from a chunked JSON store (index + chunk files) using FsUtility, - * matching the pattern used in contentstack-import entry modules. - */ -export async function readChunkedJsonItems>( - basePath: string, - indexFileName: string, - context?: Record, -): Promise { - try { - const fs = new FsUtility({ basePath, indexFileName }); - const indexer = fs.indexFileContent; - const items: T[] = []; - for (const _ in indexer) { - const chunk = await fs.readChunkFiles.next().catch((err: unknown): null => { - log.debug(`Error reading chunk: ${err}`, context); - return null; - }); - if (chunk) { - items.push(...(Object.values(chunk as Record))); - } - } - return items; - } catch (err) { - log.debug(`readChunkedJsonItems failed for ${basePath}/${indexFileName}: ${err}`, context); - return []; - } -} diff --git a/packages/contentstack-asset-management/src/utils/chunked-json-reader.ts b/packages/contentstack-asset-management/src/utils/chunked-json-reader.ts new file mode 100644 index 000000000..838cfe653 --- /dev/null +++ b/packages/contentstack-asset-management/src/utils/chunked-json-reader.ts @@ -0,0 +1,66 @@ +import { FsUtility, log } from '@contentstack/cli-utilities'; + +export type ForEachChunkedJsonStoreOptions = { + context?: Record; + /** Shown in log.debug: `Error reading