diff --git a/.talismanrc b/.talismanrc index 6cbd26fc6..e446461c2 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,4 +1,4 @@ fileignoreconfig: - filename: pnpm-lock.yaml - checksum: 78b7fca30ae03e2570a384c5432c10f0e6b023f492b68929795adcb4613e8673 + checksum: 7a2d08a029dd995917883504dd816fc7a579aca7d3e39fc5368959f0e766c7b2 version: '1.0' diff --git a/packages/contentstack-asset-management/.eslintrc b/packages/contentstack-asset-management/.eslintrc new file mode 100644 index 000000000..4d2aef653 --- /dev/null +++ b/packages/contentstack-asset-management/.eslintrc @@ -0,0 +1,54 @@ +{ + "env": { + "node": true + }, + "parser": "@typescript-eslint/parser", + "parserOptions": { + "project": "tsconfig.json", + "sourceType": "module" + }, + "extends": [ + "oclif-typescript", + "plugin:@typescript-eslint/recommended" + ], + "rules": { + "@typescript-eslint/no-unused-vars": [ + "error", + { + "args": "none" + } + ], + "@typescript-eslint/prefer-namespace-keyword": "error", + "@typescript-eslint/quotes": [ + "error", + "single", + { + "avoidEscape": true, + "allowTemplateLiterals": true + } + ], + "semi": "off", + "@typescript-eslint/type-annotation-spacing": "error", + "@typescript-eslint/no-redeclare": "off", + "eqeqeq": [ + "error", + "smart" + ], + "id-match": "error", + "no-eval": "error", + "no-var": "error", + "quotes": "off", + "indent": "off", + "camelcase": "off", + "comma-dangle": "off", + "arrow-parens": "off", + "operator-linebreak": "off", + "object-curly-spacing": "off", + "node/no-missing-import": "off", + "padding-line-between-statements": "off", + "@typescript-eslint/ban-ts-ignore": "off", + "unicorn/no-abusive-eslint-disable": "off", + "unicorn/consistent-function-scoping": "off", + "@typescript-eslint/no-use-before-define": "off" + } +} diff --git a/packages/contentstack-asset-management/.gitignore b/packages/contentstack-asset-management/.gitignore new file mode 100644 index 000000000..6cf8928fa --- /dev/null +++ b/packages/contentstack-asset-management/.gitignore @@ -0,0 +1,3 @@ +lib/ +node_modules/ +*.tsbuildinfo diff --git a/packages/contentstack-asset-management/README.md b/packages/contentstack-asset-management/README.md new file mode 100644 index 000000000..87867c247 --- /dev/null +++ b/packages/contentstack-asset-management/README.md @@ -0,0 +1,49 @@ +# @contentstack/cli-asset-management + +Asset Management 2.0 API adapter for Contentstack CLI export and import. Used by the export and import plugins when Asset Management (AM 2.0) is enabled. To learn how to export and import content in Contentstack, refer to the [Migration guide](https://www.contentstack.com/docs/developers/cli/migration/). + +[![License](https://img.shields.io/npm/l/@contentstack/cli)](https://github.com/contentstack/cli/blob/main/LICENSE) + + +* [@contentstack/cli-asset-management](#contentstackcli-asset-management) +* [Overview](#overview) +* [Usage](#usage) +* [Exports](#exports) + + +# Overview + +This package provides: + +- **AssetManagementAdapter** – HTTP client for the Asset Management API (spaces, assets, folders, fields, asset types). +- **exportSpaceStructure** – Exports space metadata and full workspace structure (metadata, folders, assets, fields, asset types) for linked workspaces. +- **Types** – `AssetManagementExportOptions`, `LinkedWorkspace`, `IAssetManagementAdapter`, and related types for export/import integration. + +# Usage + +This package is consumed by the export and import plugins. When using the export CLI with the `--asset-management` flag (or when the host app enables AM 2.0), the export plugin calls `exportSpaceStructure` with linked workspaces and options: + +```ts +import { exportSpaceStructure } from '@contentstack/cli-asset-management'; + +await exportSpaceStructure({ + linkedWorkspaces, + exportDir, + branchName: 'main', + assetManagementUrl, + org_uid, + context, + progressManager, + progressProcessName, + updateStatus, + downloadAsset, // optional +}); +``` + +# Exports + +| Export | Description | +|--------|-------------| +| `exportSpaceStructure` | Async function to export space structure for given linked workspaces. | +| `AssetManagementAdapter` | Class to call the Asset Management API (getSpace, getWorkspaceFields, getWorkspaceAssets, etc.). | +| Types from `./types` | `AssetManagementExportOptions`, `ExportSpaceOptions`, `ChunkedJsonWriteOptions`, `LinkedWorkspace`, `SpaceResponse`, `FieldsResponse`, `AssetTypesResponse`, and related API types. | diff --git a/packages/contentstack-asset-management/package.json b/packages/contentstack-asset-management/package.json new file mode 100644 index 000000000..f41f31f1d --- /dev/null +++ b/packages/contentstack-asset-management/package.json @@ -0,0 +1,58 @@ +{ + "name": "@contentstack/cli-asset-management", + "version": "1.0.0", + "description": "Asset Management 2.0 API adapter for export and import", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "files": [ + "lib", + "oclif.manifest.json" + ], + "scripts": { + "build": "pnpm compile", + "clean": "rm -rf ./lib ./node_modules tsconfig.build.tsbuildinfo", + "compile": "tsc -b tsconfig.json", + "postpack": "rm -f oclif.manifest.json", + "prepack": "pnpm compile && oclif manifest && oclif readme", + "version": "oclif readme && git add README.md", + "lint": "eslint src/**/*.ts", + "format": "eslint src/**/*.ts --fix", + "test": "nyc --extension .ts mocha --require ts-node/register --forbid-only \"test/**/*.test.ts\"", + "posttest": "npm run lint", + "test:unit": "mocha --require ts-node/register --forbid-only \"test/unit/**/*.test.ts\"", + "test:unit:report": "nyc --extension .ts mocha --require ts-node/register --forbid-only \"test/unit/**/*.test.ts\"" + }, + "keywords": [ + "contentstack", + "asset-management", + "cli" + ], + "license": "MIT", + "dependencies": { + "@contentstack/cli-utilities": "~2.0.0-beta.5" + }, + "oclif": { + "commands": "./lib/commands", + "bin": "csdx", + "devPlugins": [ + "@oclif/plugin-help" + ], + "repositoryPrefix": "<%- repo %>/blob/main/packages/contentstack-asset-management/<%- commandPath %>" + }, + "devDependencies": { + "@types/chai": "^4.3.11", + "@types/mocha": "^10.0.6", + "@types/node": "^20.17.50", + "@types/sinon": "^17.0.2", + "chai": "^4.4.1", + "eslint": "^8.57.1", + "eslint-config-oclif": "^6.0.68", + "mocha": "^10.8.2", + "nyc": "^15.1.0", + "oclif": "^4.17.46", + "sinon": "^17.0.1", + "source-map-support": "^0.5.21", + "ts-node": "^10.9.2", + "typescript": "^5.8.3" + } +} \ No newline at end of file diff --git a/packages/contentstack-asset-management/src/constants/index.ts b/packages/contentstack-asset-management/src/constants/index.ts new file mode 100644 index 000000000..9d6bca636 --- /dev/null +++ b/packages/contentstack-asset-management/src/constants/index.ts @@ -0,0 +1,98 @@ +/** Fallback when export/import do not pass `chunkFileSizeMb`. */ +export const FALLBACK_AM_CHUNK_FILE_SIZE_MB = 1; +/** Fallback when import does not pass `apiConcurrency`. */ +export const FALLBACK_AM_API_CONCURRENCY = 5; +/** @deprecated Use FALLBACK_AM_API_CONCURRENCY */ +export const DEFAULT_AM_API_CONCURRENCY = FALLBACK_AM_API_CONCURRENCY; + +/** Fallback strip lists when import options omit `fieldsImportInvalidKeys` / `assetTypesImportInvalidKeys`. */ +export const FALLBACK_FIELDS_IMPORT_INVALID_KEYS = [ + 'created_at', + 'created_by', + 'updated_at', + 'updated_by', + 'is_system', + 'asset_types_count', +] as const; +export const FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS = [ + 'created_at', + 'created_by', + 'updated_at', + 'updated_by', + 'is_system', + 'category', + 'preview_image_url', + 'category_detail', +] as const; + +/** @deprecated Use FALLBACK_AM_CHUNK_FILE_SIZE_MB */ +export const CHUNK_FILE_SIZE_MB = FALLBACK_AM_CHUNK_FILE_SIZE_MB; + +/** + * Main process name for Asset Management 2.0 export (single progress bar). + * Use this when adding/starting the process and for all ticks. + */ +export const AM_MAIN_PROCESS_NAME = 'Asset Management 2.0'; + +/** + * Process names for Asset Management 2.0 export progress (for tick labels). + */ +export const PROCESS_NAMES = { + AM_SPACE_METADATA: 'Space metadata', + AM_FOLDERS: 'Folders', + AM_ASSETS: 'Assets', + AM_FIELDS: 'Fields', + AM_ASSET_TYPES: 'Asset types', + AM_DOWNLOADS: 'Asset downloads', + // Import process names + AM_IMPORT_FIELDS: 'Import fields', + AM_IMPORT_ASSET_TYPES: 'Import asset types', + AM_IMPORT_FOLDERS: 'Import folders', + AM_IMPORT_ASSETS: 'Import assets', +} as const; + +/** + * Status messages for each process (exporting, fetching, importing, failed). + */ +export const PROCESS_STATUS = { + [PROCESS_NAMES.AM_SPACE_METADATA]: { + EXPORTING: 'Exporting space metadata...', + FAILED: 'Failed to export space metadata.', + }, + [PROCESS_NAMES.AM_FOLDERS]: { + FETCHING: 'Fetching folders...', + FAILED: 'Failed to fetch folders.', + }, + [PROCESS_NAMES.AM_ASSETS]: { + FETCHING: 'Fetching assets...', + FAILED: 'Failed to fetch assets.', + }, + [PROCESS_NAMES.AM_FIELDS]: { + FETCHING: 'Fetching fields...', + FAILED: 'Failed to fetch fields.', + }, + [PROCESS_NAMES.AM_ASSET_TYPES]: { + FETCHING: 'Fetching asset types...', + FAILED: 'Failed to fetch asset types.', + }, + [PROCESS_NAMES.AM_DOWNLOADS]: { + DOWNLOADING: 'Downloading asset files...', + FAILED: 'Failed to download assets.', + }, + [PROCESS_NAMES.AM_IMPORT_FIELDS]: { + IMPORTING: 'Importing shared fields...', + FAILED: 'Failed to import fields.', + }, + [PROCESS_NAMES.AM_IMPORT_ASSET_TYPES]: { + IMPORTING: 'Importing shared asset types...', + FAILED: 'Failed to import asset types.', + }, + [PROCESS_NAMES.AM_IMPORT_FOLDERS]: { + IMPORTING: 'Importing folders...', + FAILED: 'Failed to import folders.', + }, + [PROCESS_NAMES.AM_IMPORT_ASSETS]: { + IMPORTING: 'Importing assets...', + FAILED: 'Failed to import assets.', + }, +} as const; diff --git a/packages/contentstack-asset-management/src/export/asset-types.ts b/packages/contentstack-asset-management/src/export/asset-types.ts new file mode 100644 index 000000000..6223b38d5 --- /dev/null +++ b/packages/contentstack-asset-management/src/export/asset-types.ts @@ -0,0 +1,30 @@ +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig } from '../types/asset-management-api'; +import type { ExportContext } from '../types/export-types'; +import { AssetManagementExportAdapter } from './base'; +import { getArrayFromResponse } from '../utils/export-helpers'; +import { PROCESS_NAMES } from '../constants/index'; + +export default class ExportAssetTypes extends AssetManagementExportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) { + super(apiConfig, exportContext); + } + + async start(spaceUid: string): Promise { + await this.init(); + + log.debug('Starting shared asset types export process...', this.exportContext.context); + + const assetTypesData = await this.getWorkspaceAssetTypes(spaceUid); + const items = getArrayFromResponse(assetTypesData, 'asset_types'); + const dir = this.getAssetTypesDir(); + if (items.length === 0) { + log.info('No asset types to export, writing empty asset-types', this.exportContext.context); + } else { + log.debug(`Writing ${items.length} shared asset types`, this.exportContext.context); + } + await this.writeItemsToChunkedJson(dir, 'asset-types.json', 'asset_types', ['uid', 'title', 'category', 'file_extension'], items); + this.tick(true, PROCESS_NAMES.AM_ASSET_TYPES, null); + } +} diff --git a/packages/contentstack-asset-management/src/export/assets.ts b/packages/contentstack-asset-management/src/export/assets.ts new file mode 100644 index 000000000..acd0f1676 --- /dev/null +++ b/packages/contentstack-asset-management/src/export/assets.ts @@ -0,0 +1,142 @@ +import { resolve as pResolve } from 'node:path'; +import { Readable } from 'node:stream'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { configHandler, log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, LinkedWorkspace } from '../types/asset-management-api'; +import type { ExportContext } from '../types/export-types'; +import { AssetManagementExportAdapter } from './base'; +import { getAssetItems, writeStreamToFile } from '../utils/export-helpers'; +import { runInBatches } from '../utils/concurrent-batch'; +import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; + +export default class ExportAssets extends AssetManagementExportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) { + super(apiConfig, exportContext); + } + + async start(workspace: LinkedWorkspace, spaceDir: string): Promise { + await this.init(); + + log.debug(`Starting assets export for space ${workspace.space_uid}`, this.exportContext.context); + log.info(`Exporting asset folders, metadata, and files for space ${workspace.space_uid}`, this.exportContext.context); + + const assetsDir = pResolve(spaceDir, 'assets'); + await mkdir(assetsDir, { recursive: true }); + log.debug(`Assets directory ready: ${assetsDir}`, this.exportContext.context); + + log.debug(`Fetching folders and assets for space ${workspace.space_uid}`, this.exportContext.context); + + const [folders, assetsData] = await Promise.all([ + this.getWorkspaceFolders(workspace.space_uid, workspace.uid), + this.getWorkspaceAssets(workspace.space_uid, workspace.uid), + ]); + + await writeFile(pResolve(assetsDir, 'folders.json'), JSON.stringify(folders, null, 2)); + this.tick(true, `folders: ${workspace.space_uid}`, null); + log.debug(`Wrote folders.json for space ${workspace.space_uid}`, this.exportContext.context); + + const assetItems = getAssetItems(assetsData); + log.debug( + assetItems.length === 0 + ? `No assets for space ${workspace.space_uid}, wrote empty assets.json` + : `Writing ${assetItems.length} assets metadata for space ${workspace.space_uid}`, + this.exportContext.context, + ); + await this.writeItemsToChunkedJson( + assetsDir, + 'assets.json', + 'assets', + ['uid', 'url', 'filename', 'file_name', 'parent_uid'], + assetItems, + ); + log.debug( + `Finished writing chunked assets metadata (${assetItems.length} item(s)) under ${assetsDir}`, + this.exportContext.context, + ); + log.info( + assetItems.length === 0 + ? `Wrote empty asset metadata for space ${workspace.space_uid}` + : `Wrote ${assetItems.length} asset metadata record(s) for space ${workspace.space_uid}`, + this.exportContext.context, + ); + this.tick(true, `assets: ${workspace.space_uid} (${assetItems.length})`, null); + + log.debug(`Starting binary downloads for space ${workspace.space_uid}`, this.exportContext.context); + await this.downloadWorkspaceAssets(assetsData, assetsDir, workspace.space_uid); + } + + private async downloadWorkspaceAssets(assetsData: unknown, assetsDir: string, spaceUid: string): Promise { + const items = getAssetItems(assetsData); + if (items.length === 0) { + log.info(`No asset files to download for space ${spaceUid}`, this.exportContext.context); + log.debug('No assets to download', this.exportContext.context); + return; + } + + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].DOWNLOADING); + log.info(`Downloading asset files for space ${spaceUid} (${items.length} in metadata)`, this.exportContext.context); + log.debug(`Downloading ${items.length} asset file(s) for space ${spaceUid}...`, this.exportContext.context); + const filesDir = pResolve(assetsDir, 'files'); + await mkdir(filesDir, { recursive: true }); + log.debug(`Asset files directory ready: ${filesDir}`, this.exportContext.context); + + const securedAssets = this.exportContext.securedAssets ?? false; + const authtoken = securedAssets ? configHandler.get('authtoken') : null; + log.debug( + `Asset downloads: securedAssets=${securedAssets}, concurrency=${this.downloadAssetsBatchConcurrency}`, + this.exportContext.context, + ); + let lastError: string | null = null; + let allSuccess = true; + let downloadOk = 0; + let downloadFail = 0; + + const validItems = items.filter((asset) => Boolean(asset.url && (asset.uid ?? asset._uid))); + const skipped = items.length - validItems.length; + if (skipped > 0) { + log.debug( + `Skipping ${skipped} asset row(s) without url or uid (${validItems.length} file download(s) scheduled)`, + this.exportContext.context, + ); + } + await runInBatches(validItems, this.downloadAssetsBatchConcurrency, async (asset) => { + const uid = asset.uid ?? asset._uid; + const url = asset.url; + const filename = asset.filename ?? asset.file_name ?? 'asset'; + if (!url || !uid) return; + try { + const separator = url.includes('?') ? '&' : '?'; + const downloadUrl = securedAssets && authtoken ? `${url}${separator}authtoken=${authtoken}` : url; + const response = await fetch(downloadUrl); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const body = response.body; + if (!body) throw new Error('No response body'); + const nodeStream = Readable.fromWeb(body as Parameters[0]); + const assetFolderPath = pResolve(filesDir, uid); + await mkdir(assetFolderPath, { recursive: true }); + const filePath = pResolve(assetFolderPath, filename); + await writeStreamToFile(nodeStream, filePath); + downloadOk += 1; + log.debug(`Downloaded asset ${uid} → ${filePath}`, this.exportContext.context); + } catch (e) { + allSuccess = false; + downloadFail += 1; + lastError = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED; + log.debug(`Failed to download asset ${uid}: ${e}`, this.exportContext.context); + } + }); + + this.tick(allSuccess, `downloads: ${spaceUid}`, lastError); + log.info( + allSuccess + ? `Finished downloading ${downloadOk} asset file(s) for space ${spaceUid}` + : `Asset downloads for space ${spaceUid} completed with errors: ${downloadOk} succeeded, ${downloadFail} failed`, + this.exportContext.context, + ); + log.debug( + `Asset downloads finished for space ${spaceUid}: ok=${downloadOk}, failed=${downloadFail}, allSuccess=${allSuccess}`, + this.exportContext.context, + ); + } +} diff --git a/packages/contentstack-asset-management/src/export/base.ts b/packages/contentstack-asset-management/src/export/base.ts new file mode 100644 index 000000000..055d2d3ba --- /dev/null +++ b/packages/contentstack-asset-management/src/export/base.ts @@ -0,0 +1,108 @@ +import { resolve as pResolve } from 'node:path'; +import { writeFile } from 'node:fs/promises'; +import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig } from '../types/asset-management-api'; +import type { ExportContext } from '../types/export-types'; +import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; +import { AM_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY, FALLBACK_AM_CHUNK_FILE_SIZE_MB } from '../constants/index'; + +export type { ExportContext }; + +/** + * Base class for export modules. Extends the API adapter and adds export context, + * internal progress management, and shared write helpers. + */ +export class AssetManagementExportAdapter extends AssetManagementAdapter { + protected readonly apiConfig: AssetManagementAPIConfig; + protected readonly exportContext: ExportContext; + protected progressManager: CLIProgressManager | null = null; + protected parentProgressManager: CLIProgressManager | null = null; + protected readonly processName: string = AM_MAIN_PROCESS_NAME; + + constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) { + super(apiConfig); + this.apiConfig = apiConfig; + this.exportContext = exportContext; + } + + public setParentProgressManager(parent: CLIProgressManager): void { + this.parentProgressManager = parent; + } + + protected get progressOrParent(): CLIProgressManager | null { + return this.parentProgressManager ?? this.progressManager; + } + + protected createNestedProgress(moduleName: string): CLIProgressManager { + if (this.parentProgressManager) { + this.progressManager = this.parentProgressManager; + return this.parentProgressManager; + } + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; + this.progressManager = CLIProgressManager.createNested(moduleName, showConsoleLogs); + return this.progressManager; + } + + protected tick(success: boolean, itemName: string, error: string | null, processName?: string): void { + this.progressOrParent?.tick?.(success, itemName, error, processName ?? this.processName); + } + + protected updateStatus(message: string, processName?: string): void { + this.progressOrParent?.updateStatus?.(message, processName ?? this.processName); + } + + protected completeProcess(processName: string, success: boolean): void { + if (!this.parentProgressManager) { + this.progressManager?.completeProcess?.(processName, success); + } + } + + protected get spacesRootPath(): string { + return this.exportContext.spacesRootPath; + } + + /** Parallel AM export limit for bootstrap and default batch operations. */ + protected get apiConcurrency(): number { + return this.exportContext.apiConcurrency ?? FALLBACK_AM_API_CONCURRENCY; + } + + /** Asset download batch size; falls back to {@link apiConcurrency}. */ + protected get downloadAssetsBatchConcurrency(): number { + return this.exportContext.downloadAssetsConcurrency ?? this.apiConcurrency; + } + + protected getAssetTypesDir(): string { + return pResolve(this.exportContext.spacesRootPath, 'asset_types'); + } + + protected getFieldsDir(): string { + return pResolve(this.exportContext.spacesRootPath, 'fields'); + } + + protected async writeItemsToChunkedJson( + dir: string, + indexFileName: string, + moduleName: string, + metaPickKeys: string[], + items: unknown[], + ): Promise { + if (items.length === 0) { + await writeFile(pResolve(dir, indexFileName), '{}'); + return; + } + const chunkMb = this.exportContext.chunkFileSizeMb ?? FALLBACK_AM_CHUNK_FILE_SIZE_MB; + const fs = new FsUtility({ + basePath: dir, + indexFileName, + chunkFileSize: chunkMb, + moduleName, + fileExt: 'json', + metaPickKeys, + keepMetadata: true, + }); + fs.writeIntoFile(items as Record[], { mapKeyVal: true }); + fs.completeFile(true); + } +} diff --git a/packages/contentstack-asset-management/src/export/fields.ts b/packages/contentstack-asset-management/src/export/fields.ts new file mode 100644 index 000000000..fd997e5e5 --- /dev/null +++ b/packages/contentstack-asset-management/src/export/fields.ts @@ -0,0 +1,30 @@ +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig } from '../types/asset-management-api'; +import type { ExportContext } from '../types/export-types'; +import { AssetManagementExportAdapter } from './base'; +import { getArrayFromResponse } from '../utils/export-helpers'; +import { PROCESS_NAMES } from '../constants/index'; + +export default class ExportFields extends AssetManagementExportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) { + super(apiConfig, exportContext); + } + + async start(spaceUid: string): Promise { + await this.init(); + + log.debug('Starting shared fields export process...', this.exportContext.context); + + const fieldsData = await this.getWorkspaceFields(spaceUid); + const items = getArrayFromResponse(fieldsData, 'fields'); + const dir = this.getFieldsDir(); + if (items.length === 0) { + log.info('No field items to export, writing empty fields', this.exportContext.context); + } else { + log.debug(`Writing ${items.length} shared fields`, this.exportContext.context); + } + await this.writeItemsToChunkedJson(dir, 'fields.json', 'fields', ['uid', 'title', 'display_type'], items); + this.tick(true, PROCESS_NAMES.AM_FIELDS, null); + } +} diff --git a/packages/contentstack-asset-management/src/export/index.ts b/packages/contentstack-asset-management/src/export/index.ts new file mode 100644 index 000000000..7d71e361e --- /dev/null +++ b/packages/contentstack-asset-management/src/export/index.ts @@ -0,0 +1,7 @@ +export { ExportSpaces, exportSpaceStructure } from './spaces'; +export { default as ExportAssetTypes } from './asset-types'; +export { default as ExportFields } from './fields'; +export { default as ExportAssets } from './assets'; +export { default as ExportWorkspace } from './workspaces'; +export { AssetManagementExportAdapter } from './base'; +export type { ExportContext } from './base'; diff --git a/packages/contentstack-asset-management/src/export/spaces.ts b/packages/contentstack-asset-management/src/export/spaces.ts new file mode 100644 index 000000000..5d639a410 --- /dev/null +++ b/packages/contentstack-asset-management/src/export/spaces.ts @@ -0,0 +1,140 @@ +import { resolve as pResolve } from 'node:path'; +import { mkdir } from 'node:fs/promises'; +import { log, CLIProgressManager, configHandler, handleAndLogError } from '@contentstack/cli-utilities'; + +import type { AssetManagementExportOptions, AssetManagementAPIConfig } from '../types/asset-management-api'; +import type { ExportContext } from '../types/export-types'; +import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; +import { AM_MAIN_PROCESS_NAME, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +import ExportAssetTypes from './asset-types'; +import ExportFields from './fields'; +import ExportWorkspace from './workspaces'; + +/** + * Orchestrates the full Asset Management 2.0 export: shared asset types and fields, + * then per-workspace metadata and assets (including internal download). + * Progress and download are fully owned by this package. + */ +export class ExportSpaces { + private readonly options: AssetManagementExportOptions; + private parentProgressManager: CLIProgressManager | null = null; + private progressManager: CLIProgressManager | null = null; + + constructor(options: AssetManagementExportOptions) { + this.options = options; + } + + public setParentProgressManager(parent: CLIProgressManager): void { + this.parentProgressManager = parent; + } + + async start(): Promise { + const { + linkedWorkspaces, + exportDir, + branchName, + assetManagementUrl, + org_uid, + apiKey, + context, + securedAssets, + chunkFileSizeMb, + } = this.options; + + if (!linkedWorkspaces.length) { + log.debug('No linked workspaces to export', context); + return; + } + + log.debug('Starting Asset Management export process...', context); + log.info('Started Asset Management export', context); + log.debug(`Exporting Asset Management 2.0 (${linkedWorkspaces.length} space(s))`, context); + log.debug(`Spaces: ${linkedWorkspaces.map((ws) => ws.space_uid).join(', ')}`, context); + + const spacesRootPath = pResolve(exportDir, branchName || 'main', 'spaces'); + await mkdir(spacesRootPath, { recursive: true }); + log.debug(`Spaces root path: ${spacesRootPath}`, context); + + const totalSteps = 2 + linkedWorkspaces.length * 4; + const progress = this.createProgress(); + progress.addProcess(AM_MAIN_PROCESS_NAME, totalSteps); + progress + .startProcess(AM_MAIN_PROCESS_NAME) + .updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_FIELDS].FETCHING, AM_MAIN_PROCESS_NAME); + + const apiConfig: AssetManagementAPIConfig = { + baseURL: assetManagementUrl, + headers: { organization_uid: org_uid }, + context, + }; + const exportContext: ExportContext = { + spacesRootPath, + context, + securedAssets, + chunkFileSizeMb, + apiConcurrency: this.options.apiConcurrency, + downloadAssetsConcurrency: this.options.downloadAssetsConcurrency, + }; + + const sharedFieldsDir = pResolve(spacesRootPath, 'fields'); + const sharedAssetTypesDir = pResolve(spacesRootPath, 'asset_types'); + await mkdir(sharedFieldsDir, { recursive: true }); + await mkdir(sharedAssetTypesDir, { recursive: true }); + + const firstSpaceUid = linkedWorkspaces[0].space_uid; + try { + const exportAssetTypes = new ExportAssetTypes(apiConfig, exportContext); + exportAssetTypes.setParentProgressManager(progress); + const exportFields = new ExportFields(apiConfig, exportContext); + exportFields.setParentProgressManager(progress); + await Promise.all([exportAssetTypes.start(firstSpaceUid), exportFields.start(firstSpaceUid)]); + + for (const ws of linkedWorkspaces) { + progress.updateStatus(`Exporting space: ${ws.space_uid}...`, AM_MAIN_PROCESS_NAME); + log.debug(`Exporting space: ${ws.space_uid}`, context); + const spaceDir = pResolve(spacesRootPath, ws.space_uid); + try { + const exportWorkspace = new ExportWorkspace(apiConfig, exportContext); + exportWorkspace.setParentProgressManager(progress); + await exportWorkspace.start(ws, spaceDir, branchName || 'main'); + log.debug(`Exported workspace structure for space ${ws.space_uid}`, context); + } catch (err) { + log.debug(`Failed to export workspace for space ${ws.space_uid}: ${err}`, context); + progress.tick( + false, + `space: ${ws.space_uid}`, + (err as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_SPACE_METADATA].FAILED, + AM_MAIN_PROCESS_NAME, + ); + throw err; + } + } + + progress.completeProcess(AM_MAIN_PROCESS_NAME, true); + log.info('Asset Management export completed successfully', context); + log.debug('Asset Management 2.0 export completed', context); + } catch (err) { + progress.completeProcess(AM_MAIN_PROCESS_NAME, false); + handleAndLogError(err, { ...(context as Record) }, 'Asset Management export failed'); + throw err; + } + } + + private createProgress(): CLIProgressManager { + if (this.parentProgressManager) { + this.progressManager = this.parentProgressManager; + return this.parentProgressManager; + } + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; + this.progressManager = CLIProgressManager.createNested(AM_MAIN_PROCESS_NAME, showConsoleLogs); + return this.progressManager; + } +} + +/** + * Entry point for callers that prefer a function. Delegates to ExportSpaces. + */ +export async function exportSpaceStructure(options: AssetManagementExportOptions): Promise { + await new ExportSpaces(options).start(); +} diff --git a/packages/contentstack-asset-management/src/export/workspaces.ts b/packages/contentstack-asset-management/src/export/workspaces.ts new file mode 100644 index 000000000..c2f5bb4f1 --- /dev/null +++ b/packages/contentstack-asset-management/src/export/workspaces.ts @@ -0,0 +1,46 @@ +import { resolve as pResolve } from 'node:path'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, LinkedWorkspace } from '../types/asset-management-api'; +import type { ExportContext } from '../types/export-types'; +import { AssetManagementExportAdapter } from './base'; +import ExportAssets from './assets'; +import { PROCESS_NAMES } from '../constants/index'; + +export default class ExportWorkspace extends AssetManagementExportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) { + super(apiConfig, exportContext); + } + + async start(workspace: LinkedWorkspace, spaceDir: string, branchName: string): Promise { + await this.init(); + + log.debug(`Starting export for AM space ${workspace.space_uid}`, this.exportContext.context); + + const spaceResponse = await this.getSpace(workspace.space_uid); + const space = spaceResponse.space; + await mkdir(spaceDir, { recursive: true }); + + const metadata = { + ...space, + workspace_uid: workspace.uid, + is_default: workspace.is_default, + branch: branchName || 'main', + }; + const metadataPath = pResolve(spaceDir, 'metadata.json'); + try { + await writeFile(metadataPath, JSON.stringify(metadata, null, 2)); + } catch (e) { + log.warn(`Could not write ${metadataPath}: ${e}`, this.exportContext.context); + throw e; + } + this.tick(true, `space: ${workspace.space_uid}`, null); + log.debug(`Space metadata written for ${workspace.space_uid}`, this.exportContext.context); + + const assetsExporter = new ExportAssets(this.apiConfig, this.exportContext); + if (this.progressOrParent) assetsExporter.setParentProgressManager(this.progressOrParent); + await assetsExporter.start(workspace, spaceDir); + log.debug(`Exported workspace structure for space ${workspace.space_uid}`, this.exportContext.context); + } +} diff --git a/packages/contentstack-asset-management/src/import/asset-types.ts b/packages/contentstack-asset-management/src/import/asset-types.ts new file mode 100644 index 000000000..71f5fbbac --- /dev/null +++ b/packages/contentstack-asset-management/src/import/asset-types.ts @@ -0,0 +1,141 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import omit from 'lodash/omit'; +import isEqual from 'lodash/isEqual'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import { FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +import { runInBatches } from '../utils/concurrent-batch'; +import { forEachChunkedJsonStore } from '../utils/chunked-json-reader'; + +type AssetTypeToCreate = { uid: string; payload: Record }; + +/** + * Reads shared asset types from `spaces/asset_types/asset-types.json` and POSTs + * each to the target org-level AM endpoint (`POST /api/asset_types`). + * + * Strategy: Fetch → Diff → Create only missing, warn on conflict + * 1. Fetch asset types that already exist in the target org. + * 2. Skip entries where is_system=true (platform-owned, cannot be created via API). + * 3. If uid already exists and definition differs → warn and skip. + * 4. If uid already exists and definition matches → silently skip. + * 5. Strip read-only/computed keys from the POST body before creating new asset types. + */ +export default class ImportAssetTypes extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + async start(): Promise { + await this.init(); + + log.debug('Starting shared asset types import process...', this.importContext.context); + + const stripKeys = this.importContext.assetTypesImportInvalidKeys ?? [...FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS]; + const dir = this.getAssetTypesDir(); + const indexName = this.importContext.assetTypesFileName ?? 'asset-types.json'; + const indexPath = join(dir, indexName); + + if (!existsSync(indexPath)) { + log.info('No shared asset types to import (index missing)', this.importContext.context); + return; + } + + const existingByUid = await this.loadExistingAssetTypesMap(); + + this.updateStatus( + PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].IMPORTING, + PROCESS_NAMES.AM_IMPORT_ASSET_TYPES, + ); + + await forEachChunkedJsonStore>( + dir, + indexName, + { + context: this.importContext.context, + chunkReadLogLabel: 'asset-types', + onOpenError: (e) => log.warn(`Could not open chunked asset-types index: ${e}`, this.importContext.context), + onEmptyIndexer: () => log.debug('No shared asset types to import (empty indexer)', this.importContext.context), + }, + async (records) => { + const toCreate = this.buildAssetTypesToCreate(records, existingByUid, stripKeys); + await this.importAssetTypesCreates(toCreate); + }, + ); + } + + /** Org-level asset types keyed by uid for diff; empty map if list API fails. */ + private async loadExistingAssetTypesMap(): Promise>> { + const existingByUid = new Map>(); + try { + const existing = await this.getWorkspaceAssetTypes(''); + for (const at of existing.asset_types ?? []) { + existingByUid.set(at.uid, at as Record); + } + log.debug(`Target org has ${existingByUid.size} existing asset type(s)`, this.importContext.context); + } catch (e) { + log.debug(`Could not fetch existing asset types, will attempt to create all: ${e}`, this.importContext.context); + } + return existingByUid; + } + + private buildAssetTypesToCreate( + items: Record[], + existingByUid: Map>, + stripKeys: string[], + ): AssetTypeToCreate[] { + const toCreate: AssetTypeToCreate[] = []; + + for (const assetType of items) { + const uid = assetType.uid as string; + + if (assetType.is_system) { + log.debug(`Skipping system asset type: ${uid}`, this.importContext.context); + continue; + } + + const existing = existingByUid.get(uid); + if (existing) { + const exportedClean = omit(assetType, stripKeys); + const existingClean = omit(existing, stripKeys); + if (!isEqual(exportedClean, existingClean)) { + log.warn( + `Asset type "${uid}" already exists in the target org with a different definition. Skipping — to apply the exported definition, delete the asset type from the target org first.`, + this.importContext.context, + ); + } else { + log.debug( + `Asset type "${uid}" already exists with matching definition, skipping`, + this.importContext.context, + ); + } + this.tick(true, `asset-type: ${uid} (skipped, already exists)`, null, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + continue; + } + + toCreate.push({ uid, payload: omit(assetType, stripKeys) as Record }); + } + + return toCreate; + } + + private async importAssetTypesCreates(toCreate: AssetTypeToCreate[]): Promise { + await runInBatches(toCreate, this.apiConcurrency, async ({ uid, payload }) => { + try { + await this.createAssetType(payload as any); + this.tick(true, `asset-type: ${uid}`, null, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + log.debug(`Imported asset type: ${uid}`, this.importContext.context); + } catch (e) { + this.tick( + false, + `asset-type: ${uid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].FAILED, + PROCESS_NAMES.AM_IMPORT_ASSET_TYPES, + ); + log.debug(`Failed to import asset type ${uid}: ${e}`, this.importContext.context); + } + }); + } +} diff --git a/packages/contentstack-asset-management/src/import/assets.ts b/packages/contentstack-asset-management/src/import/assets.ts new file mode 100644 index 000000000..b69721245 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/assets.ts @@ -0,0 +1,348 @@ +import { resolve as pResolve, join } from 'node:path'; +import { existsSync, readFileSync } from 'node:fs'; +import { FsUtility, log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import { getArrayFromResponse } from '../utils/export-helpers'; +import { runInBatches } from '../utils/concurrent-batch'; +import { forEachChunkRecordsFromFs } from '../utils/chunked-json-reader'; +import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; + +type FolderRecord = { + uid: string; + title: string; + description?: string; + parent_uid?: string; +}; + +type AssetRecord = { + uid: string; + url: string; + filename?: string; + file_name?: string; + parent_uid?: string; + title?: string; + description?: string; +}; + +type UploadJob = { + asset: AssetRecord; + filePath: string; + mappedParentUid: string | undefined; + oldUid: string; +}; + +/** + * Imports folders and assets for a single AM space. + * - Reads `spaces/{spaceUid}/assets/folders.json` → creates folders, builds folderUidMap + * - Reads chunked `assets.json` → uploads each file from `files/{oldUid}/{filename}` + * - Builds UID and URL mapper entries for entries.ts consumption + * Mirrors ExportAssets. + */ +export default class ImportAssets extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + private resolveAssetsChunkedLocation(spaceDir: string): { assetsDir: string; indexName: string } | null { + const assetsDir = pResolve(spaceDir, 'assets'); + const indexName = this.importContext.assetsFileName ?? 'assets.json'; + if (!existsSync(join(assetsDir, indexName))) { + return null; + } + return { assetsDir, indexName }; + } + + /** + * Build identity uid/url mappers from export JSON only (reuse path — no upload). + * Keys and values are equal so lookupAssets contract is satisfied without remapping. + */ + async buildIdentityMappersFromExport( + spaceDir: string, + ): Promise<{ uidMap: Record; urlMap: Record }> { + const uidMap: Record = {}; + const urlMap: Record = {}; + + log.debug( + `Building identity mappers from export (reuse path, spaceDir=${spaceDir})`, + this.importContext.context, + ); + + const loc = this.resolveAssetsChunkedLocation(spaceDir); + if (!loc) { + log.debug( + `No assets.json index in ${pResolve(spaceDir, 'assets')}, identity mappers empty`, + this.importContext.context, + ); + return { uidMap, urlMap }; + } + + log.debug( + `Reading chunked assets for identity map: ${loc.assetsDir} (index: ${loc.indexName})`, + this.importContext.context, + ); + + const fs = new FsUtility({ basePath: loc.assetsDir, indexFileName: loc.indexName }); + let totalRows = 0; + + await forEachChunkRecordsFromFs( + fs, + { context: this.importContext.context, chunkReadLogLabel: 'assets' }, + async (records) => { + totalRows += records.length; + for (const asset of records) { + if (asset.uid) { + uidMap[asset.uid] = asset.uid; + } + if (asset.url) { + urlMap[asset.url] = asset.url; + } + } + }, + ); + + log.debug( + `Built identity mappers for ${totalRows} exported asset row(s): ${Object.keys(uidMap).length} uid entries, ${Object.keys(urlMap).length} url entries`, + this.importContext.context, + ); + log.info( + `Prepared identity uid/url mappers from ${totalRows} exported asset row(s) (reuse existing space)`, + this.importContext.context, + ); + + return { uidMap, urlMap }; + } + + async start( + newSpaceUid: string, + spaceDir: string, + ): Promise<{ uidMap: Record; urlMap: Record }> { + const assetsDir = pResolve(spaceDir, 'assets'); + const uidMap: Record = {}; + const urlMap: Record = {}; + + log.debug(`Starting assets and folders import for space ${newSpaceUid}`, this.importContext.context); + log.info(`Importing folders and assets into space ${newSpaceUid}`, this.importContext.context); + log.debug(`Assets directory: ${assetsDir}`, this.importContext.context); + + // ----------------------------------------------------------------------- + // 1. Import folders + // ----------------------------------------------------------------------- + const folderUidMap: Record = {}; + const foldersFileName = this.importContext.foldersFileName ?? 'folders.json'; + const foldersFilePath = join(assetsDir, foldersFileName); + + if (!existsSync(foldersFilePath)) { + log.debug(`No ${foldersFileName} at ${foldersFilePath}, skipping folder import`, this.importContext.context); + } + + if (existsSync(foldersFilePath)) { + let foldersData: unknown; + try { + foldersData = JSON.parse(readFileSync(foldersFilePath, 'utf8')); + } catch (e) { + log.warn(`Could not read ${foldersFileName}: ${e}`, this.importContext.context); + } + + if (foldersData) { + log.debug(`Reading folders from ${foldersFilePath}`, this.importContext.context); + const folders = getArrayFromResponse(foldersData, 'folders') as FolderRecord[]; + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FOLDERS].IMPORTING, PROCESS_NAMES.AM_IMPORT_FOLDERS); + log.debug( + `Importing ${folders.length} folder(s) for space ${newSpaceUid} (concurrency=${this.importFoldersBatchConcurrency})`, + this.importContext.context, + ); + await this.importFolders(newSpaceUid, folders, folderUidMap); + log.debug( + `Folder import phase complete: ${Object.keys(folderUidMap).length} exported folder uid(s) mapped to target`, + this.importContext.context, + ); + log.info( + `Finished importing ${Object.keys(folderUidMap).length} folder(s) for space ${newSpaceUid}`, + this.importContext.context, + ); + } + } + + // ----------------------------------------------------------------------- + // 2. Import assets (chunked on disk — process one chunk file at a time) + // ----------------------------------------------------------------------- + const loc = this.resolveAssetsChunkedLocation(spaceDir); + if (!loc) { + log.info( + `No asset metadata index in ${assetsDir}; skipping file uploads for space ${newSpaceUid}`, + this.importContext.context, + ); + log.debug(`No assets.json index found in ${assetsDir}, skipping asset upload`, this.importContext.context); + return { uidMap, urlMap }; + } + + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSETS].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSETS); + log.debug( + `Uploading assets for space ${newSpaceUid} from ${loc.assetsDir} (index: ${loc.indexName}, concurrency=${this.uploadAssetsBatchConcurrency})`, + this.importContext.context, + ); + + const assetFs = new FsUtility({ basePath: loc.assetsDir, indexFileName: loc.indexName }); + let exportRowCount = 0; + let uploadOk = 0; + let uploadFail = 0; + let missingFiles = 0; + + await forEachChunkRecordsFromFs( + assetFs, + { context: this.importContext.context, chunkReadLogLabel: 'assets' }, + async (assetChunk) => { + exportRowCount += assetChunk.length; + const uploadJobs: UploadJob[] = []; + + for (const asset of assetChunk) { + const oldUid = asset.uid; + const filename = asset.filename ?? asset.file_name ?? 'asset'; + const filePath = pResolve(assetsDir, 'files', oldUid, filename); + + if (!existsSync(filePath)) { + missingFiles += 1; + log.warn(`Asset file not found: ${filePath}, skipping`, this.importContext.context); + this.tick(false, `asset: ${oldUid}`, 'File not found on disk', PROCESS_NAMES.AM_IMPORT_ASSETS); + continue; + } + + const assetParent = asset.parent_uid && asset.parent_uid !== 'root' ? asset.parent_uid : undefined; + const mappedParentUid = assetParent ? folderUidMap[assetParent] ?? undefined : undefined; + + uploadJobs.push({ asset, filePath, mappedParentUid, oldUid }); + } + + const skippedInChunk = assetChunk.length - uploadJobs.length; + log.debug( + `Asset chunk: ${assetChunk.length} row(s), ${uploadJobs.length} upload job(s)${skippedInChunk ? `, ${skippedInChunk} missing on disk` : ''}`, + this.importContext.context, + ); + + await runInBatches( + uploadJobs, + this.uploadAssetsBatchConcurrency, + async ({ asset, filePath, mappedParentUid, oldUid }) => { + const filename = asset.filename ?? asset.file_name ?? 'asset'; + try { + const { asset: created } = await this.uploadAsset(newSpaceUid, filePath, { + title: asset.title ?? filename, + description: asset.description, + parent_uid: mappedParentUid, + }); + + uidMap[oldUid] = created.uid; + + if (asset.url && created.url) { + urlMap[asset.url] = created.url; + } + + this.tick(true, `asset: ${oldUid}`, null, PROCESS_NAMES.AM_IMPORT_ASSETS); + uploadOk += 1; + log.debug(`Uploaded asset ${oldUid} → ${created.uid} (${filePath})`, this.importContext.context); + } catch (e) { + uploadFail += 1; + this.tick( + false, + `asset: ${oldUid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSETS].FAILED, + PROCESS_NAMES.AM_IMPORT_ASSETS, + ); + log.debug(`Failed to upload asset ${oldUid}: ${e}`, this.importContext.context); + } + }, + ); + }, + ); + + log.debug( + `Finished asset uploads for space ${newSpaceUid}: rows=${exportRowCount}, ok=${uploadOk}, failed=${uploadFail}, missingFile=${missingFiles}`, + this.importContext.context, + ); + log.info( + uploadFail === 0 && missingFiles === 0 + ? `Finished importing ${uploadOk} asset file(s) for space ${newSpaceUid}` + : `Finished importing assets for space ${newSpaceUid}: ${uploadOk} uploaded, ${uploadFail} failed, ${missingFiles} missing on disk`, + this.importContext.context, + ); + + return { uidMap, urlMap }; + } + + /** + * Creates folders respecting hierarchy: parents before children. + * Uses multiple passes to handle arbitrary depth without requiring sorted input. + */ + private async importFolders( + newSpaceUid: string, + folders: FolderRecord[], + folderUidMap: Record, + ): Promise { + let remaining = [...folders]; + let prevLength = -1; + let pass = 0; + + while (remaining.length > 0 && remaining.length !== prevLength) { + pass += 1; + prevLength = remaining.length; + const ready: FolderRecord[] = []; + const nextPass: FolderRecord[] = []; + + for (const folder of remaining) { + const { parent_uid: parentUid } = folder; + const isRootParent = !parentUid || parentUid === 'root'; + const parentMapped = isRootParent || folderUidMap[parentUid] !== undefined; + + if (!parentMapped) { + nextPass.push(folder); + } else { + ready.push(folder); + } + } + + log.debug( + `Folder import pass ${pass}: creating ${ready.length} folder(s), ${nextPass.length} blocked on parent (${remaining.length} total remaining before this pass)`, + this.importContext.context, + ); + + await runInBatches(ready, this.importFoldersBatchConcurrency, async (folder) => { + const { parent_uid: parentUid } = folder; + const isRootParent = !parentUid || parentUid === 'root'; + try { + const { folder: created } = await this.createFolder(newSpaceUid, { + title: folder.title, + description: folder.description, + parent_uid: isRootParent ? undefined : folderUidMap[parentUid!], + }); + folderUidMap[folder.uid] = created.uid; + this.tick(true, `folder: ${folder.uid}`, null, PROCESS_NAMES.AM_IMPORT_FOLDERS); + log.debug(`Created folder ${folder.uid} → ${created.uid}`, this.importContext.context); + } catch (e) { + this.tick( + false, + `folder: ${folder.uid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FOLDERS].FAILED, + PROCESS_NAMES.AM_IMPORT_FOLDERS, + ); + log.debug(`Failed to create folder ${folder.uid}: ${e}`, this.importContext.context); + } + }); + + remaining = nextPass; + } + + log.debug( + `Folder import passes finished for space ${newSpaceUid} after ${pass} pass(es); ${Object.keys(folderUidMap).length} folder uid(s) mapped`, + this.importContext.context, + ); + + if (remaining.length > 0) { + log.warn( + `${remaining.length} folder(s) could not be imported (unresolved parent UIDs)`, + this.importContext.context, + ); + } + } +} diff --git a/packages/contentstack-asset-management/src/import/base.ts b/packages/contentstack-asset-management/src/import/base.ts new file mode 100644 index 000000000..ef1d4c0f5 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/base.ts @@ -0,0 +1,86 @@ +import { resolve as pResolve } from 'node:path'; +import { CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; +import { AM_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY } from '../constants/index'; + +export type { ImportContext }; + +/** + * Base class for all AM 2.0 import modules. Mirrors AssetManagementExportAdapter + * but carries ImportContext (spacesRootPath, apiKey, host, etc.) instead of ExportContext. + */ +export class AssetManagementImportAdapter extends AssetManagementAdapter { + protected readonly apiConfig: AssetManagementAPIConfig; + protected readonly importContext: ImportContext; + protected progressManager: CLIProgressManager | null = null; + protected parentProgressManager: CLIProgressManager | null = null; + protected readonly processName: string = AM_MAIN_PROCESS_NAME; + + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig); + this.apiConfig = apiConfig; + this.importContext = importContext; + } + + public setParentProgressManager(parent: CLIProgressManager): void { + this.parentProgressManager = parent; + } + + protected get progressOrParent(): CLIProgressManager | null { + return this.parentProgressManager ?? this.progressManager; + } + + protected createNestedProgress(moduleName: string): CLIProgressManager { + if (this.parentProgressManager) { + this.progressManager = this.parentProgressManager; + return this.parentProgressManager; + } + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; + this.progressManager = CLIProgressManager.createNested(moduleName, showConsoleLogs); + return this.progressManager; + } + + protected tick(success: boolean, itemName: string, error: string | null, processName?: string): void { + this.progressOrParent?.tick?.(success, itemName, error, processName ?? this.processName); + } + + protected updateStatus(message: string, processName?: string): void { + this.progressOrParent?.updateStatus?.(message, processName ?? this.processName); + } + + protected completeProcess(processName: string, success: boolean): void { + if (!this.parentProgressManager) { + this.progressManager?.completeProcess?.(processName, success); + } + } + + protected get spacesRootPath(): string { + return this.importContext.spacesRootPath; + } + + /** Parallel AM API limit for import batches. */ + protected get apiConcurrency(): number { + return this.importContext.apiConcurrency ?? FALLBACK_AM_API_CONCURRENCY; + } + + /** Upload batch size; falls back to {@link apiConcurrency}. */ + protected get uploadAssetsBatchConcurrency(): number { + return this.importContext.uploadAssetsConcurrency ?? this.apiConcurrency; + } + + /** Folder creation batch size; falls back to {@link apiConcurrency}. */ + protected get importFoldersBatchConcurrency(): number { + return this.importContext.importFoldersConcurrency ?? this.apiConcurrency; + } + + protected getAssetTypesDir(): string { + return pResolve(this.importContext.spacesRootPath, this.importContext.assetTypesDir ?? 'asset_types'); + } + + protected getFieldsDir(): string { + return pResolve(this.importContext.spacesRootPath, this.importContext.fieldsDir ?? 'fields'); + } +} diff --git a/packages/contentstack-asset-management/src/import/fields.ts b/packages/contentstack-asset-management/src/import/fields.ts new file mode 100644 index 000000000..9785906c2 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/fields.ts @@ -0,0 +1,135 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import omit from 'lodash/omit'; +import isEqual from 'lodash/isEqual'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import { FALLBACK_FIELDS_IMPORT_INVALID_KEYS, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +import { runInBatches } from '../utils/concurrent-batch'; +import { forEachChunkedJsonStore } from '../utils/chunked-json-reader'; + +type FieldToCreate = { uid: string; payload: Record }; + +/** + * Reads shared fields from `spaces/fields/fields.json` and POSTs each to the + * target org-level AM fields endpoint (`POST /api/fields`). + * + * Strategy: Fetch → Diff → Create only missing, warn on conflict + * 1. Fetch fields that already exist in the target org. + * 2. Skip entries where is_system=true (platform-owned, cannot be created via API). + * 3. If uid already exists and definition differs → warn and skip. + * 4. If uid already exists and definition matches → silently skip. + * 5. Strip read-only/computed keys from the POST body before creating new fields. + */ +export default class ImportFields extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + async start(): Promise { + await this.init(); + + log.debug('Starting shared fields import process...', this.importContext.context); + + const stripKeys = this.importContext.fieldsImportInvalidKeys ?? [...FALLBACK_FIELDS_IMPORT_INVALID_KEYS]; + const dir = this.getFieldsDir(); + const indexName = this.importContext.fieldsFileName ?? 'fields.json'; + const indexPath = join(dir, indexName); + + if (!existsSync(indexPath)) { + log.info('No shared fields to import (index missing)', this.importContext.context); + return; + } + + const existingByUid = await this.loadExistingFieldsMap(); + + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FIELDS].IMPORTING, PROCESS_NAMES.AM_IMPORT_FIELDS); + + await forEachChunkedJsonStore>( + dir, + indexName, + { + context: this.importContext.context, + chunkReadLogLabel: 'fields', + onOpenError: (e) => log.warn(`Could not open chunked fields index: ${e}`, this.importContext.context), + onEmptyIndexer: () => log.debug('No shared fields to import (empty indexer)', this.importContext.context), + }, + async (records) => { + const toCreate = this.buildFieldsToCreate(records, existingByUid, stripKeys); + await this.importFieldsCreates(toCreate); + }, + ); + } + + /** Org-level fields keyed by uid for diff; empty map if list API fails. */ + private async loadExistingFieldsMap(): Promise>> { + const existingByUid = new Map>(); + try { + const existing = await this.getWorkspaceFields(''); + for (const f of existing.fields ?? []) { + existingByUid.set(f.uid, f as Record); + } + log.debug(`Target org has ${existingByUid.size} existing field(s)`, this.importContext.context); + } catch (e) { + log.debug(`Could not fetch existing fields, will attempt to create all: ${e}`, this.importContext.context); + } + return existingByUid; + } + + private buildFieldsToCreate( + items: Record[], + existingByUid: Map>, + stripKeys: string[], + ): FieldToCreate[] { + const toCreate: FieldToCreate[] = []; + + for (const field of items) { + const uid = field.uid as string; + + if (field.is_system) { + log.debug(`Skipping system field: ${uid}`, this.importContext.context); + continue; + } + + const existing = existingByUid.get(uid); + if (existing) { + const exportedClean = omit(field, stripKeys); + const existingClean = omit(existing, stripKeys); + if (!isEqual(exportedClean, existingClean)) { + log.warn( + `Field "${uid}" already exists in the target org with a different definition. Skipping — to apply the exported definition, delete the field from the target org first.`, + this.importContext.context, + ); + } else { + log.debug(`Field "${uid}" already exists with matching definition, skipping`, this.importContext.context); + } + this.tick(true, `field: ${uid} (skipped, already exists)`, null, PROCESS_NAMES.AM_IMPORT_FIELDS); + continue; + } + + toCreate.push({ uid, payload: omit(field, stripKeys) as Record }); + } + + return toCreate; + } + + private async importFieldsCreates(toCreate: FieldToCreate[]): Promise { + await runInBatches(toCreate, this.apiConcurrency, async ({ uid, payload }) => { + try { + await this.createField(payload as any); + this.tick(true, `field: ${uid}`, null, PROCESS_NAMES.AM_IMPORT_FIELDS); + log.debug(`Imported field: ${uid}`, this.importContext.context); + } catch (e) { + this.tick( + false, + `field: ${uid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FIELDS].FAILED, + PROCESS_NAMES.AM_IMPORT_FIELDS, + ); + log.debug(`Failed to import field ${uid}: ${e}`, this.importContext.context); + } + }); + } +} diff --git a/packages/contentstack-asset-management/src/import/index.ts b/packages/contentstack-asset-management/src/import/index.ts new file mode 100644 index 000000000..61d8a457e --- /dev/null +++ b/packages/contentstack-asset-management/src/import/index.ts @@ -0,0 +1,7 @@ +export { ImportSpaces } from './spaces'; +export { default as ImportWorkspace } from './workspaces'; +export { default as ImportAssets } from './assets'; +export { default as ImportFields } from './fields'; +export { default as ImportAssetTypes } from './asset-types'; +export { AssetManagementImportAdapter } from './base'; +export type { ImportContext } from './base'; diff --git a/packages/contentstack-asset-management/src/import/spaces.ts b/packages/contentstack-asset-management/src/import/spaces.ts new file mode 100644 index 000000000..6f66d24be --- /dev/null +++ b/packages/contentstack-asset-management/src/import/spaces.ts @@ -0,0 +1,209 @@ +import { join, resolve as pResolve } from 'node:path'; +import { mkdirSync, readdirSync, statSync } from 'node:fs'; +import { writeFile } from 'node:fs/promises'; +import { log, CLIProgressManager, configHandler, handleAndLogError } from '@contentstack/cli-utilities'; + +import type { + AssetManagementAPIConfig, + ImportContext, + ImportResult, + ImportSpacesOptions, + SpaceMapping, +} from '../types/asset-management-api'; +import { AM_MAIN_PROCESS_NAME } from '../constants/index'; +import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; +import ImportAssetTypes from './asset-types'; +import ImportFields from './fields'; +import ImportWorkspace from './workspaces'; + +/** + * Top-level orchestrator for AM 2.0 import. + * Mirrors ExportSpaces: creates shared fields + asset types, then imports each space. + * Returns combined uidMap, urlMap, and spaceMappings for the bridge module. + */ +export class ImportSpaces { + private readonly options: ImportSpacesOptions; + private parentProgressManager: CLIProgressManager | null = null; + private progressManager: CLIProgressManager | null = null; + + constructor(options: ImportSpacesOptions) { + this.options = options; + } + + public setParentProgressManager(parent: CLIProgressManager): void { + this.parentProgressManager = parent; + } + + async start(): Promise { + const configOptions = this.options; + const spacesRootPath = pResolve(configOptions.contentDir, 'spaces'); + const org_uid = configOptions.org_uid; + const context = configOptions.context; + const importContext: ImportContext = { + spacesRootPath, + sourceApiKey: configOptions.sourceApiKey, + apiKey: configOptions.apiKey, + host: configOptions.host, + org_uid, + context, + apiConcurrency: configOptions.apiConcurrency, + uploadAssetsConcurrency: configOptions.uploadAssetsConcurrency, + importFoldersConcurrency: configOptions.importFoldersConcurrency, + spacesDirName: configOptions.spacesDirName, + fieldsDir: configOptions.fieldsDir, + assetTypesDir: configOptions.assetTypesDir, + fieldsFileName: configOptions.fieldsFileName, + assetTypesFileName: configOptions.assetTypesFileName, + foldersFileName: configOptions.foldersFileName, + assetsFileName: configOptions.assetsFileName, + fieldsImportInvalidKeys: configOptions.fieldsImportInvalidKeys, + assetTypesImportInvalidKeys: configOptions.assetTypesImportInvalidKeys, + mapperRootDir: configOptions.mapperRootDir, + mapperAssetsModuleDir: configOptions.mapperAssetsModuleDir, + mapperUidFileName: configOptions.mapperUidFileName, + mapperUrlFileName: configOptions.mapperUrlFileName, + mapperSpaceUidFileName: configOptions.mapperSpaceUidFileName, + }; + const apiConfig: AssetManagementAPIConfig = { + baseURL: configOptions.assetManagementUrl, + headers: { organization_uid: org_uid }, + context, + }; + + log.debug('Starting Asset Management import process...', context); + + // Discover space directories + let spaceDirs: string[] = []; + try { + spaceDirs = readdirSync(spacesRootPath).filter((entry) => { + try { + return statSync(join(spacesRootPath, entry)).isDirectory() && entry.startsWith('am'); + } catch { + return false; + } + }); + } catch (e) { + log.warn(`Could not read spaces root path ${spacesRootPath}: ${e}`, context); + } + + const totalSteps = 2 + spaceDirs.length * 2; + const progress = this.createProgress(); + progress.addProcess(AM_MAIN_PROCESS_NAME, totalSteps); + progress.startProcess(AM_MAIN_PROCESS_NAME); + + const allUidMap: Record = {}; + const allUrlMap: Record = {}; + const allSpaceUidMap: Record = {}; + const spaceMappings: SpaceMapping[] = []; + let hasFailures = false; + let spacesSucceeded = 0; + let spacesFailed = 0; + + // Space UIDs already present in the target org — reuse when export dir name matches a uid here. + const existingSpaceUids = new Set(); + try { + const adapterForList = new AssetManagementAdapter(apiConfig); + await adapterForList.init(); + const { spaces } = await adapterForList.listSpaces(); + for (const s of spaces) { + if (s.uid) existingSpaceUids.add(s.uid); + } + log.debug(`Found ${existingSpaceUids.size} existing space uid(s) in target org`, context); + } catch (e) { + log.debug(`Could not fetch existing spaces — reuse-by-uid disabled: ${e}`, context); + } + + try { + log.info('Started Asset Management import', context); + + // 1. Import shared fields + progress.updateStatus(`Importing shared fields...`, AM_MAIN_PROCESS_NAME); + const fieldsImporter = new ImportFields(apiConfig, importContext); + fieldsImporter.setParentProgressManager(progress); + await fieldsImporter.start(); + + // 2. Import shared asset types + progress.updateStatus('Importing shared asset types...', AM_MAIN_PROCESS_NAME); + const assetTypesImporter = new ImportAssetTypes(apiConfig, importContext); + assetTypesImporter.setParentProgressManager(progress); + await assetTypesImporter.start(); + + // 3. Import each space — continue on failure so partially-imported data is never lost + for (const spaceUid of spaceDirs) { + const spaceDir = join(spacesRootPath, spaceUid); + progress.updateStatus(`Importing space: ${spaceUid}...`, AM_MAIN_PROCESS_NAME); + log.debug(`Importing space: ${spaceUid}`, context); + + try { + const workspaceImporter = new ImportWorkspace(apiConfig, importContext); + workspaceImporter.setParentProgressManager(progress); + const result = await workspaceImporter.start(spaceUid, spaceDir, existingSpaceUids); + + // Newly created spaces get a new uid — add so later iterations in this run see it. + existingSpaceUids.add(result.newSpaceUid); + + Object.assign(allUidMap, result.uidMap); + Object.assign(allUrlMap, result.urlMap); + allSpaceUidMap[result.oldSpaceUid] = result.newSpaceUid; + spaceMappings.push({ + oldSpaceUid: result.oldSpaceUid, + newSpaceUid: result.newSpaceUid, + workspaceUid: result.workspaceUid, + isDefault: result.isDefault, + }); + + log.debug(`Imported space ${spaceUid} → ${result.newSpaceUid}`, context); + spacesSucceeded += 1; + } catch (err) { + hasFailures = true; + spacesFailed += 1; + progress.tick( + false, + `space: ${spaceUid}`, + (err as Error)?.message ?? 'Failed to import space', + AM_MAIN_PROCESS_NAME, + ); + log.warn(`Failed to import space ${spaceUid}: ${err}`, context); + } + } + + if (this.options.backupDir) { + const mapperRoot = importContext.mapperRootDir ?? 'mapper'; + const mapperAssetsMod = importContext.mapperAssetsModuleDir ?? 'assets'; + const mapperDir = join(this.options.backupDir, mapperRoot, mapperAssetsMod); + mkdirSync(mapperDir, { recursive: true }); + const uidFile = importContext.mapperUidFileName ?? 'uid-mapping.json'; + const urlFile = importContext.mapperUrlFileName ?? 'url-mapping.json'; + const spaceUidFile = importContext.mapperSpaceUidFileName ?? 'space-uid-mapping.json'; + await writeFile(join(mapperDir, uidFile), JSON.stringify(allUidMap), 'utf8'); + await writeFile(join(mapperDir, urlFile), JSON.stringify(allUrlMap), 'utf8'); + await writeFile(join(mapperDir, spaceUidFile), JSON.stringify(allSpaceUidMap), 'utf8'); + log.debug('Wrote AM 2.0 mapper files (uid, url, space-uid)', context); + } + + progress.completeProcess(AM_MAIN_PROCESS_NAME, !hasFailures); + log.info( + `Asset Management import finished: ${spacesSucceeded} space(s) succeeded, ${spacesFailed} failed, ${spaceDirs.length} attempted.`, + context, + ); + log.debug('Asset Management 2.0 import completed', context); + } catch (err) { + progress.completeProcess(AM_MAIN_PROCESS_NAME, false); + handleAndLogError(err, { ...(context as Record) }, 'Asset Management import failed'); + throw err; + } + + return { uidMap: allUidMap, urlMap: allUrlMap, spaceMappings, spaceUidMap: allSpaceUidMap }; + } + + private createProgress(): CLIProgressManager { + if (this.parentProgressManager) { + this.progressManager = this.parentProgressManager; + return this.parentProgressManager; + } + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; + this.progressManager = CLIProgressManager.createNested(AM_MAIN_PROCESS_NAME, showConsoleLogs); + return this.progressManager; + } +} diff --git a/packages/contentstack-asset-management/src/import/workspaces.ts b/packages/contentstack-asset-management/src/import/workspaces.ts new file mode 100644 index 000000000..e042b1f3b --- /dev/null +++ b/packages/contentstack-asset-management/src/import/workspaces.ts @@ -0,0 +1,84 @@ +import { join } from 'node:path'; +import { readFileSync } from 'node:fs'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext, SpaceMapping } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import ImportAssets from './assets'; +import { PROCESS_NAMES } from '../constants/index'; + +type WorkspaceResult = SpaceMapping & { + uidMap: Record; + urlMap: Record; +}; + +/** + * Handles import for a single AM 2.0 space directory. + * Reads `metadata.json`, creates the space in the target org when its uid is not + * already present, or reuses the existing space and emits identity mappers only. + * Returns the SpaceMapping plus UID/URL maps for the mapper files. + */ +export default class ImportWorkspace extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + async start( + oldSpaceUid: string, + spaceDir: string, + existingSpaceUids: Set = new Set(), + ): Promise { + await this.init(); + + log.debug(`Starting import for AM space directory ${oldSpaceUid}`, this.importContext.context); + + // Read exported metadata + const metadataPath = join(spaceDir, 'metadata.json'); + let metadata: Record = {}; + try { + metadata = JSON.parse(readFileSync(metadataPath, 'utf8')) as Record; + } catch (e) { + log.warn(`Could not read ${metadataPath} for space ${oldSpaceUid}: ${e}`, this.importContext.context); + } + + const exportedTitle = (metadata.title as string) ?? oldSpaceUid; + const description = metadata.description as string | undefined; + const isDefault = (metadata.is_default as boolean) ?? false; + const workspaceUid = 'main'; + + const assetsImporter = new ImportAssets(this.apiConfig, this.importContext); + if (this.progressOrParent) assetsImporter.setParentProgressManager(this.progressOrParent); + + // Reuse: target org already has a space with the same uid as the export directory. + if (existingSpaceUids.has(oldSpaceUid)) { + log.info( + `Reusing existing AM space "${oldSpaceUid}" (uid matches export directory); skipping create and upload.`, + this.importContext.context, + ); + const newSpaceUid = oldSpaceUid; + const { uidMap, urlMap } = await assetsImporter.buildIdentityMappersFromExport(spaceDir); + this.tick(true, `space: ${oldSpaceUid} → ${newSpaceUid} (reused)`, null, PROCESS_NAMES.AM_SPACE_METADATA); + return { + oldSpaceUid, + newSpaceUid, + workspaceUid, + isDefault, + uidMap, + urlMap, + }; + } + + // Create new space with exact exported title + log.debug(`Creating space "${exportedTitle}" (old uid: ${oldSpaceUid})`, this.importContext.context); + + const { space } = await this.createSpace({ title: exportedTitle, description }); + const newSpaceUid = space.uid; + + log.debug(`Created space ${newSpaceUid} (old: ${oldSpaceUid})`, this.importContext.context); + this.tick(true, `space: ${oldSpaceUid} → ${newSpaceUid}`, null, PROCESS_NAMES.AM_SPACE_METADATA); + + const { uidMap, urlMap } = await assetsImporter.start(newSpaceUid, spaceDir); + + return { oldSpaceUid, newSpaceUid, workspaceUid, isDefault, uidMap, urlMap }; + } +} diff --git a/packages/contentstack-asset-management/src/index.ts b/packages/contentstack-asset-management/src/index.ts new file mode 100644 index 000000000..c66c638d0 --- /dev/null +++ b/packages/contentstack-asset-management/src/index.ts @@ -0,0 +1,5 @@ +export * from './constants/index'; +export * from './types'; +export * from './utils'; +export * from './export'; +export * from './import'; diff --git a/packages/contentstack-asset-management/src/types/asset-management-api.ts b/packages/contentstack-asset-management/src/types/asset-management-api.ts new file mode 100644 index 000000000..40423da89 --- /dev/null +++ b/packages/contentstack-asset-management/src/types/asset-management-api.ts @@ -0,0 +1,311 @@ +/** + * Linked workspace from CMA branch settings (am_v2.linked_workspaces). + * Consumed by export/import after fetching branch with include_settings: true. + */ +export type LinkedWorkspace = { + uid: string; + space_uid: string; + is_default: boolean; +}; + +/** + * Space details from GET /api/spaces/{space_uid}. + */ +export type Space = { + uid: string; + title?: string; + description?: string; + org_uid?: string; + owner_uid?: string; + default_locale?: string; + default_workspace?: string; + tags?: string[]; + settings?: Record; + created_by?: string; + updated_by?: string; + created_at?: string; + updated_at?: string; + meta_info?: { + assets_count?: number; + folders_count?: number; + storage?: number; + last_modified_at?: string; + }; +}; + +/** Response shape of GET /api/spaces/{space_uid}. */ +export type SpaceResponse = { space: Space }; + +/** Response shape of GET /api/spaces (list all spaces in the org). */ +export type SpacesListResponse = { spaces: Space[]; count?: number }; + +/** + * Field structure from GET /api/fields (org-level). + */ +export type FieldStruct = { + uid: string; + title?: string; + description?: string | null; + display_type?: string; + is_system?: boolean; + is_multiple?: boolean; + is_mandatory?: boolean; + asset_types_count?: number; + created_at?: string; + created_by?: string; + updated_at?: string; + updated_by?: string; +}; + +/** Response shape of GET /api/fields. */ +export type FieldsResponse = { + count: number; + relation: string; + fields: FieldStruct[]; +}; + +/** + * Options object for asset type (from GET /api/asset_types). + */ +export type AssetTypeOptions = { + title?: string; + publishable?: boolean; + is_page?: boolean; + singleton?: boolean; + sub_title?: string[]; + url_pattern?: string; + url_prefix?: string; +}; + +/** + * Asset type structure from GET /api/asset_types (org-level). + */ +export type AssetTypeStruct = { + uid: string; + title?: string; + is_system?: boolean; + fields?: string[]; + options?: AssetTypeOptions; + description?: string; + content_type?: string; + file_extension?: string; + created_by?: string; + updated_by?: string; + created_at?: string; + updated_at?: string; + category?: string; + preview_image_url?: string; + category_detail?: string; +}; + +/** Response shape of GET /api/asset_types. */ +export type AssetTypesResponse = { + count: number; + relation: string; + asset_types: AssetTypeStruct[]; +}; + +/** + * Configuration for AssetManagementAdapter constructor. + */ +export type AssetManagementAPIConfig = { + baseURL: string; + headers?: Record; + /** Optional context for logging (e.g. exportConfig.context) */ + context?: Record; +}; + +/** + * Adapter interface for Asset Management API calls. + * Used by export and (future) import. + */ +export interface IAssetManagementAdapter { + init(): Promise; + listSpaces(): Promise; + getSpace(spaceUid: string): Promise; + getWorkspaceFields(spaceUid: string): Promise; + getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise; + getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise; + getWorkspaceAssetTypes(spaceUid: string): Promise; +} + +/** + * Options for exporting space structure (used by export app after fetching linked workspaces). + */ +export type AssetManagementExportOptions = { + linkedWorkspaces: LinkedWorkspace[]; + exportDir: string; + branchName: string; + assetManagementUrl: string; + org_uid: string; + context?: Record; + /** When true, the AM package will add authtoken to asset download URLs. */ + securedAssets?: boolean; + /** + * API key of the stack being exported. + * Saved to `spaces/export-metadata.json` so that during import the URL mapper + * can reconstruct old CMA proxy URLs (format: /v3/assets/{apiKey}/{amUid}/...). + */ + apiKey?: string; + /** + * FsUtility `chunkFileSize` in MB for AM export chunked writes. + */ + chunkFileSizeMb?: number; + /** + * Max parallel AM API/export tasks for export (shared module bootstrap default). + */ + apiConcurrency?: number; + /** + * Max parallel asset file downloads per workspace. + */ + downloadAssetsConcurrency?: number; +}; + +// --------------------------------------------------------------------------- +// Import types +// --------------------------------------------------------------------------- + +/** + * Context passed down to every import adapter class. + * Mirrors ExportContext but carries the import-specific fields needed for + * URL mapper reconstruction and API calls. + */ +export type ImportContext = { + /** Absolute path to the root `spaces/` directory inside the backup/content dir. */ + spacesRootPath: string; + /** Source stack API key — used to reconstruct old CMA proxy URLs. */ + sourceApiKey?: string; + /** Target stack API key — used to build new CMA proxy URLs. */ + apiKey: string; + /** Target CMA host (may include /v3), e.g. "https://api.contentstack.io/v3". */ + host: string; + /** Target org UID — required as `x-organization-uid` header when creating spaces. */ + org_uid: string; + /** Optional logging context (same shape as ExportConfig.context). */ + context?: Record; + /** + * Max parallel AM API calls for import (fields, asset types, and default for folders/uploads). + * Set from `ImportSpacesOptions.apiConcurrency` (or host wiring). + */ + apiConcurrency?: number; + /** Overrides parallel limit for asset uploads when set (import `modules['asset-management'].uploadAssetsConcurrency`). */ + uploadAssetsConcurrency?: number; + /** Overrides parallel limit for folder creation batches when set (import `modules['asset-management'].importFoldersConcurrency`). */ + importFoldersConcurrency?: number; + /** Relative dir under content dir for AM export root (e.g. `spaces`). */ + spacesDirName?: string; + fieldsDir?: string; + assetTypesDir?: string; + fieldsFileName?: string; + assetTypesFileName?: string; + foldersFileName?: string; + assetsFileName?: string; + fieldsImportInvalidKeys?: string[]; + assetTypesImportInvalidKeys?: string[]; + /** `{backupDir}/{mapperRootDir}/{mapperAssetsModuleDir}/` for AM mapper JSON. */ + mapperRootDir?: string; + mapperAssetsModuleDir?: string; + mapperUidFileName?: string; + mapperUrlFileName?: string; + mapperSpaceUidFileName?: string; +}; + +/** + * Single options object for `ImportSpaces` (matches the export-side pattern: one flat shape from the host, + * then AM splits API vs context internally like `ExportSpaces`). + */ +export type ImportSpacesOptions = { + /** Absolute path to the root content / backup directory. */ + contentDir: string; + /** AM 2.0 base URL (e.g. "https://am.contentstack.io"). */ + assetManagementUrl: string; + org_uid: string; + apiKey: string; + host: string; + sourceApiKey?: string; + context?: Record; + /** When set, mapper JSON is written after import under `{backupDir}/mapper/...`. */ + backupDir?: string; + apiConcurrency?: number; + uploadAssetsConcurrency?: number; + importFoldersConcurrency?: number; + spacesDirName?: string; + fieldsDir?: string; + assetTypesDir?: string; + fieldsFileName?: string; + assetTypesFileName?: string; + foldersFileName?: string; + assetsFileName?: string; + fieldsImportInvalidKeys?: string[]; + assetTypesImportInvalidKeys?: string[]; + mapperRootDir?: string; + mapperAssetsModuleDir?: string; + mapperUidFileName?: string; + mapperUrlFileName?: string; + mapperSpaceUidFileName?: string; +}; + +/** + * Maps an old source-org space UID to the newly created target-org space UID. + */ +export type SpaceMapping = { + oldSpaceUid: string; + newSpaceUid: string; + /** Workspace identifier inside the space (typically "main"). */ + workspaceUid: string; + isDefault: boolean; +}; + +/** + * The value returned by `ImportSpaces.start()`. + * When `ImportSpacesOptions.backupDir` is set, the AM package also writes these maps under + * `mapper/assets/` for `entries.ts` to resolve asset references. + */ +export type ImportResult = { + uidMap: Record; + urlMap: Record; + spaceMappings: SpaceMapping[]; + /** old space UID → new space UID, written to mapper/assets/space-uid-mapping.json */ + spaceUidMap: Record; +}; + +// --------------------------------------------------------------------------- +// Import payload types (confirmed from Postman collection) +// --------------------------------------------------------------------------- + +export type CreateSpacePayload = { + title: string; + description?: string; +}; + +export type CreateFolderPayload = { + title: string; + description?: string; + parent_uid?: string; +}; + +export type CreateAssetMetadata = { + title?: string; + description?: string; + parent_uid?: string; +}; + +export type CreateFieldPayload = { + uid: string; + title: string; + display_type?: string; + child?: unknown[]; + is_mandatory?: boolean; + is_multiple?: boolean; + [key: string]: unknown; +}; + +export type CreateAssetTypePayload = { + uid: string; + title: string; + description?: string; + content_type?: string; + file_extension?: string | string[]; + fields?: string[]; + [key: string]: unknown; +}; diff --git a/packages/contentstack-asset-management/src/types/export-types.ts b/packages/contentstack-asset-management/src/types/export-types.ts new file mode 100644 index 000000000..865302a60 --- /dev/null +++ b/packages/contentstack-asset-management/src/types/export-types.ts @@ -0,0 +1,19 @@ +export type ExportContext = { + spacesRootPath: string; + context?: Record; + securedAssets?: boolean; + chunkFileSizeMb?: number; + apiConcurrency?: number; + downloadAssetsConcurrency?: number; +}; + +/** + * Options for writing a list of items to chunked JSON files via FsUtility. + */ +export type ChunkedJsonWriteOptions = { + dir: string; + indexFileName: string; + moduleName: string; + metaPickKeys: string[]; + items: unknown[]; +}; diff --git a/packages/contentstack-asset-management/src/types/index.ts b/packages/contentstack-asset-management/src/types/index.ts new file mode 100644 index 000000000..c673e1893 --- /dev/null +++ b/packages/contentstack-asset-management/src/types/index.ts @@ -0,0 +1,2 @@ +export * from './asset-management-api'; +export * from './export-types'; diff --git a/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts b/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts new file mode 100644 index 000000000..b5398c81d --- /dev/null +++ b/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts @@ -0,0 +1,286 @@ +import { readFileSync } from 'node:fs'; +import { basename } from 'node:path'; +import { HttpClient, log, authenticationHandler, handleAndLogError } from '@contentstack/cli-utilities'; + +import type { + AssetManagementAPIConfig, + AssetTypesResponse, + CreateAssetMetadata, + CreateAssetTypePayload, + CreateFieldPayload, + CreateFolderPayload, + CreateSpacePayload, + FieldsResponse, + IAssetManagementAdapter, + Space, + SpaceResponse, + SpacesListResponse, +} from '../types/asset-management-api'; + +export class AssetManagementAdapter implements IAssetManagementAdapter { + private readonly config: AssetManagementAPIConfig; + private readonly apiClient: HttpClient; + + constructor(config: AssetManagementAPIConfig) { + this.config = config; + this.apiClient = new HttpClient(); + const baseURL = config.baseURL?.replace(/\/$/, '') ?? ''; + this.apiClient.baseUrl(baseURL); + const defaultHeaders = { Accept: 'application/json', 'x-cs-api-version': '4' }; + this.apiClient.headers(config.headers ? { ...defaultHeaders, ...config.headers } : defaultHeaders); + log.debug('AssetManagementAdapter initialized', config.context); + } + + /** + * Build query string from params. Supports string and string[] values. + * Returns empty string when params are empty so we never append "?" with no keys. + */ + private buildQueryString(params: Record): string { + const entries = Object.entries(params).filter( + ([, v]) => v !== undefined && v !== null && (typeof v === 'string' || Array.isArray(v)), + ); + if (entries.length === 0) return ''; + const parts: string[] = []; + for (const [key, value] of entries) { + if (Array.isArray(value)) { + for (const v of value) { + parts.push(`${encodeURIComponent(key)}=${encodeURIComponent(String(v))}`); + } + } else { + parts.push(`${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`); + } + } + return '?' + parts.join('&'); + } + + /** + * GET a space-level endpoint (e.g. /api/spaces/{uid}). Builds path + query string and performs the request. + */ + private async getSpaceLevel( + _spaceUid: string, + path: string, + queryParams: Record = {}, + ): Promise { + await this.init(); + const safeParams: Record = {}; + for (const [k, v] of Object.entries(queryParams)) { + let value: string | string[] | undefined; + if (typeof v === 'string') value = v; + else if (Array.isArray(v) && v.every((x) => typeof x === 'string')) value = v; + else value = undefined; + if (value !== undefined) safeParams[k] = value; + } + const queryString = this.buildQueryString(safeParams); + const fullPath = path + queryString; + log.debug(`GET ${fullPath}`, this.config.context); + const response = await this.apiClient.get(fullPath); + if (response.status < 200 || response.status >= 300) { + throw new Error(`Asset Management API error: status ${response.status}, path ${path}`); + } + return response.data as T; + } + + async init(): Promise { + try { + log.debug('Initializing Asset Management adapter...', this.config.context); + await authenticationHandler.getAuthDetails(); + const token = authenticationHandler.accessToken; + log.debug( + `Authentication type: ${authenticationHandler.isOauthEnabled ? 'OAuth' : 'Token'}`, + this.config.context, + ); + const authHeader = authenticationHandler.isOauthEnabled ? { authorization: token } : { access_token: token }; + this.apiClient.headers(this.config.headers ? { ...authHeader, ...this.config.headers } : authHeader); + log.debug('Asset Management adapter initialization completed', this.config.context); + } catch (error: unknown) { + handleAndLogError( + error as Error, + this.config.context ? { ...(this.config.context as Record) } : {}, + 'Asset Management adapter initialization failed', + ); + throw error; + } + } + + async listSpaces(): Promise { + log.debug('Fetching all spaces in org', this.config.context); + const result = await this.getSpaceLevel('', '/api/spaces', {}); + log.debug(`Fetched ${result?.count ?? result?.spaces?.length ?? '?'} space(s)`, this.config.context); + return result; + } + + async getSpace(spaceUid: string): Promise { + log.debug(`Fetching space: ${spaceUid}`, this.config.context); + const path = `/api/spaces/${spaceUid}`; + const queryParams: Record = { + addl_fields: ['meta_info', 'users'], + }; + const result = await this.getSpaceLevel(spaceUid, path, queryParams); + log.debug(`Fetched space: ${spaceUid}`, this.config.context); + return result; + } + + async getWorkspaceFields(spaceUid: string): Promise { + log.debug(`Fetching fields for space: ${spaceUid}`, this.config.context); + const result = await this.getSpaceLevel(spaceUid, '/api/fields', {}); + log.debug(`Fetched fields (count: ${result?.count ?? '?'})`, this.config.context); + return result; + } + + /** + * GET a workspace collection (assets or folders), log count, and return result. + */ + private async getWorkspaceCollection( + spaceUid: string, + path: string, + logLabel: string, + queryParams: Record = {}, + ): Promise { + log.debug(`Fetching ${logLabel} for space: ${spaceUid}`, this.config.context); + const result = await this.getSpaceLevel(spaceUid, path, queryParams); + const count = (result as { count?: number })?.count ?? (Array.isArray(result) ? result.length : '?'); + log.debug(`Fetched ${logLabel} (count: ${count})`, this.config.context); + return result; + } + + async getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise { + return this.getWorkspaceCollection( + spaceUid, + `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, + 'assets', + workspaceUid ? { workspace: workspaceUid } : {}, + ); + } + + async getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise { + return this.getWorkspaceCollection( + spaceUid, + `/api/spaces/${encodeURIComponent(spaceUid)}/folders`, + 'folders', + workspaceUid ? { workspace: workspaceUid } : {}, + ); + } + + async getWorkspaceAssetTypes(spaceUid: string): Promise { + log.debug(`Fetching asset types for space: ${spaceUid}`, this.config.context); + const result = await this.getSpaceLevel(spaceUid, '/api/asset_types', { + include_fields: 'true', + }); + log.debug(`Fetched asset types (count: ${result?.count ?? '?'})`, this.config.context); + return result; + } + + // --------------------------------------------------------------------------- + // POST helpers + // --------------------------------------------------------------------------- + + /** + * Build headers for outgoing POST requests. + */ + private async getPostHeaders(extraHeaders: Record = {}): Promise> { + await authenticationHandler.getAuthDetails(); + const token = authenticationHandler.accessToken; + const authHeader: Record = authenticationHandler.isOauthEnabled + ? { authorization: token } + : { access_token: token }; + return { + Accept: 'application/json', + 'x-cs-api-version': '4', + ...(this.config.headers ?? {}), + ...authHeader, + ...extraHeaders, + }; + } + + private async postJson(path: string, body: unknown, extraHeaders: Record = {}): Promise { + const baseUrl = this.config.baseURL?.replace(/\/$/, '') ?? ''; + const headers = await this.getPostHeaders({ 'Content-Type': 'application/json', ...extraHeaders }); + log.debug(`POST ${path}`, this.config.context); + const response = await fetch(`${baseUrl}${path}`, { + method: 'POST', + headers, + body: JSON.stringify(body), + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`AM API POST error: status ${response.status}, path ${path}, body: ${text}`); + } + return response.json() as Promise; + } + + private async postMultipart(path: string, form: FormData, extraHeaders: Record = {}): Promise { + const baseUrl = this.config.baseURL?.replace(/\/$/, '') ?? ''; + const headers = await this.getPostHeaders(extraHeaders); + log.debug(`POST (multipart) ${path}`, this.config.context); + const response = await fetch(`${baseUrl}${path}`, { + method: 'POST', + headers, + body: form, + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`AM API multipart POST error: status ${response.status}, path ${path}, body: ${text}`); + } + return response.json() as Promise; + } + + // --------------------------------------------------------------------------- + // Import API methods + // --------------------------------------------------------------------------- + + /** + * POST /api/spaces — creates a new space in the target org. + */ + async createSpace(payload: CreateSpacePayload): Promise<{ space: Space }> { + const orgUid = (this.config.headers as Record | undefined)?.organization_uid ?? ''; + return this.postJson<{ space: Space }>('/api/spaces', payload, { + 'x-organization-uid': orgUid, + }); + } + + /** + * POST /api/spaces/{spaceUid}/folders — creates a folder inside a space. + */ + async createFolder(spaceUid: string, payload: CreateFolderPayload): Promise<{ folder: { uid: string } }> { + return this.postJson<{ folder: { uid: string } }>(`/api/spaces/${encodeURIComponent(spaceUid)}/folders`, payload, { + space_key: spaceUid, + }); + } + + /** + * POST /api/spaces/{spaceUid}/assets — uploads an asset file as multipart form-data. + */ + async uploadAsset( + spaceUid: string, + filePath: string, + metadata: CreateAssetMetadata, + ): Promise<{ asset: { uid: string; url: string } }> { + const filename = basename(filePath); + const fileBuffer = readFileSync(filePath); + const blob = new Blob([fileBuffer]); + const form = new FormData(); + form.append('file', blob, filename); + if (metadata.title) form.append('title', metadata.title); + if (metadata.description) form.append('description', metadata.description); + if (metadata.parent_uid) form.append('parent_uid', metadata.parent_uid); + return this.postMultipart<{ asset: { uid: string; url: string } }>( + `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, + form, + { space_key: spaceUid }, + ); + } + + /** + * POST /api/fields — creates a shared field. + */ + async createField(payload: CreateFieldPayload): Promise<{ field: { uid: string } }> { + return this.postJson<{ field: { uid: string } }>('/api/fields', payload); + } + + /** + * POST /api/asset_types — creates a shared asset type. + */ + async createAssetType(payload: CreateAssetTypePayload): Promise<{ asset_type: { uid: string } }> { + return this.postJson<{ asset_type: { uid: string } }>('/api/asset_types', payload); + } +} diff --git a/packages/contentstack-asset-management/src/utils/chunked-json-reader.ts b/packages/contentstack-asset-management/src/utils/chunked-json-reader.ts new file mode 100644 index 000000000..838cfe653 --- /dev/null +++ b/packages/contentstack-asset-management/src/utils/chunked-json-reader.ts @@ -0,0 +1,66 @@ +import { FsUtility, log } from '@contentstack/cli-utilities'; + +export type ForEachChunkedJsonStoreOptions = { + context?: Record; + /** Shown in log.debug: `Error reading