Add hierarchical support for storage fields

docs/imagecustomizer/api/configuration/baseConfig.md

@@ -51,6 +51,29 @@ or are processed sequentially.
   - `"hard-reset"`
   - `""` (i.e. no reset)
 
+**Storage Override Fields:**
+
+Current value (if specified) fully overrides base's value
+- `.storage.disks`
+- `.storage.verity`
+- `.storage.bootType`
+- `.storage.filesystems`
+
+**Storage configuration also follows specific rules based on field combinations:**
+
+| Base\Current | None | `.storage.disks` | `.storage.resetPartitionsUuidsType` | `.storage.reinitializeVerity` |
+|--------------|------|------------------|-------------------------------------|------------------------------|
+| **None** | None | Current's `.storage.disks` | Current's `.storage.resetPartitionsUuidsType` | Current's `.storage.reinitializeVerity` |
+| **`.storage.disks`** | Base's `.storage.disks` | Current's `.storage.disks` | Error¹ | Error² |
+| **`.resetPartitionsUuidsType`** | Base's `.storage.resetPartitionsUuidsType` | Current's `.storage.disks` | Current's `.storage.resetPartitionsUuidsType` | Base's reset + Current's reinit³ |
+| **`.storage.reinitializeVerity`** | Base's `.storage.reinitializeVerity` | Current's `.storage.disks` | Base's reinit + Current's reset³ | Current's `.storage.reinitializeVerity` |
+
+¹ The current config requesting the partition UUIDs to be reset when the base config customized the partition table is somewhat pointless. Hence we error out.
+
+² Cannot reinitialize verity when base config destroys original verity setup with partition layout changes.
+
+³ Base's `.resetPartitionsUuidsType` and current's `.reinitializeVerity` are compatible and both are preserved together since UUID reset updates partition identifiers while verity reinit updates verity mapping to use new identifiers and rebuilds hash trees.
+
 ## path [string]
 
 Required.

toolkit/tools/imagecustomizerapi/config.go

@@ -88,7 +88,7 @@ func (c *Config) IsValid() (err error) {
 		return err
 	}
 
-	if c.CustomizePartitions() && !hasResetBootLoader {
+	if c.Storage.CustomizePartitions() && !hasResetBootLoader {
 		return fmt.Errorf("'os.bootloader.reset' must be specified if 'storage.disks' is specified")
 	}
 
@@ -150,7 +150,3 @@ func (c *Config) IsValid() (err error) {
 
 	return nil
 }
-
-func (c *Config) CustomizePartitions() bool {
-	return c.Storage.CustomizePartitions()
-}

toolkit/tools/pkg/imagecreatorlib/imagecreator.go

@@ -26,15 +26,13 @@ func CreateImageWithConfigFile(ctx context.Context, buildDir string, configFile
 		return fmt.Errorf("failed to unmarshal config file %s:\n%w", configFile, err)
 	}
 
-	baseConfigPath, _ := filepath.Split(configFile)
-
-	absBaseConfigPath, err := filepath.Abs(baseConfigPath)
+	absConfigFile, err := filepath.Abs(configFile)
 	if err != nil {
-		return fmt.Errorf("failed to get absolute path of config file directory:\n%w", err)
+		return fmt.Errorf("failed to get absolute path of config file:\n%w", err)
 	}
 
 	err = createNewImage(
-		ctx, buildDir, absBaseConfigPath, config, rpmsSources, outputImageFile,
+		ctx, buildDir, absConfigFile, config, rpmsSources, outputImageFile,
 		outputImageFormat, toolsTar, distro, distroVersion, packageSnapshotTime)
 	if err != nil {
 		return err
@@ -43,12 +41,12 @@ func CreateImageWithConfigFile(ctx context.Context, buildDir string, configFile
 	return nil
 }
 
-func createNewImage(ctx context.Context, buildDir string, baseConfigPath string, config imagecustomizerapi.Config,
+func createNewImage(ctx context.Context, buildDir string, configFile string, config imagecustomizerapi.Config,
 	rpmsSources []string, outputImageFile string, outputImageFormat string, toolsTar string, distro string,
 	distroVersion string, packageSnapshotTime string,
 ) error {
 	rc, err := validateConfig(
-		ctx, baseConfigPath, &config, rpmsSources, toolsTar, outputImageFile,
+		ctx, configFile, &config, rpmsSources, toolsTar, outputImageFile,
 		outputImageFormat, packageSnapshotTime, buildDir)
 	if err != nil {
 		return err
@@ -73,7 +71,7 @@ func createNewImage(ctx context.Context, buildDir string, baseConfigPath string,
 		return err
 	}
 
-	disks := rc.Config.Storage.Disks
+	disks := rc.Storage.Disks
 	diskConfig := disks[0]
 	installOSFunc := func(imageChroot *safechroot.Chroot) error {
 		return nil
@@ -86,7 +84,7 @@ func createNewImage(ctx context.Context, buildDir string, baseConfigPath string,
 
 	partIdToPartUuid, err := imagecustomizerlib.CreateNewImage(
 		distroHandler.GetTargetOs(), rc.RawImageFile,
-		diskConfig, rc.Config.Storage.FileSystems,
+		diskConfig, rc.Storage.FileSystems,
 		rc.BuildDirAbs, setupRoot, installOSFunc)
 	if err != nil {
 		return err

toolkit/tools/pkg/imagecreatorlib/validation.go

@@ -66,22 +66,22 @@ func validateSupportedOsFields(osConfig *imagecustomizerapi.OS) error {
 	return nil
 }
 
-func validateConfig(ctx context.Context, baseConfigPath string, config *imagecustomizerapi.Config, rpmsSources []string,
+func validateConfig(ctx context.Context, configFile string, config *imagecustomizerapi.Config, rpmsSources []string,
 	toolsTar string, outputImageFile, outputImageFormat string, packageSnapshotTime string, buildDir string,
 ) (*imagecustomizerlib.ResolvedConfig, error) {
 	err := validateSupportedFields(config)
 	if err != nil {
-		return nil, fmt.Errorf("invalid config file (%s):\n%w", baseConfigPath, err)
+		return nil, fmt.Errorf("invalid config file (%s):\n%w", configFile, err)
 	}
 
 	// Validate mandatory fields for creating a seed image
-	err = validateMandatoryFields(baseConfigPath, config, rpmsSources, toolsTar)
+	err = validateMandatoryFields(configFile, config, rpmsSources, toolsTar)
 	if err != nil {
 		return nil, err
 	}
 
 	// TODO: Validate for distro and release
-	rc, err := imagecustomizerlib.ValidateConfig(ctx, baseConfigPath, config, true,
+	rc, err := imagecustomizerlib.ValidateConfig(ctx, configFile, config, true,
 		imagecustomizerlib.ImageCustomizerOptions{
 			RpmsSources:         rpmsSources,
 			OutputImageFile:     outputImageFile,
@@ -100,10 +100,10 @@ func validateConfig(ctx context.Context, baseConfigPath string, config *imagecus
 	return rc, nil
 }
 
-func validateMandatoryFields(baseConfigPath string, config *imagecustomizerapi.Config, rpmsSources []string, toolsTar string) error {
+func validateMandatoryFields(configFile string, config *imagecustomizerapi.Config, rpmsSources []string, toolsTar string) error {
 	// check if storage disks is not empty for creating a seed image
 	if len(config.Storage.Disks) == 0 {
-		return fmt.Errorf("storage.disks field is required in the config file (%s)", baseConfigPath)
+		return fmt.Errorf("storage.disks field is required in the config file (%s)", configFile)
 	}
 
 	// rpmSources must not be empty for creating a seed image

toolkit/tools/pkg/imagecreatorlib/validation_test.go

@@ -97,110 +97,110 @@ func TestValidateOutput_AcceptsValidPaths(t *testing.T) {
 	packageSnapshotTime := ""
 
 	// The output image file can be sepcified as an argument without being in specified the config.
-	_, err = validateConfig(t.Context(), baseConfigPath, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
+	_, err = validateConfig(t.Context(), configFile, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
 		packageSnapshotTime, buildDir)
 	assert.NoError(t, err)
 
 	outputImageFile = outputImageFileNewRelativeCwd
 
 	// The output image file can be specified as an argument relative to the current working directory.
-	_, err = validateConfig(t.Context(), baseConfigPath, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
+	_, err = validateConfig(t.Context(), configFile, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
 		packageSnapshotTime, buildDir)
 	assert.NoError(t, err)
 
 	outputImageFile = outputImageDir
 
 	// The output image file, specified as an argument, must not be a directory.
-	_, err = validateConfig(t.Context(), baseConfigPath, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
+	_, err = validateConfig(t.Context(), configFile, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
 		packageSnapshotTime, buildDir)
 	assert.Error(t, err)
 	assert.ErrorContains(t, err, "is a directory")
 
 	outputImageFile = outputImageDirRelativeCwd
 
 	// The above is also true for relative paths.
-	_, err = validateConfig(t.Context(), baseConfigPath, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
+	_, err = validateConfig(t.Context(), configFile, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
 		packageSnapshotTime, buildDir)
 	assert.Error(t, err)
 	assert.ErrorContains(t, err, "is a directory")
 
 	outputImageFile = outputImageFileExists
 
 	// The output image file, specified as an argument, may be a file that already exists.
-	_, err = validateConfig(t.Context(), baseConfigPath, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
+	_, err = validateConfig(t.Context(), configFile, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
 		packageSnapshotTime, buildDir)
 	assert.NoError(t, err)
 
 	outputImageFile = outputImageFileExistsRelativeCwd
 
 	// The above is also true for relative paths.
-	_, err = validateConfig(t.Context(), baseConfigPath, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
+	_, err = validateConfig(t.Context(), configFile, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
 		packageSnapshotTime, buildDir)
 	assert.NoError(t, err)
 
 	outputImageFile = ""
 	config.Output.Image.Path = outputImageFileNew
 
 	// The output image file cab be specified in the config without being specified as an argument.
-	_, err = validateConfig(t.Context(), baseConfigPath, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
+	_, err = validateConfig(t.Context(), configFile, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
 		packageSnapshotTime, buildDir)
 	assert.NoError(t, err)
 
 	config.Output.Image.Path = outputImageFileNewRelativeConfig
 
 	// The output image file can be specified in the config relative to the base config path.
-	_, err = validateConfig(t.Context(), baseConfigPath, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
+	_, err = validateConfig(t.Context(), configFile, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
 		packageSnapshotTime, buildDir)
 	assert.NoError(t, err)
 
 	config.Output.Image.Path = outputImageDir
 
 	// The output image file, specified in the config, must not be a directory.
-	_, err = validateConfig(t.Context(), baseConfigPath, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
+	_, err = validateConfig(t.Context(), configFile, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
 		packageSnapshotTime, buildDir)
 	assert.Error(t, err)
 	assert.ErrorContains(t, err, "is a directory")
 
 	config.Output.Image.Path = outputImageDirRelativeConfig
 
 	// The above is also true for relative paths.
-	_, err = validateConfig(t.Context(), baseConfigPath, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
+	_, err = validateConfig(t.Context(), configFile, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
 		packageSnapshotTime, buildDir)
 	assert.Error(t, err)
 	assert.ErrorContains(t, err, "is a directory")
 
 	config.Output.Image.Path = outputImageFileExists
 
 	// The output image file, specified in the config, may be a file that already exists.
-	_, err = validateConfig(t.Context(), baseConfigPath, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
+	_, err = validateConfig(t.Context(), configFile, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
 		packageSnapshotTime, buildDir)
 	assert.NoError(t, err)
 
 	config.Output.Image.Path = outputImageFileExistsRelativeConfig
 
 	// The above is also true for relative paths.
-	_, err = validateConfig(t.Context(), baseConfigPath, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
+	_, err = validateConfig(t.Context(), configFile, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
 		packageSnapshotTime, buildDir)
 	assert.NoError(t, err)
 
 	outputImageFile = outputImageFileNew
 	config.Output.Image.Path = outputImageFileNew
 
 	// The output image file can be specified both as an argument and in the config.
-	_, err = validateConfig(t.Context(), baseConfigPath, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
+	_, err = validateConfig(t.Context(), configFile, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
 		packageSnapshotTime, buildDir)
 	assert.NoError(t, err)
 
 	config.Output.Image.Path = outputImageDir
 
 	// The output image file can even be invalid in the config if it is specified as an argument.
-	_, err = validateConfig(t.Context(), baseConfigPath, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
+	_, err = validateConfig(t.Context(), configFile, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
 		packageSnapshotTime, buildDir)
 	assert.NoError(t, err)
 }
 
 func TestValidateConfig_EmptyConfig(t *testing.T) {
-	baseConfigPath := testDir
+	configFile := filepath.Join(testDir, "empty-config.yaml")
 	config := &imagecustomizerapi.Config{}
 	rpmSources := []string{}
 
@@ -209,7 +209,7 @@ func TestValidateConfig_EmptyConfig(t *testing.T) {
 	packageSnapshotTime := ""
 	buildDir := "./"
 
-	_, err := validateConfig(t.Context(), baseConfigPath, config, rpmSources, "", outputImageFile, outputImageFormat,
+	_, err := validateConfig(t.Context(), configFile, config, rpmSources, "", outputImageFile, outputImageFormat,
 		packageSnapshotTime, buildDir)
 	assert.ErrorContains(t, err, "storage.disks field is required in the config file")
 }
@@ -236,6 +236,7 @@ func TestValidateConfig_EmptyPackagestoInstall(t *testing.T) {
 	assert.NoError(t, err)
 	// Set the packages to install to an empty slice
 	config.OS.Packages.Install = []string{}
+
 	_, err = validateConfig(t.Context(), configFile, &config, rpmSources, toolsFile, outputImageFile, outputImageFormat,
 		packageSnapshotTime, buildDir)
 	assert.ErrorContains(t, err, "no packages to install specified, please specify at least one package to install for a new image")

toolkit/tools/pkg/imagecustomizerlib/baseconfigs.go

@@ -9,27 +9,34 @@ import (
 	"github.com/microsoft/azure-linux-image-tools/toolkit/tools/internal/file"
 )
 
-type ConfigWithBasePath struct {
+type ConfigWithPath struct {
 	Config         *imagecustomizerapi.Config
-	BaseConfigPath string
+	ConfigFilePath string
 }
 
-func buildConfigChain(ctx context.Context, rc *ResolvedConfig) ([]*ConfigWithBasePath, error) {
+// ConfigDir returns the directory containing the config file
+func (c *ConfigWithPath) ConfigDir() string {
+	return filepath.Dir(c.ConfigFilePath)
+}
+
+func buildConfigChain(ctx context.Context, rc *ResolvedConfig) ([]*ConfigWithPath, error) {
 	visited := make(map[string]bool)
 	pathStack := []string{}
 
-	configChain, err := buildConfigChainHelper(ctx, rc.Config, rc.BaseConfigPath, visited, pathStack)
+	configChain, err := buildConfigChainHelper(ctx, rc.Config, rc.ConfigFilePath, visited, pathStack)
 	if err != nil {
 		return nil, err
 	}
 
 	return configChain, nil
 }
 
-func buildConfigChainHelper(ctx context.Context, cfg *imagecustomizerapi.Config, configDir string,
-	visited map[string]bool, pathStack []string) ([]*ConfigWithBasePath, error) {
+func buildConfigChainHelper(ctx context.Context, cfg *imagecustomizerapi.Config, configFilePath string,
+	visited map[string]bool, pathStack []string) ([]*ConfigWithPath, error) {
+
+	var chain []*ConfigWithPath
 
-	var chain []*ConfigWithBasePath
+	configDir := filepath.Dir(configFilePath)
 
 	for _, base := range cfg.BaseConfigs {
 		// Resolve base config path relative to current config's directory
@@ -54,8 +61,7 @@ func buildConfigChainHelper(ctx context.Context, cfg *imagecustomizerapi.Config,
 		}
 
 		// Recurse into base config
-		baseConfigDir := filepath.Dir(absPath)
-		subChain, err := buildConfigChainHelper(ctx, &baseCfg, baseConfigDir, visited, pathStack)
+		subChain, err := buildConfigChainHelper(ctx, &baseCfg, absPath, visited, pathStack)
 		if err != nil {
 			return nil, err
 		}
@@ -64,9 +70,9 @@ func buildConfigChainHelper(ctx context.Context, cfg *imagecustomizerapi.Config,
 	}
 
 	// Add the current config last
-	chain = append(chain, &ConfigWithBasePath{
+	chain = append(chain, &ConfigWithPath{
 		Config:         cfg,
-		BaseConfigPath: configDir,
+		ConfigFilePath: configFilePath,
 	})
 
 	return chain, nil