feat: added support to the DA to use a different KMS instance / key for backup encryption using new optional inputs existing_backup_kms_key_crn and existing_backup_kms_instance_crn<br>* added support to to the DA to provision and instance from a backup CRN using new input backup_crn (#511)

Author: Aashiq-J

ibm_catalog.json

@@ -39,6 +39,10 @@
         {
           "title": "Supports autoscaling",
           "description": "Provides the autoscaling to allow the database to increase resources in response to usage."
+        },
+        {
+          "title": "Supports backup restoration",
+          "description": "Provides database restoration using a backup created by a deployment with the same service ID."
         }
       ],
       "flavors": [
@@ -259,6 +263,9 @@
             },
             {
               "key": "auto_scaling"
+            },
+            {
+              "key": "backup_crn"
             }
           ]
         }

solutions/standard/main.tf

@@ -3,9 +3,8 @@
 #######################################################################################################################
 
 locals {
-  existing_kms_instance_crn_split = var.existing_kms_instance_crn != null ? split(":", var.existing_kms_instance_crn) : null
-  existing_kms_instance_guid      = var.existing_kms_instance_crn != null ? element(local.existing_kms_instance_crn_split, length(local.existing_kms_instance_crn_split) - 3) : null
-  existing_kms_instance_region    = var.existing_kms_instance_crn != null ? element(local.existing_kms_instance_crn_split, length(local.existing_kms_instance_crn_split) - 5) : null
+  existing_kms_instance_guid   = var.existing_kms_instance_crn != null ? module.kms_instance_crn_parser[0].service_instance : null
+  existing_kms_instance_region = var.existing_kms_instance_crn != null ? module.kms_instance_crn_parser[0].region : null
 
   key_name      = var.prefix != null ? "${var.prefix}-${var.key_name}" : var.key_name
   key_ring_name = var.prefix != null ? "${var.prefix}-${var.key_ring_name}" : var.key_ring_name
@@ -14,9 +13,7 @@ locals {
 
   create_cross_account_auth_policy = !var.skip_iam_authorization_policy && var.ibmcloud_kms_api_key != null
 
-  kms_service_name = local.kms_key_crn != null ? (
-    can(regex(".*kms.*", local.kms_key_crn)) ? "kms" : can(regex(".*hs-crypto.*", local.kms_key_crn)) ? "hs-crypto" : null
-  ) : null
+  kms_service_name = var.existing_kms_instance_crn != null ? module.kms_instance_crn_parser[0].service_name : null
 }
 
 #######################################################################################################################
@@ -37,6 +34,14 @@ data "ibm_iam_account_settings" "iam_account_settings" {
   count = local.create_cross_account_auth_policy ? 1 : 0
 }
 
+# If existing KMS intance CRN passed, parse details from it
+module "kms_instance_crn_parser" {
+  count   = var.existing_kms_instance_crn != null ? 1 : 0
+  source  = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser"
+  version = "1.1.0"
+  crn     = var.existing_kms_instance_crn
+}
+
 resource "ibm_iam_authorization_policy" "kms_policy" {
   count                       = local.create_cross_account_auth_policy ? 1 : 0
   provider                    = ibm.kms
@@ -84,17 +89,87 @@ module "kms" {
   ]
 }
 
+#######################################################################################################################
+# KMS backup encryption key for Postgresql
+#######################################################################################################################
+
+locals {
+  existing_backup_kms_instance_guid   = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].service_instance : null
+  existing_backup_kms_instance_region = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].region : null
+
+  backup_key_name         = var.prefix != null ? "${var.prefix}-backup-encryption-${var.key_name}" : "backup-encryption-${var.key_name}"
+  backup_key_ring_name    = var.prefix != null ? "${var.prefix}-backup-encryption-${var.key_ring_name}" : "backup-encryption-${var.key_ring_name}"
+  backup_kms_key_crn      = var.existing_backup_kms_key_crn != null ? var.existing_backup_kms_key_crn : var.existing_backup_kms_instance_crn != null ? module.backup_kms[0].keys[format("%s.%s", local.backup_key_ring_name, local.backup_key_name)].crn : null
+  backup_kms_service_name = var.existing_backup_kms_instance_crn != null ? module.backup_kms_instance_crn_parser[0].service_name : null
+}
+
+# If existing KMS intance CRN passed, parse details from it
+module "backup_kms_instance_crn_parser" {
+  count   = var.existing_backup_kms_instance_crn != null ? 1 : 0
+  source  = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser"
+  version = "1.1.0"
+  crn     = var.existing_backup_kms_instance_crn
+}
+
+resource "ibm_iam_authorization_policy" "backup_kms_policy" {
+  count                       = local.existing_backup_kms_instance_guid == local.existing_kms_instance_guid ? 0 : var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? !var.skip_iam_authorization_policy ? 1 : 0 : 0
+  provider                    = ibm.kms
+  source_service_account      = local.create_cross_account_auth_policy ? data.ibm_iam_account_settings.iam_account_settings[0].account_id : null
+  source_service_name         = "databases-for-postgresql"
+  source_resource_group_id    = module.resource_group.resource_group_id
+  target_service_name         = local.backup_kms_service_name
+  target_resource_instance_id = local.existing_backup_kms_instance_guid
+  roles                       = ["Reader"]
+  description                 = "Allow all Postgresql instances in the resource group ${module.resource_group.resource_group_id} to read from the ${local.backup_kms_service_name} instance GUID ${local.existing_backup_kms_instance_guid}"
+}
+
+# workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478
+resource "time_sleep" "wait_for_backup_kms_authorization_policy" {
+  depends_on      = [ibm_iam_authorization_policy.backup_kms_policy]
+  create_duration = "30s"
+}
+
+module "backup_kms" {
+  providers = {
+    ibm = ibm.kms
+  }
+  count                       = var.existing_backup_kms_key_crn != null ? 0 : var.existing_backup_kms_instance_crn != null ? 1 : 0
+  source                      = "terraform-ibm-modules/kms-all-inclusive/ibm"
+  version                     = "4.15.13"
+  create_key_protect_instance = false
+  region                      = local.existing_backup_kms_instance_region
+  existing_kms_instance_crn   = var.existing_backup_kms_instance_crn
+  key_ring_endpoint_type      = var.kms_endpoint_type
+  key_endpoint_type           = var.kms_endpoint_type
+  keys = [
+    {
+      key_ring_name         = local.backup_key_ring_name
+      existing_key_ring     = false
+      force_delete_key_ring = true
+      keys = [
+        {
+          key_name                 = local.backup_key_name
+          standard_key             = false
+          rotation_interval_month  = 3
+          dual_auth_delete_enabled = false
+          force_delete             = true
+        }
+      ]
+    }
+  ]
+}
+
 #######################################################################################################################
 # Postgresql
 #######################################################################################################################
 
 module "postgresql_db" {
   source                        = "../../modules/fscloud"
-  depends_on                    = [time_sleep.wait_for_authorization_policy]
+  depends_on                    = [time_sleep.wait_for_authorization_policy, time_sleep.wait_for_backup_kms_authorization_policy]
   resource_group_id             = module.resource_group.resource_group_id
   name                          = var.prefix != null ? "${var.prefix}-${var.name}" : var.name
   region                        = var.region
-  skip_iam_authorization_policy = var.skip_iam_authorization_policy
+  skip_iam_authorization_policy = local.create_cross_account_auth_policy ? true : var.skip_iam_authorization_policy
   pg_version                    = var.pg_version
   existing_kms_instance_guid    = local.existing_kms_instance_guid
   kms_key_crn                   = local.kms_key_crn
@@ -110,4 +185,6 @@ module "postgresql_db" {
   auto_scaling                  = var.auto_scaling
   configuration                 = var.configuration
   service_credential_names      = var.service_credential_names
+  backup_encryption_key_crn     = local.backup_kms_key_crn
+  backup_crn                    = var.backup_crn
 }

solutions/standard/variables.tf

@@ -50,6 +50,12 @@ variable "pg_version" {
   default     = null
 }
 
+variable "backup_crn" {
+  type        = string
+  description = "The CRN of a backup resource to restore from. The backup is created by a database deployment with the same service ID. The backup is loaded after provisioning and the new deployment starts up that uses that data. A backup CRN is in the format crn:v1:<…>:backup:. If omitted, the database is provisioned empty."
+  default     = null
+}
+
 ##############################################################################
 # ICD hosting model properties
 ##############################################################################
@@ -221,12 +227,12 @@ variable "kms_endpoint_type" {
 
 variable "existing_kms_key_crn" {
   type        = string
-  description = "The CRN of a Hyper Protect Crypto Services or Key Protect root key to use for disk encryption. If not specified, a root key is created in the KMS instance."
+  description = "The CRN of an Hyper Protect Crypto Services or Key Protect encryption key that you want to use to use for both disk and backup encryption. If no value is passed, a new key ring and key will be created in the instance provided in the `existing_kms_instance_crn` input. Backup encryption is only supported is some regions ([learn more](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok)), so if you need to use a key from a different region for backup encryption, use the `existing_backup_kms_key_crn` input."
   default     = null
 }
 
 variable "existing_kms_instance_crn" {
-  description = "The CRN of a Hyper Protect Crypto Services or Key Protect instance in the same account as the PostgreSQL instance. This value is used to create an authorization policy if `skip_iam_authorization_policy` is false. If not specified, a root key is created."
+  description = "The CRN of an Hyper Protect Crypto Services or Key Protect instance that you want to use for both disk and backup encryption. Backup encryption is only supported is some regions ([learn more](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok)), so if you need to use a different instance for backup encryption from a supported region, use the `existing_backup_kms_instance_crn` input."
   type        = string
   default     = null
 }
@@ -236,3 +242,18 @@ variable "skip_iam_authorization_policy" {
   description = "Whether to create an IAM authorization policy that permits all PostgreSQL instances in the resource group to read the encryption key from the Hyper Protect Crypto Services instance specified in the `existing_kms_instance_crn` variable."
   default     = false
 }
+
+##############################################################
+# Backup Encryption
+##############################################################
+variable "existing_backup_kms_key_crn" {
+  type        = string
+  description = "The CRN of an Hyper Protect Crypto Services or Key Protect encryption key that you want to use to encrypt database backups. If no value is passed, the value of `existing_kms_key_crn` is used. If no value is passed for that, a new key will be created in the provided KMS instance and used for both disk encryption, and backup encryption."
+  default     = null
+}
+
+variable "existing_backup_kms_instance_crn" {
+  description = "The CRN of an Hyper Protect Crypto Services or Key Protect instance that you want to use to encrypt database backups. If no value is passed, the value of the `existing_kms_instance_crn` input will be used, however backup encryption is only supported in certain regions so you need to ensure the KMS for backup is coming from one of the supported regions. [Learn more](https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect&interface=ui#key-byok)"
+  type        = string
+  default     = null
+}

tests/pr_test.go

@@ -109,10 +109,11 @@ func TestRunStandardSolution(t *testing.T) {
 	})
 
 	options.TerraformVars = map[string]interface{}{
-		"pg_version":                "16", // Always lock this test into the latest supported PostgreSQL version
-		"existing_kms_instance_crn": permanentResources["hpcs_south_crn"],
-		"kms_endpoint_type":         "public",
-		"resource_group_name":       options.Prefix,
+		"pg_version":                  "16", // Always lock this test into the latest supported PostgreSQL version
+		"existing_kms_instance_crn":   permanentResources["hpcs_south_crn"],
+		"kms_endpoint_type":           "public",
+		"existing_backup_kms_key_crn": permanentResources["hpcs_south_root_key_crn"],
+		"resource_group_name":         options.Prefix,
 	}
 
 	output, err := options.RunTestConsistency()