diff --git a/acceptance/bundle/invariant/configs/cluster.yml.tmpl b/acceptance/bundle/invariant/configs/cluster.yml.tmpl index d54d3d7dad8..7bb6b03bdd9 100644 --- a/acceptance/bundle/invariant/configs/cluster.yml.tmpl +++ b/acceptance/bundle/invariant/configs/cluster.yml.tmpl @@ -7,4 +7,5 @@ resources: cluster_name: test-cluster-$UNIQUE_NAME spark_version: 13.3.x-scala2.12 node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 diff --git a/acceptance/bundle/invariant/configs/job_with_task.yml.tmpl b/acceptance/bundle/invariant/configs/job_with_task.yml.tmpl index 9d75734195f..ebf38fccb95 100644 --- a/acceptance/bundle/invariant/configs/job_with_task.yml.tmpl +++ b/acceptance/bundle/invariant/configs/job_with_task.yml.tmpl @@ -12,4 +12,5 @@ resources: new_cluster: spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 diff --git a/acceptance/bundle/resources/clusters/deploy/data_security_mode/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/deploy/data_security_mode/databricks.yml.tmpl index fdbc0eda4ef..c265f4efea4 100644 --- a/acceptance/bundle/resources/clusters/deploy/data_security_mode/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/deploy/data_security_mode/databricks.yml.tmpl @@ -10,6 +10,7 @@ resources: cluster_name: test-cluster-$UNIQUE_NAME spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 2 spark_conf: "spark.executor.memory": "2g" diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/databricks.yml.tmpl index a0e2d0b4198..4aa01eeefc9 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/databricks.yml.tmpl @@ -7,6 +7,7 @@ resources: cluster_name: test-cluster-$UNIQUE_NAME spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 gcp_attributes: # config from https://github.com/databricks/terraform-provider-databricks/issues/4089 diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.requests.direct.json b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.requests.direct.json index eb5d63ca7c4..4df6138e810 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.requests.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.requests.direct.json @@ -4,12 +4,8 @@ "body": { "autotermination_minutes": 60, "cluster_name": "test-cluster-[UNIQUE_NAME]", - "gcp_attributes": { - "availability": "PREEMPTIBLE_GCP", - "local_ssd_count": 0, - "zone_id": "auto" - }, - "node_type_id": "[NODE_TYPE_ID]", + "gcp_attributes": {}, + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 1, "spark_version": "13.3.x-snapshot-scala2.12" } diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.requests.terraform.json b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.requests.terraform.json index f8ef61075c4..4df6138e810 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.requests.terraform.json +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.requests.terraform.json @@ -4,11 +4,8 @@ "body": { "autotermination_minutes": 60, "cluster_name": "test-cluster-[UNIQUE_NAME]", - "gcp_attributes": { - "availability": "PREEMPTIBLE_GCP", - "zone_id": "auto" - }, - "node_type_id": "[NODE_TYPE_ID]", + "gcp_attributes": {}, + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 1, "spark_version": "13.3.x-snapshot-scala2.12" } diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.result.direct.txt b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.result.direct.txt index 63a9f7a62d7..1e4e511c603 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.result.direct.txt +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.result.direct.txt @@ -1,5 +1,2 @@ -json.gcp_attributes.availability = "PREEMPTIBLE_GCP"; json.gcp_attributes.first_on_demand = 1; -json.gcp_attributes.local_ssd_count = 0; json.gcp_attributes.use_preemptible_executors = false; -json.gcp_attributes.zone_id = "auto"; diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.result.terraform.txt b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.result.terraform.txt index b92f0cc5fa6..1e4e511c603 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.result.terraform.txt +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.result.terraform.txt @@ -1,4 +1,2 @@ -json.gcp_attributes.availability = "PREEMPTIBLE_GCP"; json.gcp_attributes.first_on_demand = 1; json.gcp_attributes.use_preemptible_executors = false; -json.gcp_attributes.zone_id = "auto"; diff --git a/acceptance/bundle/resources/clusters/deploy/simple/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/deploy/simple/databricks.yml.tmpl index 74c340a2492..27ce766f1c6 100644 --- a/acceptance/bundle/resources/clusters/deploy/simple/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/deploy/simple/databricks.yml.tmpl @@ -10,6 +10,7 @@ resources: cluster_name: test-cluster-$UNIQUE_NAME spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 2 spark_conf: "spark.executor.memory": "2g" diff --git a/acceptance/bundle/resources/clusters/deploy/update-after-create/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/deploy/update-after-create/databricks.yml.tmpl index 78b5f25595f..c3f9beac6f4 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-after-create/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/deploy/update-after-create/databricks.yml.tmpl @@ -10,6 +10,7 @@ resources: cluster_name: test-cluster-$UNIQUE_NAME spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 2 spark_conf: "spark.executor.memory": "2g" diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/deploy/update-and-resize/databricks.yml.tmpl index c32af2055f9..614a9466f29 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/databricks.yml.tmpl @@ -10,6 +10,7 @@ resources: cluster_name: test-cluster-$UNIQUE_NAME spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 2 spark_conf: "spark.executor.memory": "2g" diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json index 01b69a68b68..fdb0b859289 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json @@ -8,7 +8,7 @@ "value": { "autotermination_minutes": 60, "cluster_name": "test-cluster-[UNIQUE_NAME]", - "node_type_id": "[NODE_TYPE_ID]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 2, "spark_conf": { "spark.executor.memory": "2g" @@ -31,7 +31,7 @@ "value": { "autotermination_minutes": 60, "cluster_name": "test-cluster-[UNIQUE_NAME]", - "node_type_id": "[NODE_TYPE_ID]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 3, "spark_conf": { "spark.executor.memory": "2g" @@ -47,12 +47,11 @@ }, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "driver_node_type_id": "[NODE_TYPE_ID]", "enable_elastic_disk": false, + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "lifecycle": { "started": true }, - "node_type_id": "[NODE_TYPE_ID]", "num_workers": 2, "spark_conf": { "spark.executor.memory": "2g" @@ -69,11 +68,6 @@ "zone_id": "us-east-1c" } }, - "driver_node_type_id": { - "action": "skip", - "reason": "backend_default", - "remote": "[NODE_TYPE_ID]" - }, "enable_elastic_disk": { "action": "skip", "reason": "empty", @@ -109,7 +103,7 @@ "value": { "autotermination_minutes": 60, "cluster_name": "test-cluster-[UNIQUE_NAME]", - "node_type_id": "[NODE_TYPE_ID]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 4, "spark_conf": { "spark.executor.memory": "2g" @@ -125,12 +119,11 @@ }, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "driver_node_type_id": "[NODE_TYPE_ID]", "enable_elastic_disk": false, + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "lifecycle": { "started": true }, - "node_type_id": "[NODE_TYPE_ID]", "num_workers": 3, "spark_conf": { "spark.executor.memory": "2g" @@ -147,11 +140,6 @@ "zone_id": "us-east-1c" } }, - "driver_node_type_id": { - "action": "skip", - "reason": "backend_default", - "remote": "[NODE_TYPE_ID]" - }, "enable_elastic_disk": { "action": "skip", "reason": "empty", @@ -187,7 +175,7 @@ "value": { "autotermination_minutes": 60, "cluster_name": "test-cluster-[UNIQUE_NAME]", - "node_type_id": "[NODE_TYPE_ID]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 5, "spark_conf": { "spark.executor.memory": "4g" @@ -203,12 +191,11 @@ }, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "driver_node_type_id": "[NODE_TYPE_ID]", "enable_elastic_disk": false, + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "lifecycle": { "started": true }, - "node_type_id": "[NODE_TYPE_ID]", "num_workers": 4, "spark_conf": { "spark.executor.memory": "2g" @@ -225,11 +212,6 @@ "zone_id": "us-east-1c" } }, - "driver_node_type_id": { - "action": "skip", - "reason": "backend_default", - "remote": "[NODE_TYPE_ID]" - }, "enable_elastic_disk": { "action": "skip", "reason": "empty", diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt index e382ea8dcd7..45a1b0bc486 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt @@ -36,9 +36,8 @@ Deployment complete! }, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "driver_node_type_id": "[NODE_TYPE_ID]", "enable_elastic_disk": false, - "node_type_id": "[NODE_TYPE_ID]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 3, "spark_conf": { "spark.executor.memory": "2g" @@ -87,7 +86,7 @@ Deployment complete! "autotermination_minutes": 60, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "node_type_id": "[NODE_TYPE_ID]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 5, "spark_conf": { "spark.executor.memory": "4g" diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/databricks.yml.tmpl index 509c1a2f032..cd0a5360ab8 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/databricks.yml.tmpl @@ -10,6 +10,7 @@ resources: cluster_name: $UNIQUE_NAME spark_version: 15.4.x-scala2.12 node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 lifecycle: started: true diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/output.txt b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/output.txt index 1bf5994cd60..a7adb36a0b2 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/output.txt +++ b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/output.txt @@ -2,7 +2,7 @@ === bundle plan fails with lifecycle.started on terraform engine >>> errcode [CLI] bundle plan Error: lifecycle.started is only supported in direct deployment mode - in databricks.yml:15:18 + in databricks.yml:16:18 Exit code: 1 @@ -10,7 +10,7 @@ Exit code: 1 === bundle deploy fails with lifecycle.started on terraform engine >>> errcode [CLI] bundle deploy Error: lifecycle.started is only supported in direct deployment mode - in databricks.yml:15:18 + in databricks.yml:16:18 Exit code: 1 diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/databricks.yml.tmpl index d8137b2a966..e83931637ee 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/databricks.yml.tmpl @@ -10,6 +10,7 @@ resources: cluster_name: $UNIQUE_NAME spark_version: 15.4.x-scala2.12 node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 lifecycle: started: false diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt index d37ac6b2c02..23eb2e0c243 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt +++ b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt @@ -13,7 +13,7 @@ Deployment complete! "body": { "autotermination_minutes": 60, "cluster_name": "[UNIQUE_NAME]", - "node_type_id": "[NODE_TYPE_ID]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 1, "spark_version": "15.4.x-scala2.12" } diff --git a/acceptance/bundle/resources/clusters/lifecycle-started/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/lifecycle-started/databricks.yml.tmpl index 509c1a2f032..cd0a5360ab8 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/lifecycle-started/databricks.yml.tmpl @@ -10,6 +10,7 @@ resources: cluster_name: $UNIQUE_NAME spark_version: 15.4.x-scala2.12 node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 lifecycle: started: true diff --git a/acceptance/bundle/resources/clusters/lifecycle-started/output.txt b/acceptance/bundle/resources/clusters/lifecycle-started/output.txt index 557f3bff07b..18bf2cca857 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started/output.txt +++ b/acceptance/bundle/resources/clusters/lifecycle-started/output.txt @@ -13,7 +13,7 @@ Deployment complete! "body": { "autotermination_minutes": 60, "cluster_name": "[UNIQUE_NAME]", - "node_type_id": "[NODE_TYPE_ID]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 1, "spark_version": "15.4.x-scala2.12" } diff --git a/acceptance/bundle/run/app-with-job/databricks.yml.tmpl b/acceptance/bundle/run/app-with-job/databricks.yml.tmpl index 1c98ab3e77d..6715bf28a90 100644 --- a/acceptance/bundle/run/app-with-job/databricks.yml.tmpl +++ b/acceptance/bundle/run/app-with-job/databricks.yml.tmpl @@ -30,6 +30,7 @@ resources: num_workers: 1 spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID data_security_mode: USER_ISOLATION spark_python_task: python_file: ./hello_world.py