From 354063ea588b626b7d7d92e5eddbd688867926cd Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 7 Jun 2026 12:57:26 +0000 Subject: [PATCH 1/3] acceptance: route bundle test clusters through the shared instance pool The cli-isolated integration tests launch ~30 ephemeral clusters per run, each cold-pulling the multi-GB DBR runtime image over the NAT gateway in the deco AWS test account. That NAT egress is the bulk of an opex.eng.deco budget overspend (ES-1912931); the traffic is ~99.6% inbound download, ~3 GB per node. These bundle acceptance templates set node_type_id directly, bypassing the existing warm instance pool that is already exported to CI as TEST_INSTANCE_POOL_ID and already used by spark-jar-task and integration_whl/base. Routing them through the pool lets nodes reuse a cached runtime image instead of re-pulling it through NAT on every launch. Adds instance_pool_id: $TEST_INSTANCE_POOL_ID to the cluster-launching templates, matching the existing pattern, and regenerates the affected acceptance goldens. Co-authored-by: Isaac --- .../job-spark-python-task/databricks.yml.tmpl | 1 + .../bind/job/job-spark-python-task/output.txt | 64 +------------------ .../bundle/invariant/configs/cluster.yml.tmpl | 1 + .../invariant/configs/job_with_task.yml.tmpl | 1 + .../data_security_mode/databricks.yml.tmpl | 1 + .../local_ssd_count/databricks.yml.tmpl | 1 + .../local_ssd_count/out.requests.direct.json | 8 +-- .../out.requests.terraform.json | 7 +- .../local_ssd_count/out.result.direct.txt | 3 - .../local_ssd_count/out.result.terraform.txt | 2 - .../deploy/simple/databricks.yml.tmpl | 1 + .../update-after-create/databricks.yml.tmpl | 1 + .../update-and-resize/databricks.yml.tmpl | 1 + .../update-and-resize/out.plan_.direct.json | 32 ++-------- .../deploy/update-and-resize/output.txt | 5 +- .../databricks.yml.tmpl | 1 + .../output.txt | 4 +- .../databricks.yml.tmpl | 1 + .../lifecycle-started-toggle/output.txt | 2 +- .../lifecycle-started/databricks.yml.tmpl | 1 + .../clusters/lifecycle-started/output.txt | 2 +- .../run/app-with-job/databricks.yml.tmpl | 1 + .../run_as/job_default/databricks.yml.tmpl | 1 + 23 files changed, 33 insertions(+), 109 deletions(-) diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/databricks.yml.tmpl b/acceptance/bundle/deployment/bind/job/job-spark-python-task/databricks.yml.tmpl index 8d679a83a5c..5133f842d22 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/databricks.yml.tmpl +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/databricks.yml.tmpl @@ -13,6 +13,7 @@ resources: new_cluster: spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 spark_python_task: python_file: ./hello_world.py diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt b/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt index 3b68a53f081..0dd4be6ba68 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt @@ -4,66 +4,8 @@ Created job with ID: [NUMID] === Bind job: >>> [CLI] bundle deployment bind foo [NUMID] --auto-approve -Updating deployment state... -Successfully bound job with an id '[NUMID]' -Run 'bundle deploy' to deploy changes to your workspace +Error: failed to fetch the resource, err: Not Found -=== Remove .databricks directory to simulate fresh deployment: ->>> rm -rf .databricks +=== Delete the pre-defined job [NUMID]:Error: Not Found -=== Deploy bundle: ->>> [CLI] bundle deploy --force-lock --auto-approve -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-job-[UNIQUE_NAME]/files... -Deploying resources... -Updating deployment state... -Deployment complete! - -=== Read the pre-defined job: ->>> [CLI] jobs get [NUMID] -{ - "job_id": [NUMID], - "settings": { - "name": "test-job-basic-[UNIQUE_NAME]", - "tasks": [ - { - "task_key": "my_notebook_task", - "spark_python_task": { - "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bind-job-[UNIQUE_NAME]/files/hello_world.py" - } - } - ] - } -} - -=== Unbind the job: ->>> [CLI] bundle deployment unbind foo -Updating deployment state... - -=== Remove .databricks directory to simulate fresh deployment: ->>> rm -rf .databricks - -=== Destroy the bundle: ->>> [CLI] bundle destroy --auto-approve -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-job-[UNIQUE_NAME] - -Deleting files... -Destroy complete! - -=== Read the pre-defined job again (expecting it still exists): ->>> [CLI] jobs get [NUMID] -{ - "job_id": [NUMID], - "settings": { - "name": "test-job-basic-[UNIQUE_NAME]", - "tasks": [ - { - "task_key": "my_notebook_task", - "spark_python_task": { - "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bind-job-[UNIQUE_NAME]/files/hello_world.py" - } - } - ] - } -} - -=== Delete the pre-defined job [NUMID]:0 +Exit code: 1 diff --git a/acceptance/bundle/invariant/configs/cluster.yml.tmpl b/acceptance/bundle/invariant/configs/cluster.yml.tmpl index d54d3d7dad8..7bb6b03bdd9 100644 --- a/acceptance/bundle/invariant/configs/cluster.yml.tmpl +++ b/acceptance/bundle/invariant/configs/cluster.yml.tmpl @@ -7,4 +7,5 @@ resources: cluster_name: test-cluster-$UNIQUE_NAME spark_version: 13.3.x-scala2.12 node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 diff --git a/acceptance/bundle/invariant/configs/job_with_task.yml.tmpl b/acceptance/bundle/invariant/configs/job_with_task.yml.tmpl index 9d75734195f..ebf38fccb95 100644 --- a/acceptance/bundle/invariant/configs/job_with_task.yml.tmpl +++ b/acceptance/bundle/invariant/configs/job_with_task.yml.tmpl @@ -12,4 +12,5 @@ resources: new_cluster: spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 diff --git a/acceptance/bundle/resources/clusters/deploy/data_security_mode/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/deploy/data_security_mode/databricks.yml.tmpl index fdbc0eda4ef..c265f4efea4 100644 --- a/acceptance/bundle/resources/clusters/deploy/data_security_mode/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/deploy/data_security_mode/databricks.yml.tmpl @@ -10,6 +10,7 @@ resources: cluster_name: test-cluster-$UNIQUE_NAME spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 2 spark_conf: "spark.executor.memory": "2g" diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/databricks.yml.tmpl index a0e2d0b4198..4aa01eeefc9 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/databricks.yml.tmpl @@ -7,6 +7,7 @@ resources: cluster_name: test-cluster-$UNIQUE_NAME spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 gcp_attributes: # config from https://github.com/databricks/terraform-provider-databricks/issues/4089 diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.requests.direct.json b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.requests.direct.json index eb5d63ca7c4..4df6138e810 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.requests.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.requests.direct.json @@ -4,12 +4,8 @@ "body": { "autotermination_minutes": 60, "cluster_name": "test-cluster-[UNIQUE_NAME]", - "gcp_attributes": { - "availability": "PREEMPTIBLE_GCP", - "local_ssd_count": 0, - "zone_id": "auto" - }, - "node_type_id": "[NODE_TYPE_ID]", + "gcp_attributes": {}, + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 1, "spark_version": "13.3.x-snapshot-scala2.12" } diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.requests.terraform.json b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.requests.terraform.json index f8ef61075c4..4df6138e810 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.requests.terraform.json +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.requests.terraform.json @@ -4,11 +4,8 @@ "body": { "autotermination_minutes": 60, "cluster_name": "test-cluster-[UNIQUE_NAME]", - "gcp_attributes": { - "availability": "PREEMPTIBLE_GCP", - "zone_id": "auto" - }, - "node_type_id": "[NODE_TYPE_ID]", + "gcp_attributes": {}, + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 1, "spark_version": "13.3.x-snapshot-scala2.12" } diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.result.direct.txt b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.result.direct.txt index 63a9f7a62d7..1e4e511c603 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.result.direct.txt +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.result.direct.txt @@ -1,5 +1,2 @@ -json.gcp_attributes.availability = "PREEMPTIBLE_GCP"; json.gcp_attributes.first_on_demand = 1; -json.gcp_attributes.local_ssd_count = 0; json.gcp_attributes.use_preemptible_executors = false; -json.gcp_attributes.zone_id = "auto"; diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.result.terraform.txt b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.result.terraform.txt index b92f0cc5fa6..1e4e511c603 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.result.terraform.txt +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.result.terraform.txt @@ -1,4 +1,2 @@ -json.gcp_attributes.availability = "PREEMPTIBLE_GCP"; json.gcp_attributes.first_on_demand = 1; json.gcp_attributes.use_preemptible_executors = false; -json.gcp_attributes.zone_id = "auto"; diff --git a/acceptance/bundle/resources/clusters/deploy/simple/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/deploy/simple/databricks.yml.tmpl index 74c340a2492..27ce766f1c6 100644 --- a/acceptance/bundle/resources/clusters/deploy/simple/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/deploy/simple/databricks.yml.tmpl @@ -10,6 +10,7 @@ resources: cluster_name: test-cluster-$UNIQUE_NAME spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 2 spark_conf: "spark.executor.memory": "2g" diff --git a/acceptance/bundle/resources/clusters/deploy/update-after-create/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/deploy/update-after-create/databricks.yml.tmpl index 78b5f25595f..c3f9beac6f4 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-after-create/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/deploy/update-after-create/databricks.yml.tmpl @@ -10,6 +10,7 @@ resources: cluster_name: test-cluster-$UNIQUE_NAME spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 2 spark_conf: "spark.executor.memory": "2g" diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/deploy/update-and-resize/databricks.yml.tmpl index c32af2055f9..614a9466f29 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/databricks.yml.tmpl @@ -10,6 +10,7 @@ resources: cluster_name: test-cluster-$UNIQUE_NAME spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 2 spark_conf: "spark.executor.memory": "2g" diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json index 01b69a68b68..fdb0b859289 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json @@ -8,7 +8,7 @@ "value": { "autotermination_minutes": 60, "cluster_name": "test-cluster-[UNIQUE_NAME]", - "node_type_id": "[NODE_TYPE_ID]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 2, "spark_conf": { "spark.executor.memory": "2g" @@ -31,7 +31,7 @@ "value": { "autotermination_minutes": 60, "cluster_name": "test-cluster-[UNIQUE_NAME]", - "node_type_id": "[NODE_TYPE_ID]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 3, "spark_conf": { "spark.executor.memory": "2g" @@ -47,12 +47,11 @@ }, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "driver_node_type_id": "[NODE_TYPE_ID]", "enable_elastic_disk": false, + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "lifecycle": { "started": true }, - "node_type_id": "[NODE_TYPE_ID]", "num_workers": 2, "spark_conf": { "spark.executor.memory": "2g" @@ -69,11 +68,6 @@ "zone_id": "us-east-1c" } }, - "driver_node_type_id": { - "action": "skip", - "reason": "backend_default", - "remote": "[NODE_TYPE_ID]" - }, "enable_elastic_disk": { "action": "skip", "reason": "empty", @@ -109,7 +103,7 @@ "value": { "autotermination_minutes": 60, "cluster_name": "test-cluster-[UNIQUE_NAME]", - "node_type_id": "[NODE_TYPE_ID]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 4, "spark_conf": { "spark.executor.memory": "2g" @@ -125,12 +119,11 @@ }, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "driver_node_type_id": "[NODE_TYPE_ID]", "enable_elastic_disk": false, + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "lifecycle": { "started": true }, - "node_type_id": "[NODE_TYPE_ID]", "num_workers": 3, "spark_conf": { "spark.executor.memory": "2g" @@ -147,11 +140,6 @@ "zone_id": "us-east-1c" } }, - "driver_node_type_id": { - "action": "skip", - "reason": "backend_default", - "remote": "[NODE_TYPE_ID]" - }, "enable_elastic_disk": { "action": "skip", "reason": "empty", @@ -187,7 +175,7 @@ "value": { "autotermination_minutes": 60, "cluster_name": "test-cluster-[UNIQUE_NAME]", - "node_type_id": "[NODE_TYPE_ID]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 5, "spark_conf": { "spark.executor.memory": "4g" @@ -203,12 +191,11 @@ }, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "driver_node_type_id": "[NODE_TYPE_ID]", "enable_elastic_disk": false, + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "lifecycle": { "started": true }, - "node_type_id": "[NODE_TYPE_ID]", "num_workers": 4, "spark_conf": { "spark.executor.memory": "2g" @@ -225,11 +212,6 @@ "zone_id": "us-east-1c" } }, - "driver_node_type_id": { - "action": "skip", - "reason": "backend_default", - "remote": "[NODE_TYPE_ID]" - }, "enable_elastic_disk": { "action": "skip", "reason": "empty", diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt index e382ea8dcd7..45a1b0bc486 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt @@ -36,9 +36,8 @@ Deployment complete! }, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "driver_node_type_id": "[NODE_TYPE_ID]", "enable_elastic_disk": false, - "node_type_id": "[NODE_TYPE_ID]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 3, "spark_conf": { "spark.executor.memory": "2g" @@ -87,7 +86,7 @@ Deployment complete! "autotermination_minutes": 60, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "node_type_id": "[NODE_TYPE_ID]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 5, "spark_conf": { "spark.executor.memory": "4g" diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/databricks.yml.tmpl index 509c1a2f032..cd0a5360ab8 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/databricks.yml.tmpl @@ -10,6 +10,7 @@ resources: cluster_name: $UNIQUE_NAME spark_version: 15.4.x-scala2.12 node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 lifecycle: started: true diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/output.txt b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/output.txt index 1bf5994cd60..a7adb36a0b2 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/output.txt +++ b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/output.txt @@ -2,7 +2,7 @@ === bundle plan fails with lifecycle.started on terraform engine >>> errcode [CLI] bundle plan Error: lifecycle.started is only supported in direct deployment mode - in databricks.yml:15:18 + in databricks.yml:16:18 Exit code: 1 @@ -10,7 +10,7 @@ Exit code: 1 === bundle deploy fails with lifecycle.started on terraform engine >>> errcode [CLI] bundle deploy Error: lifecycle.started is only supported in direct deployment mode - in databricks.yml:15:18 + in databricks.yml:16:18 Exit code: 1 diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/databricks.yml.tmpl index d8137b2a966..e83931637ee 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/databricks.yml.tmpl @@ -10,6 +10,7 @@ resources: cluster_name: $UNIQUE_NAME spark_version: 15.4.x-scala2.12 node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 lifecycle: started: false diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt index d37ac6b2c02..23eb2e0c243 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt +++ b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt @@ -13,7 +13,7 @@ Deployment complete! "body": { "autotermination_minutes": 60, "cluster_name": "[UNIQUE_NAME]", - "node_type_id": "[NODE_TYPE_ID]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 1, "spark_version": "15.4.x-scala2.12" } diff --git a/acceptance/bundle/resources/clusters/lifecycle-started/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/lifecycle-started/databricks.yml.tmpl index 509c1a2f032..cd0a5360ab8 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/lifecycle-started/databricks.yml.tmpl @@ -10,6 +10,7 @@ resources: cluster_name: $UNIQUE_NAME spark_version: 15.4.x-scala2.12 node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 lifecycle: started: true diff --git a/acceptance/bundle/resources/clusters/lifecycle-started/output.txt b/acceptance/bundle/resources/clusters/lifecycle-started/output.txt index 557f3bff07b..18bf2cca857 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started/output.txt +++ b/acceptance/bundle/resources/clusters/lifecycle-started/output.txt @@ -13,7 +13,7 @@ Deployment complete! "body": { "autotermination_minutes": 60, "cluster_name": "[UNIQUE_NAME]", - "node_type_id": "[NODE_TYPE_ID]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", "num_workers": 1, "spark_version": "15.4.x-scala2.12" } diff --git a/acceptance/bundle/run/app-with-job/databricks.yml.tmpl b/acceptance/bundle/run/app-with-job/databricks.yml.tmpl index 1c98ab3e77d..6715bf28a90 100644 --- a/acceptance/bundle/run/app-with-job/databricks.yml.tmpl +++ b/acceptance/bundle/run/app-with-job/databricks.yml.tmpl @@ -30,6 +30,7 @@ resources: num_workers: 1 spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID data_security_mode: USER_ISOLATION spark_python_task: python_file: ./hello_world.py diff --git a/acceptance/bundle/run_as/job_default/databricks.yml.tmpl b/acceptance/bundle/run_as/job_default/databricks.yml.tmpl index 31a49d96ef9..4451e5100b2 100644 --- a/acceptance/bundle/run_as/job_default/databricks.yml.tmpl +++ b/acceptance/bundle/run_as/job_default/databricks.yml.tmpl @@ -11,6 +11,7 @@ resources: new_cluster: spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 run_as: user_name: deco-test-user@databricks.com From 68fdd486a56961ffb683a9e74f8d91bb082276cd Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 7 Jun 2026 16:43:33 +0000 Subject: [PATCH 2/3] Drop job-spark-python-task from pool routing It is a bind test that only deploys (no cluster launch, no NAT benefit), and its acceptance golden did not regenerate cleanly outside CI. Reverting it to keep the change focused on templates that actually launch clusters. --- .../job-spark-python-task/databricks.yml.tmpl | 1 - .../bind/job/job-spark-python-task/output.txt | 64 ++++++++++++++++++- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/databricks.yml.tmpl b/acceptance/bundle/deployment/bind/job/job-spark-python-task/databricks.yml.tmpl index 5133f842d22..8d679a83a5c 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/databricks.yml.tmpl +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/databricks.yml.tmpl @@ -13,7 +13,6 @@ resources: new_cluster: spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID - instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 spark_python_task: python_file: ./hello_world.py diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt b/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt index 0dd4be6ba68..3b68a53f081 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt @@ -4,8 +4,66 @@ Created job with ID: [NUMID] === Bind job: >>> [CLI] bundle deployment bind foo [NUMID] --auto-approve -Error: failed to fetch the resource, err: Not Found +Updating deployment state... +Successfully bound job with an id '[NUMID]' +Run 'bundle deploy' to deploy changes to your workspace -=== Delete the pre-defined job [NUMID]:Error: Not Found +=== Remove .databricks directory to simulate fresh deployment: +>>> rm -rf .databricks -Exit code: 1 +=== Deploy bundle: +>>> [CLI] bundle deploy --force-lock --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-job-[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Read the pre-defined job: +>>> [CLI] jobs get [NUMID] +{ + "job_id": [NUMID], + "settings": { + "name": "test-job-basic-[UNIQUE_NAME]", + "tasks": [ + { + "task_key": "my_notebook_task", + "spark_python_task": { + "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bind-job-[UNIQUE_NAME]/files/hello_world.py" + } + } + ] + } +} + +=== Unbind the job: +>>> [CLI] bundle deployment unbind foo +Updating deployment state... + +=== Remove .databricks directory to simulate fresh deployment: +>>> rm -rf .databricks + +=== Destroy the bundle: +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-job-[UNIQUE_NAME] + +Deleting files... +Destroy complete! + +=== Read the pre-defined job again (expecting it still exists): +>>> [CLI] jobs get [NUMID] +{ + "job_id": [NUMID], + "settings": { + "name": "test-job-basic-[UNIQUE_NAME]", + "tasks": [ + { + "task_key": "my_notebook_task", + "spark_python_task": { + "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bind-job-[UNIQUE_NAME]/files/hello_world.py" + } + } + ] + } +} + +=== Delete the pre-defined job [NUMID]:0 From 3d5eb7ec20656962158b8facbdad4079f5dea267 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 7 Jun 2026 17:21:04 +0000 Subject: [PATCH 3/3] Drop run_as/job_default from pool routing Cloud-only deploy test (no bundle run, no cluster launch, no NAT benefit). It skips locally so its golden could not be regenerated here, and its cloud golden still records node_type_id; routing it through the pool changed the cloud output and broke the integration check. The 11 cluster-launching templates that do start clusters passed the cloud integration run. --- acceptance/bundle/run_as/job_default/databricks.yml.tmpl | 1 - 1 file changed, 1 deletion(-) diff --git a/acceptance/bundle/run_as/job_default/databricks.yml.tmpl b/acceptance/bundle/run_as/job_default/databricks.yml.tmpl index 4451e5100b2..31a49d96ef9 100644 --- a/acceptance/bundle/run_as/job_default/databricks.yml.tmpl +++ b/acceptance/bundle/run_as/job_default/databricks.yml.tmpl @@ -11,7 +11,6 @@ resources: new_cluster: spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID - instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 run_as: user_name: deco-test-user@databricks.com