Add instance_pools resource type to bundles (direct engine only)

.nextchanges/bundles/instance-pools.md

@@ -0,0 +1 @@
+* Add support for the `instance_pools` resource type in Declarative Automation Bundles. Instance pools are only supported in direct deployment mode.

acceptance/bundle/invariant/configs/instance_pool.yml.tmpl

@@ -0,0 +1,11 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  instance_pools:
+    foo:
+      instance_pool_name: test-instance-pool-$UNIQUE_NAME
+      node_type_id: $NODE_TYPE_ID
+      permissions:
+        - level: CAN_ATTACH_TO
+          group_name: users

acceptance/bundle/invariant/migrate/test.toml

@@ -11,6 +11,8 @@ EnvMatrixExclude.no_catalog = ["INPUT_CONFIG=catalog.yml.tmpl"]
 EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmpl"]
 # Genie spaces are direct-only too; the terraform deploy that seeds the migration fails for them.
 EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"]
+# Instance pools are direct-only; the terraform deploy that seeds the migration fails for them.
+EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"]
 
 # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level})
 # don't work in terraform mode: the terraform interpolator converts the path to

acceptance/bundle/invariant/test.toml

@@ -42,6 +42,7 @@ EnvMatrix.INPUT_CONFIG = [
     "experiment.yml.tmpl",
     "external_location.yml.tmpl",
     "genie_space.yml.tmpl",
+    "instance_pool.yml.tmpl",
     "job.yml.tmpl",
     "job_pydabs_10_tasks.yml.tmpl",
     "job_pydabs_1000_tasks.yml.tmpl",

acceptance/bundle/refschema/out.fields.txt

@@ -756,6 +756,72 @@ resources.genie_spaces.*.permissions[*].group_name	string	ALL
 resources.genie_spaces.*.permissions[*].level	iam.PermissionLevel	ALL
 resources.genie_spaces.*.permissions[*].service_principal_name	string	ALL
 resources.genie_spaces.*.permissions[*].user_name	string	ALL
+resources.instance_pools.*.aws_attributes	*compute.InstancePoolAwsAttributes	ALL
+resources.instance_pools.*.aws_attributes.availability	compute.InstancePoolAwsAttributesAvailability	ALL
+resources.instance_pools.*.aws_attributes.instance_profile_arn	string	ALL
+resources.instance_pools.*.aws_attributes.spot_bid_price_percent	int	ALL
+resources.instance_pools.*.aws_attributes.zone_id	string	ALL
+resources.instance_pools.*.azure_attributes	*compute.InstancePoolAzureAttributes	ALL
+resources.instance_pools.*.azure_attributes.availability	compute.InstancePoolAzureAttributesAvailability	ALL
+resources.instance_pools.*.azure_attributes.spot_bid_max_price	float64	ALL
+resources.instance_pools.*.custom_tags	map[string]string	ALL
+resources.instance_pools.*.custom_tags.*	string	ALL
+resources.instance_pools.*.default_tags	map[string]string	REMOTE
+resources.instance_pools.*.default_tags.*	string	REMOTE
+resources.instance_pools.*.disk_spec	*compute.DiskSpec	ALL
+resources.instance_pools.*.disk_spec.disk_count	int	ALL
+resources.instance_pools.*.disk_spec.disk_iops	int	ALL
+resources.instance_pools.*.disk_spec.disk_size	int	ALL
+resources.instance_pools.*.disk_spec.disk_throughput	int	ALL
+resources.instance_pools.*.disk_spec.disk_type	*compute.DiskType	ALL
+resources.instance_pools.*.disk_spec.disk_type.azure_disk_volume_type	compute.DiskTypeAzureDiskVolumeType	ALL
+resources.instance_pools.*.disk_spec.disk_type.ebs_volume_type	compute.DiskTypeEbsVolumeType	ALL
+resources.instance_pools.*.enable_elastic_disk	bool	ALL
+resources.instance_pools.*.gcp_attributes	*compute.InstancePoolGcpAttributes	ALL
+resources.instance_pools.*.gcp_attributes.gcp_availability	compute.GcpAvailability	ALL
+resources.instance_pools.*.gcp_attributes.local_ssd_count	int	ALL
+resources.instance_pools.*.gcp_attributes.zone_id	string	ALL
+resources.instance_pools.*.id	string	INPUT
+resources.instance_pools.*.idle_instance_autotermination_minutes	int	ALL
+resources.instance_pools.*.instance_pool_id	string	REMOTE
+resources.instance_pools.*.instance_pool_name	string	ALL
+resources.instance_pools.*.lifecycle	resources.Lifecycle	INPUT
+resources.instance_pools.*.lifecycle.prevent_destroy	bool	INPUT
+resources.instance_pools.*.max_capacity	int	ALL
+resources.instance_pools.*.min_idle_instances	int	ALL
+resources.instance_pools.*.modified_status	string	INPUT
+resources.instance_pools.*.node_type_flexibility	*compute.NodeTypeFlexibility	ALL
+resources.instance_pools.*.node_type_flexibility.alternate_node_type_ids	[]string	ALL
+resources.instance_pools.*.node_type_flexibility.alternate_node_type_ids[*]	string	ALL
+resources.instance_pools.*.node_type_id	string	ALL
+resources.instance_pools.*.preloaded_docker_images	[]compute.DockerImage	ALL
+resources.instance_pools.*.preloaded_docker_images[*]	compute.DockerImage	ALL
+resources.instance_pools.*.preloaded_docker_images[*].basic_auth	*compute.DockerBasicAuth	ALL
+resources.instance_pools.*.preloaded_docker_images[*].basic_auth.password	string	ALL
+resources.instance_pools.*.preloaded_docker_images[*].basic_auth.username	string	ALL
+resources.instance_pools.*.preloaded_docker_images[*].url	string	ALL
+resources.instance_pools.*.preloaded_spark_versions	[]string	ALL
+resources.instance_pools.*.preloaded_spark_versions[*]	string	ALL
+resources.instance_pools.*.remote_disk_throughput	int	ALL
+resources.instance_pools.*.state	compute.InstancePoolState	REMOTE
+resources.instance_pools.*.stats	*compute.InstancePoolStats	REMOTE
+resources.instance_pools.*.stats.idle_count	int	REMOTE
+resources.instance_pools.*.stats.pending_idle_count	int	REMOTE
+resources.instance_pools.*.stats.pending_used_count	int	REMOTE
+resources.instance_pools.*.stats.used_count	int	REMOTE
+resources.instance_pools.*.status	*compute.InstancePoolStatus	REMOTE
+resources.instance_pools.*.status.pending_instance_errors	[]compute.PendingInstanceError	REMOTE
+resources.instance_pools.*.status.pending_instance_errors[*]	compute.PendingInstanceError	REMOTE
+resources.instance_pools.*.status.pending_instance_errors[*].instance_id	string	REMOTE
+resources.instance_pools.*.status.pending_instance_errors[*].message	string	REMOTE
+resources.instance_pools.*.total_initial_remote_disk_size	int	ALL
+resources.instance_pools.*.url	string	INPUT
+resources.instance_pools.*.permissions.object_id	string	ALL
+resources.instance_pools.*.permissions[*]	dresources.StatePermission	ALL
+resources.instance_pools.*.permissions[*].group_name	string	ALL
+resources.instance_pools.*.permissions[*].level	iam.PermissionLevel	ALL
+resources.instance_pools.*.permissions[*].service_principal_name	string	ALL
+resources.instance_pools.*.permissions[*].user_name	string	ALL
 resources.job_runs.*.dbt_commands	[]string	ALL
 resources.job_runs.*.dbt_commands[*]	string	ALL
 resources.job_runs.*.id	string	INPUT

acceptance/bundle/resources/instance_pools/databricks.yml

@@ -0,0 +1,11 @@
+bundle:
+  name: test_instance_pool
+
+resources:
+  instance_pools:
+    test_instance_pool:
+      instance_pool_name: my_instance_pool
+      node_type_id: Standard_DS3_v2
+      min_idle_instances: 0
+      max_capacity: 5
+      idle_instance_autotermination_minutes: 60

acceptance/bundle/resources/instance_pools/output.txt

@@ -0,0 +1,135 @@
+
+>>> [CLI] bundle validate
+Name: test_instance_pool
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default
+
+Validation OK!
+
+>>> [CLI] bundle validate -o json
+{
+  "test_instance_pool": {
+    "idle_instance_autotermination_minutes": 60,
+    "instance_pool_name": "my_instance_pool",
+    "max_capacity": 5,
+    "min_idle_instances": 0,
+    "node_type_id": "Standard_DS3_v2"
+  }
+}
+
+>>> [CLI] bundle summary
+Name: test_instance_pool
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default
+Resources:
+  Instance Pools:
+    test_instance_pool:
+      Name: my_instance_pool
+      URL:  (not deployed)
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Verify the create request
+>>> jq select(.method == "POST" and (.path | contains("/instance-pools/create"))) out.requests.txt
+{
+  "method": "POST",
+  "path": "/api/2.0/instance-pools/create",
+  "body": {
+    "idle_instance_autotermination_minutes": 60,
+    "instance_pool_name": "my_instance_pool",
+    "max_capacity": 5,
+    "min_idle_instances": 0,
+    "node_type_id": "Standard_DS3_v2"
+  }
+}
+
+>>> [CLI] bundle summary
+Name: test_instance_pool
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default
+Resources:
+  Instance Pools:
+    test_instance_pool:
+      Name: my_instance_pool
+      URL:  [DATABRICKS_URL]/compute/instance-pools/[UUID]?w=[NUMID]
+
+=== Update the instance pool name
+>>> update_file.py databricks.yml my_instance_pool my_instance_pool_2
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Verify the update request
+>>> jq select(.method == "POST" and (.path | contains("/instance-pools/edit"))) out.requests.txt
+{
+  "method": "POST",
+  "path": "/api/2.0/instance-pools/edit",
+  "body": {
+    "idle_instance_autotermination_minutes": 60,
+    "instance_pool_id": "[UUID]",
+    "instance_pool_name": "my_instance_pool_2",
+    "max_capacity": 5,
+    "min_idle_instances": 0,
+    "node_type_id": "Standard_DS3_v2"
+  }
+}
+
+>>> [CLI] bundle summary
+Name: test_instance_pool
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default
+Resources:
+  Instance Pools:
+    test_instance_pool:
+      Name: my_instance_pool_2
+      URL:  [DATABRICKS_URL]/compute/instance-pools/[UUID]?w=[NUMID]
+
+=== Destroy the instance pool
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.instance_pools.test_instance_pool
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default
+
+Deleting files...
+Destroy complete!
+
+=== Verify the destroy request
+>>> jq select(.method == "POST" and (.path | contains("/instance-pools/delete"))) out.requests.txt
+{
+  "method": "POST",
+  "path": "/api/2.0/instance-pools/delete",
+  "body": {
+    "instance_pool_id": "[UUID]"
+  }
+}
+
+>>> [CLI] bundle summary
+Name: test_instance_pool
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default
+Resources:
+  Instance Pools:
+    test_instance_pool:
+      Name: my_instance_pool_2
+      URL:  (not deployed)
+
+>>> [CLI] bundle destroy --auto-approve
+No active deployment found to destroy!

acceptance/bundle/resources/instance_pools/script

@@ -0,0 +1,33 @@
+trace $CLI bundle validate
+trace $CLI bundle validate -o json | jq ".resources.instance_pools"
+
+trace $CLI bundle summary
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm out.requests.txt
+}
+trap cleanup EXIT
+trace $CLI bundle deploy
+
+title "Verify the create request"
+trace jq 'select(.method == "POST" and (.path | contains("/instance-pools/create")))' out.requests.txt
+
+trace $CLI bundle summary
+
+title "Update the instance pool name"
+trace update_file.py databricks.yml my_instance_pool my_instance_pool_2
+trace $CLI bundle deploy
+
+title "Verify the update request"
+trace jq 'select(.method == "POST" and (.path | contains("/instance-pools/edit")))' out.requests.txt
+
+trace $CLI bundle summary
+
+title "Destroy the instance pool"
+trace $CLI bundle destroy --auto-approve
+
+title "Verify the destroy request"
+trace jq 'select(.method == "POST" and (.path | contains("/instance-pools/delete")))' out.requests.txt
+
+trace $CLI bundle summary

acceptance/bundle/resources/instance_pools/test.toml

@@ -0,0 +1,7 @@
+Local = true
+Cloud = false
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+Ignore = [
+    "databricks.yml",
+]

acceptance/experimental/open/output.txt

@@ -9,7 +9,7 @@
 
 === unknown resource type
 >>> [CLI] experimental open --url unknown 123
-Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses
+Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses
 
 === test auto-completion handler
 >>> [CLI] __complete experimental open ,
@@ -22,6 +22,7 @@ database_catalogs
 database_instances
 experiments
 genie_spaces
+instance_pools
 jobs
 model_serving_endpoints
 models

bundle/config/mutator/resourcemutator/apply_bundle_permissions.go

@@ -89,6 +89,10 @@ var (
 			permissions.CAN_MANAGE: "CAN_MANAGE",
 			permissions.CAN_VIEW:   "CAN_USE",
 		},
+		"instance_pools": {
+			permissions.CAN_MANAGE: "CAN_MANAGE",
+			permissions.CAN_VIEW:   "CAN_ATTACH_TO",
+		},
 	}
 )
 

bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go

@@ -89,6 +89,10 @@ func TestApplyBundlePermissions(t *testing.T) {
 					"vs_1": {},
 					"vs_2": {},
 				},
+				InstancePools: map[string]*resources.InstancePool{
+					"instance_pool_1": {},
+					"instance_pool_2": {},
+				},
 			},
 		},
 	}
@@ -157,6 +161,10 @@ func TestApplyBundlePermissions(t *testing.T) {
 	require.Len(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, 2)
 	require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"})
 	require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.Permission{Level: "CAN_USE", GroupName: "TestGroup"})
+
+	require.Len(t, b.Config.Resources.InstancePools["instance_pool_1"].Permissions, 2)
+	require.Contains(t, b.Config.Resources.InstancePools["instance_pool_1"].Permissions, resources.InstancePoolPermission{Level: "CAN_MANAGE", UserName: "TestUser"})
+	require.Contains(t, b.Config.Resources.InstancePools["instance_pool_1"].Permissions, resources.InstancePoolPermission{Level: "CAN_ATTACH_TO", GroupName: "TestGroup"})
 }
 
 func TestWarningOnOverlapPermission(t *testing.T) {

bundle/config/mutator/resourcemutator/apply_presets.go

@@ -232,6 +232,26 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 		}
 	}
 
+	// Instance Pools: Prefix, Tags
+	for _, pool := range r.InstancePools {
+		if pool == nil {
+			continue
+		}
+		pool.InstancePoolName = prefix + pool.InstancePoolName
+		if len(tags) > 0 {
+			if pool.CustomTags == nil {
+				pool.CustomTags = make(map[string]string, len(tags))
+			}
+			for _, tag := range tags {
+				k := b.Tagging.NormalizeKey(tag.Key)
+				v := b.Tagging.NormalizeValue(tag.Value)
+				if _, ok := pool.CustomTags[k]; !ok {
+					pool.CustomTags[k] = v
+				}
+			}
+		}
+	}
+
 	// Dashboards: Prefix
 	for _, dashboard := range r.Dashboards {
 		if dashboard == nil {