From 215fe4d9e1be267fc83592564bcc5a4fbc1b7770 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 15 Jul 2026 14:15:30 +0000 Subject: [PATCH 1/3] Add instance_pools resource type to bundles (direct engine only) Adds support for defining Databricks instance pools as a bundle resource. Instance pools are implemented as a direct-engine-only resource: they are registered in the direct engine's SupportedResources but not wired to Terraform, so ValidateDirectOnlyResources rejects them under the terraform engine. Includes the resource definition and ConfigResource implementation, the direct-engine adapter (create/read/edit/delete + RemapState), presets and bundle-permission wiring, workspace URL generation, testserver CRUD handlers, schema annotations, and unit + acceptance test coverage. --- .nextchanges/bundles/instance-pools.md | 1 + acceptance/bundle/refschema/out.fields.txt | 66 +++ .../resources/instance_pools/databricks.yml | 11 + .../resources/instance_pools/out.test.toml | 5 + .../resources/instance_pools/output.txt | 135 ++++++ .../bundle/resources/instance_pools/script | 33 ++ .../bundle/resources/instance_pools/test.toml | 7 + .../apply_bundle_permissions.go | 4 + .../apply_bundle_permissions_test.go | 8 + .../mutator/resourcemutator/apply_presets.go | 20 + .../resourcemutator/apply_target_mode_test.go | 7 + .../mutator/resourcemutator/run_as_test.go | 2 + bundle/config/resources.go | 3 + bundle/config/resources/instance_pools.go | 59 +++ bundle/config/resources/permission_types.go | 1 + bundle/config/resources_test.go | 4 + bundle/direct/dresources/all.go | 2 + bundle/direct/dresources/all_test.go | 18 + .../direct/dresources/apitypes.generated.yml | 2 + bundle/direct/dresources/instance_pool.go | 77 ++++ bundle/direct/dresources/permissions.go | 1 + .../direct/dresources/resources.generated.yml | 2 + bundle/direct/dresources/resources.yml | 21 + bundle/internal/schema/annotations.yml | 53 +++ bundle/schema/jsonschema.json | 391 ++++++++++++++++++ libs/testserver/fake_workspace.go | 2 + libs/testserver/handlers.go | 6 + libs/testserver/instance_pools.go | 81 ++++ libs/workspaceurls/urls.go | 1 + 29 files changed, 1023 insertions(+) create mode 100644 .nextchanges/bundles/instance-pools.md create mode 100644 acceptance/bundle/resources/instance_pools/databricks.yml create mode 100644 acceptance/bundle/resources/instance_pools/out.test.toml create mode 100644 acceptance/bundle/resources/instance_pools/output.txt create mode 100644 acceptance/bundle/resources/instance_pools/script create mode 100644 acceptance/bundle/resources/instance_pools/test.toml create mode 100644 bundle/config/resources/instance_pools.go create mode 100644 bundle/direct/dresources/instance_pool.go create mode 100644 libs/testserver/instance_pools.go diff --git a/.nextchanges/bundles/instance-pools.md b/.nextchanges/bundles/instance-pools.md new file mode 100644 index 00000000000..ea0d59855f7 --- /dev/null +++ b/.nextchanges/bundles/instance-pools.md @@ -0,0 +1 @@ +* Add support for the `instance_pools` resource type in Declarative Automation Bundles. Instance pools are only supported in direct deployment mode. diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 3b66f214fb1..d313abbb82f 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -756,6 +756,72 @@ resources.genie_spaces.*.permissions[*].group_name string ALL resources.genie_spaces.*.permissions[*].level iam.PermissionLevel ALL resources.genie_spaces.*.permissions[*].service_principal_name string ALL resources.genie_spaces.*.permissions[*].user_name string ALL +resources.instance_pools.*.aws_attributes *compute.InstancePoolAwsAttributes ALL +resources.instance_pools.*.aws_attributes.availability compute.InstancePoolAwsAttributesAvailability ALL +resources.instance_pools.*.aws_attributes.instance_profile_arn string ALL +resources.instance_pools.*.aws_attributes.spot_bid_price_percent int ALL +resources.instance_pools.*.aws_attributes.zone_id string ALL +resources.instance_pools.*.azure_attributes *compute.InstancePoolAzureAttributes ALL +resources.instance_pools.*.azure_attributes.availability compute.InstancePoolAzureAttributesAvailability ALL +resources.instance_pools.*.azure_attributes.spot_bid_max_price float64 ALL +resources.instance_pools.*.custom_tags map[string]string ALL +resources.instance_pools.*.custom_tags.* string ALL +resources.instance_pools.*.default_tags map[string]string REMOTE +resources.instance_pools.*.default_tags.* string REMOTE +resources.instance_pools.*.disk_spec *compute.DiskSpec ALL +resources.instance_pools.*.disk_spec.disk_count int ALL +resources.instance_pools.*.disk_spec.disk_iops int ALL +resources.instance_pools.*.disk_spec.disk_size int ALL +resources.instance_pools.*.disk_spec.disk_throughput int ALL +resources.instance_pools.*.disk_spec.disk_type *compute.DiskType ALL +resources.instance_pools.*.disk_spec.disk_type.azure_disk_volume_type compute.DiskTypeAzureDiskVolumeType ALL +resources.instance_pools.*.disk_spec.disk_type.ebs_volume_type compute.DiskTypeEbsVolumeType ALL +resources.instance_pools.*.enable_elastic_disk bool ALL +resources.instance_pools.*.gcp_attributes *compute.InstancePoolGcpAttributes ALL +resources.instance_pools.*.gcp_attributes.gcp_availability compute.GcpAvailability ALL +resources.instance_pools.*.gcp_attributes.local_ssd_count int ALL +resources.instance_pools.*.gcp_attributes.zone_id string ALL +resources.instance_pools.*.id string INPUT +resources.instance_pools.*.idle_instance_autotermination_minutes int ALL +resources.instance_pools.*.instance_pool_id string REMOTE +resources.instance_pools.*.instance_pool_name string ALL +resources.instance_pools.*.lifecycle resources.Lifecycle INPUT +resources.instance_pools.*.lifecycle.prevent_destroy bool INPUT +resources.instance_pools.*.max_capacity int ALL +resources.instance_pools.*.min_idle_instances int ALL +resources.instance_pools.*.modified_status string INPUT +resources.instance_pools.*.node_type_flexibility *compute.NodeTypeFlexibility ALL +resources.instance_pools.*.node_type_flexibility.alternate_node_type_ids []string ALL +resources.instance_pools.*.node_type_flexibility.alternate_node_type_ids[*] string ALL +resources.instance_pools.*.node_type_id string ALL +resources.instance_pools.*.preloaded_docker_images []compute.DockerImage ALL +resources.instance_pools.*.preloaded_docker_images[*] compute.DockerImage ALL +resources.instance_pools.*.preloaded_docker_images[*].basic_auth *compute.DockerBasicAuth ALL +resources.instance_pools.*.preloaded_docker_images[*].basic_auth.password string ALL +resources.instance_pools.*.preloaded_docker_images[*].basic_auth.username string ALL +resources.instance_pools.*.preloaded_docker_images[*].url string ALL +resources.instance_pools.*.preloaded_spark_versions []string ALL +resources.instance_pools.*.preloaded_spark_versions[*] string ALL +resources.instance_pools.*.remote_disk_throughput int ALL +resources.instance_pools.*.state compute.InstancePoolState REMOTE +resources.instance_pools.*.stats *compute.InstancePoolStats REMOTE +resources.instance_pools.*.stats.idle_count int REMOTE +resources.instance_pools.*.stats.pending_idle_count int REMOTE +resources.instance_pools.*.stats.pending_used_count int REMOTE +resources.instance_pools.*.stats.used_count int REMOTE +resources.instance_pools.*.status *compute.InstancePoolStatus REMOTE +resources.instance_pools.*.status.pending_instance_errors []compute.PendingInstanceError REMOTE +resources.instance_pools.*.status.pending_instance_errors[*] compute.PendingInstanceError REMOTE +resources.instance_pools.*.status.pending_instance_errors[*].instance_id string REMOTE +resources.instance_pools.*.status.pending_instance_errors[*].message string REMOTE +resources.instance_pools.*.total_initial_remote_disk_size int ALL +resources.instance_pools.*.url string INPUT +resources.instance_pools.*.permissions.object_id string ALL +resources.instance_pools.*.permissions[*] dresources.StatePermission ALL +resources.instance_pools.*.permissions[*].group_name string ALL +resources.instance_pools.*.permissions[*].level iam.PermissionLevel ALL +resources.instance_pools.*.permissions[*].service_principal_name string ALL +resources.instance_pools.*.permissions[*].user_name string ALL resources.job_runs.*.dbt_commands []string ALL resources.job_runs.*.dbt_commands[*] string ALL resources.job_runs.*.id string INPUT diff --git a/acceptance/bundle/resources/instance_pools/databricks.yml b/acceptance/bundle/resources/instance_pools/databricks.yml new file mode 100644 index 00000000000..dd58ba4b0d0 --- /dev/null +++ b/acceptance/bundle/resources/instance_pools/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test_instance_pool + +resources: + instance_pools: + test_instance_pool: + instance_pool_name: my_instance_pool + node_type_id: Standard_DS3_v2 + min_idle_instances: 0 + max_capacity: 5 + idle_instance_autotermination_minutes: 60 diff --git a/acceptance/bundle/resources/instance_pools/out.test.toml b/acceptance/bundle/resources/instance_pools/out.test.toml new file mode 100644 index 00000000000..1a3e24fa574 --- /dev/null +++ b/acceptance/bundle/resources/instance_pools/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false +GOOSOnPR.darwin = false +GOOSOnPR.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/instance_pools/output.txt b/acceptance/bundle/resources/instance_pools/output.txt new file mode 100644 index 00000000000..30381ea5b70 --- /dev/null +++ b/acceptance/bundle/resources/instance_pools/output.txt @@ -0,0 +1,135 @@ + +>>> [CLI] bundle validate +Name: test_instance_pool +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default + +Validation OK! + +>>> [CLI] bundle validate -o json +{ + "test_instance_pool": { + "idle_instance_autotermination_minutes": 60, + "instance_pool_name": "my_instance_pool", + "max_capacity": 5, + "min_idle_instances": 0, + "node_type_id": "Standard_DS3_v2" + } +} + +>>> [CLI] bundle summary +Name: test_instance_pool +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default +Resources: + Instance Pools: + test_instance_pool: + Name: my_instance_pool + URL: (not deployed) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Verify the create request +>>> jq select(.method == "POST" and (.path | contains("/instance-pools/create"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/instance-pools/create", + "body": { + "idle_instance_autotermination_minutes": 60, + "instance_pool_name": "my_instance_pool", + "max_capacity": 5, + "min_idle_instances": 0, + "node_type_id": "Standard_DS3_v2" + } +} + +>>> [CLI] bundle summary +Name: test_instance_pool +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default +Resources: + Instance Pools: + test_instance_pool: + Name: my_instance_pool + URL: [DATABRICKS_URL]/compute/instance-pools/[UUID]?w=[NUMID] + +=== Update the instance pool name +>>> update_file.py databricks.yml my_instance_pool my_instance_pool_2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Verify the update request +>>> jq select(.method == "POST" and (.path | contains("/instance-pools/edit"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/instance-pools/edit", + "body": { + "idle_instance_autotermination_minutes": 60, + "instance_pool_id": "[UUID]", + "instance_pool_name": "my_instance_pool_2", + "max_capacity": 5, + "min_idle_instances": 0, + "node_type_id": "Standard_DS3_v2" + } +} + +>>> [CLI] bundle summary +Name: test_instance_pool +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default +Resources: + Instance Pools: + test_instance_pool: + Name: my_instance_pool_2 + URL: [DATABRICKS_URL]/compute/instance-pools/[UUID]?w=[NUMID] + +=== Destroy the instance pool +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.instance_pools.test_instance_pool + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default + +Deleting files... +Destroy complete! + +=== Verify the destroy request +>>> jq select(.method == "POST" and (.path | contains("/instance-pools/delete"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/instance-pools/delete", + "body": { + "instance_pool_id": "[UUID]" + } +} + +>>> [CLI] bundle summary +Name: test_instance_pool +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default +Resources: + Instance Pools: + test_instance_pool: + Name: my_instance_pool_2 + URL: (not deployed) + +>>> [CLI] bundle destroy --auto-approve +No active deployment found to destroy! diff --git a/acceptance/bundle/resources/instance_pools/script b/acceptance/bundle/resources/instance_pools/script new file mode 100644 index 00000000000..208a61abc1b --- /dev/null +++ b/acceptance/bundle/resources/instance_pools/script @@ -0,0 +1,33 @@ +trace $CLI bundle validate +trace $CLI bundle validate -o json | jq ".resources.instance_pools" + +trace $CLI bundle summary + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm out.requests.txt +} +trap cleanup EXIT +trace $CLI bundle deploy + +title "Verify the create request" +trace jq 'select(.method == "POST" and (.path | contains("/instance-pools/create")))' out.requests.txt + +trace $CLI bundle summary + +title "Update the instance pool name" +trace update_file.py databricks.yml my_instance_pool my_instance_pool_2 +trace $CLI bundle deploy + +title "Verify the update request" +trace jq 'select(.method == "POST" and (.path | contains("/instance-pools/edit")))' out.requests.txt + +trace $CLI bundle summary + +title "Destroy the instance pool" +trace $CLI bundle destroy --auto-approve + +title "Verify the destroy request" +trace jq 'select(.method == "POST" and (.path | contains("/instance-pools/delete")))' out.requests.txt + +trace $CLI bundle summary diff --git a/acceptance/bundle/resources/instance_pools/test.toml b/acceptance/bundle/resources/instance_pools/test.toml new file mode 100644 index 00000000000..96ce9798d0c --- /dev/null +++ b/acceptance/bundle/resources/instance_pools/test.toml @@ -0,0 +1,7 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "databricks.yml", +] diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go index c81d452d641..f2a6d857090 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go @@ -89,6 +89,10 @@ var ( permissions.CAN_MANAGE: "CAN_MANAGE", permissions.CAN_VIEW: "CAN_USE", }, + "instance_pools": { + permissions.CAN_MANAGE: "CAN_MANAGE", + permissions.CAN_VIEW: "CAN_ATTACH_TO", + }, } ) diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 912d70bdd40..85f30c952f2 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -89,6 +89,10 @@ func TestApplyBundlePermissions(t *testing.T) { "vs_1": {}, "vs_2": {}, }, + InstancePools: map[string]*resources.InstancePool{ + "instance_pool_1": {}, + "instance_pool_2": {}, + }, }, }, } @@ -157,6 +161,10 @@ func TestApplyBundlePermissions(t *testing.T) { require.Len(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, 2) require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.Permission{Level: "CAN_USE", GroupName: "TestGroup"}) + + require.Len(t, b.Config.Resources.InstancePools["instance_pool_1"].Permissions, 2) + require.Contains(t, b.Config.Resources.InstancePools["instance_pool_1"].Permissions, resources.InstancePoolPermission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.InstancePools["instance_pool_1"].Permissions, resources.InstancePoolPermission{Level: "CAN_ATTACH_TO", GroupName: "TestGroup"}) } func TestWarningOnOverlapPermission(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index 70cc2f66cbc..72817d13566 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -232,6 +232,26 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } } + // Instance Pools: Prefix, Tags + for _, pool := range r.InstancePools { + if pool == nil { + continue + } + pool.InstancePoolName = prefix + pool.InstancePoolName + if len(tags) > 0 { + if pool.CustomTags == nil { + pool.CustomTags = make(map[string]string, len(tags)) + } + for _, tag := range tags { + k := b.Tagging.NormalizeKey(tag.Key) + v := b.Tagging.NormalizeValue(tag.Value) + if _, ok := pool.CustomTags[k]; !ok { + pool.CustomTags[k] = v + } + } + } + } + // Dashboards: Prefix for _, dashboard := range r.Dashboards { if dashboard == nil { diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index da9fcab9770..b97113028b8 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -150,6 +150,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle { Clusters: map[string]*resources.Cluster{ "cluster1": {ClusterSpec: compute.ClusterSpec{ClusterName: "cluster1", SparkVersion: "13.2.x", NumWorkers: 1}}, }, + InstancePools: map[string]*resources.InstancePool{ + "instance_pool1": {CreateInstancePool: compute.CreateInstancePool{InstancePoolName: "instance_pool1", NodeTypeId: "i3.xlarge"}}, + }, Dashboards: map[string]*resources.Dashboard{ "dashboard1": { DashboardConfig: resources.DashboardConfig{ @@ -376,6 +379,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Clusters assert.Equal(t, "[dev lennart] cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName) + // Instance pools + assert.Equal(t, "[dev lennart] instance_pool1", b.Config.Resources.InstancePools["instance_pool1"].InstancePoolName) + // Dashboards assert.Equal(t, "[dev lennart] dashboard1", b.Config.Resources.Dashboards["dashboard1"].DisplayName) @@ -452,6 +458,7 @@ func TestProcessTargetModeDefault(t *testing.T) { assert.Equal(t, "schema1", b.Config.Resources.Schemas["schema1"].Name) assert.Equal(t, "volume1", b.Config.Resources.Volumes["volume1"].Name) assert.Equal(t, "cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName) + assert.Equal(t, "instance_pool1", b.Config.Resources.InstancePools["instance_pool1"].InstancePoolName) assert.Equal(t, "sql_warehouse1", b.Config.Resources.SqlWarehouses["sql_warehouse1"].Name) } diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 94db7ed67bb..5faed5f7b1b 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -42,6 +42,7 @@ func allResourceTypes(t *testing.T) []string { "experiments", "external_locations", "genie_spaces", + "instance_pools", "job_runs", "jobs", "model_serving_endpoints", @@ -190,6 +191,7 @@ var allowList = []string{ "registered_models", "experiments", "genie_spaces", + "instance_pools", "job_runs", "schemas", "secret_scopes", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index cfbfbe9fff5..3ab06f49918 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -43,6 +43,7 @@ type Resources struct { PostgresSyncedTables map[string]*resources.PostgresSyncedTable `json:"postgres_synced_tables,omitempty"` VectorSearchEndpoints map[string]*resources.VectorSearchEndpoint `json:"vector_search_endpoints,omitempty"` VectorSearchIndexes map[string]*resources.VectorSearchIndex `json:"vector_search_indexes,omitempty"` + InstancePools map[string]*resources.InstancePool `json:"instance_pools,omitempty"` } type ConfigResource interface { @@ -127,6 +128,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["postgres_synced_tables"], r.PostgresSyncedTables), collectResourceMap(descriptions["vector_search_endpoints"], r.VectorSearchEndpoints), collectResourceMap(descriptions["vector_search_indexes"], r.VectorSearchIndexes), + collectResourceMap(descriptions["instance_pools"], r.InstancePools), } } @@ -163,6 +165,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "pipelines": (&resources.Pipeline{}).ResourceDescription(), "models": (&resources.MlflowModel{}).ResourceDescription(), "experiments": (&resources.MlflowExperiment{}).ResourceDescription(), + "instance_pools": (&resources.InstancePool{}).ResourceDescription(), "model_serving_endpoints": (&resources.ModelServingEndpoint{}).ResourceDescription(), "registered_models": (&resources.RegisteredModel{}).ResourceDescription(), "quality_monitors": (&resources.QualityMonitor{}).ResourceDescription(), diff --git a/bundle/config/resources/instance_pools.go b/bundle/config/resources/instance_pools.go new file mode 100644 index 00000000000..31c4ee45a69 --- /dev/null +++ b/bundle/config/resources/instance_pools.go @@ -0,0 +1,59 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +type InstancePool struct { + BaseResource + compute.CreateInstancePool + Permissions []InstancePoolPermission `json:"permissions,omitempty"` +} + +func (s *InstancePool) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s InstancePool) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func (s *InstancePool) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.InstancePools.GetByInstancePoolId(ctx, id) + if err != nil { + log.Debugf(ctx, "instance pool %s does not exist", id) + return false, err + } + return true, nil +} + +func (*InstancePool) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "instance_pool", + PluralName: "instance_pools", + SingularTitle: "Instance Pool", + PluralTitle: "Instance Pools", + } +} + +func (s *InstancePool) InitializeURL(baseURL url.URL) { + if s.ID == "" { + return + } + s.URL = workspaceurls.ResourceURL(baseURL, "instance_pools", s.ID) +} + +func (s *InstancePool) GetName() string { + return s.InstancePoolName +} + +func (s *InstancePool) GetURL() string { + return s.URL +} diff --git a/bundle/config/resources/permission_types.go b/bundle/config/resources/permission_types.go index 3029ee40b8c..b73d0b878ed 100644 --- a/bundle/config/resources/permission_types.go +++ b/bundle/config/resources/permission_types.go @@ -26,6 +26,7 @@ func (p Permission) String() string { type ( AppPermission PermissionT[apps.AppPermissionLevel] ClusterPermission PermissionT[compute.ClusterPermissionLevel] + InstancePoolPermission PermissionT[compute.InstancePoolPermissionLevel] JobPermission PermissionT[jobs.JobPermissionLevel] MlflowExperimentPermission PermissionT[ml.ExperimentPermissionLevel] MlflowModelPermission PermissionT[ml.RegisteredModelPermissionLevel] diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 978d5799f45..de9caa09dc2 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -201,6 +201,9 @@ func TestResourcesBindSupport(t *testing.T) { Clusters: map[string]*resources.Cluster{ "my_cluster": {}, }, + InstancePools: map[string]*resources.InstancePool{ + "my_instance_pool": {}, + }, Dashboards: map[string]*resources.Dashboard{ "my_dashboard": {}, }, @@ -353,6 +356,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockExternalLocationsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockSchemasAPI().EXPECT().GetByFullName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockClustersAPI().EXPECT().GetByClusterId(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockInstancePoolsAPI().EXPECT().GetByInstancePoolId(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockLakeviewAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockGenieAPI().EXPECT().GetSpace(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVolumesAPI().EXPECT().Read(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 9856e6f979d..d6bcd03f73c 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -38,6 +38,7 @@ var SupportedResources = map[string]any{ "quality_monitors": (*ResourceQualityMonitor)(nil), "vector_search_endpoints": (*ResourceVectorSearchEndpoint)(nil), "vector_search_indexes": (*ResourceVectorSearchIndex)(nil), + "instance_pools": (*ResourceInstancePool)(nil), // Permissions "jobs.permissions": (*ResourcePermissions)(nil), @@ -55,6 +56,7 @@ var SupportedResources = map[string]any{ "dashboards.permissions": (*ResourcePermissions)(nil), "genie_spaces.permissions": (*ResourcePermissions)(nil), "vector_search_endpoints.permissions": (*ResourcePermissions)(nil), + "instance_pools.permissions": (*ResourcePermissions)(nil), // Grants "catalogs.grants": (*ResourceGrants)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index d79c80c61b2..a960e168fec 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -19,6 +19,7 @@ import ( "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/dashboards" "github.com/databricks/databricks-sdk-go/service/database" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -79,6 +80,13 @@ var testConfig map[string]any = map[string]any{ }, }, + "instance_pools": &resources.InstancePool{ + CreateInstancePool: compute.CreateInstancePool{ + InstancePoolName: "my-instance-pool", + NodeTypeId: "i3.xlarge", + }, + }, + "synced_database_tables": &resources.SyncedDatabaseTable{ SyncedDatabaseTable: database.SyncedDatabaseTable{ Name: "main.myschema.my_synced_table", @@ -434,6 +442,16 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "instance_pools.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + return &PermissionsState{ + ObjectID: "/instance-pools/pool-permissions", + EmbeddedSlice: []StatePermission{{ + Level: "CAN_MANAGE", + UserName: "user@example.com", + }}, + }, nil + }, + "apps.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { waiter, err := client.Apps.Create(ctx, apps.CreateAppRequest{ App: apps.App{ diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index 8fc2eec34a0..6da6f3555ec 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -20,6 +20,8 @@ external_locations: catalog.CreateExternalLocation genie_spaces: dashboards.GenieUpdateSpaceRequest +instance_pools: compute.CreateInstancePool + job_runs: jobs.RunNow jobs: jobs.JobSettings diff --git a/bundle/direct/dresources/instance_pool.go b/bundle/direct/dresources/instance_pool.go new file mode 100644 index 00000000000..37a1aeab447 --- /dev/null +++ b/bundle/direct/dresources/instance_pool.go @@ -0,0 +1,77 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/utils" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +type ResourceInstancePool struct { + client *databricks.WorkspaceClient +} + +func (*ResourceInstancePool) New(client *databricks.WorkspaceClient) *ResourceInstancePool { + return &ResourceInstancePool{client: client} +} + +func (*ResourceInstancePool) PrepareState(input *resources.InstancePool) *compute.CreateInstancePool { + return &input.CreateInstancePool +} + +// RemapState copies the config fields shared by GetInstancePool and CreateInstancePool; +// output-only fields (state, stats, default_tags, instance_pool_id) are not in the state. +func (*ResourceInstancePool) RemapState(remote *compute.GetInstancePool) *compute.CreateInstancePool { + return &compute.CreateInstancePool{ + AwsAttributes: remote.AwsAttributes, + AzureAttributes: remote.AzureAttributes, + CustomTags: remote.CustomTags, + DiskSpec: remote.DiskSpec, + EnableElasticDisk: remote.EnableElasticDisk, + GcpAttributes: remote.GcpAttributes, + IdleInstanceAutoterminationMinutes: remote.IdleInstanceAutoterminationMinutes, + InstancePoolName: remote.InstancePoolName, + MaxCapacity: remote.MaxCapacity, + MinIdleInstances: remote.MinIdleInstances, + NodeTypeFlexibility: remote.NodeTypeFlexibility, + NodeTypeId: remote.NodeTypeId, + PreloadedDockerImages: remote.PreloadedDockerImages, + PreloadedSparkVersions: remote.PreloadedSparkVersions, + RemoteDiskThroughput: remote.RemoteDiskThroughput, + TotalInitialRemoteDiskSize: remote.TotalInitialRemoteDiskSize, + ForceSendFields: utils.FilterFields[compute.CreateInstancePool](remote.ForceSendFields), + } +} + +func (r *ResourceInstancePool) DoRead(ctx context.Context, id string) (*compute.GetInstancePool, error) { + return r.client.InstancePools.GetByInstancePoolId(ctx, id) +} + +func (r *ResourceInstancePool) DoCreate(ctx context.Context, config *compute.CreateInstancePool) (string, *compute.GetInstancePool, error) { + resp, err := r.client.InstancePools.Create(ctx, *config) + if err != nil { + return "", nil, err + } + return resp.InstancePoolId, nil, nil +} + +func (r *ResourceInstancePool) DoUpdate(ctx context.Context, id string, config *compute.CreateInstancePool, _ *PlanEntry) (*compute.GetInstancePool, error) { + return nil, r.client.InstancePools.Edit(ctx, compute.EditInstancePool{ + InstancePoolId: id, + InstancePoolName: config.InstancePoolName, + NodeTypeId: config.NodeTypeId, + MinIdleInstances: config.MinIdleInstances, + MaxCapacity: config.MaxCapacity, + IdleInstanceAutoterminationMinutes: config.IdleInstanceAutoterminationMinutes, + CustomTags: config.CustomTags, + RemoteDiskThroughput: config.RemoteDiskThroughput, + TotalInitialRemoteDiskSize: config.TotalInitialRemoteDiskSize, + ForceSendFields: utils.FilterFields[compute.EditInstancePool](config.ForceSendFields), + }) +} + +func (r *ResourceInstancePool) DoDelete(ctx context.Context, id string, _ *compute.CreateInstancePool) error { + return r.client.InstancePools.DeleteByInstancePoolId(ctx, id) +} diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index 6e1fa79d811..e99311757a2 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -17,6 +17,7 @@ var permissionResourceToObjectType = map[string]string{ "alerts": "/alertsv2/", "apps": "/apps/", "clusters": "/clusters/", + "instance_pools": "/instance-pools/", "dashboards": "/dashboards/", "genie_spaces": "/genie/", "database_instances": "/database-instances/", diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index e4154180970..13b7d48acc8 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -176,6 +176,8 @@ resources: - field: etag reason: spec:output_only + # instance_pools: no api field behaviors + # job_runs: no api field behaviors # jobs: no api field behaviors diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index c0f640f9c46..267402f060b 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -598,6 +598,27 @@ resources: # DataSecurityModeDiffSuppressFunc: suppress when old != "" && new == "" #- field: data_security_mode + instance_pools: + # Field behaviors follow the TF provider tags cross-referenced with the edit API (compute.EditInstancePool): + # https://github.com/databricks/terraform-provider-databricks/blob/main/pools/resource_instance_pool.go + ignore_remote_changes: + # Backend fills cloud defaults the user omits; treated as managed like clusters above. + - field: aws_attributes + reason: managed + - field: azure_attributes + reason: managed + - field: gcp_attributes + reason: managed + recreate_on_changes: + # force_new and not accepted by /instance-pools/edit. + - field: disk_spec + - field: node_type_flexibility + - field: preloaded_spark_versions + - field: preloaded_docker_images + backend_defaults: + # Defaults to true server-side. + - field: enable_elastic_disk + sql_warehouses: ignore_remote_changes: # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/sql/resource_sql_endpoint.go#L62 diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 247c1684977..10832fe04a6 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -912,6 +912,59 @@ resources: "warehouse_id": "description": |- ID of the SQL warehouse used to run queries for this Genie space. + "instance_pools": + "description": |- + The instance pool definitions for the bundle, where each key is the name of the instance pool. + "markdown_description": |- + The instance pool definitions for the bundle, where each key is the name of the instance pool. See [\_](/dev-tools/bundles/resources.md#instance_pools). + "$type": + "markdown_description": |- + The instance pool resource defines an [instance pool](/api/workspace/instancepools/create), a set of idle, ready-to-use cloud instances that reduce cluster start and auto-scaling times. + "markdown_examples": |- + The following example creates an instance pool named `my_instance_pool`: + + ```yaml + resources: + instance_pools: + my_instance_pool: + instance_pool_name: my_instance_pool + node_type_id: i3.xlarge + min_idle_instances: 2 + max_capacity: 10 + idle_instance_autotermination_minutes: 60 + ``` + "$fields": + "disk_spec": + "$fields": + "disk_iops": + "description": |- + The number of IOPS to provision for each attached disk. + "disk_throughput": + "description": |- + The disk throughput to provision for each attached disk, in MB per second. + "lifecycle": + "description": |- + Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. + "permissions": + "description": |- + The permissions to apply to this resource. + "markdown_description": |- + A Sequence of permissions to apply to this resource, where each item grants a permission `level` to a single `user_name`, `group_name`, or `service_principal_name`. A principal cannot be set in both a resource's `permissions` and the top-level `permissions` mapping. + + See [\_](/dev-tools/bundles/settings.md#permissions) and [\_](/dev-tools/bundles/permissions.md). + "$fields": + "group_name": + "description": |- + The name of the group granted the permission level. + "level": + "description": |- + The permission level to apply. The allowed levels depend on the resource type. + "service_principal_name": + "description": |- + The name of the service principal granted the permission level. + "user_name": + "description": |- + The name of the user granted the permission level. "job_runs": "description": |- The job run definitions for the bundle, where each key is the name of the job run. Each job run triggers a run of an existing job as part of bundle deployment. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 2fe8eecb720..dde66a9ad0b 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -879,6 +879,131 @@ } ] }, + "resources.InstancePool": { + "oneOf": [ + { + "type": "object", + "properties": { + "aws_attributes": { + "description": "Attributes related to instance pools running on Amazon Web Services.\nIf not specified at pool creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.InstancePoolAwsAttributes" + }, + "azure_attributes": { + "description": "Attributes related to instance pools running on Azure.\nIf not specified at pool creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.InstancePoolAzureAttributes" + }, + "custom_tags": { + "description": "Additional tags for pool resources. Databricks will tag all pool resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags", + "$ref": "#/$defs/map/string" + }, + "disk_spec": { + "description": "Defines the specification of the disks that will be attached to all spark containers.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DiskSpec" + }, + "enable_elastic_disk": { + "description": "Autoscaling Local Storage: when enabled, this instances in this pool will dynamically acquire\nadditional disk space when its Spark workers are running low on disk space. In AWS, this\nfeature requires specific AWS permissions to function correctly - refer to the User Guide for\nmore details.", + "$ref": "#/$defs/bool" + }, + "gcp_attributes": { + "description": "Attributes related to instance pools running on Google Cloud Platform.\nIf not specified at pool creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.InstancePoolGcpAttributes" + }, + "idle_instance_autotermination_minutes": { + "description": "Automatically terminates the extra instances in the pool cache after they are inactive for this\ntime in minutes if min_idle_instances requirement is already met. If not set, the extra pool\ninstances will be automatically terminated after a default timeout. If specified, the\nthreshold must be between 0 and 10000 minutes.\nUsers can also set this value to 0 to instantly remove idle instances from the cache if\nmin cache size could still hold.", + "$ref": "#/$defs/int" + }, + "instance_pool_name": { + "description": "Pool name requested by the user. Pool name must be unique. Length must be between 1 and 100\ncharacters.", + "$ref": "#/$defs/string" + }, + "lifecycle": { + "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "max_capacity": { + "description": "Maximum number of outstanding instances to keep in the pool, including both instances used by\nclusters and idle instances. Clusters that require further instance provisioning will fail during\nupsize requests.", + "$ref": "#/$defs/int" + }, + "min_idle_instances": { + "description": "Minimum number of idle instances to keep in the instance pool", + "$ref": "#/$defs/int" + }, + "node_type_flexibility": { + "description": "Flexible node type configuration for the pool.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.NodeTypeFlexibility" + }, + "node_type_id": { + "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the [clusters/listNodeTypes](https://docs.databricks.com/api/workspace/clusters/listnodetypes) API call.", + "$ref": "#/$defs/string" + }, + "permissions": { + "description": "The permissions to apply to this resource.", + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.InstancePoolPermission", + "markdownDescription": "A Sequence of permissions to apply to this resource, where each item grants a permission `level` to a single `user_name`, `group_name`, or `service_principal_name`. A principal cannot be set in both a resource's `permissions` and the top-level `permissions` mapping.\n\nSee [permissions](https://docs.databricks.com/dev-tools/bundles/settings.html#permissions) and [link](https://docs.databricks.com/dev-tools/bundles/permissions.html)." + }, + "preloaded_docker_images": { + "description": "Custom Docker Image BYOC", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.DockerImage" + }, + "preloaded_spark_versions": { + "description": "A list containing at most one preloaded Spark image version for the pool. Pool-backed clusters started\nwith the preloaded Spark version will start faster. A list of available Spark versions\ncan be retrieved by using the [clusters/sparkVersions](https://docs.databricks.com/api/workspace/clusters/sparkversions) API call.", + "$ref": "#/$defs/slice/string" + }, + "remote_disk_throughput": { + "description": "If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only supported for GCP HYPERDISK_BALANCED types.", + "$ref": "#/$defs/int" + }, + "total_initial_remote_disk_size": { + "description": "If set, what the total initial volume size (in GB) of the remote disks should be. Currently only supported for GCP HYPERDISK_BALANCED types.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false, + "required": [ + "instance_pool_name", + "node_type_id" + ], + "markdownDescription": "The instance pool resource defines an [instance pool](https://docs.databricks.com/api/workspace/instancepools/create), a set of idle, ready-to-use cloud instances that reduce cluster start and auto-scaling times." + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.InstancePoolPermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "description": "The name of the group granted the permission level.", + "$ref": "#/$defs/string" + }, + "level": { + "description": "The permission level to apply. The allowed levels depend on the resource type.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.InstancePoolPermissionLevel" + }, + "service_principal_name": { + "description": "The name of the service principal granted the permission level.", + "$ref": "#/$defs/string" + }, + "user_name": { + "description": "The name of the user granted the permission level.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "oneOf": [ { @@ -3096,6 +3221,11 @@ "genie_spaces": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.GenieSpace" }, + "instance_pools": { + "description": "The instance pool definitions for the bundle, where each key is the name of the instance pool.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.InstancePool", + "markdownDescription": "The instance pool definitions for the bundle, where each key is the name of the instance pool. See [instance_pools](https://docs.databricks.com/dev-tools/bundles/resources.html#instance_pools)." + }, "job_runs": { "description": "The job run definitions for the bundle, where each key is the name of the job run. Each job run triggers a run of an existing job as part of bundle deployment.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.JobRun" @@ -5391,6 +5521,96 @@ } ] }, + "compute.DiskSpec": { + "oneOf": [ + { + "type": "object", + "description": "Describes the disks that are launched for each instance in the spark cluster.\nFor example, if the cluster has 3 instances, each instance is configured to launch\n2 disks, 100 GiB each, then Databricks will launch a total of 6 disks,\n100 GiB each, for this cluster.", + "properties": { + "disk_count": { + "description": "The number of disks launched for each instance:\n- This feature is only enabled for supported node types.\n- Users can choose up to the limit of the disks supported by the node type.\n- For node types with no OS disk, at least one disk must be specified;\notherwise, cluster creation will fail.\n\nIf disks are attached, Databricks will configure Spark to use only the disks for\nscratch storage, because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no disks are attached, Databricks will configure Spark to use\ninstance store disks.\n\nNote: If disks are specified, then the Spark configuration\n`spark.local.dir` will be overridden.\n\nDisks will be mounted at:\n- For AWS: `/ebs0`, `/ebs1`, and etc.\n- For Azure: `/remote_volume0`, `/remote_volume1`, and etc.", + "$ref": "#/$defs/int" + }, + "disk_iops": { + "description": "The number of IOPS to provision for each attached disk.", + "$ref": "#/$defs/int" + }, + "disk_size": { + "description": "The size of each disk (in GiB) launched for each instance.\nValues must fall into the supported range for a particular instance type.\n\nFor AWS:\n- General Purpose SSD: 100 - 4096 GiB\n- Throughput Optimized HDD: 500 - 4096 GiB\n\nFor Azure:\n- Premium LRS (SSD): 1 - 1023 GiB\n- Standard LRS (HDD): 1- 1023 GiB", + "$ref": "#/$defs/int" + }, + "disk_throughput": { + "description": "The disk throughput to provision for each attached disk, in MB per second.", + "$ref": "#/$defs/int" + }, + "disk_type": { + "description": "The type of disks that will be launched with this cluster.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DiskType" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.DiskType": { + "oneOf": [ + { + "type": "object", + "description": "Describes the disk type.", + "properties": { + "azure_disk_volume_type": { + "description": "All Azure Disk types that Databricks supports.\nSee https://docs.microsoft.com/en-us/azure/storage/storage-about-disks-and-vhds-linux#types-of-disks", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DiskTypeAzureDiskVolumeType" + }, + "ebs_volume_type": { + "description": "All EBS volume types that Databricks supports.\nSee https://aws.amazon.com/ebs/details/ for details.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DiskTypeEbsVolumeType" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.DiskTypeAzureDiskVolumeType": { + "oneOf": [ + { + "type": "string", + "description": "All Azure Disk types that Databricks supports.\nSee https://docs.microsoft.com/en-us/azure/storage/storage-about-disks-and-vhds-linux#types-of-disks", + "enum": [ + "PREMIUM_LRS", + "STANDARD_LRS" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.DiskTypeEbsVolumeType": { + "oneOf": [ + { + "type": "string", + "description": "All EBS volume types that Databricks supports.\nSee https://aws.amazon.com/ebs/details/ for details.", + "enum": [ + "GENERAL_PURPOSE_SSD", + "THROUGHPUT_OPTIMIZED_HDD" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "compute.DockerBasicAuth": { "oneOf": [ { @@ -5643,6 +5863,135 @@ } ] }, + "compute.InstancePoolAwsAttributes": { + "oneOf": [ + { + "type": "object", + "description": "Attributes set during instance pool creation which are related to Amazon Web Services.", + "properties": { + "availability": { + "description": "Availability type used for the spot nodes.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.InstancePoolAwsAttributesAvailability" + }, + "instance_profile_arn": { + "description": "[Beta] All AWS instances belonging to the instance pool will have this instance profile. If omitted, instances\nwill initially be launched with the workspace's default instance profile. If defined, clusters that use the\npool will inherit the instance profile, and must not specify their own instance profile on cluster creation or\nupdate. If the pool does not specify an instance profile, clusters using the pool may specify any instance profile.\nThe instance profile must have previously been added to the Databricks environment by an account administrator.\n\nThis feature may only be available to certain customer plans.", + "$ref": "#/$defs/string" + }, + "spot_bid_price_percent": { + "description": "Calculates the bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.", + "$ref": "#/$defs/int" + }, + "zone_id": { + "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.InstancePoolAwsAttributesAvailability": { + "oneOf": [ + { + "type": "string", + "description": "The set of AWS availability types supported when setting up nodes for a cluster.", + "enum": [ + "SPOT", + "ON_DEMAND" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.InstancePoolAzureAttributes": { + "oneOf": [ + { + "type": "object", + "description": "Attributes set during instance pool creation which are related to Azure.", + "properties": { + "availability": { + "description": "Availability type used for the spot nodes.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.InstancePoolAzureAttributesAvailability" + }, + "spot_bid_max_price": { + "description": "With variable pricing, you have option to set a max price, in US dollars (USD)\nFor example, the value 2 would be a max price of $2.00 USD per hour.\nIf you set the max price to be -1, the VM won't be evicted based on price.\nThe price for the VM will be the current price for spot or the price for a standard VM,\nwhich ever is less, as long as there is capacity and quota available.", + "$ref": "#/$defs/float64" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.InstancePoolAzureAttributesAvailability": { + "oneOf": [ + { + "type": "string", + "description": "The set of Azure availability types supported when setting up nodes for a cluster.", + "enum": [ + "SPOT_AZURE", + "ON_DEMAND_AZURE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.InstancePoolGcpAttributes": { + "oneOf": [ + { + "type": "object", + "description": "Attributes set during instance pool creation which are related to GCP.", + "properties": { + "gcp_availability": { + "description": "This field determines whether the instance pool will contain preemptible\nVMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcpAvailability" + }, + "local_ssd_count": { + "description": "If provided, each node in the instance pool will have this number of local SSDs attached.\nEach local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds)\nfor the supported number of local SSDs for each instance type.", + "$ref": "#/$defs/int" + }, + "zone_id": { + "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west1-a\". The provided availability\nzone must be in the same region as the Databricks workspace. For example, \"us-west1-a\"\nis not a valid zone id if the Databricks workspace resides in the \"us-east1\" region.\nThis is an optional field at instance pool creation, and if not specified, a default zone will be used.\n\nThis field can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones (e.g. \"us-west1-a\").\n\nIf empty, Databricks picks an availability zone to schedule the cluster on.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.InstancePoolPermissionLevel": { + "oneOf": [ + { + "type": "string", + "description": "Permission level", + "enum": [ + "CAN_MANAGE", + "CAN_ATTACH_TO" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "compute.Kind": { "oneOf": [ { @@ -13768,6 +14117,20 @@ } ] }, + "resources.InstancePool": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.InstancePool" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "oneOf": [ { @@ -14228,6 +14591,20 @@ } ] }, + "resources.InstancePoolPermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.InstancePoolPermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.JobPermission": { "oneOf": [ { @@ -14457,6 +14834,20 @@ } ] }, + "compute.DockerImage": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DockerImage" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "compute.InitScriptInfo": { "oneOf": [ { diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index df9499c7e91..8d6e8ee0dd3 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -185,6 +185,7 @@ type FakeWorkspace struct { ModelRegistryModels map[string]ml.Model ModelRegistryModelIDs map[string]string // model name -> numeric ID Clusters map[string]compute.ClusterDetails + InstancePools map[string]compute.GetInstancePool Catalogs map[string]catalog.CatalogInfo ExternalLocations map[string]catalog.ExternalLocationInfo RegisteredModels map[string]catalog.RegisteredModelInfo @@ -393,6 +394,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { SingleUserName: TestUser.UserName, }, }, + InstancePools: map[string]compute.GetInstancePool{}, } } diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 338eb7e3e50..5d7c6fc5e54 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -55,6 +55,12 @@ func AddDefaultHandlers(server *Server) { }, } }) + server.Handle("POST", "/api/2.0/instance-pools/create", func(req Request) any { return req.Workspace.InstancePoolsCreate(req) }) + server.Handle("POST", "/api/2.0/instance-pools/edit", func(req Request) any { return req.Workspace.InstancePoolsEdit(req) }) + server.Handle("POST", "/api/2.0/instance-pools/delete", func(req Request) any { return req.Workspace.InstancePoolsDelete(req) }) + server.Handle("GET", "/api/2.0/instance-pools/get", func(req Request) any { + return req.Workspace.InstancePoolsGet(req, req.URL.Query().Get("instance_pool_id")) + }) server.Handle("GET", "/api/2.1/clusters/list", func(req Request) any { return compute.ListClustersResponse{ diff --git a/libs/testserver/instance_pools.go b/libs/testserver/instance_pools.go new file mode 100644 index 00000000000..f6279fbfc3f --- /dev/null +++ b/libs/testserver/instance_pools.go @@ -0,0 +1,81 @@ +package testserver + +import ( + "encoding/json" + "fmt" + + "github.com/databricks/databricks-sdk-go/service/compute" +) + +func (s *FakeWorkspace) InstancePoolsCreate(req Request) any { + // Unmarshal into the stored (GET) type directly: CreateInstancePool and + // GetInstancePool share JSON field names, so every config field is carried over. + var pool compute.GetInstancePool + if err := json.Unmarshal(req.Body, &pool); err != nil { + return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + id := nextUUID() + pool.InstancePoolId = id + s.InstancePools[id] = pool + + return Response{Body: compute.CreateInstancePoolResponse{InstancePoolId: id}} +} + +func (s *FakeWorkspace) InstancePoolsGet(req Request, instancePoolId string) any { + defer s.LockUnlock()() + + pool, ok := s.InstancePools[instancePoolId] + if !ok { + return Response{StatusCode: 404} + } + + return Response{Body: pool} +} + +func (s *FakeWorkspace) InstancePoolsEdit(req Request) any { + var request compute.EditInstancePool + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + pool, ok := s.InstancePools[request.InstancePoolId] + if !ok { + return Response{StatusCode: 404} + } + + // Edit only accepts a subset of fields; overwrite those and keep the rest + // (immutable fields like aws_attributes, disk_spec) as stored. + pool.InstancePoolName = request.InstancePoolName + pool.NodeTypeId = request.NodeTypeId + pool.MinIdleInstances = request.MinIdleInstances + pool.MaxCapacity = request.MaxCapacity + pool.IdleInstanceAutoterminationMinutes = request.IdleInstanceAutoterminationMinutes + pool.CustomTags = request.CustomTags + pool.RemoteDiskThroughput = request.RemoteDiskThroughput + pool.TotalInitialRemoteDiskSize = request.TotalInitialRemoteDiskSize + s.InstancePools[request.InstancePoolId] = pool + + return Response{} +} + +func (s *FakeWorkspace) InstancePoolsDelete(req Request) any { + var request compute.DeleteInstancePool + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + if _, ok := s.InstancePools[request.InstancePoolId]; !ok { + return Response{StatusCode: 404} + } + + delete(s.InstancePools, request.InstancePoolId) + + return Response{} +} diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index be19bd31e75..a1bf973801f 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -33,6 +33,7 @@ var resourceURLPatterns = map[string]string{ "vector_search_indexes": "explore/data/%s", "volumes": "explore/data/volumes/%s", "warehouses": "sql/warehouses/%s", + "instance_pools": "compute/instance-pools/%s", } // resourceAliases lets callers use bundle-config plural names as synonyms for From 1f79351f397d7a2fe6909eafad5950f0900464a8 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 16 Jul 2026 07:54:00 +0000 Subject: [PATCH 2/3] Wire instance_pools into resource-type enumerations and regenerate validation Fixes CI failures from adding the instance_pools resource: update the workspace-open supported-type lists, skip instance_pools (direct-only) in the Terraform lifecycle test, extend the statemgmt StateToBundle tests for full resource coverage, and regenerate the validation enum/required fields. --- bundle/deploy/terraform/lifecycle_test.go | 1 + .../validation/generated/enum_fields.go | 7 ++++ .../validation/generated/required_fields.go | 3 ++ bundle/statemgmt/state_load_test.go | 35 +++++++++++++++++++ cmd/experimental/workspace_open_test.go | 5 +-- 5 files changed, 49 insertions(+), 2 deletions(-) diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index 248a66c21a9..9fb59329ffd 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -18,6 +18,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { "catalogs", "external_locations", "genie_spaces", + "instance_pools", "job_runs", "vector_search_endpoints", "vector_search_indexes", diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index f223ab7ef52..975369e8190 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -64,6 +64,13 @@ var EnumFields = map[string][]string{ "resources.genie_spaces.*.permissions[*].level": {"CAN_ATTACH_TO", "CAN_BIND", "CAN_CREATE", "CAN_CREATE_APP", "CAN_EDIT", "CAN_EDIT_METADATA", "CAN_MANAGE", "CAN_MANAGE_PRODUCTION_VERSIONS", "CAN_MANAGE_RUN", "CAN_MANAGE_STAGING_VERSIONS", "CAN_MONITOR", "CAN_MONITOR_ONLY", "CAN_QUERY", "CAN_READ", "CAN_RESTART", "CAN_RUN", "CAN_USE", "CAN_VIEW", "CAN_VIEW_METADATA", "IS_OWNER"}, + "resources.instance_pools.*.aws_attributes.availability": {"ON_DEMAND", "SPOT"}, + "resources.instance_pools.*.azure_attributes.availability": {"ON_DEMAND_AZURE", "SPOT_AZURE"}, + "resources.instance_pools.*.disk_spec.disk_type.azure_disk_volume_type": {"PREMIUM_LRS", "STANDARD_LRS"}, + "resources.instance_pools.*.disk_spec.disk_type.ebs_volume_type": {"GENERAL_PURPOSE_SSD", "THROUGHPUT_OPTIMIZED_HDD"}, + "resources.instance_pools.*.gcp_attributes.gcp_availability": {"ON_DEMAND_GCP", "PREEMPTIBLE_GCP", "PREEMPTIBLE_WITH_FALLBACK_GCP"}, + "resources.instance_pools.*.permissions[*].level": {"CAN_ATTACH_TO", "CAN_MANAGE"}, + "resources.job_runs.*.performance_target": {"PERFORMANCE_OPTIMIZED", "STANDARD"}, "resources.jobs.*.continuous.pause_status": {"PAUSED", "UNPAUSED"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 1eb3d7443c0..9a6dc75b68a 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -66,6 +66,9 @@ var RequiredFields = map[string][]string{ "resources.genie_spaces.*.permissions[*]": {"level"}, + "resources.instance_pools.*": {"instance_pool_name", "node_type_id"}, + "resources.instance_pools.*.permissions[*]": {"level"}, + "resources.job_runs.*": {"job_id"}, "resources.job_runs.*.queue": {"enabled"}, diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 313348f30d2..1ec6dc82999 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -57,6 +57,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.postgres_synced_tables.test_postgres_synced_table": {ID: "synced_tables/main.public.test_synced_table"}, "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, "resources.vector_search_indexes.test_vector_search_index": {ID: "vs-index-1"}, + "resources.instance_pools.test_instance_pool": {ID: "1"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -149,6 +150,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "vs-index-1", config.Resources.VectorSearchIndexes["test_vector_search_index"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.VectorSearchIndexes["test_vector_search_index"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.InstancePools["test_instance_pool"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.InstancePools["test_instance_pool"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -378,6 +382,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + InstancePools: map[string]*resources.InstancePool{ + "test_instance_pool": { + CreateInstancePool: compute.CreateInstancePool{ + InstancePoolName: "test_instance_pool", + }, + }, + }, }, } @@ -477,6 +488,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Empty(t, config.Resources.VectorSearchIndexes["test_vector_search_index"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchIndexes["test_vector_search_index"].ModifiedStatus) + assert.Empty(t, config.Resources.InstancePools["test_instance_pool"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.InstancePools["test_instance_pool"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -856,6 +870,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + InstancePools: map[string]*resources.InstancePool{ + "test_instance_pool": { + CreateInstancePool: compute.CreateInstancePool{ + InstancePoolName: "test_instance_pool", + }, + }, + "test_instance_pool_new": { + CreateInstancePool: compute.CreateInstancePool{ + InstancePoolName: "test_instance_pool_new", + }, + }, + }, }, } state := ExportedResourcesMap{ @@ -913,6 +939,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.vector_search_endpoints.test_vector_search_endpoint_old": {ID: "vs-endpoint-old"}, "resources.vector_search_indexes.test_vector_search_index": {ID: "vs-index-1"}, "resources.vector_search_indexes.test_vector_search_index_old": {ID: "vs-index-old"}, + "resources.instance_pools.test_instance_pool": {ID: "1"}, + "resources.instance_pools.test_instance_pool_old": {ID: "2"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -1108,6 +1136,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Empty(t, config.Resources.VectorSearchIndexes["test_vector_search_index_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchIndexes["test_vector_search_index_new"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.InstancePools["test_instance_pool"].ID) + assert.Empty(t, config.Resources.InstancePools["test_instance_pool"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.InstancePools["test_instance_pool_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.InstancePools["test_instance_pool_old"].ModifiedStatus) + assert.Empty(t, config.Resources.InstancePools["test_instance_pool_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.InstancePools["test_instance_pool_new"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } diff --git a/cmd/experimental/workspace_open_test.go b/cmd/experimental/workspace_open_test.go index 9589b48a126..861dab1eda4 100644 --- a/cmd/experimental/workspace_open_test.go +++ b/cmd/experimental/workspace_open_test.go @@ -67,7 +67,7 @@ func TestBuildWorkspaceURLFragmentBasedResources(t *testing.T) { func TestBuildWorkspaceURLUnknownResourceType(t *testing.T) { _, err := workspaceurls.BuildResourceURL("https://myworkspace.databricks.com", "unknown", "123", "") assert.ErrorContains(t, err, "unknown resource type \"unknown\"") - assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") + assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") } func TestBuildWorkspaceURLHostWithTrailingSlash(t *testing.T) { @@ -116,6 +116,7 @@ func TestWorkspaceOpenCommandCompletion(t *testing.T) { "database_instances", "experiments", "genie_spaces", + "instance_pools", "jobs", "model_serving_endpoints", "models", @@ -146,7 +147,7 @@ func TestWorkspaceOpenCommandCompletionSecondArg(t *testing.T) { func TestWorkspaceOpenCommandHelpText(t *testing.T) { cmd := newWorkspaceOpenCommand() - assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") + assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") assert.Contains(t, cmd.Long, "databricks experimental open jobs 123456789") assert.Contains(t, cmd.Long, "databricks experimental open notebooks /Users/user@example.com/my-notebook") assert.Contains(t, cmd.Long, "databricks experimental open registered_models catalog.schema.my_model") From e17841bd7f4efed65479a4643d78fc4866954d22 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 16 Jul 2026 08:17:12 +0000 Subject: [PATCH 3/3] acc: add instance_pools invariant config and refresh golden outputs Adds an instance_pools invariant test config (with permissions) so TestInvariantConfigsCoverage passes, wires it into the invariant matrix, and excludes it from the migrate variant since instance_pools is direct-only and the migrate seed deploy uses Terraform. Also refreshes the experimental/open acceptance golden output to include instance_pools. --- .../bundle/invariant/configs/instance_pool.yml.tmpl | 11 +++++++++++ .../bundle/invariant/continue_293/out.test.toml | 1 + acceptance/bundle/invariant/migrate/test.toml | 2 ++ acceptance/bundle/invariant/no_drift/out.test.toml | 1 + acceptance/bundle/invariant/test.toml | 1 + acceptance/experimental/open/output.txt | 3 ++- 6 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/invariant/configs/instance_pool.yml.tmpl diff --git a/acceptance/bundle/invariant/configs/instance_pool.yml.tmpl b/acceptance/bundle/invariant/configs/instance_pool.yml.tmpl new file mode 100644 index 00000000000..4a8d6bc5afd --- /dev/null +++ b/acceptance/bundle/invariant/configs/instance_pool.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + instance_pools: + foo: + instance_pool_name: test-instance-pool-$UNIQUE_NAME + node_type_id: $NODE_TYPE_ID + permissions: + - level: CAN_ATTACH_TO + group_name: users diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 8fd536c2b85..2f7187a8a9a 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -18,6 +18,7 @@ EnvMatrix.INPUT_CONFIG = [ "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", + "instance_pool.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_run_job_ref.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index af25cbb62dd..a91f990df3b 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -11,6 +11,8 @@ EnvMatrixExclude.no_catalog = ["INPUT_CONFIG=catalog.yml.tmpl"] EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmpl"] # Genie spaces are direct-only too; the terraform deploy that seeds the migration fails for them. EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] +# Instance pools are direct-only; the terraform deploy that seeds the migration fails for them. +EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"] # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level}) # don't work in terraform mode: the terraform interpolator converts the path to diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 6c269426855..0c8462419cc 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -19,6 +19,7 @@ EnvMatrix.INPUT_CONFIG = [ "experiment.yml.tmpl", "external_location.yml.tmpl", "genie_space.yml.tmpl", + "instance_pool.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index dd1b0f9104a..cf7b71f5404 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -42,6 +42,7 @@ EnvMatrix.INPUT_CONFIG = [ "experiment.yml.tmpl", "external_location.yml.tmpl", "genie_space.yml.tmpl", + "instance_pool.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", diff --git a/acceptance/experimental/open/output.txt b/acceptance/experimental/open/output.txt index f634d630fc8..08d3a757c83 100644 --- a/acceptance/experimental/open/output.txt +++ b/acceptance/experimental/open/output.txt @@ -9,7 +9,7 @@ === unknown resource type >>> [CLI] experimental open --url unknown 123 -Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses +Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses === test auto-completion handler >>> [CLI] __complete experimental open , @@ -22,6 +22,7 @@ database_catalogs database_instances experiments genie_spaces +instance_pools jobs model_serving_endpoints models