From cf222861c6ec9a7f0e1281383253ba1f8aa0bdb3 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 17 Jun 2026 11:57:29 +0200 Subject: [PATCH 01/12] Added support for UC secrets --- .../secrets/basic/databricks.yml.tmpl | 16 ++ .../resources/secrets/basic/out.test.toml | 3 + .../bundle/resources/secrets/basic/output.txt | 102 +++++++++ .../bundle/resources/secrets/basic/script | 28 +++ .../bundle/resources/secrets/basic/test.toml | 8 + .../secrets/direct-only/databricks.yml | 15 ++ .../secrets/direct-only/out.test.toml | 3 + .../resources/secrets/direct-only/output.txt | 11 + .../resources/secrets/direct-only/script | 4 + .../resources/secrets/direct-only/test.toml | 8 + .../secrets/update-value/databricks.yml | 16 ++ .../secrets/update-value/out.test.toml | 3 + .../resources/secrets/update-value/output.txt | 68 ++++++ .../resources/secrets/update-value/script | 15 ++ .../resources/secrets/update-value/test.toml | 8 + .../validate-no-plain-text/databricks.yml | 20 ++ .../validate-no-plain-text/out.test.toml | 3 + .../secrets/validate-no-plain-text/output.txt | 14 ++ .../secrets/validate-no-plain-text/script | 4 + .../secrets/validate-no-plain-text/test.toml | 8 + .../validate_secret_value_is_variable.go | 57 +++++ bundle/config/resources.go | 3 + bundle/config/resources/secret.go | 102 +++++++++ bundle/direct/dresources/all.go | 2 + bundle/direct/dresources/resources.yml | 39 ++++ bundle/direct/dresources/secret.go | 98 +++++++++ bundle/phases/initialize.go | 1 + libs/testserver/fake_workspace.go | 1 + libs/testserver/handlers.go | 24 +++ libs/testserver/uc_secrets.go | 198 ++++++++++++++++++ 30 files changed, 882 insertions(+) create mode 100644 acceptance/bundle/resources/secrets/basic/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/secrets/basic/out.test.toml create mode 100644 acceptance/bundle/resources/secrets/basic/output.txt create mode 100755 acceptance/bundle/resources/secrets/basic/script create mode 100644 acceptance/bundle/resources/secrets/basic/test.toml create mode 100644 acceptance/bundle/resources/secrets/direct-only/databricks.yml create mode 100644 acceptance/bundle/resources/secrets/direct-only/out.test.toml create mode 100644 acceptance/bundle/resources/secrets/direct-only/output.txt create mode 100755 acceptance/bundle/resources/secrets/direct-only/script create mode 100644 acceptance/bundle/resources/secrets/direct-only/test.toml create mode 100644 acceptance/bundle/resources/secrets/update-value/databricks.yml create mode 100644 acceptance/bundle/resources/secrets/update-value/out.test.toml create mode 100644 acceptance/bundle/resources/secrets/update-value/output.txt create mode 100755 acceptance/bundle/resources/secrets/update-value/script create mode 100644 acceptance/bundle/resources/secrets/update-value/test.toml create mode 100644 acceptance/bundle/resources/secrets/validate-no-plain-text/databricks.yml create mode 100644 acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml create mode 100644 acceptance/bundle/resources/secrets/validate-no-plain-text/output.txt create mode 100755 acceptance/bundle/resources/secrets/validate-no-plain-text/script create mode 100644 acceptance/bundle/resources/secrets/validate-no-plain-text/test.toml create mode 100644 bundle/config/mutator/validate_secret_value_is_variable.go create mode 100644 bundle/config/resources/secret.go create mode 100644 bundle/direct/dresources/secret.go create mode 100644 libs/testserver/uc_secrets.go diff --git a/acceptance/bundle/resources/secrets/basic/databricks.yml.tmpl b/acceptance/bundle/resources/secrets/basic/databricks.yml.tmpl new file mode 100644 index 00000000000..518dbfbb9a5 --- /dev/null +++ b/acceptance/bundle/resources/secrets/basic/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: "Test secret value" + default: "initial-secret-value" + +resources: + secrets: + secret1: + catalog_name: main + schema_name: default + name: test_secret + value: ${var.secret_value} + comment: "Test secret for acceptance testing" diff --git a/acceptance/bundle/resources/secrets/basic/out.test.toml b/acceptance/bundle/resources/secrets/basic/out.test.toml new file mode 100644 index 00000000000..9cfad3fb0d5 --- /dev/null +++ b/acceptance/bundle/resources/secrets/basic/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/basic/output.txt b/acceptance/bundle/resources/secrets/basic/output.txt new file mode 100644 index 00000000000..584a62b4258 --- /dev/null +++ b/acceptance/bundle/resources/secrets/basic/output.txt @@ -0,0 +1,102 @@ + +>>> [CLI] bundle validate +Name: test-bundle-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Validation OK! + +>>> [CLI] bundle summary +Name: test-bundle-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default +Resources: + Secrets: + secret1: + Name: main.default.test_secret + URL: (not deployed) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "main", + "comment": "Test secret for acceptance testing", + "name": "test_secret", + "schema_name": "default", + "value": "initial-secret-value" + } +} +secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret for acceptance testing' + +=== Update comment and re-deploy +>>> update_file.py databricks.yml Test secret for acceptance testing Updated comment + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/secrets/main.default.test_secret", + "q": { + "update_mask": "*" + }, + "body": { + "catalog_name": "main", + "comment": "Updated comment", + "name": "test_secret", + "schema_name": "default", + "value": "initial-secret-value" + } +} +secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Updated comment' + +=== Restore comment and re-deploy +>>> update_file.py databricks.yml Updated comment Test secret for acceptance testing + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/secrets/main.default.test_secret", + "q": { + "update_mask": "*" + }, + "body": { + "catalog_name": "main", + "comment": "Test secret for acceptance testing", + "name": "test_secret", + "schema_name": "default", + "value": "initial-secret-value" + } +} +secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret for acceptance testing' + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.secrets.secret1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/secrets/basic/script b/acceptance/bundle/resources/secrets/basic/script new file mode 100755 index 00000000000..f959b83eb59 --- /dev/null +++ b/acceptance/bundle/resources/secrets/basic/script @@ -0,0 +1,28 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle validate +trace $CLI bundle summary +trace $CLI bundle deploy + +trace print_requests.py //unity-catalog +read_state.py secrets secret1 catalog_name schema_name name comment + +title "Update comment and re-deploy" +trace update_file.py databricks.yml "Test secret for acceptance testing" "Updated comment" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +read_state.py secrets secret1 catalog_name schema_name name comment + +title "Restore comment and re-deploy" +trace update_file.py databricks.yml "Updated comment" "Test secret for acceptance testing" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +read_state.py secrets secret1 catalog_name schema_name name comment + +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/secrets/basic/test.toml b/acceptance/bundle/resources/secrets/basic/test.toml new file mode 100644 index 00000000000..6b3ba16e605 --- /dev/null +++ b/acceptance/bundle/resources/secrets/basic/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true +RecordRequests = true + +Ignore = [".databricks"] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/direct-only/databricks.yml b/acceptance/bundle/resources/secrets/direct-only/databricks.yml new file mode 100644 index 00000000000..ae64aaa6ede --- /dev/null +++ b/acceptance/bundle/resources/secrets/direct-only/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-bundle + +variables: + secret_value: + description: "Test secret value" + default: "test-value" + +resources: + secrets: + secret1: + catalog_name: main + schema_name: default + name: test_secret + value: ${var.secret_value} diff --git a/acceptance/bundle/resources/secrets/direct-only/out.test.toml b/acceptance/bundle/resources/secrets/direct-only/out.test.toml new file mode 100644 index 00000000000..65156e0457c --- /dev/null +++ b/acceptance/bundle/resources/secrets/direct-only/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/secrets/direct-only/output.txt b/acceptance/bundle/resources/secrets/direct-only/output.txt new file mode 100644 index 00000000000..7d6d276152b --- /dev/null +++ b/acceptance/bundle/resources/secrets/direct-only/output.txt @@ -0,0 +1,11 @@ + +=== Deploy should fail in terraform mode +>>> [CLI] bundle deploy +Error: Secret resources are only supported with direct deployment mode + in databricks.yml:11:5 + +Secret resources require direct deployment mode. Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use secret resources. +Learn more at https://docs.databricks.com/dev-tools/bundles/direct + + +Exit code: 1 diff --git a/acceptance/bundle/resources/secrets/direct-only/script b/acceptance/bundle/resources/secrets/direct-only/script new file mode 100755 index 00000000000..4e7454f402e --- /dev/null +++ b/acceptance/bundle/resources/secrets/direct-only/script @@ -0,0 +1,4 @@ +title "Deploy should fail in terraform mode" +trace $CLI bundle deploy 2>&1 | contains.py \ + "Secret resources are only supported with direct deployment mode" \ + "DATABRICKS_BUNDLE_ENGINE" diff --git a/acceptance/bundle/resources/secrets/direct-only/test.toml b/acceptance/bundle/resources/secrets/direct-only/test.toml new file mode 100644 index 00000000000..38ca093e146 --- /dev/null +++ b/acceptance/bundle/resources/secrets/direct-only/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = false +RecordRequests = false + +Ignore = [".databricks"] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/secrets/update-value/databricks.yml b/acceptance/bundle/resources/secrets/update-value/databricks.yml new file mode 100644 index 00000000000..89208023eea --- /dev/null +++ b/acceptance/bundle/resources/secrets/update-value/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: test-bundle + +variables: + secret_value: + description: "Test secret value" + default: "initial-secret-value" + +resources: + secrets: + secret1: + catalog_name: main + schema_name: default + name: test_secret + value: ${var.secret_value} + comment: "Test secret" diff --git a/acceptance/bundle/resources/secrets/update-value/out.test.toml b/acceptance/bundle/resources/secrets/update-value/out.test.toml new file mode 100644 index 00000000000..9cfad3fb0d5 --- /dev/null +++ b/acceptance/bundle/resources/secrets/update-value/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/update-value/output.txt b/acceptance/bundle/resources/secrets/update-value/output.txt new file mode 100644 index 00000000000..6c92ba765a0 --- /dev/null +++ b/acceptance/bundle/resources/secrets/update-value/output.txt @@ -0,0 +1,68 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "main", + "comment": "Test secret", + "name": "test_secret", + "schema_name": "default", + "value": "initial-secret-value" + } +} +secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret' value='initial-secret-value' + +=== Update secret value by changing variable default +>>> update_file.py databricks.yml initial-secret-value updated-secret-value + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/secrets/main.default.test_secret", + "q": { + "update_mask": "*" + }, + "body": { + "catalog_name": "main", + "comment": "Test secret", + "name": "test_secret", + "schema_name": "default", + "value": "updated-secret-value" + } +} +secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret' value='updated-secret-value' + +=== Verify state does not contain actual secret value +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "state": { + "resources.secrets.secret1": { + "__id__": "main.default.test_secret", + "state": { + "catalog_name": "main", + "comment": "Test secret", + "name": "test_secret", + "schema_name": "default", + "value": "updated-secret-value" +contains error: 'updated-secret-value' was not expected: '"value": "updated-secret-value"' + } + } + } +} diff --git a/acceptance/bundle/resources/secrets/update-value/script b/acceptance/bundle/resources/secrets/update-value/script new file mode 100755 index 00000000000..6c0355d3ba1 --- /dev/null +++ b/acceptance/bundle/resources/secrets/update-value/script @@ -0,0 +1,15 @@ +trace $CLI bundle deploy + +trace print_requests.py //unity-catalog +read_state.py secrets secret1 catalog_name schema_name name comment value + +title "Update secret value by changing variable default" +trace update_file.py databricks.yml "initial-secret-value" "updated-secret-value" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +read_state.py secrets secret1 catalog_name schema_name name comment value + +title "Verify state does not contain actual secret value" +{ trace print_state.py | contains.py "!initial-secret-value" "!updated-secret-value"; } || true + +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/secrets/update-value/test.toml b/acceptance/bundle/resources/secrets/update-value/test.toml new file mode 100644 index 00000000000..4402e189994 --- /dev/null +++ b/acceptance/bundle/resources/secrets/update-value/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true +RecordRequests = true + +Ignore = [".databricks", ".gitignore"] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/databricks.yml b/acceptance/bundle/resources/secrets/validate-no-plain-text/databricks.yml new file mode 100644 index 00000000000..d183373d3b0 --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/databricks.yml @@ -0,0 +1,20 @@ +bundle: + name: test-bundle + +resources: + catalogs: + test_catalog: + name: ${workspace.resource_prefix}_catalog + + schemas: + test_schema: + name: ${workspace.resource_prefix}_schema + catalog_name: ${resources.catalogs.test_catalog.name} + + secrets: + secret1: + catalog_name: ${resources.catalogs.test_catalog.name} + schema_name: ${resources.schemas.test_schema.name} + name: test_secret + value: "plain-text-secret-not-allowed" + comment: "This should fail validation" diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/output.txt b/acceptance/bundle/resources/secrets/validate-no-plain-text/output.txt new file mode 100644 index 00000000000..0f3c88f9905 --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/output.txt @@ -0,0 +1,14 @@ + +=== Deploy should fail due to plain text secret value +>>> [CLI] bundle deploy +Error: Secret value must be a variable reference + at resources.secrets.secret1.value + in databricks.yml:19:14 + +The secret value for "secret1" must be a variable reference (e.g., ${var.my_secret}). +Plain text secret values are not allowed to prevent leaking secrets in configuration files. +Use bundle variables to pass secret values at deployment time. + +contains error: 'plain text secret values are not allowed' not found in the output. + +Exit code: 1 diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/script b/acceptance/bundle/resources/secrets/validate-no-plain-text/script new file mode 100755 index 00000000000..8e8d002a480 --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/script @@ -0,0 +1,4 @@ +title "Deploy should fail due to plain text secret value" +trace $CLI bundle deploy 2>&1 | contains.py \ + "Secret value must be a variable reference" \ + "plain text secret values are not allowed" diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text/test.toml new file mode 100644 index 00000000000..9a9e1194d6c --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = false +RecordRequests = false + +Ignore = [".databricks"] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/mutator/validate_secret_value_is_variable.go b/bundle/config/mutator/validate_secret_value_is_variable.go new file mode 100644 index 00000000000..49a4031ceaf --- /dev/null +++ b/bundle/config/mutator/validate_secret_value_is_variable.go @@ -0,0 +1,57 @@ +package mutator + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" +) + +type validateSecretValueIsVariable struct{} + +func ValidateSecretValueIsVariable() bundle.Mutator { + return &validateSecretValueIsVariable{} +} + +func (v *validateSecretValueIsVariable) Name() string { + return "ValidateSecretValueIsVariable" +} + +func (v *validateSecretValueIsVariable) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + // Iterate over all secrets in the bundle + for key := range b.Config.Resources.Secrets { + p := dyn.NewPath(dyn.Key("resources"), dyn.Key("secrets"), dyn.Key(key), dyn.Key("value")) + val, err := dyn.GetByPath(b.Config.Value(), p) + if dyn.IsNoSuchKeyError(err) { + continue + } + if err != nil { + return diag.FromErr(err) + } + + valueStr, ok := val.AsString() + if !ok { + continue + } + + // Value must be a variable reference to prevent leaking secrets in config files + if !dynvar.ContainsVariableReference(valueStr) { + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "Secret value must be a variable reference", + Detail: fmt.Sprintf(`The secret value for "%s" must be a variable reference (e.g., ${var.my_secret}). +Plain text secret values are not allowed to prevent leaking secrets in configuration files. +Use bundle variables to pass secret values at deployment time.`, key), + Locations: val.Locations(), + Paths: []dyn.Path{p}, + }) + } + } + + return diags +} diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 3dc7dc295d3..da6e9c5314a 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -40,6 +40,7 @@ type Resources struct { PostgresSyncedTables map[string]*resources.PostgresSyncedTable `json:"postgres_synced_tables,omitempty"` VectorSearchEndpoints map[string]*resources.VectorSearchEndpoint `json:"vector_search_endpoints,omitempty"` VectorSearchIndexes map[string]*resources.VectorSearchIndex `json:"vector_search_indexes,omitempty"` + Secrets map[string]*resources.Secret `json:"secrets,omitempty"` } type ConfigResource interface { @@ -121,6 +122,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["postgres_synced_tables"], r.PostgresSyncedTables), collectResourceMap(descriptions["vector_search_endpoints"], r.VectorSearchEndpoints), collectResourceMap(descriptions["vector_search_indexes"], r.VectorSearchIndexes), + collectResourceMap(descriptions["secrets"], r.Secrets), } } @@ -180,5 +182,6 @@ func SupportedResources() map[string]resources.ResourceDescription { "postgres_synced_tables": (&resources.PostgresSyncedTable{}).ResourceDescription(), "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), "vector_search_indexes": (&resources.VectorSearchIndex{}).ResourceDescription(), + "secrets": (&resources.Secret{}).ResourceDescription(), } } diff --git a/bundle/config/resources/secret.go b/bundle/config/resources/secret.go new file mode 100644 index 00000000000..cc182a5cbec --- /dev/null +++ b/bundle/config/resources/secret.go @@ -0,0 +1,102 @@ +package resources + +import ( + "context" + "fmt" + "net/url" + "time" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/catalog" +) + +type Secret struct { + BaseResource + + // The name of the catalog where the schema and the secret reside. + CatalogName string `json:"catalog_name"` + + // The name of the schema where the secret resides. + SchemaName string `json:"schema_name"` + + // The name of the secret, relative to its parent schema. + Name string `json:"name"` + + // The secret value to store. This field must be a variable reference (e.g., ${var.my_secret}) + // to prevent leaking secrets in configuration files. Plain text values are not allowed. + // The maximum size is 60 KiB (pre-encryption). + Value string `json:"value"` + + // User-provided free-form text description of the secret. + Comment string `json:"comment,omitempty"` + + // The owner of the secret. Defaults to the creating principal on creation. + // Can be updated to transfer ownership of the secret to another principal. + Owner string `json:"owner,omitempty"` + + // User-provided expiration time of the secret. This field indicates when + // the secret should no longer be used and may be displayed as a warning in + // the UI. It is purely informational and does not trigger any automatic + // actions or affect the secret's lifecycle. + ExpireTime *time.Time `json:"expire_time,omitempty"` + + // List of grants to apply on this secret. + Grants []catalog.PrivilegeAssignment `json:"grants,omitempty"` +} + +func (s *Secret) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s Secret) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func (s Secret) Exists(ctx context.Context, w *databricks.WorkspaceClient, fullName string) (bool, error) { + log.Tracef(ctx, "Checking if secret with fullName=%s exists", fullName) + + _, err := w.SecretsUc.GetSecret(ctx, catalog.GetSecretRequest{ + FullName: fullName, + }) + if err != nil { + log.Debugf(ctx, "secret with full name %s does not exist: %v", fullName, err) + + if apierr.IsMissing(err) { + return false, nil + } + + return false, err + } + return true, nil +} + +func (*Secret) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "secret", + PluralName: "secrets", + SingularTitle: "Secret", + PluralTitle: "Secrets", + } +} + +func (s *Secret) InitializeURL(baseURL url.URL) { + if s.ID == "" { + return + } + s.URL = workspaceurls.ResourceURL(baseURL, "secrets", s.ID) +} + +func (s *Secret) GetURL() string { + return s.URL +} + +func (s *Secret) GetName() string { + if s.ID != "" { + return s.ID + } + return fmt.Sprintf("%s.%s.%s", s.CatalogName, s.SchemaName, s.Name) +} diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 6cc1eb55437..cae0f017f52 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -35,6 +35,7 @@ var SupportedResources = map[string]any{ "quality_monitors": (*ResourceQualityMonitor)(nil), "vector_search_endpoints": (*ResourceVectorSearchEndpoint)(nil), "vector_search_indexes": (*ResourceVectorSearchIndex)(nil), + "secrets": (*ResourceSecret)(nil), // Permissions "jobs.permissions": (*ResourcePermissions)(nil), @@ -60,6 +61,7 @@ var SupportedResources = map[string]any{ "volumes.grants": (*ResourceGrants)(nil), "registered_models.grants": (*ResourceGrants)(nil), "vector_search_indexes.grants": (*ResourceGrants)(nil), + "secrets.grants": (*ResourceGrants)(nil), } func InitAll(client *databricks.WorkspaceClient) (map[string]*Adapter, error) { diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 9c12da9f39b..8ad71a19511 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -651,3 +651,42 @@ resources: backend_defaults: # The Vector Search API assigns index_subtype when the config omits it - field: index_subtype + + secrets: + recreate_on_changes: + # Secrets are three-level namespace objects (catalog.schema.secret) + # and cannot be renamed or moved between schemas + - field: catalog_name + reason: immutable + - field: schema_name + reason: immutable + - field: name + reason: immutable + ignore_remote_changes: + # These fields are computed by the backend and not settable by the user + - field: browse_only + reason: output_only + - field: create_time + reason: output_only + - field: created_by + reason: output_only + - field: effective_owner + reason: output_only + - field: effective_value + reason: output_only + - field: external_secret_id + reason: output_only + - field: full_name + reason: output_only + - field: metastore_id + reason: output_only + - field: update_time + reason: output_only + - field: updated_by + reason: output_only + # The actual secret value is never stored in state for security reasons. + # RemapState always returns empty string for value to prevent persisting + # secrets in the deployment state file. Remote changes to the value are + # ignored since we can't read the plaintext value back from the API. + - field: value + reason: security - value not persisted in state diff --git a/bundle/direct/dresources/secret.go b/bundle/direct/dresources/secret.go new file mode 100644 index 00000000000..1c303f9429f --- /dev/null +++ b/bundle/direct/dresources/secret.go @@ -0,0 +1,98 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/utils" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/common/types/fieldmask" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/service/catalog" +) + +// Terraform provider implementation: +// https://github.com/databricks/terraform-provider-databricks/blob/main/catalog/resource_secret.go + +type ResourceSecret struct { + client *databricks.WorkspaceClient +} + +func (*ResourceSecret) New(client *databricks.WorkspaceClient) *ResourceSecret { + return &ResourceSecret{client: client} +} + +func (*ResourceSecret) PrepareState(input *resources.Secret) *catalog.Secret { + var expireTime *sdktime.Time + if input.ExpireTime != nil { + expireTime = sdktime.New(*input.ExpireTime) + } + return &catalog.Secret{ + CatalogName: input.CatalogName, + SchemaName: input.SchemaName, + Name: input.Name, + Value: input.Value, + Comment: input.Comment, + Owner: input.Owner, + ExpireTime: expireTime, + } +} + +func (*ResourceSecret) RemapState(remote *catalog.Secret) *catalog.Secret { + // Do not store the actual secret value in state for security reasons. + // The API does not return the plaintext value in GetSecret responses (only metadata), + // and we should not persist secrets in the deployment state file. + // We store an empty value and rely on update_time changes to detect drift. + return &catalog.Secret{ + CatalogName: remote.CatalogName, + SchemaName: remote.SchemaName, + Name: remote.Name, + Value: "", // Never store actual secret value in state + Comment: remote.Comment, + Owner: remote.Owner, + ExpireTime: remote.ExpireTime, + ForceSendFields: utils.FilterFields[catalog.Secret](remote.ForceSendFields), + } +} + +// DoRead fetches the secret by full name. +func (r *ResourceSecret) DoRead(ctx context.Context, id string) (*catalog.Secret, error) { + return r.client.SecretsUc.GetSecret(ctx, catalog.GetSecretRequest{ + FullName: id, + }) +} + +// DoCreate creates a new UC secret. +func (r *ResourceSecret) DoCreate(ctx context.Context, state *catalog.Secret) (string, *catalog.Secret, error) { + response, err := r.client.SecretsUc.CreateSecret(ctx, catalog.CreateSecretRequest{ + Secret: *state, + }) + if err != nil || response == nil { + return "", nil, err + } + return response.FullName, response, nil +} + +// DoUpdate updates the secret in place and returns remote state. +func (r *ResourceSecret) DoUpdate(ctx context.Context, id string, state *catalog.Secret, _ *PlanEntry) (*catalog.Secret, error) { + // The UpdateSecret API accepts a field mask specifying which fields to update. + // Supported fields: value, comment, owner, expire_time + response, err := r.client.SecretsUc.UpdateSecret(ctx, catalog.UpdateSecretRequest{ + FullName: id, + Secret: *state, + UpdateMask: fieldmask.FieldMask{ + Paths: []string{"*"}, + }, + }) + if err != nil { + return nil, err + } + return response, nil +} + +// DoDelete deletes the secret. +func (r *ResourceSecret) DoDelete(ctx context.Context, id string, _ *catalog.Secret) error { + return r.client.SecretsUc.DeleteSecret(ctx, catalog.DeleteSecretRequest{ + FullName: id, + }) +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 1b86e37dcee..3d74d6bf8cd 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -33,6 +33,7 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { validate.NoInterpolationInBundleName(), validate.ValidateEngine(), validate.Scripts(), + mutator.ValidateSecretValueIsVariable(), // Updates (dynamic): sync.{paths,include,exclude} (makes them relative to bundle root rather than to definition file) // Rewrites sync paths to be relative to the bundle root instead of the file they were defined in. diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 6c3766ccaeb..3e9875ca175 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -158,6 +158,7 @@ type FakeWorkspace struct { SecretScopes map[string]workspace.SecretScope Secrets map[string]map[string]string // scope -> key -> value Acls map[string][]workspace.AclItem + UCSecrets map[string]catalog.Secret // full_name -> secret (Unity Catalog secrets) // Generic permissions storage: key is "{object_type}:{object_id}" Permissions map[string]iam.ObjectPermissions diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 3b0a154a6ee..d0a7ad2def0 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -671,6 +671,30 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.SecretsAclsDelete(req) }) + // Unity Catalog base endpoint (used for UC availability check): + server.Handle("GET", "/api/2.1/unity-catalog", func(req Request) any { + return map[string]any{ + "metastore_id": "test-metastore-id", + } + }) + + // Unity Catalog Secrets: + server.Handle("POST", "/api/2.1/unity-catalog/secrets", func(req Request) any { + return req.Workspace.SecretsUcCreateSecret(req) + }) + + server.Handle("GET", "/api/2.1/unity-catalog/secrets/{full_name}", func(req Request) any { + return req.Workspace.SecretsUcGetSecret(req) + }) + + server.Handle("PATCH", "/api/2.1/unity-catalog/secrets/{full_name}", func(req Request) any { + return req.Workspace.SecretsUcUpdateSecret(req) + }) + + server.Handle("DELETE", "/api/2.1/unity-catalog/secrets/{full_name}", func(req Request) any { + return req.Workspace.SecretsUcDeleteSecret(req) + }) + // Groups: server.Handle("POST", "/api/2.0/preview/scim/v2/Groups", func(req Request) any { return req.Workspace.GroupsCreate(req) diff --git a/libs/testserver/uc_secrets.go b/libs/testserver/uc_secrets.go new file mode 100644 index 00000000000..98537afed35 --- /dev/null +++ b/libs/testserver/uc_secrets.go @@ -0,0 +1,198 @@ +package testserver + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/service/catalog" +) + +// SecretsUcCreateSecret handles POST /api/2.1/unity-catalog/secrets +func (s *FakeWorkspace) SecretsUcCreateSecret(req Request) Response { + defer s.LockUnlock()() + + // The API accepts flat fields, not wrapped in a "secret" object + var inputSecret catalog.Secret + if err := json.Unmarshal(req.Body, &inputSecret); err != nil { + return Response{ + StatusCode: 400, + Body: map[string]string{"message": fmt.Sprintf("Failed to parse request: %s", err)}, + } + } + + if s.UCSecrets == nil { + s.UCSecrets = make(map[string]catalog.Secret) + } + + fullName := fmt.Sprintf("%s.%s.%s", + inputSecret.CatalogName, + inputSecret.SchemaName, + inputSecret.Name) + + // Check if secret already exists + if _, exists := s.UCSecrets[fullName]; exists { + return Response{ + StatusCode: 409, + Body: map[string]any{ + "error_code": "RESOURCE_ALREADY_EXISTS", + "message": fmt.Sprintf("Secret %s already exists", fullName), + }, + } + } + + now := sdktime.New(time.Now()) + secret := catalog.Secret{ + CatalogName: inputSecret.CatalogName, + SchemaName: inputSecret.SchemaName, + Name: inputSecret.Name, + FullName: fullName, + Value: inputSecret.Value, + Comment: inputSecret.Comment, + Owner: inputSecret.Owner, + ExpireTime: inputSecret.ExpireTime, + CreateTime: now, + UpdateTime: now, + CreatedBy: "test-user@databricks.com", + UpdatedBy: "test-user@databricks.com", + EffectiveOwner: inputSecret.Owner, + MetastoreId: "test-metastore-id", + } + + if secret.Owner == "" { + secret.Owner = "test-user@databricks.com" + secret.EffectiveOwner = "test-user@databricks.com" + } + + s.UCSecrets[fullName] = secret + + return Response{ + Body: secret, + } +} + +// SecretsUcGetSecret handles GET /api/2.1/unity-catalog/secrets/{full_name} +func (s *FakeWorkspace) SecretsUcGetSecret(req Request) Response { + defer s.LockUnlock()() + + // Extract full_name from path parameter + fullName := req.Vars["full_name"] + if fullName == "" { + // Fallback: extract from path + parts := strings.Split(req.URL.Path, "/") + if len(parts) >= 6 { + fullName = parts[5] + } + } + + secret, exists := s.UCSecrets[fullName] + if !exists { + return Response{ + StatusCode: 404, + Body: map[string]any{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": fmt.Sprintf("Secret %s not found", fullName), + }, + } + } + + // Return secret without the actual value (only metadata) + // The real API doesn't return the value unless specifically requested + returnSecret := secret + returnSecret.EffectiveValue = "" // Don't expose value in GET + returnSecret.Value = "" + + return Response{ + Body: returnSecret, + } +} + +// SecretsUcUpdateSecret handles PATCH /api/2.1/unity-catalog/secrets/{full_name} +func (s *FakeWorkspace) SecretsUcUpdateSecret(req Request) Response { + defer s.LockUnlock()() + + // Extract full_name from path parameter + fullName := req.Vars["full_name"] + if fullName == "" { + // Fallback: extract from path + parts := strings.Split(req.URL.Path, "/") + if len(parts) >= 6 { + fullName = parts[5] + } + } + + // The API accepts flat fields + var updateSecret catalog.Secret + if err := json.Unmarshal(req.Body, &updateSecret); err != nil { + return Response{ + StatusCode: 400, + Body: map[string]string{"message": fmt.Sprintf("Failed to parse request: %s", err)}, + } + } + + secret, exists := s.UCSecrets[fullName] + if !exists { + return Response{ + StatusCode: 404, + Body: map[string]any{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": fmt.Sprintf("Secret %s not found", fullName), + }, + } + } + + // Update fields based on update mask + if updateSecret.Value != "" { + secret.Value = updateSecret.Value + } + if updateSecret.Comment != "" { + secret.Comment = updateSecret.Comment + } + if updateSecret.Owner != "" { + secret.Owner = updateSecret.Owner + secret.EffectiveOwner = updateSecret.Owner + } + if updateSecret.ExpireTime != nil { + secret.ExpireTime = updateSecret.ExpireTime + } + + secret.UpdateTime = sdktime.New(time.Now()) + secret.UpdatedBy = "test-user@databricks.com" + + s.UCSecrets[fullName] = secret + + return Response{ + Body: secret, + } +} + +// SecretsUcDeleteSecret handles DELETE /api/2.1/unity-catalog/secrets/{full_name} +func (s *FakeWorkspace) SecretsUcDeleteSecret(req Request) Response { + defer s.LockUnlock()() + + // Extract full_name from path parameter + fullName := req.Vars["full_name"] + if fullName == "" { + // Fallback: extract from path + parts := strings.Split(req.URL.Path, "/") + if len(parts) >= 6 { + fullName = parts[5] + } + } + + if _, exists := s.UCSecrets[fullName]; !exists { + return Response{ + StatusCode: 404, + Body: map[string]any{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": fmt.Sprintf("Secret %s not found", fullName), + }, + } + } + + delete(s.UCSecrets, fullName) + + return Response{} +} From 668279511681b70ad165925296d1adf264f91129 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 9 Jul 2026 10:47:32 +0200 Subject: [PATCH 02/12] fixes --- .../apply_bundle_permissions_test.go | 1 + .../resourcemutator/apply_target_mode_test.go | 8 +++ .../config/mutator/resourcemutator/run_as.go | 10 ++++ .../mutator/resourcemutator/run_as_test.go | 1 + bundle/config/resources/secret.go | 4 +- bundle/config/resources_test.go | 8 +++ bundle/deploy/terraform/lifecycle_test.go | 1 + bundle/direct/dresources/all_test.go | 19 +++++++ bundle/direct/dresources/grants.go | 1 + bundle/direct/dresources/secret.go | 55 +++++++++++++------ bundle/internal/schema/annotations.yml | 33 +++++++++++ bundle/statemgmt/state_load_test.go | 35 ++++++++++++ libs/workspaceurls/urls.go | 2 + 13 files changed, 160 insertions(+), 18 deletions(-) diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 3fa97c41a7f..196917f17b7 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -31,6 +31,7 @@ var unsupportedResources = []string{ "postgres_catalogs", "postgres_synced_tables", "vector_search_indexes", + "secrets", } func TestApplyBundlePermissions(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 440221001a7..ca79928db2f 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -173,6 +173,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { Name: "secretScope1", }, }, + Secrets: map[string]*resources.Secret{ + "secret1": { + CatalogName: "main", + SchemaName: "default", + Name: "secret1", + }, + }, SqlWarehouses: map[string]*resources.SqlWarehouse{ "sql_warehouse1": { CreateWarehouseRequest: sql.CreateWarehouseRequest{ @@ -473,6 +480,7 @@ func TestAppropriateResourcesAreRenamed(t *testing.T) { notUserNamed := []string{ "Apps", "SecretScopes", + "Secrets", "DatabaseInstances", "DatabaseCatalogs", "SyncedDatabaseTables", diff --git a/bundle/config/mutator/resourcemutator/run_as.go b/bundle/config/mutator/resourcemutator/run_as.go index 4f5e3ce9036..9f13b31eabe 100644 --- a/bundle/config/mutator/resourcemutator/run_as.go +++ b/bundle/config/mutator/resourcemutator/run_as.go @@ -126,6 +126,16 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics { )) } + // UC secrets do not support run_as in the API. + if len(b.Config.Resources.Secrets) > 0 { + diags = diags.Extend(reportRunAsNotSupported( + "secrets", + b.Config.GetLocation("resources.secrets"), + b.Config.Workspace.CurrentUser.UserName, + identity, + )) + } + return diags } diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index af1470848d7..88311780b94 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -55,6 +55,7 @@ func allResourceTypes(t *testing.T) []string { "registered_models", "schemas", "secret_scopes", + "secrets", "sql_warehouses", "synced_database_tables", "vector_search_endpoints", diff --git a/bundle/config/resources/secret.go b/bundle/config/resources/secret.go index cc182a5cbec..15832f51aaa 100644 --- a/bundle/config/resources/secret.go +++ b/bundle/config/resources/secret.go @@ -52,11 +52,11 @@ func (s *Secret) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, s) } -func (s Secret) MarshalJSON() ([]byte, error) { +func (s *Secret) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } -func (s Secret) Exists(ctx context.Context, w *databricks.WorkspaceClient, fullName string) (bool, error) { +func (s *Secret) Exists(ctx context.Context, w *databricks.WorkspaceClient, fullName string) (bool, error) { log.Tracef(ctx, "Checking if secret with fullName=%s exists", fullName) _, err := w.SecretsUc.GetSecret(ctx, catalog.GetSecretRequest{ diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index d56a24ced46..85a410a661c 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -226,6 +226,13 @@ func TestResourcesBindSupport(t *testing.T) { Name: "0", }, }, + Secrets: map[string]*resources.Secret{ + "my_secret": { + CatalogName: "main", + SchemaName: "default", + Name: "my_secret", + }, + }, SqlWarehouses: map[string]*resources.SqlWarehouse{ "my_sql_warehouse": { CreateWarehouseRequest: sql.CreateWarehouseRequest{}, @@ -343,6 +350,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockPostgresAPI().EXPECT().GetSyncedTable(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVectorSearchEndpointsAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVectorSearchIndexesAPI().EXPECT().GetIndexByIndexName(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockSecretsUcAPI().EXPECT().GetSecret(mock.Anything, mock.Anything).Return(&catalog.Secret{FullName: "0"}, nil) allResources := supportedResources.AllResources() for _, group := range allResources { diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index b60bff612c7..4a3c5280891 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -18,6 +18,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { "catalogs", "external_locations", "genie_spaces", + "secrets", "vector_search_endpoints", "vector_search_indexes", } diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 30adb4640cc..a0d85548a2a 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -121,6 +121,14 @@ var testConfig map[string]any = map[string]any{ }, }, + "secrets": &resources.Secret{ + CatalogName: "main", + SchemaName: "default", + Name: "my_secret", + Value: "my_secret_value", + Comment: "Test secret", + }, + "secret_scopes": &resources.SecretScope{ Name: "my_secret_scope", BackendType: workspace.ScopeBackendTypeAzureKeyvault, @@ -665,6 +673,17 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "secrets.grants": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + return &GrantsState{ + SecurableType: "secret", + FullName: "main.default.my_secret", + EmbeddedSlice: []catalog.PrivilegeAssignment{{ + Privileges: []catalog.Privilege{catalog.PrivilegeSelect}, + Principal: "user@example.com", + }}, + }, nil + }, + "secret_scopes.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { err := client.Secrets.CreateScope(ctx, workspace.CreateScope{ Scope: "permissions_test_scope", diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index 596346f1614..8f10c17f917 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -18,6 +18,7 @@ var grantResourceToSecurableType = map[string]string{ "external_locations": "external_location", "volumes": "volume", "registered_models": "function", + "secrets": "secret", "vector_search_indexes": "table", } diff --git a/bundle/direct/dresources/secret.go b/bundle/direct/dresources/secret.go index 1c303f9429f..78720a2f67c 100644 --- a/bundle/direct/dresources/secret.go +++ b/bundle/direct/dresources/secret.go @@ -28,13 +28,24 @@ func (*ResourceSecret) PrepareState(input *resources.Secret) *catalog.Secret { expireTime = sdktime.New(*input.ExpireTime) } return &catalog.Secret{ - CatalogName: input.CatalogName, - SchemaName: input.SchemaName, - Name: input.Name, - Value: input.Value, - Comment: input.Comment, - Owner: input.Owner, - ExpireTime: expireTime, + CatalogName: input.CatalogName, + SchemaName: input.SchemaName, + Name: input.Name, + Value: input.Value, + Comment: input.Comment, + Owner: input.Owner, + ExpireTime: expireTime, + BrowseOnly: false, + CreateTime: nil, + CreatedBy: "", + EffectiveOwner: "", + EffectiveValue: "", + ExternalSecretId: "", + FullName: "", + MetastoreId: "", + UpdateTime: nil, + UpdatedBy: "", + ForceSendFields: nil, } } @@ -44,21 +55,33 @@ func (*ResourceSecret) RemapState(remote *catalog.Secret) *catalog.Secret { // and we should not persist secrets in the deployment state file. // We store an empty value and rely on update_time changes to detect drift. return &catalog.Secret{ - CatalogName: remote.CatalogName, - SchemaName: remote.SchemaName, - Name: remote.Name, - Value: "", // Never store actual secret value in state - Comment: remote.Comment, - Owner: remote.Owner, - ExpireTime: remote.ExpireTime, - ForceSendFields: utils.FilterFields[catalog.Secret](remote.ForceSendFields), + CatalogName: remote.CatalogName, + SchemaName: remote.SchemaName, + Name: remote.Name, + Value: "", // Never store actual secret value in state + Comment: remote.Comment, + Owner: remote.Owner, + ExpireTime: remote.ExpireTime, + BrowseOnly: false, + CreateTime: nil, + CreatedBy: "", + EffectiveOwner: "", + EffectiveValue: "", + ExternalSecretId: "", + FullName: "", + MetastoreId: "", + UpdateTime: nil, + UpdatedBy: "", + ForceSendFields: utils.FilterFields[catalog.Secret](remote.ForceSendFields), } } // DoRead fetches the secret by full name. func (r *ResourceSecret) DoRead(ctx context.Context, id string) (*catalog.Secret, error) { return r.client.SecretsUc.GetSecret(ctx, catalog.GetSecretRequest{ - FullName: id, + FullName: id, + IncludeBrowse: false, + ForceSendFields: nil, }) } diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 69d7c9d025d..69da6acc1b0 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -253,6 +253,11 @@ github.com/databricks/cli/bundle/config.Resources: The secret scope definitions for the bundle, where each key is the name of the secret scope. "markdown_description": |- The secret scope definitions for the bundle, where each key is the name of the secret scope. See [\_](/dev-tools/bundles/resources.md#secret_scopes). + "secrets": + "description": |- + The Unity Catalog secret definitions for the bundle, where each key is the name of the secret. + "markdown_description": |- + The Unity Catalog secret definitions for the bundle, where each key is the name of the secret. See [\_](/dev-tools/bundles/resources.md#secrets). "sql_warehouses": "description": |- The SQL warehouse definitions for the bundle, where each key is the name of the warehouse. @@ -1012,6 +1017,34 @@ github.com/databricks/cli/bundle/config/resources.PostgresSyncedTable: "timeseries_key": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.Secret: + "catalog_name": + "description": |- + The name of the catalog where the schema and the secret reside. + "comment": + "description": |- + User-provided free-form text description of the secret. + "expire_time": + "description": |- + User-provided expiration time of the secret. Purely informational; does not trigger automatic actions. + "grants": + "description": |- + The grants to apply on this secret. + "lifecycle": + "description": |- + Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed. + "name": + "description": |- + The name of the secret, relative to its parent schema. + "owner": + "description": |- + The owner of the secret. Defaults to the creating principal on creation. + "schema_name": + "description": |- + The name of the schema where the secret resides. + "value": + "description": |- + The secret value to store. Must be a variable reference (e.g. ${var.my_secret}) to prevent plain-text secrets in configuration files. github.com/databricks/cli/bundle/config/resources.SecretScope: "backend_type": "description": |- diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 672cd9855b2..2c6611a2ff6 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -54,6 +54,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.postgres_synced_tables.test_postgres_synced_table": {ID: "synced_tables/main.public.test_synced_table"}, "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, "resources.vector_search_indexes.test_vector_search_index": {ID: "vs-index-1"}, + "resources.secrets.test_secret": {ID: "main.default.test_secret"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -134,6 +135,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "vs-index-1", config.Resources.VectorSearchIndexes["test_vector_search_index"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.VectorSearchIndexes["test_vector_search_index"].ModifiedStatus) + assert.Equal(t, "main.default.test_secret", config.Resources.Secrets["test_secret"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Secrets["test_secret"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -251,6 +255,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { Name: "test_secret_scope", }, }, + Secrets: map[string]*resources.Secret{ + "test_secret": { + CatalogName: "main", + SchemaName: "default", + Name: "test_secret", + }, + }, SqlWarehouses: map[string]*resources.SqlWarehouse{ "test_sql_warehouse": { CreateWarehouseRequest: sql.CreateWarehouseRequest{ @@ -394,6 +405,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Empty(t, config.Resources.SecretScopes["test_secret_scope"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.SecretScopes["test_secret_scope"].ModifiedStatus) + assert.Empty(t, config.Resources.Secrets["test_secret"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Secrets["test_secret"].ModifiedStatus) + assert.Empty(t, config.Resources.SqlWarehouses["test_sql_warehouse"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.SqlWarehouses["test_sql_warehouse"].ModifiedStatus) @@ -617,6 +631,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { Name: "test_secret_scope_new", }, }, + Secrets: map[string]*resources.Secret{ + "test_secret": { + CatalogName: "main", + SchemaName: "default", + Name: "test_secret", + }, + "test_secret_new": { + CatalogName: "main", + SchemaName: "default", + Name: "test_secret_new", + }, + }, SqlWarehouses: map[string]*resources.SqlWarehouse{ "test_sql_warehouse": { CreateWarehouseRequest: sql.CreateWarehouseRequest{ @@ -817,6 +843,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.vector_search_endpoints.test_vector_search_endpoint_old": {ID: "vs-endpoint-old"}, "resources.vector_search_indexes.test_vector_search_index": {ID: "vs-index-1"}, "resources.vector_search_indexes.test_vector_search_index_old": {ID: "vs-index-old"}, + "resources.secrets.test_secret": {ID: "main.default.test_secret"}, + "resources.secrets.test_secret_old": {ID: "main.default.test_secret_old"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -991,6 +1019,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Empty(t, config.Resources.VectorSearchIndexes["test_vector_search_index_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchIndexes["test_vector_search_index_new"].ModifiedStatus) + assert.Equal(t, "main.default.test_secret", config.Resources.Secrets["test_secret"].ID) + assert.Empty(t, config.Resources.Secrets["test_secret"].ModifiedStatus) + assert.Equal(t, "main.default.test_secret_old", config.Resources.Secrets["test_secret_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Secrets["test_secret_old"].ModifiedStatus) + assert.Empty(t, config.Resources.Secrets["test_secret_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Secrets["test_secret_new"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index 46930c92239..53e308c532a 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -28,6 +28,7 @@ var resourceURLPatterns = map[string]string{ "queries": "sql/editor/%s", "registered_models": "explore/data/models/%s", "schemas": "explore/data/%s", + "secrets": "explore/data/%s", "synced_database_tables": "explore/data/%s", "vector_search_endpoints": "compute/vector-search/%s", "vector_search_indexes": "explore/data/%s", @@ -53,6 +54,7 @@ var dotSeparatedResources = map[string]bool{ "quality_monitors": true, "registered_models": true, "schemas": true, + "secrets": true, "vector_search_indexes": true, "volumes": true, } From 4a52fd3c2d90bc5d3326333ea116318ae2c19622 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 9 Jul 2026 11:24:12 +0200 Subject: [PATCH 03/12] fixed test + lint --- .../bundle/invariant/configs/secrets.yml.tmpl | 16 +++++ acceptance/bundle/invariant/test.toml | 1 + bundle/direct/dresources/resources.yml | 4 -- bundle/direct/dresources/secret.go | 58 +++++++++---------- bundle/internal/schema/annotations.yml | 33 +++++++++++ 5 files changed, 76 insertions(+), 36 deletions(-) create mode 100644 acceptance/bundle/invariant/configs/secrets.yml.tmpl diff --git a/acceptance/bundle/invariant/configs/secrets.yml.tmpl b/acceptance/bundle/invariant/configs/secrets.yml.tmpl new file mode 100644 index 00000000000..96aa784e16f --- /dev/null +++ b/acceptance/bundle/invariant/configs/secrets.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: "Test secret value" + default: "initial-secret-value" + +resources: + secrets: + secret1: + catalog_name: main + schema_name: default + name: test_secret_$UNIQUE_NAME + value: ${var.secret_value} + comment: "Test secret" diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index ec4233c8c59..706292504e4 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -63,6 +63,7 @@ EnvMatrix.INPUT_CONFIG = [ "schema_uppercase_name.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", + "secrets.yml.tmpl", "sql_warehouse.yml.tmpl", "synced_database_table.yml.tmpl", "vector_search_endpoint.yml.tmpl", diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 10b63f2f9cf..06c27b3272f 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -786,8 +786,6 @@ resources: reason: immutable ignore_remote_changes: # These fields are computed by the backend and not settable by the user - - field: browse_only - reason: output_only - field: create_time reason: output_only - field: created_by @@ -796,8 +794,6 @@ resources: reason: output_only - field: effective_value reason: output_only - - field: external_secret_id - reason: output_only - field: full_name reason: output_only - field: metastore_id diff --git a/bundle/direct/dresources/secret.go b/bundle/direct/dresources/secret.go index 78720a2f67c..4de72f1efb2 100644 --- a/bundle/direct/dresources/secret.go +++ b/bundle/direct/dresources/secret.go @@ -35,17 +35,15 @@ func (*ResourceSecret) PrepareState(input *resources.Secret) *catalog.Secret { Comment: input.Comment, Owner: input.Owner, ExpireTime: expireTime, - BrowseOnly: false, - CreateTime: nil, - CreatedBy: "", - EffectiveOwner: "", - EffectiveValue: "", - ExternalSecretId: "", - FullName: "", - MetastoreId: "", - UpdateTime: nil, - UpdatedBy: "", - ForceSendFields: nil, + CreateTime: nil, + CreatedBy: "", + EffectiveOwner: "", + EffectiveValue: "", + FullName: "", + MetastoreId: "", + UpdateTime: nil, + UpdatedBy: "", + ForceSendFields: nil, } } @@ -55,33 +53,29 @@ func (*ResourceSecret) RemapState(remote *catalog.Secret) *catalog.Secret { // and we should not persist secrets in the deployment state file. // We store an empty value and rely on update_time changes to detect drift. return &catalog.Secret{ - CatalogName: remote.CatalogName, - SchemaName: remote.SchemaName, - Name: remote.Name, - Value: "", // Never store actual secret value in state - Comment: remote.Comment, - Owner: remote.Owner, - ExpireTime: remote.ExpireTime, - BrowseOnly: false, - CreateTime: nil, - CreatedBy: "", - EffectiveOwner: "", - EffectiveValue: "", - ExternalSecretId: "", - FullName: "", - MetastoreId: "", - UpdateTime: nil, - UpdatedBy: "", - ForceSendFields: utils.FilterFields[catalog.Secret](remote.ForceSendFields), + CatalogName: remote.CatalogName, + SchemaName: remote.SchemaName, + Name: remote.Name, + Value: "", // Never store actual secret value in state + Comment: remote.Comment, + Owner: remote.Owner, + ExpireTime: remote.ExpireTime, + CreateTime: nil, + CreatedBy: "", + EffectiveOwner: "", + EffectiveValue: "", + FullName: "", + MetastoreId: "", + UpdateTime: nil, + UpdatedBy: "", + ForceSendFields: utils.FilterFields[catalog.Secret](remote.ForceSendFields), } } // DoRead fetches the secret by full name. func (r *ResourceSecret) DoRead(ctx context.Context, id string) (*catalog.Secret, error) { return r.client.SecretsUc.GetSecret(ctx, catalog.GetSecretRequest{ - FullName: id, - IncludeBrowse: false, - ForceSendFields: nil, + FullName: id, }) } diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 7fafe145330..112f219c3d1 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1797,6 +1797,39 @@ resources: "user_name": "description": |- The name of the user granted the permission level. This field translates to a `principal` field in secret scope ACL. + "secrets": + "description": |- + The Unity Catalog secret definitions for the bundle, where each key is the name of the secret. + "markdown_description": |- + The Unity Catalog secret definitions for the bundle, where each key is the name of the secret. See [\_](/dev-tools/bundles/resources.md#secrets). + "$fields": + "catalog_name": + "description": |- + The name of the catalog where the schema and the secret reside. + "comment": + "description": |- + User-provided free-form text description of the secret. + "expire_time": + "description": |- + User-provided expiration time of the secret. Purely informational; does not trigger automatic actions. + "grants": + "description": |- + The grants to apply on this secret. + "lifecycle": + "description": |- + Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. + "name": + "description": |- + The name of the secret, relative to its parent schema. + "owner": + "description": |- + The owner of the secret. Defaults to the creating principal on creation. + "schema_name": + "description": |- + The name of the schema where the secret resides. + "value": + "description": |- + The secret value to store. Must be a variable reference (e.g. ${var.my_secret}) to prevent plain-text secrets in configuration files. "sql_warehouses": "description": |- The SQL warehouse definitions for the bundle, where each key is the name of the warehouse. From e8c51ad2660733d2bad1b7920f1ca75cae188d37 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 9 Jul 2026 11:50:21 +0200 Subject: [PATCH 04/12] fixed tests --- cmd/experimental/workspace_open_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/experimental/workspace_open_test.go b/cmd/experimental/workspace_open_test.go index 9589b48a126..cc2255d580a 100644 --- a/cmd/experimental/workspace_open_test.go +++ b/cmd/experimental/workspace_open_test.go @@ -67,7 +67,7 @@ func TestBuildWorkspaceURLFragmentBasedResources(t *testing.T) { func TestBuildWorkspaceURLUnknownResourceType(t *testing.T) { _, err := workspaceurls.BuildResourceURL("https://myworkspace.databricks.com", "unknown", "123", "") assert.ErrorContains(t, err, "unknown resource type \"unknown\"") - assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") + assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") } func TestBuildWorkspaceURLHostWithTrailingSlash(t *testing.T) { @@ -127,6 +127,7 @@ func TestWorkspaceOpenCommandCompletion(t *testing.T) { "queries", "registered_models", "schemas", + "secrets", "synced_database_tables", "vector_search_endpoints", "vector_search_indexes", @@ -146,7 +147,7 @@ func TestWorkspaceOpenCommandCompletionSecondArg(t *testing.T) { func TestWorkspaceOpenCommandHelpText(t *testing.T) { cmd := newWorkspaceOpenCommand() - assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") + assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") assert.Contains(t, cmd.Long, "databricks experimental open jobs 123456789") assert.Contains(t, cmd.Long, "databricks experimental open notebooks /Users/user@example.com/my-notebook") assert.Contains(t, cmd.Long, "databricks experimental open registered_models catalog.schema.my_model") From 2ed69603d507eb4f1dfd4e8ea58eb96f60651dcd Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 9 Jul 2026 12:03:09 +0200 Subject: [PATCH 05/12] updated generation --- acceptance/bundle/refschema/out.fields.txt | 26 ++++++ .../direct/dresources/apitypes.generated.yml | 2 + .../direct/dresources/resources.generated.yml | 33 +++++++ .../validation/generated/enum_fields.go | 2 + .../validation/generated/required_fields.go | 2 + bundle/schema/jsonschema.json | 87 +++++++++++++++++++ 6 files changed, 152 insertions(+) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 7f93d586b83..616aec700d3 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3344,6 +3344,32 @@ resources.secret_scopes.*.permissions.acls[*] workspace.AclItem ALL resources.secret_scopes.*.permissions.acls[*].permission workspace.AclPermission ALL resources.secret_scopes.*.permissions.acls[*].principal string ALL resources.secret_scopes.*.permissions.scope_name string ALL +resources.secrets.*.catalog_name string ALL +resources.secrets.*.comment string ALL +resources.secrets.*.create_time *time.Time REMOTE STATE +resources.secrets.*.created_by string REMOTE STATE +resources.secrets.*.effective_owner string REMOTE STATE +resources.secrets.*.effective_value string REMOTE STATE +resources.secrets.*.expire_time *time.Time ALL +resources.secrets.*.full_name string REMOTE STATE +resources.secrets.*.id string INPUT +resources.secrets.*.lifecycle resources.Lifecycle INPUT +resources.secrets.*.lifecycle.prevent_destroy bool INPUT +resources.secrets.*.metastore_id string REMOTE STATE +resources.secrets.*.modified_status string INPUT +resources.secrets.*.name string ALL +resources.secrets.*.owner string ALL +resources.secrets.*.schema_name string ALL +resources.secrets.*.update_time *time.Time REMOTE STATE +resources.secrets.*.updated_by string REMOTE STATE +resources.secrets.*.url string INPUT +resources.secrets.*.value string ALL +resources.secrets.*.grants.full_name string ALL +resources.secrets.*.grants.securable_type string ALL +resources.secrets.*.grants[*] catalog.PrivilegeAssignment ALL +resources.secrets.*.grants[*].principal string ALL +resources.secrets.*.grants[*].privileges []catalog.Privilege ALL +resources.secrets.*.grants[*].privileges[*] catalog.Privilege ALL resources.sql_warehouses.*.auto_stop_mins int ALL resources.sql_warehouses.*.channel *sql.Channel ALL resources.sql_warehouses.*.channel.dbsql_version string ALL diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index 8fc2eec34a0..90f4a7b3815 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -52,6 +52,8 @@ schemas: catalog.CreateSchema secret_scopes: workspace.CreateScope +secrets: catalog.Secret + sql_warehouses: sql.EditWarehouseRequest synced_database_tables: database.SyncedDatabaseTable diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index e4154180970..c5e662626f8 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -338,6 +338,39 @@ resources: # secret_scopes: no api field behaviors + secrets: + + recreate_on_changes: + - field: catalog_name + reason: spec:immutable + - field: name + reason: spec:immutable + - field: schema_name + reason: spec:immutable + + ignore_remote_changes: + - field: owner + reason: spec:input_only + - field: value + reason: spec:input_only + + - field: create_time + reason: spec:output_only + - field: created_by + reason: spec:output_only + - field: effective_owner + reason: spec:output_only + - field: effective_value + reason: spec:output_only + - field: full_name + reason: spec:output_only + - field: metastore_id + reason: spec:output_only + - field: update_time + reason: spec:output_only + - field: updated_by + reason: spec:output_only + # sql_warehouses: no api field behaviors synced_database_tables: diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index f223ab7ef52..ec2e6077993 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -217,6 +217,8 @@ var EnumFields = map[string][]string{ "resources.secret_scopes.*.backend_type": {"AZURE_KEYVAULT", "DATABRICKS"}, + "resources.secrets.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_METADATA", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, + "resources.sql_warehouses.*.channel.name": {"CHANNEL_NAME_CURRENT", "CHANNEL_NAME_CUSTOM", "CHANNEL_NAME_PREVIEW", "CHANNEL_NAME_PREVIOUS"}, "resources.sql_warehouses.*.permissions[*].level": {"CAN_MANAGE", "CAN_MONITOR", "CAN_USE", "CAN_VIEW", "IS_OWNER"}, "resources.sql_warehouses.*.spot_instance_policy": {"COST_OPTIMIZED", "POLICY_UNSPECIFIED", "RELIABILITY_OPTIMIZED"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 1eb3d7443c0..3ced7963c1a 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -259,6 +259,8 @@ var RequiredFields = map[string][]string{ "resources.secret_scopes.*.keyvault_metadata": {"dns_name", "resource_id"}, "resources.secret_scopes.*.permissions[*]": {"level"}, + "resources.secrets.*": {"catalog_name", "schema_name", "name", "value"}, + "resources.sql_warehouses.*.permissions[*]": {"level"}, "resources.synced_database_tables.*": {"name"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 503ddbf2ec4..841635e30ed 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2203,6 +2203,62 @@ } ] }, + "resources.Secret": { + "oneOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "The name of the catalog where the schema and the secret reside.", + "$ref": "#/$defs/string" + }, + "comment": { + "description": "User-provided free-form text description of the secret.", + "$ref": "#/$defs/string" + }, + "expire_time": { + "description": "User-provided expiration time of the secret. Purely informational; does not trigger automatic actions.", + "$ref": "#/$defs/time.Time" + }, + "grants": { + "description": "The grants to apply on this secret.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.PrivilegeAssignment" + }, + "lifecycle": { + "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "name": { + "description": "The name of the secret, relative to its parent schema.", + "$ref": "#/$defs/string" + }, + "owner": { + "description": "The owner of the secret. Defaults to the creating principal on creation.", + "$ref": "#/$defs/string" + }, + "schema_name": { + "description": "The name of the schema where the secret resides.", + "$ref": "#/$defs/string" + }, + "value": { + "description": "The secret value to store. Must be a variable reference (e.g. ${var.my_secret}) to prevent plain-text secrets in configuration files.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "catalog_name", + "schema_name", + "name", + "value" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.SecretScope": { "oneOf": [ { @@ -3152,6 +3208,11 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.SecretScope", "markdownDescription": "The secret scope definitions for the bundle, where each key is the name of the secret scope. See [secret_scopes](https://docs.databricks.com/dev-tools/bundles/resources.html#secret_scopes)." }, + "secrets": { + "description": "The Unity Catalog secret definitions for the bundle, where each key is the name of the secret.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Secret", + "markdownDescription": "The Unity Catalog secret definitions for the bundle, where each key is the name of the secret. See [secrets](https://docs.databricks.com/dev-tools/bundles/resources.html#secrets)." + }, "sql_warehouses": { "description": "The SQL warehouse definitions for the bundle, where each key is the name of the warehouse.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.SqlWarehouse", @@ -13977,6 +14038,20 @@ } ] }, + "resources.Secret": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Secret" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.SecretScope": { "oneOf": [ { @@ -15037,6 +15112,18 @@ }, "string": { "type": "string" + }, + "time.Time": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] } }, "type": "object", From e1b0b77fed13ed558efc77b0047f6ab442493f10 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 9 Jul 2026 12:49:56 +0200 Subject: [PATCH 06/12] fixed resources.yml --- bundle/direct/dresources/resources.yml | 35 -------------------------- 1 file changed, 35 deletions(-) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 06c27b3272f..8baa03536a2 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -773,38 +773,3 @@ resources: backend_defaults: # The Vector Search API assigns index_subtype when the config omits it - field: index_subtype - - secrets: - recreate_on_changes: - # Secrets are three-level namespace objects (catalog.schema.secret) - # and cannot be renamed or moved between schemas - - field: catalog_name - reason: immutable - - field: schema_name - reason: immutable - - field: name - reason: immutable - ignore_remote_changes: - # These fields are computed by the backend and not settable by the user - - field: create_time - reason: output_only - - field: created_by - reason: output_only - - field: effective_owner - reason: output_only - - field: effective_value - reason: output_only - - field: full_name - reason: output_only - - field: metastore_id - reason: output_only - - field: update_time - reason: output_only - - field: updated_by - reason: output_only - # The actual secret value is never stored in state for security reasons. - # RemapState always returns empty string for value to prevent persisting - # secrets in the deployment state file. Remote changes to the value are - # ignored since we can't read the plaintext value back from the API. - - field: value - reason: security - value not persisted in state From 44965b1fb2e23ca6b34f5a64f50ee91f3a4afaf8 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 9 Jul 2026 13:20:20 +0200 Subject: [PATCH 07/12] fixed acc output --- acceptance/bundle/invariant/migrate/test.toml | 3 +++ acceptance/bundle/resources/secrets/update-value/output.txt | 2 +- acceptance/experimental/open/output.txt | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index af25cbb62dd..a8c26c46109 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -34,3 +34,6 @@ EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml # migrate deploys via Terraform first, and the TF provider rejects an uppercase # volume schema_name ("inconsistent final plan"). Covered by no_drift on direct. EnvMatrixExclude.no_volume_uppercase = ["INPUT_CONFIG=volume_uppercase_name.yml.tmpl"] + +# Secrets are not supported in terraform mode, so the terraform deploy that seeds the migration fails for them. +EnvMatrixExclude.no_secret = ["INPUT_CONFIG=secrets.yml.tmpl"] diff --git a/acceptance/bundle/resources/secrets/update-value/output.txt b/acceptance/bundle/resources/secrets/update-value/output.txt index 6c92ba765a0..03aadc3e5bc 100644 --- a/acceptance/bundle/resources/secrets/update-value/output.txt +++ b/acceptance/bundle/resources/secrets/update-value/output.txt @@ -49,7 +49,7 @@ secrets secret1 catalog_name='main' schema_name='default' name='test_secret' com >>> print_state.py { "state_version": 2, - "cli_version": "[DEV_VERSION]", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, "state": { diff --git a/acceptance/experimental/open/output.txt b/acceptance/experimental/open/output.txt index f634d630fc8..d60765969d3 100644 --- a/acceptance/experimental/open/output.txt +++ b/acceptance/experimental/open/output.txt @@ -9,7 +9,7 @@ === unknown resource type >>> [CLI] experimental open --url unknown 123 -Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses +Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses === test auto-completion handler >>> [CLI] __complete experimental open , @@ -33,6 +33,7 @@ quality_monitors queries registered_models schemas +secrets synced_database_tables vector_search_endpoints vector_search_indexes From 1f56144ceb1efee7c550b9c33d10611c11cd7840 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 9 Jul 2026 13:43:34 +0200 Subject: [PATCH 08/12] add schema.grants to invariant --- acceptance/bundle/invariant/configs/secrets.yml.tmpl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/acceptance/bundle/invariant/configs/secrets.yml.tmpl b/acceptance/bundle/invariant/configs/secrets.yml.tmpl index 96aa784e16f..b402d8a8cdc 100644 --- a/acceptance/bundle/invariant/configs/secrets.yml.tmpl +++ b/acceptance/bundle/invariant/configs/secrets.yml.tmpl @@ -14,3 +14,7 @@ resources: name: test_secret_$UNIQUE_NAME value: ${var.secret_value} comment: "Test secret" + grants: + - principal: account users + privileges: + - READ_SECRET From 3821f05865e375219be07ffe269be957f8d13fff Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 9 Jul 2026 14:35:57 +0200 Subject: [PATCH 09/12] updated out.test.toml --- acceptance/bundle/invariant/continue_293/out.test.toml | 1 + acceptance/bundle/invariant/no_drift/out.test.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 407bac1bf19..795910c9ab7 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -37,6 +37,7 @@ EnvMatrix.INPUT_CONFIG = [ "schema_uppercase_name.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", + "secrets.yml.tmpl", "sql_warehouse.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl", diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index e61d97cc42c..267f9e56cec 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -45,6 +45,7 @@ EnvMatrix.INPUT_CONFIG = [ "schema_uppercase_name.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", + "secrets.yml.tmpl", "sql_warehouse.yml.tmpl", "synced_database_table.yml.tmpl", "vector_search_endpoint.yml.tmpl", From e37a7e569eaab38d533321052756d76ab0e3661c Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 14 Jul 2026 11:51:19 +0200 Subject: [PATCH 10/12] do not allow default --- .../secrets/basic/databricks.yml.tmpl | 1 - .../bundle/resources/secrets/basic/output.txt | 12 +++--- .../bundle/resources/secrets/basic/script | 12 +++--- .../secrets/direct-only/databricks.yml | 1 - .../resources/secrets/direct-only/output.txt | 4 +- .../resources/secrets/direct-only/script | 2 +- .../secrets/update-value/databricks.yml | 1 - .../resources/secrets/update-value/output.txt | 8 ++-- .../resources/secrets/update-value/script | 7 ++-- .../databricks.yml | 15 +++++++ .../out.test.toml | 3 ++ .../validate-no-plain-text-default/output.txt | 14 +++++++ .../validate-no-plain-text-default/script | 4 ++ .../validate-no-plain-text-default/test.toml | 8 ++++ .../validate_secret_value_is_variable.go | 40 +++++++++++++++++++ 15 files changed, 105 insertions(+), 27 deletions(-) create mode 100644 acceptance/bundle/resources/secrets/validate-no-plain-text-default/databricks.yml create mode 100644 acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml create mode 100644 acceptance/bundle/resources/secrets/validate-no-plain-text-default/output.txt create mode 100644 acceptance/bundle/resources/secrets/validate-no-plain-text-default/script create mode 100644 acceptance/bundle/resources/secrets/validate-no-plain-text-default/test.toml diff --git a/acceptance/bundle/resources/secrets/basic/databricks.yml.tmpl b/acceptance/bundle/resources/secrets/basic/databricks.yml.tmpl index 518dbfbb9a5..fb24de6b961 100644 --- a/acceptance/bundle/resources/secrets/basic/databricks.yml.tmpl +++ b/acceptance/bundle/resources/secrets/basic/databricks.yml.tmpl @@ -4,7 +4,6 @@ bundle: variables: secret_value: description: "Test secret value" - default: "initial-secret-value" resources: secrets: diff --git a/acceptance/bundle/resources/secrets/basic/output.txt b/acceptance/bundle/resources/secrets/basic/output.txt index 584a62b4258..09e33fdd49c 100644 --- a/acceptance/bundle/resources/secrets/basic/output.txt +++ b/acceptance/bundle/resources/secrets/basic/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle validate +>>> [CLI] bundle validate --var secret_value=initial-secret-value Name: test-bundle-[UNIQUE_NAME] Target: default Workspace: @@ -8,7 +8,7 @@ Workspace: Validation OK! ->>> [CLI] bundle summary +>>> [CLI] bundle summary --var secret_value=initial-secret-value Name: test-bundle-[UNIQUE_NAME] Target: default Workspace: @@ -20,7 +20,7 @@ Resources: Name: main.default.test_secret URL: (not deployed) ->>> [CLI] bundle deploy +>>> [CLI] bundle deploy --var secret_value=initial-secret-value Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Deploying resources... Updating deployment state... @@ -43,7 +43,7 @@ secrets secret1 catalog_name='main' schema_name='default' name='test_secret' com === Update comment and re-deploy >>> update_file.py databricks.yml Test secret for acceptance testing Updated comment ->>> [CLI] bundle deploy +>>> [CLI] bundle deploy --var secret_value=initial-secret-value Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Deploying resources... Updating deployment state... @@ -69,7 +69,7 @@ secrets secret1 catalog_name='main' schema_name='default' name='test_secret' com === Restore comment and re-deploy >>> update_file.py databricks.yml Updated comment Test secret for acceptance testing ->>> [CLI] bundle deploy +>>> [CLI] bundle deploy --var secret_value=initial-secret-value Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Deploying resources... Updating deployment state... @@ -92,7 +92,7 @@ Deployment complete! } secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret for acceptance testing' ->>> [CLI] bundle destroy --auto-approve +>>> [CLI] bundle destroy --auto-approve --var secret_value=initial-secret-value The following resources will be deleted: delete resources.secrets.secret1 diff --git a/acceptance/bundle/resources/secrets/basic/script b/acceptance/bundle/resources/secrets/basic/script index f959b83eb59..d808e9033e3 100755 --- a/acceptance/bundle/resources/secrets/basic/script +++ b/acceptance/bundle/resources/secrets/basic/script @@ -1,27 +1,27 @@ envsubst < databricks.yml.tmpl > databricks.yml cleanup() { - trace $CLI bundle destroy --auto-approve + trace $CLI bundle destroy --auto-approve --var secret_value=initial-secret-value rm out.requests.txt } trap cleanup EXIT -trace $CLI bundle validate -trace $CLI bundle summary -trace $CLI bundle deploy +trace $CLI bundle validate --var secret_value=initial-secret-value +trace $CLI bundle summary --var secret_value=initial-secret-value +trace $CLI bundle deploy --var secret_value=initial-secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 catalog_name schema_name name comment title "Update comment and re-deploy" trace update_file.py databricks.yml "Test secret for acceptance testing" "Updated comment" -trace $CLI bundle deploy +trace $CLI bundle deploy --var secret_value=initial-secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 catalog_name schema_name name comment title "Restore comment and re-deploy" trace update_file.py databricks.yml "Updated comment" "Test secret for acceptance testing" -trace $CLI bundle deploy +trace $CLI bundle deploy --var secret_value=initial-secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 catalog_name schema_name name comment diff --git a/acceptance/bundle/resources/secrets/direct-only/databricks.yml b/acceptance/bundle/resources/secrets/direct-only/databricks.yml index ae64aaa6ede..6edb6f3cf13 100644 --- a/acceptance/bundle/resources/secrets/direct-only/databricks.yml +++ b/acceptance/bundle/resources/secrets/direct-only/databricks.yml @@ -4,7 +4,6 @@ bundle: variables: secret_value: description: "Test secret value" - default: "test-value" resources: secrets: diff --git a/acceptance/bundle/resources/secrets/direct-only/output.txt b/acceptance/bundle/resources/secrets/direct-only/output.txt index 7d6d276152b..d3bef4d3b23 100644 --- a/acceptance/bundle/resources/secrets/direct-only/output.txt +++ b/acceptance/bundle/resources/secrets/direct-only/output.txt @@ -1,8 +1,8 @@ === Deploy should fail in terraform mode ->>> [CLI] bundle deploy +>>> [CLI] bundle deploy --var secret_value=test-value Error: Secret resources are only supported with direct deployment mode - in databricks.yml:11:5 + in databricks.yml:10:5 Secret resources require direct deployment mode. Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use secret resources. Learn more at https://docs.databricks.com/dev-tools/bundles/direct diff --git a/acceptance/bundle/resources/secrets/direct-only/script b/acceptance/bundle/resources/secrets/direct-only/script index 4e7454f402e..4ec9195eef9 100755 --- a/acceptance/bundle/resources/secrets/direct-only/script +++ b/acceptance/bundle/resources/secrets/direct-only/script @@ -1,4 +1,4 @@ title "Deploy should fail in terraform mode" -trace $CLI bundle deploy 2>&1 | contains.py \ +trace $CLI bundle deploy --var secret_value=test-value 2>&1 | contains.py \ "Secret resources are only supported with direct deployment mode" \ "DATABRICKS_BUNDLE_ENGINE" diff --git a/acceptance/bundle/resources/secrets/update-value/databricks.yml b/acceptance/bundle/resources/secrets/update-value/databricks.yml index 89208023eea..314e22db6ed 100644 --- a/acceptance/bundle/resources/secrets/update-value/databricks.yml +++ b/acceptance/bundle/resources/secrets/update-value/databricks.yml @@ -4,7 +4,6 @@ bundle: variables: secret_value: description: "Test secret value" - default: "initial-secret-value" resources: secrets: diff --git a/acceptance/bundle/resources/secrets/update-value/output.txt b/acceptance/bundle/resources/secrets/update-value/output.txt index 03aadc3e5bc..c8b12bbedde 100644 --- a/acceptance/bundle/resources/secrets/update-value/output.txt +++ b/acceptance/bundle/resources/secrets/update-value/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle deploy +>>> [CLI] bundle deploy --var secret_value=initial-secret-value Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Deploying resources... Updating deployment state... @@ -19,10 +19,8 @@ Deployment complete! } secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret' value='initial-secret-value' -=== Update secret value by changing variable default ->>> update_file.py databricks.yml initial-secret-value updated-secret-value - ->>> [CLI] bundle deploy +=== Update secret value by passing a different variable value +>>> [CLI] bundle deploy --var secret_value=updated-secret-value Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Deploying resources... Updating deployment state... diff --git a/acceptance/bundle/resources/secrets/update-value/script b/acceptance/bundle/resources/secrets/update-value/script index 6c0355d3ba1..c312db607d2 100755 --- a/acceptance/bundle/resources/secrets/update-value/script +++ b/acceptance/bundle/resources/secrets/update-value/script @@ -1,11 +1,10 @@ -trace $CLI bundle deploy +trace $CLI bundle deploy --var secret_value=initial-secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 catalog_name schema_name name comment value -title "Update secret value by changing variable default" -trace update_file.py databricks.yml "initial-secret-value" "updated-secret-value" -trace $CLI bundle deploy +title "Update secret value by passing a different variable value" +trace $CLI bundle deploy --var secret_value=updated-secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 catalog_name schema_name name comment value diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/databricks.yml b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/databricks.yml new file mode 100644 index 00000000000..aee031ac950 --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-bundle + +variables: + secret_value: + description: "Test secret value" + default: "plain-text-default-not-allowed" + +resources: + secrets: + secret1: + catalog_name: main + schema_name: default + name: test_secret + value: ${var.secret_value} diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/output.txt b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/output.txt new file mode 100644 index 00000000000..a8242e1fec1 --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/output.txt @@ -0,0 +1,14 @@ + +=== Deploy should fail when variable used for secret has a default value +>>> [CLI] bundle deploy +Error: Variable used for secret value must not have a default value + at resources.secrets.secret1.value + in databricks.yml:7:14 + databricks.yml:15:14 + +The variable "secret_value" used for the secret "secret1" must not have a default value. +A default value is stored in plain text in the configuration file, which defeats the purpose of using a variable reference for a secret. +Remove the default value and pass the secret value at deployment time using "--var", the BUNDLE_VAR_secret_value environment variable, or a variable overrides file. + + +Exit code: 1 diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/script b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/script new file mode 100644 index 00000000000..f4095fe7141 --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/script @@ -0,0 +1,4 @@ +title "Deploy should fail when variable used for secret has a default value" +trace $CLI bundle deploy 2>&1 | contains.py \ + "Variable used for secret value must not have a default value" \ + "default value is stored in plain text" diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/test.toml new file mode 100644 index 00000000000..9a9e1194d6c --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = false +RecordRequests = false + +Ignore = [".databricks"] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/mutator/validate_secret_value_is_variable.go b/bundle/config/mutator/validate_secret_value_is_variable.go index 49a4031ceaf..a784d418b76 100644 --- a/bundle/config/mutator/validate_secret_value_is_variable.go +++ b/bundle/config/mutator/validate_secret_value_is_variable.go @@ -50,8 +50,48 @@ Use bundle variables to pass secret values at deployment time.`, key), Locations: val.Locations(), Paths: []dyn.Path{p}, }) + continue } + + // The value is a variable reference. If it is a pure ${var.} reference, + // check that the referenced variable does not have a default value set. A default + // value would be stored in plain text in the config file, defeating the purpose + // of using a variable reference. + diags = append(diags, v.checkVariableDefault(b, key, valueStr, p, val)...) } return diags } + +// checkVariableDefault emits an error if valueStr is a pure ${var.} reference +// and the referenced variable has a default value set. +func (v *validateSecretValueIsVariable) checkVariableDefault(b *bundle.Bundle, secretKey, valueStr string, p dyn.Path, val dyn.Value) diag.Diagnostics { + refPath, ok := dynvar.PureReferenceToPath(valueStr) + if !ok || len(refPath) < 2 || refPath[0].Key() != "var" { + return nil + } + + varName := refPath[1].Key() + variable, exists := b.Config.Variables[varName] + if !exists || variable == nil || !variable.HasDefault() { + return nil + } + + // The default path in the dynamic config for the variable's default field. + defaultPath := dyn.NewPath(dyn.Key("variables"), dyn.Key(varName), dyn.Key("default")) + defaultVal, err := dyn.GetByPath(b.Config.Value(), defaultPath) + locations := val.Locations() + if err == nil { + locations = append(defaultVal.Locations(), locations...) + } + + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: "Variable used for secret value must not have a default value", + Detail: fmt.Sprintf(`The variable "%s" used for the secret "%s" must not have a default value. +A default value is stored in plain text in the configuration file, which defeats the purpose of using a variable reference for a secret. +Remove the default value and pass the secret value at deployment time using "--var", the BUNDLE_VAR_%s environment variable, or a variable overrides file.`, varName, secretKey, varName), + Locations: locations, + Paths: []dyn.Path{p}, + }} +} From 3e8298d0c0859353b521f61c08f80e86c1a51168 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 14 Jul 2026 17:15:40 +0200 Subject: [PATCH 11/12] fixed invariant test setup --- acceptance/bundle/invariant/configs/secrets.yml.tmpl | 1 - acceptance/bundle/invariant/configs/secrets.yml.tmpl-init.sh | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100755 acceptance/bundle/invariant/configs/secrets.yml.tmpl-init.sh diff --git a/acceptance/bundle/invariant/configs/secrets.yml.tmpl b/acceptance/bundle/invariant/configs/secrets.yml.tmpl index b402d8a8cdc..9aa926a1e5e 100644 --- a/acceptance/bundle/invariant/configs/secrets.yml.tmpl +++ b/acceptance/bundle/invariant/configs/secrets.yml.tmpl @@ -4,7 +4,6 @@ bundle: variables: secret_value: description: "Test secret value" - default: "initial-secret-value" resources: secrets: diff --git a/acceptance/bundle/invariant/configs/secrets.yml.tmpl-init.sh b/acceptance/bundle/invariant/configs/secrets.yml.tmpl-init.sh new file mode 100755 index 00000000000..4ef9d49d782 --- /dev/null +++ b/acceptance/bundle/invariant/configs/secrets.yml.tmpl-init.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +export BUNDLE_VAR_secret_value="initial-secret-value" From 4ae6be95a70383ba3d07bb22cfc3e86da05d909e Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 15 Jul 2026 17:31:21 +0200 Subject: [PATCH 12/12] use typed leaf --- libs/dyn/convert/from_typed.go | 9 ++++ libs/dyn/convert/normalize.go | 4 ++ libs/dyn/convert/struct_info.go | 10 ++++ libs/dyn/convert/struct_info_test.go | 76 ++++++++++++++++++++++++++++ libs/dyn/dynvar/resolve.go | 16 ++++++ libs/dyn/dynvar/resolve_test.go | 30 +++++++++++ libs/dyn/jsonsaver/marshal.go | 10 ++++ libs/dyn/jsonsaver/marshal_test.go | 18 +++++++ libs/dyn/kind.go | 2 +- libs/dyn/sensitive.go | 32 ++++++++++++ libs/dyn/value.go | 5 ++ libs/dyn/value_test.go | 32 ++++++++++++ libs/dyn/value_underlying.go | 17 ++++++- libs/dyn/yamlsaver/saver.go | 4 ++ libs/structs/structtag/bundletag.go | 6 +++ 15 files changed, 268 insertions(+), 3 deletions(-) create mode 100644 libs/dyn/sensitive.go diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go index 66451124293..7247e5e07ff 100644 --- a/libs/dyn/convert/from_typed.go +++ b/libs/dyn/convert/from_typed.go @@ -133,6 +133,15 @@ func fromTypedStruct(src reflect.Value, ref dyn.Value, options ...fromTypedOptio return dyn.InvalidValue, err } + // If the field carries the bundle:"sensitive" tag and resolved to a + // non-empty string, wrap it as a sensitive value so that all downstream + // serializers (JSON, YAML) redact it automatically. + if info.Sensitive[k] && nv.Kind() == dyn.KindString { + if s, ok := nv.AsString(); ok && s != "" { + nv = dyn.NewSensitiveValue(s, nv.Locations()) + } + } + // Either if the key was set in the reference, the field is not zero-valued, OR it's forced if ok || nv.Kind() != dyn.KindNil || isForced { // If v isZero, it could be because it's a variable reference; so we check that nv is zero as well diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index 79cfee37441..3cc94909992 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -285,6 +285,10 @@ func (n normalizeOptions) normalizeString(typ reflect.Type, src dyn.Value, path switch src.Kind() { case dyn.KindString: + // Preserve sensitive strings; NewValue would strip the secretString wrapper. + if src.IsSensitive() { + return src, nil + } return dyn.NewValue(src.MustString(), src.Locations()), nil case dyn.KindBool: return dyn.NewValue(strconv.FormatBool(src.MustBool()), src.Locations()), nil diff --git a/libs/dyn/convert/struct_info.go b/libs/dyn/convert/struct_info.go index 69e8f868ed5..ab53b5ccc55 100644 --- a/libs/dyn/convert/struct_info.go +++ b/libs/dyn/convert/struct_info.go @@ -28,6 +28,10 @@ type structInfo struct { // Maps JSON-name of the field to Golang struct name GolangNames map[string]string + // Sensitive tracks fields tagged `bundle:"sensitive"` by their JSON name. + // Values for these fields should be masked in display output. + Sensitive map[string]bool + // ForceSendFieldsIndex maps the JSON-name of the field to the index path (for // use with [reflect.Value.FieldByIndex]) of the ForceSendFields slice that // governs it: the one declared by the struct that also declares the field. @@ -67,6 +71,7 @@ func buildStructInfo(typ reflect.Type) structInfo { Fields: make(map[string][]int), ForceEmpty: make(map[string]bool), GolangNames: make(map[string]string), + Sensitive: make(map[string]bool), ForceSendFieldsIndex: make(map[string][]int), } @@ -134,6 +139,11 @@ func buildStructInfo(typ reflect.Type) structInfo { } out.GolangNames[name] = sf.Name + btag := structtag.BundleTag(sf.Tag.Get("bundle")) + if btag.Sensitive() { + out.Sensitive[name] = true + } + // The field is declared directly in this struct, so it is governed by // this struct's ForceSendFields (if it has one). if forceSendFieldsIndex != nil { diff --git a/libs/dyn/convert/struct_info_test.go b/libs/dyn/convert/struct_info_test.go index f921523c391..73a2a825837 100644 --- a/libs/dyn/convert/struct_info_test.go +++ b/libs/dyn/convert/struct_info_test.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestStructInfoPlain(t *testing.T) { @@ -226,3 +227,78 @@ func TestStructInfoValueFieldMultiple(t *testing.T) { getStructInfo(reflect.TypeFor[Tmp]()) }) } + +func TestStructInfoSensitivePlain(t *testing.T) { + type Tmp struct { + Name string `json:"name"` + Token string `json:"token" bundle:"sensitive"` + } + + si := getStructInfo(reflect.TypeFor[Tmp]()) + assert.True(t, si.Sensitive["token"]) + assert.False(t, si.Sensitive["name"]) +} + +func TestStructInfoSensitiveEmbeddedByValue(t *testing.T) { + type Inner struct { + Token string `json:"token" bundle:"sensitive"` + } + type Outer struct { + Name string `json:"name"` + Inner + } + + si := getStructInfo(reflect.TypeFor[Outer]()) + assert.True(t, si.Sensitive["token"]) + assert.False(t, si.Sensitive["name"]) +} + +func TestStructInfoSensitiveEmbeddedByPointer(t *testing.T) { + type Inner struct { + Token string `json:"token" bundle:"sensitive"` + } + type Outer struct { + Name string `json:"name"` + *Inner + } + + si := getStructInfo(reflect.TypeFor[Outer]()) + assert.True(t, si.Sensitive["token"]) +} + +func TestStructInfoSensitiveTopLevelPrecedence(t *testing.T) { + // A non-sensitive top-level field shadows a sensitive embedded field. + type Inner struct { + Token string `json:"token" bundle:"sensitive"` + } + type Outer struct { + Token string `json:"token"` // not sensitive; shadows Inner.Token + Inner + } + + si := getStructInfo(reflect.TypeFor[Outer]()) + assert.False(t, si.Sensitive["token"]) +} + +func TestFromTypedSensitiveField(t *testing.T) { + type Resource struct { + Name string `json:"name"` + Token string `json:"token" bundle:"sensitive"` + } + + src := Resource{Name: "my-resource", Token: "s3cr3t"} + v, err := FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + tok, err := dyn.GetByPath(v, dyn.NewPath(dyn.Key("token"))) + require.NoError(t, err) + assert.True(t, tok.IsSensitive(), "token field should be sensitive") + assert.Equal(t, "s3cr3t", tok.MustString(), "MustString should return the real value") + assert.Equal(t, dyn.SensitiveValueRedacted, tok.AsAny(), "AsAny should return the redacted placeholder") + + // Non-sensitive fields are unaffected. + name, err := dyn.GetByPath(v, dyn.NewPath(dyn.Key("name"))) + require.NoError(t, err) + assert.False(t, name.IsSensitive()) + assert.Equal(t, "my-resource", name.MustString()) +} diff --git a/libs/dyn/dynvar/resolve.go b/libs/dyn/dynvar/resolve.go index 6f9269f4974..e5acccfc2d2 100644 --- a/libs/dyn/dynvar/resolve.go +++ b/libs/dyn/dynvar/resolve.go @@ -153,10 +153,18 @@ func (r *resolver) resolveRef(ref Ref, seen []string) (dyn.Value, error) { // of where it is used. This also means that relative path resolution is done // relative to where a variable is used, not where it is defined. // + // Preserve sensitivity: NewValue strips the secretString wrapper, so use + // NewSensitiveValue when the resolved value is sensitive. + if resolved[0].IsSensitive() { + s, _ := resolved[0].AsString() + return dyn.NewSensitiveValue(s, ref.Value.Locations()), nil + } return dyn.NewValue(resolved[0].Value(), ref.Value.Locations()), nil } // Not pure; perform string interpolation. + // If any resolved value is sensitive the result string must also be sensitive. + anySensitive := false for j := range ref.Matches { // The value is invalid if resolution returned [ErrSkipResolution]. // We must skip those and leave the original variable reference in place. @@ -164,7 +172,12 @@ func (r *resolver) resolveRef(ref Ref, seen []string) (dyn.Value, error) { continue } + if resolved[j].IsSensitive() { + anySensitive = true + } + // Try to turn the resolved value into a string. + // Use AsString (not AsAny) to get the real value even for sensitive strings. s, ok := resolved[j].AsString() if !ok { // Only allow primitive types to be converted to string. @@ -179,6 +192,9 @@ func (r *resolver) resolveRef(ref Ref, seen []string) (dyn.Value, error) { ref.Str = strings.Replace(ref.Str, ref.Matches[j][0], s, 1) } + if anySensitive { + return dyn.NewSensitiveValue(ref.Str, ref.Value.Locations()), nil + } return dyn.NewValue(ref.Str, ref.Value.Locations()), nil } diff --git a/libs/dyn/dynvar/resolve_test.go b/libs/dyn/dynvar/resolve_test.go index c2bbaae74f1..5e66d9c1c74 100644 --- a/libs/dyn/dynvar/resolve_test.go +++ b/libs/dyn/dynvar/resolve_test.go @@ -393,3 +393,33 @@ func TestResolveSequenceVariable(t *testing.T) { assert.Equal(t, "value1", seq[0].MustString()) assert.Equal(t, "value2", seq[1].MustString()) } + +func TestResolveSensitivePureSubstitution(t *testing.T) { + in := dyn.V(map[string]dyn.Value{ + "secret": dyn.NewSensitiveValue("s3cr3t", nil), + "ref": dyn.V("${secret}"), + }) + + out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) + require.NoError(t, err) + + ref := getByPath(t, out, "ref") + assert.True(t, ref.IsSensitive(), "pure substitution of sensitive value must be sensitive") + assert.Equal(t, "s3cr3t", ref.MustString(), "MustString must return the real value") + assert.Equal(t, dyn.SensitiveValueRedacted, ref.AsAny(), "AsAny must return the redaction placeholder") +} + +func TestResolveSensitiveStringInterpolation(t *testing.T) { + in := dyn.V(map[string]dyn.Value{ + "secret": dyn.NewSensitiveValue("s3cr3t", nil), + "ref": dyn.V("prefix-${secret}-suffix"), + }) + + out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) + require.NoError(t, err) + + ref := getByPath(t, out, "ref") + assert.True(t, ref.IsSensitive(), "string interpolation containing a sensitive value must be sensitive") + // The real interpolated string is still accessible. + assert.Equal(t, "prefix-s3cr3t-suffix", ref.MustString()) +} diff --git a/libs/dyn/jsonsaver/marshal.go b/libs/dyn/jsonsaver/marshal.go index a78a68f2998..c8c3cd2fac3 100644 --- a/libs/dyn/jsonsaver/marshal.go +++ b/libs/dyn/jsonsaver/marshal.go @@ -39,6 +39,16 @@ func (w wrap) MarshalJSON() ([]byte, error) { // marshalValue recursively writes JSON for a [dyn.Value] to the buffer. func marshalValue(buf *bytes.Buffer, v dyn.Value) error { + if v.IsSensitive() { + out, err := marshalNoEscape(dyn.SensitiveValueRedacted) + if err != nil { + return err + } + out = out[:len(out)-1] + buf.Write(out) + return nil + } + switch v.Kind() { case dyn.KindString, dyn.KindBool, dyn.KindInt, dyn.KindFloat, dyn.KindTime, dyn.KindNil: out, err := marshalNoEscape(v.AsAny()) diff --git a/libs/dyn/jsonsaver/marshal_test.go b/libs/dyn/jsonsaver/marshal_test.go index 68c2017397f..98d49fcb06c 100644 --- a/libs/dyn/jsonsaver/marshal_test.go +++ b/libs/dyn/jsonsaver/marshal_test.go @@ -64,6 +64,24 @@ func TestMarshal_Sequence(t *testing.T) { } } +func TestMarshal_Sensitive(t *testing.T) { + b, err := Marshal(dyn.NewSensitiveValue("s3cr3t", nil)) + if assert.NoError(t, err) { + assert.JSONEq(t, `"`+dyn.SensitiveValueRedacted+`"`, string(b)) + } +} + +func TestMarshal_SensitiveInMap(t *testing.T) { + m := dyn.NewMapping() + m.SetLoc("name", nil, dyn.V("resource")) + m.SetLoc("token", nil, dyn.NewSensitiveValue("s3cr3t", nil)) + + b, err := Marshal(dyn.V(m)) + if assert.NoError(t, err) { + assert.JSONEq(t, `{"name":"resource","token":"`+dyn.SensitiveValueRedacted+`"}`, string(b)) + } +} + func TestMarshal_Complex(t *testing.T) { map1 := dyn.NewMapping() map1.SetLoc("str1", nil, dyn.V("value1")) diff --git a/libs/dyn/kind.go b/libs/dyn/kind.go index 1890e9e2ccd..684239503b3 100644 --- a/libs/dyn/kind.go +++ b/libs/dyn/kind.go @@ -25,7 +25,7 @@ func kindOf(v any) Kind { return KindMap case []Value: return KindSequence - case string: + case string, secretString: return KindString case bool: return KindBool diff --git a/libs/dyn/sensitive.go b/libs/dyn/sensitive.go new file mode 100644 index 00000000000..7376e99f43a --- /dev/null +++ b/libs/dyn/sensitive.go @@ -0,0 +1,32 @@ +package dyn + +import "slices" + +// SensitiveValueRedacted is the placeholder emitted in JSON/YAML output +// whenever a sensitive string value is serialized. +const SensitiveValueRedacted = "" + +// secretString is the internal storage type for sensitive string values. +// Storing a distinct Go type (rather than a plain string) makes sensitivity +// impossible to strip by accident: every code path that copies v.v preserves +// it, and every switch on v.v that handles `string` but not `secretString` +// fails to compile or silently falls through — making omissions auditable. +// +// Kind() still returns KindString for a secretString value so that all +// existing switch-on-Kind logic continues to work unchanged. Only the +// leaf accessors (AsAny, AsString, MustString) are aware of the type. +type secretString struct { + value string +} + +// NewSensitiveValue returns a new KindString Value whose content is treated as +// sensitive. JSON and YAML serializers replace it with [SensitiveValueRedacted]; +// AsString / MustString still return the real value for use in the deployment +// pipeline. +func NewSensitiveValue(s string, loc []Location) Value { + return Value{ + v: secretString{s}, + k: KindString, + l: slices.Clone(loc), + } +} diff --git a/libs/dyn/value.go b/libs/dyn/value.go index 72803511b80..555ae507e70 100644 --- a/libs/dyn/value.go +++ b/libs/dyn/value.go @@ -120,6 +120,11 @@ func (v Value) AsAny() any { case KindNil: return v.v case KindString: + // secretString holds a sensitive value; return the redaction placeholder. + if ss, ok := v.v.(secretString); ok { + _ = ss + return SensitiveValueRedacted + } return v.v case KindBool: return v.v diff --git a/libs/dyn/value_test.go b/libs/dyn/value_test.go index 8717aae620e..6c440241322 100644 --- a/libs/dyn/value_test.go +++ b/libs/dyn/value_test.go @@ -89,3 +89,35 @@ func TestIsZero(t *testing.T) { assert.True(t, dyn.V([]dyn.Value{}).IsZero(), "Sequence") assert.False(t, dyn.V([]dyn.Value{dyn.V(0)}).IsZero(), "Sequence") } + +func TestSensitiveValue(t *testing.T) { + v := dyn.NewSensitiveValue("s3cr3t", nil) + + // Kind is still KindString. + assert.Equal(t, dyn.KindString, v.Kind()) + + // IsSensitive detects the wrapper. + assert.True(t, v.IsSensitive()) + + // AsString / MustString return the real value. + s, ok := v.AsString() + assert.True(t, ok) + assert.Equal(t, "s3cr3t", s) + assert.Equal(t, "s3cr3t", v.MustString()) + + // AsAny returns the redaction placeholder. + assert.Equal(t, dyn.SensitiveValueRedacted, v.AsAny()) +} + +func TestPlainStringNotSensitive(t *testing.T) { + v := dyn.V("hello") + assert.False(t, v.IsSensitive()) + assert.Equal(t, "hello", v.AsAny()) +} + +func TestSensitivePreservedByWithLocations(t *testing.T) { + locs := []dyn.Location{{File: "f", Line: 1}} + v := dyn.NewSensitiveValue("s3cr3t", nil).WithLocations(locs) + assert.True(t, v.IsSensitive()) + assert.Equal(t, locs, v.Locations()) +} diff --git a/libs/dyn/value_underlying.go b/libs/dyn/value_underlying.go index a33ecd38ed8..ccdca65a026 100644 --- a/libs/dyn/value_underlying.go +++ b/libs/dyn/value_underlying.go @@ -40,9 +40,22 @@ func (v Value) MustSequence() []Value { // AsString returns the underlying string if this value is a string, // the zero value and false otherwise. +// For sensitive values (stored as [secretString]) it returns the real plaintext. func (v Value) AsString() (string, bool) { - vv, ok := v.v.(string) - return vv, ok + switch vv := v.v.(type) { + case string: + return vv, true + case secretString: + return vv.value, true + default: + return "", false + } +} + +// IsSensitive reports whether this value carries a sensitive string. +func (v Value) IsSensitive() bool { + _, ok := v.v.(secretString) + return ok } // MustString returns the underlying string if this value is a string, diff --git a/libs/dyn/yamlsaver/saver.go b/libs/dyn/yamlsaver/saver.go index 4c302e26f0d..01b345923d2 100644 --- a/libs/dyn/yamlsaver/saver.go +++ b/libs/dyn/yamlsaver/saver.go @@ -72,6 +72,10 @@ func (s *saver) toYamlNode(v dyn.Value) (*yaml.Node, error) { } func (s *saver) toYamlNodeWithStyle(v dyn.Value, style yaml.Style) (*yaml.Node, error) { + if v.IsSensitive() { + return &yaml.Node{Kind: yaml.ScalarNode, Value: dyn.SensitiveValueRedacted, Style: style}, nil + } + switch v.Kind() { case dyn.KindMap: m, _ := v.AsMap() diff --git a/libs/structs/structtag/bundletag.go b/libs/structs/structtag/bundletag.go index 9b7bb2d0ac2..2fd5f19a29d 100644 --- a/libs/structs/structtag/bundletag.go +++ b/libs/structs/structtag/bundletag.go @@ -11,3 +11,9 @@ func (tag BundleTag) ReadOnly() bool { func (tag BundleTag) Internal() bool { return hasOption(string(tag), "internal") } + +// Sensitive reports whether the field holds a value that must be redacted +// when rendering configuration or plan output (e.g. secret values). +func (tag BundleTag) Sensitive() bool { + return hasOption(string(tag), "sensitive") +}