diff --git a/.nextchanges/bundles/empty-alias-minimal.md b/.nextchanges/bundles/empty-alias-minimal.md new file mode 100644 index 00000000000..c7cd17477c5 --- /dev/null +++ b/.nextchanges/bundles/empty-alias-minimal.md @@ -0,0 +1 @@ +Simplified the `default-minimal` bundle template and added an alias `databricks bundle init empty` ([#5899](https://github.com/databricks/cli/pull/5899)). diff --git a/acceptance/bundle/templates/default-minimal/python/input.json b/acceptance/bundle/templates/default-minimal/python/input.json new file mode 100644 index 00000000000..c620c3159b5 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/python/input.json @@ -0,0 +1,4 @@ +{ + "project_name": "my_default_minimal", + "language_choice": "python" +} diff --git a/acceptance/bundle/templates/default-minimal/out.test.toml b/acceptance/bundle/templates/default-minimal/python/out.test.toml similarity index 100% rename from acceptance/bundle/templates/default-minimal/out.test.toml rename to acceptance/bundle/templates/default-minimal/python/out.test.toml diff --git a/acceptance/bundle/templates/default-minimal/output.txt b/acceptance/bundle/templates/default-minimal/python/output.txt similarity index 100% rename from acceptance/bundle/templates/default-minimal/output.txt rename to acceptance/bundle/templates/default-minimal/python/output.txt diff --git a/acceptance/bundle/templates/default-minimal/output/my_default_minimal/.vscode/__builtins__.pyi b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/.vscode/__builtins__.pyi similarity index 100% rename from acceptance/bundle/templates/default-minimal/output/my_default_minimal/.vscode/__builtins__.pyi rename to acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/.vscode/__builtins__.pyi diff --git a/acceptance/bundle/templates/default-minimal/output/my_default_minimal/.vscode/extensions.json b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/.vscode/extensions.json similarity index 53% rename from acceptance/bundle/templates/default-minimal/output/my_default_minimal/.vscode/extensions.json rename to acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/.vscode/extensions.json index 5ba48e79c9b..b958aacfcb4 100644 --- a/acceptance/bundle/templates/default-minimal/output/my_default_minimal/.vscode/extensions.json +++ b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ + "charliermarsh.ruff", "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/bundle/templates/default-minimal/output/my_default_minimal/.vscode/settings.json b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/.vscode/settings.json similarity index 100% rename from acceptance/bundle/templates/default-minimal/output/my_default_minimal/.vscode/settings.json rename to acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/.vscode/settings.json diff --git a/acceptance/bundle/templates/default-minimal/output/my_default_minimal/README.md b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/README.md similarity index 87% rename from acceptance/bundle/templates/default-minimal/output/my_default_minimal/README.md rename to acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/README.md index 4304687ae33..8af9ac7e000 100644 --- a/acceptance/bundle/templates/default-minimal/output/my_default_minimal/README.md +++ b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/README.md @@ -2,8 +2,11 @@ The 'my_default_minimal' project was generated by using the default-minimal template. -* `src/`: SQL source code for this project. +* `src/`: Python source code for this project. * `resources/`: Resource configurations (jobs, pipelines, etc.) +* `tests/`: Unit tests for the shared Python code. +* `fixtures/`: Fixtures for data sets (primarily used for testing). + ## Getting started @@ -52,3 +55,8 @@ with this project. It's also possible to interact with it directly using the CLI ``` $ databricks bundle run ``` + +5. Finally, to run tests locally, use `pytest`: + ``` + $ uv run pytest + ``` diff --git a/acceptance/bundle/templates/default-minimal/output/my_default_minimal/databricks.yml b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/databricks.yml similarity index 100% rename from acceptance/bundle/templates/default-minimal/output/my_default_minimal/databricks.yml rename to acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/databricks.yml diff --git a/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/fixtures/.gitkeep b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/fixtures/.gitkeep new file mode 100644 index 00000000000..77a906614cb --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/fixtures/.gitkeep @@ -0,0 +1,9 @@ +# Test fixtures directory + +Add JSON or CSV files here. In tests, use them with `load_fixture()`: + +``` +def test_using_fixture(load_fixture): + data = load_fixture("my_data.json") + assert len(data) >= 1 +``` diff --git a/acceptance/bundle/templates/default-minimal/output/my_default_minimal/out.gitignore b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/out.gitignore similarity index 100% rename from acceptance/bundle/templates/default-minimal/output/my_default_minimal/out.gitignore rename to acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/out.gitignore diff --git a/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/pyproject.toml b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/pyproject.toml new file mode 100644 index 00000000000..4678f9c4368 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "my_default_minimal" +version = "0.0.1" +authors = [{ name = "[USERNAME]" }] +requires-python = ">=3.10,<3.13" +dependencies = [ + # Any dependencies for jobs and pipelines in this project can be added here + # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies + # + # LIMITATION: for pipelines, dependencies are cached during development; + # add dependencies to the 'environment' section of your pipeline.yml file instead +] + +[dependency-groups] +dev = [ + "pytest", + "ruff", + "databricks-dlt", + "databricks-connect>=15.4,<15.5", + "ipykernel", +] + +[project.scripts] +main = "my_default_minimal.main:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src"] + +[tool.ruff] +line-length = 120 diff --git a/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/tests/conftest.py b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/tests/conftest.py new file mode 100644 index 00000000000..72ebfeb5666 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/tests/conftest.py @@ -0,0 +1,94 @@ +"""This file configures pytest, initializes Databricks Connect, and provides fixtures for Spark and loading test data.""" + +import os, sys, pathlib +from contextlib import contextmanager + + +try: + from databricks.connect import DatabricksSession + from databricks.sdk import WorkspaceClient + from pyspark.sql import SparkSession + import pytest + import json + import csv + import os +except ImportError: + raise ImportError( + "Test dependencies not found.\n\nRun tests using 'uv run pytest'. See http://docs.astral.sh/uv to learn more about uv." + ) + + +@pytest.fixture() +def spark() -> SparkSession: + """Provide a SparkSession fixture for tests. + + Minimal example: + def test_uses_spark(spark): + df = spark.createDataFrame([(1,)], ["x"]) + assert df.count() == 1 + """ + return DatabricksSession.builder.getOrCreate() + + +@pytest.fixture() +def load_fixture(spark: SparkSession): + """Provide a callable to load JSON or CSV from fixtures/ directory. + + Example usage: + + def test_using_fixture(load_fixture): + data = load_fixture("my_data.json") + assert data.count() >= 1 + """ + + def _loader(filename: str): + path = pathlib.Path(__file__).parent.parent / "fixtures" / filename + suffix = path.suffix.lower() + if suffix == ".json": + rows = json.loads(path.read_text()) + return spark.createDataFrame(rows) + if suffix == ".csv": + with path.open(newline="") as f: + rows = list(csv.DictReader(f)) + return spark.createDataFrame(rows) + raise ValueError(f"Unsupported fixture type for: {filename}") + + return _loader + + +def _enable_fallback_compute(): + """Enable serverless compute if no compute is specified.""" + conf = WorkspaceClient().config + if conf.serverless_compute_id or conf.cluster_id or os.environ.get("SPARK_REMOTE"): + return + + url = "https://docs.databricks.com/dev-tools/databricks-connect/cluster-config" + print("☁️ no compute specified, falling back to serverless compute", file=sys.stderr) + print(f" see {url} for manual configuration", file=sys.stdout) + + os.environ["DATABRICKS_SERVERLESS_COMPUTE_ID"] = "auto" + + +@contextmanager +def _allow_stderr_output(config: pytest.Config): + """Temporarily disable pytest output capture.""" + capman = config.pluginmanager.get_plugin("capturemanager") + if capman: + with capman.global_and_fixture_disabled(): + yield + else: + yield + + +def pytest_configure(config: pytest.Config): + """Configure pytest session.""" + with _allow_stderr_output(config): + _enable_fallback_compute() + + # Initialize Spark session eagerly, so it is available even when + # SparkSession.builder.getOrCreate() is used. For DB Connect 15+, + # we validate version compatibility with the remote cluster. + if hasattr(DatabricksSession.builder, "validateSession"): + DatabricksSession.builder.validateSession().getOrCreate() + else: + DatabricksSession.builder.getOrCreate() diff --git a/acceptance/bundle/templates/default-minimal/script b/acceptance/bundle/templates/default-minimal/python/script similarity index 100% rename from acceptance/bundle/templates/default-minimal/script rename to acceptance/bundle/templates/default-minimal/python/script diff --git a/acceptance/bundle/templates/default-minimal/skip/input.json b/acceptance/bundle/templates/default-minimal/skip/input.json new file mode 100644 index 00000000000..b66e679822d --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/input.json @@ -0,0 +1,4 @@ +{ + "project_name": "my_default_minimal", + "language_choice": "skip" +} diff --git a/acceptance/bundle/templates/default-minimal/skip/out.test.toml b/acceptance/bundle/templates/default-minimal/skip/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/skip/output.txt b/acceptance/bundle/templates/default-minimal/skip/output.txt new file mode 100644 index 00000000000..06eb69b9a11 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/output.txt @@ -0,0 +1,33 @@ + +>>> [CLI] bundle init default-minimal --config-file ./input.json --output-dir output +Welcome to the minimal Declarative Automation Bundle template! + +This template creates a minimal project structure without sample code, ideal for advanced users. +(For getting started with Python or SQL code, use the default-python or default-sql templates instead.) + +Your workspace at [DATABRICKS_URL] is used for initialization. +(See https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile.) + +✨ Your new project has been created in the 'my_default_minimal' directory! + +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. + +>>> [CLI] bundle validate -t dev +Name: my_default_minimal +Target: dev +Workspace: + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_minimal/dev + +Validation OK! + +>>> [CLI] bundle validate -t prod +Name: my_default_minimal +Target: prod +Workspace: + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_minimal/prod + +Validation OK! diff --git a/libs/template/templates/default/template/{{.project_name}}/.vscode/extensions.json b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/.vscode/extensions.json similarity index 53% rename from libs/template/templates/default/template/{{.project_name}}/.vscode/extensions.json rename to acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/.vscode/extensions.json index 5ba48e79c9b..1f39c330871 100644 --- a/libs/template/templates/default/template/{{.project_name}}/.vscode/extensions.json +++ b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/.vscode/extensions.json @@ -1,7 +1,6 @@ { "recommendations": [ "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/.vscode/settings.json b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/.vscode/settings.json new file mode 100644 index 00000000000..d3e814885cb --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "files.associations": { + "**/.gitkeep": "markdown" + }, +} diff --git a/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/README.md b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/README.md new file mode 100644 index 00000000000..420f11cf53b --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/README.md @@ -0,0 +1,47 @@ +# my_default_minimal + +The 'my_default_minimal' project was generated by using the default-minimal template. + +* `src/`: Any source code for this project. +* `resources/`: Resource configurations (jobs, pipelines, etc.) + +## Getting started + +Choose how you want to work on this project: + +(a) Directly in your Databricks workspace, see + https://docs.databricks.com/dev-tools/bundles/workspace. + +(b) Locally with an IDE like Cursor or VS Code, see + https://docs.databricks.com/dev-tools/vscode-ext.html. + +(c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html + +# Using this project using the CLI + +The Databricks workspace and IDE extensions provide a graphical interface for working +with this project. It's also possible to interact with it directly using the CLI: + +1. Authenticate to your Databricks workspace, if you have not done so already: + ``` + $ databricks configure + ``` + +2. To deploy a development copy of this project, type: + ``` + $ databricks bundle deploy --target dev + ``` + (Note that "dev" is the default target, so the `--target` parameter + is optional here.) + + This deploys everything that's defined for this project. + +3. Similarly, to deploy a production copy, type: + ``` + $ databricks bundle deploy --target prod + ``` + +4. To run a job or pipeline, use the "run" command: + ``` + $ databricks bundle run + ``` diff --git a/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/databricks.yml b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/databricks.yml new file mode 100644 index 00000000000..ec72bde8b31 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/databricks.yml @@ -0,0 +1,41 @@ +# This is a Declarative Automation Bundle definition for my_default_minimal. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +bundle: + name: my_default_minimal + uuid: [UUID] + +include: + - resources/*.yml + +# Variable declarations. These variables are assigned in the dev/prod targets below. +variables: + catalog: + description: The catalog to use + schema: + description: The schema to use + +targets: + dev: + # The default target uses 'mode: development' to create a development copy. + # - Deployed resources get prefixed with '[dev my_user_name]' + # - Any job schedules and triggers are paused by default. + # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. + mode: development + default: true + workspace: + host: [DATABRICKS_URL] + variables: + catalog: hive_metastore + schema: ${workspace.current_user.short_name} + prod: + mode: production + workspace: + host: [DATABRICKS_URL] + # We explicitly deploy to /Workspace/Users/[USERNAME] to make sure we only have a single copy. + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + variables: + catalog: hive_metastore + schema: prod + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE diff --git a/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/out.gitignore b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/out.gitignore new file mode 100644 index 00000000000..622a8811a7b --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/out.gitignore @@ -0,0 +1,5 @@ +.databricks/ +scratch/** +!scratch/README.md +**/explorations/** +**/!explorations/README.md diff --git a/acceptance/bundle/templates/default-minimal/skip/script b/acceptance/bundle/templates/default-minimal/skip/script new file mode 100644 index 00000000000..bce20c6ed9d --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/script @@ -0,0 +1,16 @@ +trace $CLI bundle init default-minimal --config-file ./input.json --output-dir output + +cd output/my_default_minimal + +# Verify that empty directories are preserved +[ -d "src" ] || exit 1 +[ -d "resources" ] || exit 1 + +trace $CLI bundle validate -t dev +trace $CLI bundle validate -t prod + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore +rm -r .databricks + +cd ../../ diff --git a/acceptance/bundle/templates/default-minimal/input.json b/acceptance/bundle/templates/default-minimal/sql/input.json similarity index 100% rename from acceptance/bundle/templates/default-minimal/input.json rename to acceptance/bundle/templates/default-minimal/sql/input.json diff --git a/acceptance/bundle/templates/default-minimal/sql/out.test.toml b/acceptance/bundle/templates/default-minimal/sql/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/sql/output.txt b/acceptance/bundle/templates/default-minimal/sql/output.txt new file mode 100644 index 00000000000..06eb69b9a11 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/output.txt @@ -0,0 +1,33 @@ + +>>> [CLI] bundle init default-minimal --config-file ./input.json --output-dir output +Welcome to the minimal Declarative Automation Bundle template! + +This template creates a minimal project structure without sample code, ideal for advanced users. +(For getting started with Python or SQL code, use the default-python or default-sql templates instead.) + +Your workspace at [DATABRICKS_URL] is used for initialization. +(See https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile.) + +✨ Your new project has been created in the 'my_default_minimal' directory! + +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. + +>>> [CLI] bundle validate -t dev +Name: my_default_minimal +Target: dev +Workspace: + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_minimal/dev + +Validation OK! + +>>> [CLI] bundle validate -t prod +Name: my_default_minimal +Target: prod +Workspace: + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_minimal/prod + +Validation OK! diff --git a/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/.vscode/extensions.json b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/.vscode/extensions.json new file mode 100644 index 00000000000..1f39c330871 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/.vscode/extensions.json @@ -0,0 +1,6 @@ +{ + "recommendations": [ + "databricks.databricks", + "redhat.vscode-yaml" + ] +} diff --git a/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/.vscode/settings.json b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/.vscode/settings.json new file mode 100644 index 00000000000..d3e814885cb --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "files.associations": { + "**/.gitkeep": "markdown" + }, +} diff --git a/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/README.md b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/README.md new file mode 100644 index 00000000000..ace4bdc7d74 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/README.md @@ -0,0 +1,47 @@ +# my_default_minimal + +The 'my_default_minimal' project was generated by using the default-minimal template. + +* `src/`: SQL source code for this project. +* `resources/`: Resource configurations (jobs, pipelines, etc.) + +## Getting started + +Choose how you want to work on this project: + +(a) Directly in your Databricks workspace, see + https://docs.databricks.com/dev-tools/bundles/workspace. + +(b) Locally with an IDE like Cursor or VS Code, see + https://docs.databricks.com/dev-tools/vscode-ext.html. + +(c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html + +# Using this project using the CLI + +The Databricks workspace and IDE extensions provide a graphical interface for working +with this project. It's also possible to interact with it directly using the CLI: + +1. Authenticate to your Databricks workspace, if you have not done so already: + ``` + $ databricks configure + ``` + +2. To deploy a development copy of this project, type: + ``` + $ databricks bundle deploy --target dev + ``` + (Note that "dev" is the default target, so the `--target` parameter + is optional here.) + + This deploys everything that's defined for this project. + +3. Similarly, to deploy a production copy, type: + ``` + $ databricks bundle deploy --target prod + ``` + +4. To run a job or pipeline, use the "run" command: + ``` + $ databricks bundle run + ``` diff --git a/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/databricks.yml b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/databricks.yml new file mode 100644 index 00000000000..ec72bde8b31 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/databricks.yml @@ -0,0 +1,41 @@ +# This is a Declarative Automation Bundle definition for my_default_minimal. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +bundle: + name: my_default_minimal + uuid: [UUID] + +include: + - resources/*.yml + +# Variable declarations. These variables are assigned in the dev/prod targets below. +variables: + catalog: + description: The catalog to use + schema: + description: The schema to use + +targets: + dev: + # The default target uses 'mode: development' to create a development copy. + # - Deployed resources get prefixed with '[dev my_user_name]' + # - Any job schedules and triggers are paused by default. + # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. + mode: development + default: true + workspace: + host: [DATABRICKS_URL] + variables: + catalog: hive_metastore + schema: ${workspace.current_user.short_name} + prod: + mode: production + workspace: + host: [DATABRICKS_URL] + # We explicitly deploy to /Workspace/Users/[USERNAME] to make sure we only have a single copy. + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + variables: + catalog: hive_metastore + schema: prod + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE diff --git a/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/out.gitignore b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/out.gitignore new file mode 100644 index 00000000000..622a8811a7b --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/out.gitignore @@ -0,0 +1,5 @@ +.databricks/ +scratch/** +!scratch/README.md +**/explorations/** +**/!explorations/README.md diff --git a/acceptance/bundle/templates/default-minimal/sql/script b/acceptance/bundle/templates/default-minimal/sql/script new file mode 100644 index 00000000000..bce20c6ed9d --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/script @@ -0,0 +1,16 @@ +trace $CLI bundle init default-minimal --config-file ./input.json --output-dir output + +cd output/my_default_minimal + +# Verify that empty directories are preserved +[ -d "src" ] || exit 1 +[ -d "resources" ] || exit 1 + +trace $CLI bundle validate -t dev +trace $CLI bundle validate -t prod + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore +rm -r .databricks + +cd ../../ diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/extensions.json b/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/extensions.json index 5ba48e79c9b..b958aacfcb4 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/extensions.json +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ + "charliermarsh.ruff", "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json index 5ba48e79c9b..b958aacfcb4 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ + "charliermarsh.ruff", "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/.vscode/extensions.json b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/.vscode/extensions.json index 5ba48e79c9b..b958aacfcb4 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/.vscode/extensions.json +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ + "charliermarsh.ruff", "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/__builtins__.pyi b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/__builtins__.pyi deleted file mode 100644 index 0edd5181bc5..00000000000 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/__builtins__.pyi +++ /dev/null @@ -1,3 +0,0 @@ -# Typings for Pylance in Visual Studio Code -# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md -from databricks.sdk.runtime import * diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/extensions.json b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/extensions.json index 5ba48e79c9b..1f39c330871 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/extensions.json +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/extensions.json @@ -1,7 +1,6 @@ { "recommendations": [ "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/settings.json b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/settings.json index d73c73b5705..d3e814885cb 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/settings.json +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/settings.json @@ -1,39 +1,5 @@ { - "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", - "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", - "python.testing.pytestArgs": [ - "." - ], - "files.exclude": { - "**/*.egg-info": true, - "**/__pycache__": true, - ".pytest_cache": true, - "dist": true, - }, "files.associations": { "**/.gitkeep": "markdown" }, - - // Pylance settings (VS Code) - // Set typeCheckingMode to "basic" to enable type checking! - "python.analysis.typeCheckingMode": "off", - "python.analysis.extraPaths": ["src", "lib", "resources"], - "python.analysis.diagnosticMode": "workspace", - "python.analysis.stubPath": ".vscode", - - // Pyright settings (Cursor) - // Set typeCheckingMode to "basic" to enable type checking! - "cursorpyright.analysis.typeCheckingMode": "off", - "cursorpyright.analysis.extraPaths": ["src", "lib", "resources"], - "cursorpyright.analysis.diagnosticMode": "workspace", - "cursorpyright.analysis.stubPath": ".vscode", - - // General Python settings - "python.defaultInterpreterPath": "./.venv/bin/python", - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, - "[python]": { - "editor.defaultFormatter": "charliermarsh.ruff", - "editor.formatOnSave": true, - }, } diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/out.gitignore b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/out.gitignore index e566c51f740..622a8811a7b 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/out.gitignore +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/out.gitignore @@ -1,9 +1,4 @@ .databricks/ -build/ -dist/ -__pycache__/ -*.egg-info -.venv/ scratch/** !scratch/README.md **/explorations/** diff --git a/acceptance/pipelines/e2e/output/lakeflow_project/.vscode/extensions.json b/acceptance/pipelines/e2e/output/lakeflow_project/.vscode/extensions.json index 5ba48e79c9b..b958aacfcb4 100644 --- a/acceptance/pipelines/e2e/output/lakeflow_project/.vscode/extensions.json +++ b/acceptance/pipelines/e2e/output/lakeflow_project/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ + "charliermarsh.ruff", "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/pipelines/init/python/output/my_python_project/.vscode/extensions.json b/acceptance/pipelines/init/python/output/my_python_project/.vscode/extensions.json index 5ba48e79c9b..b958aacfcb4 100644 --- a/acceptance/pipelines/init/python/output/my_python_project/.vscode/extensions.json +++ b/acceptance/pipelines/init/python/output/my_python_project/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ + "charliermarsh.ruff", "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/__builtins__.pyi b/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/__builtins__.pyi deleted file mode 100644 index 0edd5181bc5..00000000000 --- a/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/__builtins__.pyi +++ /dev/null @@ -1,3 +0,0 @@ -# Typings for Pylance in Visual Studio Code -# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md -from databricks.sdk.runtime import * diff --git a/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/extensions.json b/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/extensions.json index 5ba48e79c9b..1f39c330871 100644 --- a/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/extensions.json +++ b/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/extensions.json @@ -1,7 +1,6 @@ { "recommendations": [ "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/settings.json b/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/settings.json index d73c73b5705..d3e814885cb 100644 --- a/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/settings.json +++ b/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/settings.json @@ -1,39 +1,5 @@ { - "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", - "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", - "python.testing.pytestArgs": [ - "." - ], - "files.exclude": { - "**/*.egg-info": true, - "**/__pycache__": true, - ".pytest_cache": true, - "dist": true, - }, "files.associations": { "**/.gitkeep": "markdown" }, - - // Pylance settings (VS Code) - // Set typeCheckingMode to "basic" to enable type checking! - "python.analysis.typeCheckingMode": "off", - "python.analysis.extraPaths": ["src", "lib", "resources"], - "python.analysis.diagnosticMode": "workspace", - "python.analysis.stubPath": ".vscode", - - // Pyright settings (Cursor) - // Set typeCheckingMode to "basic" to enable type checking! - "cursorpyright.analysis.typeCheckingMode": "off", - "cursorpyright.analysis.extraPaths": ["src", "lib", "resources"], - "cursorpyright.analysis.diagnosticMode": "workspace", - "cursorpyright.analysis.stubPath": ".vscode", - - // General Python settings - "python.defaultInterpreterPath": "./.venv/bin/python", - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, - "[python]": { - "editor.defaultFormatter": "charliermarsh.ruff", - "editor.formatOnSave": true, - }, } diff --git a/acceptance/pipelines/init/sql/output/my_sql_project/out.gitignore b/acceptance/pipelines/init/sql/output/my_sql_project/out.gitignore index e566c51f740..622a8811a7b 100644 --- a/acceptance/pipelines/init/sql/output/my_sql_project/out.gitignore +++ b/acceptance/pipelines/init/sql/output/my_sql_project/out.gitignore @@ -1,9 +1,4 @@ .databricks/ -build/ -dist/ -__pycache__/ -*.egg-info -.venv/ scratch/** !scratch/README.md **/explorations/** diff --git a/libs/template/template.go b/libs/template/template.go index b8448936bea..cb2e1a8e3a2 100644 --- a/libs/template/template.go +++ b/libs/template/template.go @@ -56,6 +56,7 @@ var databricksTemplates = []Template{ { name: DefaultMinimal, description: "The minimal template, for advanced users", + aliases: []string{"empty"}, Reader: &builtinReader{name: string(DefaultMinimal)}, Writer: &writerWithFullTelemetry{defaultWriter: defaultWriter{name: DefaultMinimal}}, }, diff --git a/libs/template/template_test.go b/libs/template/template_test.go index 4692f0acb2a..b0a5b6c641b 100644 --- a/libs/template/template_test.go +++ b/libs/template/template_test.go @@ -84,4 +84,5 @@ func TestTemplateGetDatabricksTemplate(t *testing.T) { // Assert aliases work. assert.Equal(t, MlopsStacks, GetDatabricksTemplate(TemplateName("mlops-stack")).name) + assert.Equal(t, DefaultMinimal, GetDatabricksTemplate(TemplateName("empty")).name) } diff --git a/libs/template/templates/default-minimal/databricks_template_schema.json b/libs/template/templates/default-minimal/databricks_template_schema.json index 7faec4a5cb7..ba74e1160e2 100644 --- a/libs/template/templates/default-minimal/databricks_template_schema.json +++ b/libs/template/templates/default-minimal/databricks_template_schema.json @@ -63,13 +63,15 @@ "order": 6 }, "personal_schemas": { + "//": "Always 'yes' for default-minimal: development uses a personal schema based on the current user name", + "skip_prompt_if": {}, "type": "string", - "description": "Use a personal schema for each user working on this project\n(this is recommended, your personal schema will be '{{.default_catalog}}.{{short_name}}')", "default": "yes", "enum": [ "yes", "no, I will customize the schema configuration later in databricks.yml" ], + "description": "Use a personal schema for each user working on this project", "order": 7 }, "language_choice": { @@ -79,15 +81,15 @@ "enum": [ "python", "sql", - "other" + "skip" ], "order": 8 }, "language": { - "//": "Derived from language_choice: for Python, pyproject.toml and tests are included; for others they are excluded", + "//": "Mirrors language_choice: 'python' includes pyproject.toml and tests, 'sql' includes SQL sources, 'skip' includes neither", "skip_prompt_if": {}, "type": "string", - "default": "{{if eq .language_choice \"python\"}}python{{else}}sql{{end}}", + "default": "{{.language_choice}}", "description": "Derived language property for template logic", "order": 9 }, diff --git a/libs/template/templates/default/template/__preamble.tmpl b/libs/template/templates/default/template/__preamble.tmpl index b3f1d25039c..5bbdc888e2b 100644 --- a/libs/template/templates/default/template/__preamble.tmpl +++ b/libs/template/templates/default/template/__preamble.tmpl @@ -8,6 +8,7 @@ This file only contains template directives; it is skipped for the actual output {{$notebook_job := eq .include_job "yes"}} {{$python_package := eq .include_python "yes"}} {{$sql_language := eq .language "sql"}} +{{$python_language := eq .language "python"}} {{$lakeflow_only := eq .lakeflow_only "yes"}} {{$pydabs := eq .enable_pydabs "yes"}} {{$has_python_package_dir := and (or $python_package $notebook_job $pipeline) (not $lakeflow_only)}} @@ -35,15 +36,16 @@ This file only contains template directives; it is skipped for the actual output {{skip "{{.project_name}}/src/{{.project_name}}_etl/transformations/*.sql"}} {{end}} -{{if or $sql_language $lakeflow_only}} +{{if or (not $python_language) $lakeflow_only}} {{skip "{{.project_name}}/tests"}} {{skip "{{.project_name}}/fixtures"}} {{else if not (or $python_package $notebook_job $pipeline)}} {{skip "{{.project_name}}/tests/sample_*.py"}} {{end}} -{{if $sql_language}} +{{if not $python_language}} {{skip "{{.project_name}}/pyproject.toml"}} + {{skip "{{.project_name}}/.vscode/__builtins__.pyi"}} {{end}} {{if $pydabs}} diff --git a/libs/template/templates/default/template/{{.project_name}}/.gitignore.tmpl b/libs/template/templates/default/template/{{.project_name}}/.gitignore.tmpl index e566c51f740..e400f0256cc 100644 --- a/libs/template/templates/default/template/{{.project_name}}/.gitignore.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/.gitignore.tmpl @@ -1,9 +1,11 @@ .databricks/ +{{- if eq .language "python"}} build/ dist/ __pycache__/ *.egg-info .venv/ +{{- end}} scratch/** !scratch/README.md **/explorations/** diff --git a/libs/template/templates/default/template/{{.project_name}}/.vscode/extensions.json.tmpl b/libs/template/templates/default/template/{{.project_name}}/.vscode/extensions.json.tmpl new file mode 100644 index 00000000000..f1cf722e7fc --- /dev/null +++ b/libs/template/templates/default/template/{{.project_name}}/.vscode/extensions.json.tmpl @@ -0,0 +1,9 @@ +{ + "recommendations": [ +{{- if eq .language "python"}} + "charliermarsh.ruff", +{{- end}} + "databricks.databricks", + "redhat.vscode-yaml" + ] +} diff --git a/libs/template/templates/default/template/{{.project_name}}/.vscode/settings.json b/libs/template/templates/default/template/{{.project_name}}/.vscode/settings.json.tmpl similarity index 94% rename from libs/template/templates/default/template/{{.project_name}}/.vscode/settings.json rename to libs/template/templates/default/template/{{.project_name}}/.vscode/settings.json.tmpl index d73c73b5705..e1dae324628 100644 --- a/libs/template/templates/default/template/{{.project_name}}/.vscode/settings.json +++ b/libs/template/templates/default/template/{{.project_name}}/.vscode/settings.json.tmpl @@ -1,4 +1,5 @@ { +{{- if eq .language "python"}} "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", "python.testing.pytestArgs": [ @@ -10,9 +11,11 @@ ".pytest_cache": true, "dist": true, }, +{{- end}} "files.associations": { "**/.gitkeep": "markdown" }, +{{- if eq .language "python"}} // Pylance settings (VS Code) // Set typeCheckingMode to "basic" to enable type checking! @@ -36,4 +39,5 @@ "editor.defaultFormatter": "charliermarsh.ruff", "editor.formatOnSave": true, }, +{{- end}} } diff --git a/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl index bca68edc2fb..486df502f4e 100644 --- a/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl @@ -1,12 +1,12 @@ # {{.project_name}} -{{- $skip_tests := or (eq .language "sql") (eq .lakeflow_only "yes")}} +{{- $skip_tests := or (ne .language "python") (eq .lakeflow_only "yes")}} {{- $has_python_package_dir := and (or (eq .include_python "yes") (eq .include_job "yes") (eq .include_pipeline "yes")) (not (eq .lakeflow_only "yes"))}} {{- $is_lakeflow := eq .lakeflow_only "yes"}} The '{{.project_name}}' project was generated by using the {{.template_name}} template. -* `src/`: {{if eq .language "sql"}}SQL{{else}}Python{{end}} source code for this project. +* `src/`: {{if eq .language "sql"}}SQL{{else if eq .language "python"}}Python{{else}}Any{{end}} source code for this project. {{- if $has_python_package_dir}} * `src/{{.project_name}}/`: Shared Python code that can be used by jobs and pipelines. {{- end}} @@ -28,7 +28,7 @@ Choose how you want to work on this project: (c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html -{{- if not $is_lakeflow}} +{{- if and (eq .language "python") (not $is_lakeflow)}} If you're developing with an IDE, dependencies for this project should be installed using uv: