From ec78345480d52257a8f5b8df8125ab68849d00ff Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Mon, 13 Oct 2025 13:40:26 +0200 Subject: [PATCH 01/24] [DECO-25663] Replace default-python template with improved version - Update default-python to use template_dir referencing the shared 'default' template - Add comprehensive properties: project_name_short, include_job, include_pipeline, default_catalog, personal_schemas, language, and lakeflow_only - Update welcome and success messages to provide better guidance - Remove experimental-default-python-vnext template - Update default template README to reference default-python instead of experimental variant --- NEXT_CHANGELOG.md | 1 + libs/template/template.go | 28 +++----- .../databricks_template_schema.json | 71 +++++++++++++++---- libs/template/templates/default/README.md | 2 +- 4 files changed, 71 insertions(+), 31 deletions(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 454dcc43a5..714bfe74a4 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -11,6 +11,7 @@ ### Dependency updates ### Bundles +* Updated the default-python template with improved structure and functionality. * Updated the internal lakeflow-pipelines template to use an "src" layout ([#3671](https://github.com/databricks/cli/pull/3671)). ### API Changes diff --git a/libs/template/template.go b/libs/template/template.go index 74de09a857..9fa7657785 100644 --- a/libs/template/template.go +++ b/libs/template/template.go @@ -24,17 +24,16 @@ type Template struct { type TemplateName string const ( - DefaultPython TemplateName = "default-python" - ExperimentalDefaultPython TemplateName = "experimental-default-python-vnext" - DefaultSql TemplateName = "default-sql" - LakeflowPipelines TemplateName = "lakeflow-pipelines" - CLIPipelines TemplateName = "cli-pipelines" - DbtSql TemplateName = "dbt-sql" - MlopsStacks TemplateName = "mlops-stacks" - DefaultPydabs TemplateName = "default-pydabs" - Custom TemplateName = "custom" - ExperimentalJobsAsCode TemplateName = "experimental-jobs-as-code" - Default TemplateName = "default" + DefaultPython TemplateName = "default-python" + DefaultSql TemplateName = "default-sql" + LakeflowPipelines TemplateName = "lakeflow-pipelines" + CLIPipelines TemplateName = "cli-pipelines" + DbtSql TemplateName = "dbt-sql" + MlopsStacks TemplateName = "mlops-stacks" + DefaultPydabs TemplateName = "default-pydabs" + Custom TemplateName = "custom" + ExperimentalJobsAsCode TemplateName = "experimental-jobs-as-code" + Default TemplateName = "default" ) var databricksTemplates = []Template{ @@ -44,13 +43,6 @@ var databricksTemplates = []Template{ Reader: &builtinReader{name: string(DefaultPython)}, Writer: &writerWithFullTelemetry{defaultWriter: defaultWriter{name: DefaultPython}}, }, - { - name: ExperimentalDefaultPython, - hidden: true, - description: "The next version of the default Python template (experimental)", - Reader: &builtinReader{name: string(ExperimentalDefaultPython)}, - Writer: &writerWithFullTelemetry{defaultWriter: defaultWriter{name: ExperimentalDefaultPython}}, - }, { name: DefaultSql, description: "The default SQL template for .sql files that run with Databricks SQL", diff --git a/libs/template/templates/default-python/databricks_template_schema.json b/libs/template/templates/default-python/databricks_template_schema.json index c4207a3b35..a323e4360b 100644 --- a/libs/template/templates/default-python/databricks_template_schema.json +++ b/libs/template/templates/default-python/databricks_template_schema.json @@ -1,42 +1,89 @@ { - "welcome_message": "\nWelcome to the default Python template for Databricks Asset Bundles!", + "template_dir": "../default", + "welcome_message": "Welcome to the default Python template for Databricks Asset Bundles!\n\nPlease answer the below to tailor your project to your preferences.\nYou can always change your mind and change your configuration in the databricks.yml file later.\n\nNote that {{workspace_host}} is used for initialization\n(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile).", "properties": { "project_name": { "type": "string", "default": "my_project", - "description": "Please provide the following details to tailor the template to your preferences.\n\nUnique name for this project", + "description": "\nUnique name for this project", "order": 1, "pattern": "^[A-Za-z0-9_]+$", "pattern_match_failure_message": "Name must consist of letters, numbers, and underscores." }, - "include_notebook": { + "project_name_short": { + "//": "This is a derived property based on project_name (it replaces my_project with sample and strips _project|_app|_service)", + "skip_prompt_if": {}, "type": "string", - "default": "yes", - "enum": ["yes", "no"], - "description": "Include a stub (sample) notebook in '{{.project_name}}{{path_separator}}src'", + "default": "{{if eq .project_name \"my_project\"}}sample{{else}}{{with (regexp \"^(my_)?(.*)(_project|_app|_service)?$\").FindStringSubmatch .project_name}}{{index . 2}}{{else}}{{.project_name}}{{end}}{{end}}", + "description": "Short name for the project", "order": 2 }, - "include_dlt": { + "include_job": { "type": "string", "default": "yes", "enum": ["yes", "no"], - "description": "Include a stub (sample) Lakeflow Declarative Pipeline in '{{.project_name}}{{path_separator}}src'", + "description": "Include a Lakeflow job that runs a notebook", "order": 3 }, - "include_python": { + "include_pipeline": { "type": "string", "default": "yes", "enum": ["yes", "no"], - "description": "Include a stub (sample) Python package in '{{.project_name}}{{path_separator}}src'", + "description": "Include a Lakeflow ETL pipeline", "order": 4 }, + "include_python": { + "type": "string", + "default": "yes", + "enum": ["yes", "no"], + "description": "Include a sample Python package that is built to a wheel file", + "order": 5 + }, "serverless": { "type": "string", "default": "yes", "enum": ["yes", "no"], "description": "Use serverless compute", - "order": 5 + "order": 6 + }, + "default_catalog": { + "type": "string", + "default": "{{default_catalog}}", + "pattern": "^\\w*$", + "pattern_match_failure_message": "Invalid catalog name.", + "description": "Default catalog for any tables created by this project{{if eq (default_catalog) \"\"}} (leave blank when not using Unity Catalog){{end}}", + "order": 7 + }, + "personal_schemas": { + "type": "string", + "description": "Use a personal schema for each user working on this project\n(this is recommended, your personal schema will be '{{.default_catalog}}.{{short_name}}')", + "default": "yes", + "enum": [ + "yes", + "no (advanced: I will customize the schema configuration later in databricks.yml)" + ], + "order": 8 + }, + "language": { + "//": "This property is always set to 'python'", + "skip_prompt_if": {}, + "type": "string", + "default": "python", + "description": "Initial language for this project", + "enum": [ + "python", + "sql" + ], + "order": 9 + }, + "lakeflow_only": { + "//": "This property is always set to 'no' for default-python", + "skip_prompt_if": {}, + "type": "string", + "default": "no", + "description": "Internal flag for lakeflow-only templates", + "order": 10 } }, - "success_message": "Workspace to use (auto-detected, edit in '{{.project_name}}/databricks.yml'): {{workspace_host}}\n\n✨ Your new project has been created in the '{{.project_name}}' directory!\n\nPlease refer to the README.md file for \"getting started\" instructions.\nSee also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html." + "success_message": "\n✨ Your new project has been created in the '{{.project_name}}' directory!\n\nPlease refer to the README.md file for \"getting started\" instructions.\nSee also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html." } diff --git a/libs/template/templates/default/README.md b/libs/template/templates/default/README.md index dcc2136736..87d4e0b656 100644 --- a/libs/template/templates/default/README.md +++ b/libs/template/templates/default/README.md @@ -1,4 +1,4 @@ # Default template This template directory acts as a default template that is referenced -from the lakeflow-pipelines and experimental-default-python-vnext templates. +from the lakeflow-pipelines and default-python templates. From f4479fc35866108e691a32354c05204d741e39ec Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Mon, 13 Oct 2025 13:47:58 +0200 Subject: [PATCH 02/24] Update acceptance test outputs for default-python template --- .../classic/out.compare-vs-serverless.diff | 77 +++---- .../classic/out.plan_dev.direct-exp.json | 76 ++++--- .../classic/out.plan_dev.terraform.json | 4 +- .../classic/out.plan_prod.direct-exp.json | 74 ++++--- .../classic/out.plan_prod.terraform.json | 4 +- .../classic/out.requests.dev.direct-exp.txt | 71 +++++-- .../classic/out.requests.dev.terraform.txt | 73 +++++-- .../classic/out.requests.prod.direct-exp.txt | 69 ++++-- .../classic/out.requests.prod.terraform.txt | 71 +++++-- .../default-python/classic/output.txt | 16 +- .../my_default_python/.vscode/extensions.json | 4 +- .../my_default_python/.vscode/settings.json | 31 ++- .../output/my_default_python/README.md | 26 ++- .../output/my_default_python/databricks.yml | 25 ++- .../my_default_python/fixtures/.gitkeep | 23 +- .../output/my_default_python/out.gitignore | 2 + .../output/my_default_python/pyproject.toml | 28 +-- .../resources/default_python_etl.pipeline.yml | 20 ++ .../resources/my_default_python.pipeline.yml | 14 -- ...ault_python.job.yml => sample_job.job.yml} | 45 ++-- .../my_default_python/scratch/README.md | 4 - .../scratch/exploration.ipynb | 61 ------ .../src/default_python_etl/README.md | 20 ++ .../explorations/sample_exploration.ipynb} | 40 ++-- .../sample_trips_my_default_python.py | 12 ++ .../sample_zones_my_default_python.py | 17 ++ .../src/my_default_python/main.py | 18 +- .../src/my_default_python/taxis.py | 7 + .../my_default_python/src/pipeline.ipynb | 90 -------- .../src/sample_notebook.ipynb | 149 +++++++++++++ .../my_default_python/tests/conftest.py | 63 ++++-- .../my_default_python/tests/main_test.py | 6 - .../tests/sample_taxis_test.py | 8 + .../combinations/classic/output.txt | 8 +- .../combinations/serverless/output.txt | 8 +- .../default-python/fail-missing-uv/output.txt | 8 +- .../integration_classic/out.validate.dev.json | 84 +++++--- .../integration_classic/output.txt | 199 ++++++++++++------ .../templates/default-python/no-uc/output.txt | 8 +- .../serverless-customcatalog/output.txt | 70 +++++- .../default-python/serverless/output.txt | 8 +- .../my_default_python/.vscode/extensions.json | 4 +- .../my_default_python/.vscode/settings.json | 31 ++- .../output/my_default_python/README.md | 26 ++- .../output/my_default_python/databricks.yml | 20 +- .../my_default_python/fixtures/.gitkeep | 23 +- .../output/my_default_python/out.gitignore | 2 + .../output/my_default_python/pyproject.toml | 28 +-- .../resources/default_python_etl.pipeline.yml | 21 ++ .../resources/my_default_python.job.yml | 45 ---- .../resources/my_default_python.pipeline.yml | 15 -- .../resources/sample_job.job.yml | 55 +++++ .../my_default_python/scratch/README.md | 4 - .../scratch/exploration.ipynb | 61 ------ .../src/default_python_etl/README.md | 20 ++ .../explorations/sample_exploration.ipynb} | 40 ++-- .../sample_trips_my_default_python.py | 12 ++ .../sample_zones_my_default_python.py | 17 ++ .../src/my_default_python/main.py | 18 +- .../src/my_default_python/taxis.py | 7 + .../my_default_python/src/pipeline.ipynb | 90 -------- .../src/sample_notebook.ipynb | 149 +++++++++++++ .../my_default_python/tests/conftest.py | 63 ++++-- .../my_default_python/tests/main_test.py | 6 - .../tests/sample_taxis_test.py | 8 + .../default-python/out.databricks.yml | 25 ++- .../telemetry/default-python/out.requests.txt | 6 +- .../telemetry/default-python/output.txt | 22 +- 68 files changed, 1534 insertions(+), 925 deletions(-) create mode 100644 acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/default_python_etl.pipeline.yml delete mode 100644 acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml rename acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/{my_default_python.job.yml => sample_job.job.yml} (58%) delete mode 100644 acceptance/bundle/templates/default-python/classic/output/my_default_python/scratch/README.md delete mode 100644 acceptance/bundle/templates/default-python/classic/output/my_default_python/scratch/exploration.ipynb create mode 100644 acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/README.md rename acceptance/bundle/templates/default-python/classic/output/my_default_python/src/{notebook.ipynb => default_python_etl/explorations/sample_exploration.ipynb} (50%) create mode 100644 acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/transformations/sample_trips_my_default_python.py create mode 100644 acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/transformations/sample_zones_my_default_python.py create mode 100644 acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python/taxis.py delete mode 100644 acceptance/bundle/templates/default-python/classic/output/my_default_python/src/pipeline.ipynb create mode 100644 acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb delete mode 100644 acceptance/bundle/templates/default-python/classic/output/my_default_python/tests/main_test.py create mode 100644 acceptance/bundle/templates/default-python/classic/output/my_default_python/tests/sample_taxis_test.py create mode 100644 acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/default_python_etl.pipeline.yml delete mode 100644 acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.job.yml delete mode 100644 acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml create mode 100644 acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml delete mode 100644 acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/README.md delete mode 100644 acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/exploration.ipynb create mode 100644 acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/README.md rename acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/{notebook.ipynb => default_python_etl/explorations/sample_exploration.ipynb} (50%) create mode 100644 acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/transformations/sample_trips_my_default_python.py create mode 100644 acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/transformations/sample_zones_my_default_python.py create mode 100644 acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/taxis.py delete mode 100644 acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/pipeline.ipynb create mode 100644 acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb delete mode 100644 acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/main_test.py create mode 100644 acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/sample_taxis_test.py diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff index 21370439b4..d3850c8fc0 100644 --- a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff +++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff @@ -1,68 +1,69 @@ --- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/databricks.yml +++ output/my_default_python/databricks.yml -@@ -25,4 +25,11 @@ - host: [DATABRICKS_URL] - +@@ -34,4 +34,6 @@ + catalog: hive_metastore + schema: ${workspace.current_user.short_name} + presets: -+ # Set dynamic_version: true on all artifacts of type "whl". -+ # This makes "bundle deploy" add a timestamp to wheel's version before uploading, -+ # new wheel takes over the previous installation even if actual wheel version is unchanged. -+ # See https://docs.databricks.com/aws/en/dev-tools/bundles/settings + artifacts_dynamic_version: true -+ prod: mode: production ---- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/resources/my_default_python.job.yml -+++ output/my_default_python/resources/my_default_python.job.yml -@@ -17,4 +17,5 @@ - tasks: - - task_key: notebook_task -+ job_cluster_key: job_cluster +--- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/resources/default_python_etl.pipeline.yml ++++ output/my_default_python/resources/default_python_etl.pipeline.yml +@@ -5,8 +5,7 @@ + default_python_etl: + name: default_python_etl +- ## Catalog is required for serverless compute +- catalog: ${var.catalog} ++ ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: ++ # catalog: ${var.catalog} + schema: ${var.schema} +- serverless: true + root_path: "../src/default_python_etl" + +--- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/resources/sample_job.job.yml ++++ output/my_default_python/resources/sample_job.job.yml +@@ -26,5 +26,10 @@ notebook_task: - notebook_path: ../src/notebook.ipynb -@@ -29,17 +30,21 @@ + notebook_path: ../src/sample_notebook.ipynb +- environment_key: default ++ job_cluster_key: job_cluster ++ libraries: ++ # By default we just include the .whl file generated for the my_default_python package. ++ # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html ++ # for more information on how to add other libraries. ++ - whl: ../dist/*.whl + - task_key: python_wheel_task depends_on: - - task_key: refresh_pipeline +@@ -38,5 +43,10 @@ + - "--schema" + - "${var.schema}" - environment_key: default + job_cluster_key: job_cluster - python_wheel_task: - package_name: my_default_python - entry_point: main + libraries: + # By default we just include the .whl file generated for the my_default_python package. + # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html + # for more information on how to add other libraries. + - whl: ../dist/*.whl + - task_key: refresh_pipeline + depends_on: +@@ -45,11 +55,11 @@ + pipeline_id: ${resources.pipelines.default_python_etl.id} -- # A list of task execution environment specifications that can be referenced by tasks of this job. - environments: - - environment_key: default -- -- # Full documentation of this spec can be found at: -- # https://docs.databricks.com/api/workspace/jobs/create#environments-spec - spec: - environment_version: "2" - dependencies: +- # By default we just include the .whl file generated for the my_default_python package. +- # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html +- # for more information on how to add other libraries. - - ../dist/*.whl + job_clusters: + - job_cluster_key: job_cluster + new_cluster: -+ spark_version: 15.4.x-scala2.12 ++ spark_version: 16.4.x-scala2.12 + node_type_id: [NODE_TYPE_ID] + data_security_mode: SINGLE_USER + autoscale: + min_workers: 1 + max_workers: 4 ---- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/resources/my_default_python.pipeline.yml -+++ output/my_default_python/resources/my_default_python.pipeline.yml -@@ -4,8 +4,7 @@ - my_default_python_pipeline: - name: my_default_python_pipeline -- ## Catalog is required for serverless compute -- catalog: main -+ ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: -+ # catalog: catalog_name - schema: my_default_python_${bundle.target} -- serverless: true - libraries: - - notebook: diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct-exp.json b/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct-exp.json index b46b9a42a2..a51ef092ea 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct-exp.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct-exp.json @@ -1,10 +1,10 @@ { "plan": { - "resources.jobs.my_default_python_job": { + "resources.jobs.sample_job": { "depends_on": [ { - "node": "resources.pipelines.my_default_python_pipeline", - "label": "${resources.pipelines.my_default_python_pipeline.id}" + "node": "resources.pipelines.default_python_etl", + "label": "${resources.pipelines.default_python_etl.id}" } ], "action": "create", @@ -27,12 +27,22 @@ "data_security_mode": "SINGLE_USER", "node_type_id": "[NODE_TYPE_ID]", "num_workers": 0, - "spark_version": "15.4.x-scala2.12" + "spark_version": "16.4.x-scala2.12" } } ], "max_concurrent_runs": 4, - "name": "[dev [USERNAME]] my_default_python_job", + "name": "[dev [USERNAME]] sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], "queue": { "enabled": true }, @@ -40,30 +50,41 @@ "dev": "[USERNAME]" }, "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook" + }, + "task_key": "notebook_task" + }, { "depends_on": [ { - "task_key": "refresh_pipeline" + "task_key": "notebook_task" } ], "job_cluster_key": "job_cluster", "libraries": [ { - "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl" + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" } ], "python_wheel_task": { "entry_point": "main", - "package_name": "my_default_python" - }, - "task_key": "main_task" - }, - { - "job_cluster_key": "job_cluster", - "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/notebook" + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] }, - "task_key": "notebook_task" + "task_key": "python_wheel_task" }, { "depends_on": [ @@ -72,7 +93,7 @@ } ], "pipeline_task": { - "pipeline_id": "${resources.pipelines.my_default_python_pipeline.id}" + "pipeline_id": "${resources.pipelines.default_python_etl.id}" }, "task_key": "refresh_pipeline" } @@ -86,33 +107,36 @@ } }, "vars": { - "tasks[2].pipeline_task.pipeline_id": "${resources.pipelines.my_default_python_pipeline.id}" + "tasks[2].pipeline_task.pipeline_id": "${resources.pipelines.default_python_etl.id}" } } }, - "resources.pipelines.my_default_python_pipeline": { + "resources.pipelines.default_python_etl": { "action": "create", "new_state": { "config": { "channel": "CURRENT", - "configuration": { - "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src" - }, "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" }, "development": true, "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files" + ] + }, "libraries": [ { - "notebook": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/pipeline" + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/default_python_etl/transformations/**" } } ], - "name": "[dev [USERNAME]] my_default_python_pipeline", - "schema": "my_default_python_dev", + "name": "[dev [USERNAME]] default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/default_python_etl", + "schema": "[USERNAME]", "tags": { "dev": "[USERNAME]" } diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_dev.terraform.json b/acceptance/bundle/templates/default-python/classic/out.plan_dev.terraform.json index fb3b63b282..56f43e5adb 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_dev.terraform.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_dev.terraform.json @@ -1,9 +1,9 @@ { "plan": { - "resources.jobs.my_default_python_job": { + "resources.jobs.sample_job": { "action": "create" }, - "resources.pipelines.my_default_python_pipeline": { + "resources.pipelines.default_python_etl": { "action": "create" } } diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct-exp.json b/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct-exp.json index 95b847d7fa..e9d40f366b 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct-exp.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct-exp.json @@ -1,10 +1,10 @@ { "plan": { - "resources.jobs.my_default_python_job": { + "resources.jobs.sample_job": { "depends_on": [ { - "node": "resources.pipelines.my_default_python_pipeline", - "label": "${resources.pipelines.my_default_python_pipeline.id}" + "node": "resources.pipelines.default_python_etl", + "label": "${resources.pipelines.default_python_etl.id}" } ], "action": "create", @@ -27,20 +27,42 @@ "data_security_mode": "SINGLE_USER", "node_type_id": "[NODE_TYPE_ID]", "num_workers": 0, - "spark_version": "15.4.x-scala2.12" + "spark_version": "16.4.x-scala2.12" } } ], "max_concurrent_runs": 1, - "name": "my_default_python_job", + "name": "sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "prod", + "name": "schema" + } + ], "queue": { "enabled": true }, "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/sample_notebook" + }, + "task_key": "notebook_task" + }, { "depends_on": [ { - "task_key": "refresh_pipeline" + "task_key": "notebook_task" } ], "job_cluster_key": "job_cluster", @@ -51,16 +73,15 @@ ], "python_wheel_task": { "entry_point": "main", - "package_name": "my_default_python" - }, - "task_key": "main_task" - }, - { - "job_cluster_key": "job_cluster", - "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/notebook" + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "prod" + ] }, - "task_key": "notebook_task" + "task_key": "python_wheel_task" }, { "depends_on": [ @@ -69,7 +90,7 @@ } ], "pipeline_task": { - "pipeline_id": "${resources.pipelines.my_default_python_pipeline.id}" + "pipeline_id": "${resources.pipelines.default_python_etl.id}" }, "task_key": "refresh_pipeline" } @@ -83,32 +104,35 @@ } }, "vars": { - "tasks[2].pipeline_task.pipeline_id": "${resources.pipelines.my_default_python_pipeline.id}" + "tasks[2].pipeline_task.pipeline_id": "${resources.pipelines.default_python_etl.id}" } } }, - "resources.pipelines.my_default_python_pipeline": { + "resources.pipelines.default_python_etl": { "action": "create", "new_state": { "config": { "channel": "CURRENT", - "configuration": { - "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src" - }, "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" }, "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files" + ] + }, "libraries": [ { - "notebook": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/pipeline" + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/default_python_etl/transformations/**" } } ], - "name": "my_default_python_pipeline", - "schema": "my_default_python_prod" + "name": "default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/default_python_etl", + "schema": "prod" } } } diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_prod.terraform.json b/acceptance/bundle/templates/default-python/classic/out.plan_prod.terraform.json index fb3b63b282..56f43e5adb 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_prod.terraform.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_prod.terraform.json @@ -1,9 +1,9 @@ { "plan": { - "resources.jobs.my_default_python_job": { + "resources.jobs.sample_job": { "action": "create" }, - "resources.pipelines.my_default_python_pipeline": { + "resources.pipelines.default_python_etl": { "action": "create" } } diff --git a/acceptance/bundle/templates/default-python/classic/out.requests.dev.direct-exp.txt b/acceptance/bundle/templates/default-python/classic/out.requests.dev.direct-exp.txt index b0f0f5f665..0f89b32629 100644 --- a/acceptance/bundle/templates/default-python/classic/out.requests.dev.direct-exp.txt +++ b/acceptance/bundle/templates/default-python/classic/out.requests.dev.direct-exp.txt @@ -3,24 +3,27 @@ "path": "/api/2.0/pipelines", "body": { "channel": "CURRENT", - "configuration": { - "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src" - }, "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" }, "development": true, "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files" + ] + }, "libraries": [ { - "notebook": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/pipeline" + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/default_python_etl/transformations/**" } } ], - "name": "[dev [USERNAME]] my_default_python_pipeline", - "schema": "my_default_python_dev", + "name": "[dev [USERNAME]] default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/default_python_etl", + "schema": "[USERNAME]", "tags": { "dev": "[USERNAME]" } @@ -62,6 +65,13 @@ "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/resources" } } +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/default_python_etl/transformations" + } +} { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -97,12 +107,22 @@ "data_security_mode": "SINGLE_USER", "node_type_id": "[NODE_TYPE_ID]", "num_workers": 0, - "spark_version": "15.4.x-scala2.12" + "spark_version": "16.4.x-scala2.12" } } ], "max_concurrent_runs": 4, - "name": "[dev [USERNAME]] my_default_python_job", + "name": "[dev [USERNAME]] sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], "queue": { "enabled": true }, @@ -110,30 +130,41 @@ "dev": "[USERNAME]" }, "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook" + }, + "task_key": "notebook_task" + }, { "depends_on": [ { - "task_key": "refresh_pipeline" + "task_key": "notebook_task" } ], "job_cluster_key": "job_cluster", "libraries": [ { - "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl" + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" } ], "python_wheel_task": { "entry_point": "main", - "package_name": "my_default_python" + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] }, - "task_key": "main_task" - }, - { - "job_cluster_key": "job_cluster", - "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/notebook" - }, - "task_key": "notebook_task" + "task_key": "python_wheel_task" }, { "depends_on": [ diff --git a/acceptance/bundle/templates/default-python/classic/out.requests.dev.terraform.txt b/acceptance/bundle/templates/default-python/classic/out.requests.dev.terraform.txt index 2f312f3e3a..91e9828451 100644 --- a/acceptance/bundle/templates/default-python/classic/out.requests.dev.terraform.txt +++ b/acceptance/bundle/templates/default-python/classic/out.requests.dev.terraform.txt @@ -3,24 +3,27 @@ "path": "/api/2.0/pipelines", "body": { "channel": "CURRENT", - "configuration": { - "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src" - }, "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" }, "development": true, "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files" + ] + }, "libraries": [ { - "notebook": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/pipeline" + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/default_python_etl/transformations/**" } } ], - "name": "[dev [USERNAME]] my_default_python_pipeline", - "schema": "my_default_python_dev", + "name": "[dev [USERNAME]] default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/default_python_etl", + "schema": "[USERNAME]", "tags": { "dev": "[USERNAME]" } @@ -62,6 +65,13 @@ "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/resources" } } +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/default_python_etl/transformations" + } +} { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -96,12 +106,22 @@ }, "data_security_mode": "SINGLE_USER", "node_type_id": "[NODE_TYPE_ID]", - "spark_version": "15.4.x-scala2.12" + "spark_version": "16.4.x-scala2.12" } } ], "max_concurrent_runs": 4, - "name": "[dev [USERNAME]] my_default_python_job", + "name": "[dev [USERNAME]] sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], "queue": { "enabled": true }, @@ -109,31 +129,42 @@ "dev": "[USERNAME]" }, "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook", + "source": "WORKSPACE" + }, + "task_key": "notebook_task" + }, { "depends_on": [ { - "task_key": "refresh_pipeline" + "task_key": "notebook_task" } ], "job_cluster_key": "job_cluster", "libraries": [ { - "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl" + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" } ], "python_wheel_task": { "entry_point": "main", - "package_name": "my_default_python" - }, - "task_key": "main_task" - }, - { - "job_cluster_key": "job_cluster", - "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/notebook", - "source": "WORKSPACE" + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] }, - "task_key": "notebook_task" + "task_key": "python_wheel_task" }, { "depends_on": [ diff --git a/acceptance/bundle/templates/default-python/classic/out.requests.prod.direct-exp.txt b/acceptance/bundle/templates/default-python/classic/out.requests.prod.direct-exp.txt index 11ce067ba4..55311f4093 100644 --- a/acceptance/bundle/templates/default-python/classic/out.requests.prod.direct-exp.txt +++ b/acceptance/bundle/templates/default-python/classic/out.requests.prod.direct-exp.txt @@ -3,23 +3,26 @@ "path": "/api/2.0/pipelines", "body": { "channel": "CURRENT", - "configuration": { - "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src" - }, "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" }, "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files" + ] + }, "libraries": [ { - "notebook": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/pipeline" + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/default_python_etl/transformations/**" } } ], - "name": "my_default_python_pipeline", - "schema": "my_default_python_prod" + "name": "default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/default_python_etl", + "schema": "prod" } } { @@ -65,6 +68,13 @@ "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/resources" } } +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/default_python_etl/transformations" + } +} { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -100,20 +110,42 @@ "data_security_mode": "SINGLE_USER", "node_type_id": "[NODE_TYPE_ID]", "num_workers": 0, - "spark_version": "15.4.x-scala2.12" + "spark_version": "16.4.x-scala2.12" } } ], "max_concurrent_runs": 1, - "name": "my_default_python_job", + "name": "sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "prod", + "name": "schema" + } + ], "queue": { "enabled": true }, "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/sample_notebook" + }, + "task_key": "notebook_task" + }, { "depends_on": [ { - "task_key": "refresh_pipeline" + "task_key": "notebook_task" } ], "job_cluster_key": "job_cluster", @@ -124,16 +156,15 @@ ], "python_wheel_task": { "entry_point": "main", - "package_name": "my_default_python" + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "prod" + ] }, - "task_key": "main_task" - }, - { - "job_cluster_key": "job_cluster", - "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/notebook" - }, - "task_key": "notebook_task" + "task_key": "python_wheel_task" }, { "depends_on": [ diff --git a/acceptance/bundle/templates/default-python/classic/out.requests.prod.terraform.txt b/acceptance/bundle/templates/default-python/classic/out.requests.prod.terraform.txt index 4ce80655b4..cf913ee737 100644 --- a/acceptance/bundle/templates/default-python/classic/out.requests.prod.terraform.txt +++ b/acceptance/bundle/templates/default-python/classic/out.requests.prod.terraform.txt @@ -3,23 +3,26 @@ "path": "/api/2.0/pipelines", "body": { "channel": "CURRENT", - "configuration": { - "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src" - }, "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" }, "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files" + ] + }, "libraries": [ { - "notebook": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/pipeline" + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/default_python_etl/transformations/**" } } ], - "name": "my_default_python_pipeline", - "schema": "my_default_python_prod" + "name": "default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/default_python_etl", + "schema": "prod" } } { @@ -65,6 +68,13 @@ "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/resources" } } +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/default_python_etl/transformations" + } +} { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -99,20 +109,43 @@ }, "data_security_mode": "SINGLE_USER", "node_type_id": "[NODE_TYPE_ID]", - "spark_version": "15.4.x-scala2.12" + "spark_version": "16.4.x-scala2.12" } } ], "max_concurrent_runs": 1, - "name": "my_default_python_job", + "name": "sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "prod", + "name": "schema" + } + ], "queue": { "enabled": true }, "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/sample_notebook", + "source": "WORKSPACE" + }, + "task_key": "notebook_task" + }, { "depends_on": [ { - "task_key": "refresh_pipeline" + "task_key": "notebook_task" } ], "job_cluster_key": "job_cluster", @@ -123,17 +156,15 @@ ], "python_wheel_task": { "entry_point": "main", - "package_name": "my_default_python" - }, - "task_key": "main_task" - }, - { - "job_cluster_key": "job_cluster", - "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/notebook", - "source": "WORKSPACE" + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "prod" + ] }, - "task_key": "notebook_task" + "task_key": "python_wheel_task" }, { "depends_on": [ diff --git a/acceptance/bundle/templates/default-python/classic/output.txt b/acceptance/bundle/templates/default-python/classic/output.txt index cc9622b30b..e78cc4b8c2 100644 --- a/acceptance/bundle/templates/default-python/classic/output.txt +++ b/acceptance/bundle/templates/default-python/classic/output.txt @@ -1,8 +1,12 @@ >>> [CLI] bundle init default-python --config-file ./input.json --output-dir output - Welcome to the default Python template for Databricks Asset Bundles! -Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): [DATABRICKS_URL] + +Please answer the below to tailor your project to your preferences. +You can always change your mind and change your configuration in the databricks.yml file later. + +Note that [DATABRICKS_URL] is used for initialization +(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). ✨ Your new project has been created in the 'my_default_python' directory! @@ -31,15 +35,15 @@ Validation OK! >>> [CLI] bundle plan -t dev Building python_artifact... -create jobs.my_default_python_job -create pipelines.my_default_python_pipeline +create jobs.sample_job +create pipelines.default_python_etl Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged >>> [CLI] bundle plan -t prod Building python_artifact... -create jobs.my_default_python_job -create pipelines.my_default_python_pipeline +create jobs.sample_job +create pipelines.default_python_etl Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged Building python_artifact... diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/extensions.json b/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/extensions.json index 5d15eba363..75a111a6a9 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/extensions.json +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ "databricks.databricks", - "ms-python.vscode-pylance", - "redhat.vscode-yaml" + "redhat.vscode-yaml", + "ms-python.black-formatter" ] } diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/settings.json b/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/settings.json index 8ee87c30d4..c49593bc59 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/settings.json +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/settings.json @@ -1,16 +1,39 @@ { - "python.analysis.stubPath": ".vscode", "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", "python.testing.pytestArgs": [ "." ], - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, - "python.analysis.extraPaths": ["src"], "files.exclude": { "**/*.egg-info": true, "**/__pycache__": true, ".pytest_cache": true, + "dist": true, + }, + "files.associations": { + "**/.gitkeep": "markdown" + }, + + // Pylance settings (VS Code) + // Set typeCheckingMode to "basic" to enable type checking! + "python.analysis.typeCheckingMode": "off", + "python.analysis.extraPaths": ["src", "lib", "resources"], + "python.analysis.diagnosticMode": "workspace", + "python.analysis.stubPath": ".vscode", + + // Pyright settings (Cursor) + // Set typeCheckingMode to "basic" to enable type checking! + "cursorpyright.analysis.typeCheckingMode": "off", + "cursorpyright.analysis.extraPaths": ["src", "lib", "resources"], + "cursorpyright.analysis.diagnosticMode": "workspace", + "cursorpyright.analysis.stubPath": ".vscode", + + // General Python settings + "python.defaultInterpreterPath": "./.venv/bin/python", + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnSave": true, }, } diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md index e01be4259d..75a9fb55b7 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md @@ -1,9 +1,13 @@ # my_default_python -The 'my_default_python' project was generated by using the default-python template. +The 'my_default_python' project was generated by using the default template. + +* `src/`: Python source code for this project. + * `src/my_default_python/`: Shared Python code that can be used by jobs and pipelines. +* `resources/`: Resource configurations (jobs, pipelines, etc.) +* `tests/`: Unit tests for the shared Python code. +* `fixtures/`: Fixtures for data sets (primarily used for testing). -For documentation on the Databricks Asset Bundles format use for this project, -and for CI/CD configuration, see https://docs.databricks.com/aws/en/dev-tools/bundles. ## Getting started @@ -17,13 +21,13 @@ Choose how you want to work on this project: (c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html - -Dependencies for this project should be installed using uv: +If you're developing with an IDE, dependencies for this project should be installed using uv: * Make sure you have the UV package manager installed. It's an alternative to tools like pip: https://docs.astral.sh/uv/getting-started/installation/. * Run `uv sync --dev` to install the project's dependencies. + # Using this project using the CLI The Databricks workspace and IDE extensions provide a graphical interface for working @@ -42,17 +46,16 @@ with this project. It's also possible to interact with it directly using the CLI is optional here.) This deploys everything that's defined for this project. - For example, the default template would deploy a job called - `[dev yourname] my_default_python_job` to your workspace. - You can find that job by opening your workpace and clicking on **Jobs & Pipelines**. + For example, the default template would deploy a pipeline called + `[dev yourname] default_python_etl` to your workspace. + You can find that resource by opening your workpace and clicking on **Jobs & Pipelines**. 3. Similarly, to deploy a production copy, type: ``` $ databricks bundle deploy --target prod ``` - - Note that the default job from the template has a schedule that runs every day - (defined in resources/my_default_python.job.yml). The schedule + Note the default template has a includes a job that runs the pipeline every day + (defined in resources/sample_job.job.yml). The schedule is paused when deploying in development mode (see https://docs.databricks.com/dev-tools/bundles/deployment-modes.html). @@ -65,3 +68,4 @@ with this project. It's also possible to interact with it directly using the CLI ``` $ uv run pytest ``` + diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/databricks.yml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/databricks.yml index ed3d53b999..cc6079c53c 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/databricks.yml +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/databricks.yml @@ -4,14 +4,21 @@ bundle: name: my_default_python uuid: [UUID] +include: + - resources/*.yml + - resources/*/*.yml + artifacts: python_artifact: type: whl build: uv build --wheel -include: - - resources/*.yml - - resources/*/*.yml +# Variable declarations. These variables are assigned in the dev/prod targets below. +variables: + catalog: + description: The catalog to use + schema: + description: The schema to use targets: dev: @@ -23,20 +30,20 @@ targets: default: true workspace: host: [DATABRICKS_URL] - + variables: + catalog: hive_metastore + schema: ${workspace.current_user.short_name} presets: - # Set dynamic_version: true on all artifacts of type "whl". - # This makes "bundle deploy" add a timestamp to wheel's version before uploading, - # new wheel takes over the previous installation even if actual wheel version is unchanged. - # See https://docs.databricks.com/aws/en/dev-tools/bundles/settings artifacts_dynamic_version: true - prod: mode: production workspace: host: [DATABRICKS_URL] # We explicitly deploy to /Workspace/Users/[USERNAME] to make sure we only have a single copy. root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + variables: + catalog: hive_metastore + schema: prod permissions: - user_name: [USERNAME] level: CAN_MANAGE diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/fixtures/.gitkeep b/acceptance/bundle/templates/default-python/classic/output/my_default_python/fixtures/.gitkeep index fa25d2745e..77a906614c 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/fixtures/.gitkeep +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/fixtures/.gitkeep @@ -1,22 +1,9 @@ -# Fixtures +# Test fixtures directory -This folder is reserved for fixtures, such as CSV files. - -Below is an example of how to load fixtures as a data frame: +Add JSON or CSV files here. In tests, use them with `load_fixture()`: ``` -import pandas as pd -import os - -def get_absolute_path(*relative_parts): - if 'dbutils' in globals(): - base_dir = os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) # type: ignore - path = os.path.normpath(os.path.join(base_dir, *relative_parts)) - return path if path.startswith("/Workspace") else "/Workspace" + path - else: - return os.path.join(*relative_parts) - -csv_file = get_absolute_path("..", "fixtures", "mycsv.csv") -df = pd.read_csv(csv_file) -display(df) +def test_using_fixture(load_fixture): + data = load_fixture("my_data.json") + assert len(data) >= 1 ``` diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/out.gitignore b/acceptance/bundle/templates/default-python/classic/output/my_default_python/out.gitignore index 0dab7f4995..e566c51f74 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/out.gitignore +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/out.gitignore @@ -6,3 +6,5 @@ __pycache__/ .venv/ scratch/** !scratch/README.md +**/explorations/** +**/!explorations/README.md diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml index ef43b9429f..607ea88364 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml @@ -3,33 +3,27 @@ name = "my_default_python" version = "0.0.1" authors = [{ name = "[USERNAME]" }] requires-python = ">=3.10,<=3.13" +dependencies = [ + # Any dependencies for jobs and pipelines in this project can be added here + # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies + # + # LIMITATION: for pipelines, dependencies are cached during development; + # add dependencies to the 'environment' section of pipeline.yml file instead +] [dependency-groups] dev = [ "pytest", - - # Code completion support for Lakeflow Declarative Pipelines, also install databricks-connect "databricks-dlt", - - # databricks-connect can be used to run parts of this project locally. - # Note that for local development, you should use a version that is not newer - # than the remote cluster or serverless compute you connect to. - # See also https://docs.databricks.com/dev-tools/databricks-connect.html. "databricks-connect>=15.4,<15.5", ] -[tool.pytest.ini_options] -pythonpath = "src" -testpaths = [ - "tests", -] +[project.scripts] +main = "my_default_python.main:main" [build-system] requires = ["hatchling"] build-backend = "hatchling.build" -[tool.hatch.build.targets.wheel] -packages = ["src/my_default_python"] - -[project.scripts] -main = "my_default_python.main:main" +[tool.black] +line-length = 125 diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/default_python_etl.pipeline.yml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/default_python_etl.pipeline.yml new file mode 100644 index 0000000000..82df765028 --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/default_python_etl.pipeline.yml @@ -0,0 +1,20 @@ +# The main pipeline for my_default_python + +resources: + pipelines: + default_python_etl: + name: default_python_etl + ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: + # catalog: ${var.catalog} + schema: ${var.schema} + root_path: "../src/default_python_etl" + + libraries: + - glob: + include: ../src/default_python_etl/transformations/** + + environment: + dependencies: + # We include every dependency defined by pyproject.toml by defining an editable environment + # that points to the folder where pyproject.toml is deployed. + - --editable ${workspace.file_path} diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml deleted file mode 100644 index 6e49947083..0000000000 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml +++ /dev/null @@ -1,14 +0,0 @@ -# The main pipeline for my_default_python -resources: - pipelines: - my_default_python_pipeline: - name: my_default_python_pipeline - ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: - # catalog: catalog_name - schema: my_default_python_${bundle.target} - libraries: - - notebook: - path: ../src/pipeline.ipynb - - configuration: - bundle.sourcePath: ${workspace.file_path}/src diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.job.yml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/sample_job.job.yml similarity index 58% rename from acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.job.yml rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/sample_job.job.yml index 30b579f500..b91da70d42 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.job.yml +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/sample_job.job.yml @@ -1,8 +1,9 @@ -# The main job for my_default_python. +# A sample job for my_default_python. + resources: jobs: - my_default_python_job: - name: my_default_python_job + sample_job: + name: sample_job trigger: # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger @@ -14,35 +15,49 @@ resources: # on_failure: # - your_email@example.com + parameters: + - name: catalog + default: ${var.catalog} + - name: schema + default: ${var.schema} + tasks: - task_key: notebook_task - job_cluster_key: job_cluster notebook_task: - notebook_path: ../src/notebook.ipynb - - - task_key: refresh_pipeline + notebook_path: ../src/sample_notebook.ipynb + job_cluster_key: job_cluster + libraries: + # By default we just include the .whl file generated for the my_default_python package. + # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html + # for more information on how to add other libraries. + - whl: ../dist/*.whl + - task_key: python_wheel_task depends_on: - task_key: notebook_task - pipeline_task: - pipeline_id: ${resources.pipelines.my_default_python_pipeline.id} - - - task_key: main_task - depends_on: - - task_key: refresh_pipeline - job_cluster_key: job_cluster python_wheel_task: package_name: my_default_python entry_point: main + parameters: + - "--catalog" + - "${var.catalog}" + - "--schema" + - "${var.schema}" + job_cluster_key: job_cluster libraries: # By default we just include the .whl file generated for the my_default_python package. # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html # for more information on how to add other libraries. - whl: ../dist/*.whl + - task_key: refresh_pipeline + depends_on: + - task_key: notebook_task + pipeline_task: + pipeline_id: ${resources.pipelines.default_python_etl.id} job_clusters: - job_cluster_key: job_cluster new_cluster: - spark_version: 15.4.x-scala2.12 + spark_version: 16.4.x-scala2.12 node_type_id: [NODE_TYPE_ID] data_security_mode: SINGLE_USER autoscale: diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/scratch/README.md b/acceptance/bundle/templates/default-python/classic/output/my_default_python/scratch/README.md deleted file mode 100644 index e6cfb81b46..0000000000 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/scratch/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# scratch - -This folder is reserved for personal, exploratory notebooks. -By default these are not committed to Git, as 'scratch' is listed in .gitignore. diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/scratch/exploration.ipynb b/acceptance/bundle/templates/default-python/classic/output/my_default_python/scratch/exploration.ipynb deleted file mode 100644 index 3f589fed74..0000000000 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/scratch/exploration.ipynb +++ /dev/null @@ -1,61 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "import sys\n", - "\n", - "sys.path.append(\"../src\")\n", - "from my_default_python import main\n", - "\n", - "main.get_taxis().show(10)" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "ipynb-notebook", - "widgets": {} - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/README.md b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/README.md new file mode 100644 index 0000000000..b59a51fffb --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/README.md @@ -0,0 +1,20 @@ +# my_default_python + +This folder defines all source code for the my_default_python pipeline: + +- `explorations/`: Ad-hoc notebooks used to explore the data processed by this pipeline. +- `transformations/`: All dataset definitions and transformations. +- `utilities/` (optional): Utility functions and Python modules used in this pipeline. +- `data_sources/` (optional): View definitions describing the source data for this pipeline. + +## Getting Started + +To get started, go to the `transformations` folder -- most of the relevant source code lives there: + +* By convention, every dataset under `transformations` is in a separate file. +* Take a look at the sample called "sample_trips_my_default_python.py" to get familiar with the syntax. + Read more about the syntax at https://docs.databricks.com/dlt/python-ref.html. +* If you're using the workspace UI, use `Run file` to run and preview a single transformation. +* If you're using the CLI, use `databricks bundle run default_python_etl --select sample_trips_my_default_python` to run a single transformation. + +For more tutorials and reference material, see https://docs.databricks.com/dlt. diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/notebook.ipynb b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/explorations/sample_exploration.ipynb similarity index 50% rename from acceptance/bundle/templates/default-python/classic/output/my_default_python/src/notebook.ipynb rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/explorations/sample_exploration.ipynb index 27c3f19e34..e326a35169 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/notebook.ipynb +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/explorations/sample_exploration.ipynb @@ -8,23 +8,16 @@ "inputWidgets": {}, "nuid": "[UUID]", "showTitle": false, + "tableResultSettingsMap": {}, "title": "" } }, "source": [ - "# Default notebook\n", + "### Example Exploratory Notebook\n", "\n", - "This default notebook is executed using Databricks Workflows as defined in resources/my_default_python.job.yml." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" + "Use this notebook to explore the data generated by the pipeline in your preferred programming language.\n", + "\n", + "**Note**: This notebook is not executed as part of the pipeline." ] }, { @@ -32,42 +25,37 @@ "execution_count": 0, "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, + "cellMetadata": {}, "inputWidgets": {}, "nuid": "[UUID]", "showTitle": false, + "tableResultSettingsMap": {}, "title": "" } }, "outputs": [], "source": [ - "from my_default_python import main\n", + "# !!! Before performing any data analysis, make sure to run the pipeline to materialize the sample datasets. The tables referenced in this notebook depend on that step.\n", "\n", - "main.find_all_taxis().show(10)" + "display(spark.sql(\"SELECT * FROM hive_metastore.[USERNAME].sample_trips_my_default_python\"))" ] } ], "metadata": { "application/vnd.databricks.v1+notebook": { + "computePreferences": null, "dashboards": [], + "environmentMetadata": null, + "inputWidgetPreferences": null, "language": "python", "notebookMetadata": { "pythonIndentUnit": 2 }, - "notebookName": "notebook", + "notebookName": "sample_exploration", "widgets": {} }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, "language_info": { - "name": "python", - "version": "3.11.4" + "name": "python" } }, "nbformat": 4, diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/transformations/sample_trips_my_default_python.py b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/transformations/sample_trips_my_default_python.py new file mode 100644 index 0000000000..eb18701f08 --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/transformations/sample_trips_my_default_python.py @@ -0,0 +1,12 @@ +from pyspark import pipelines as dp +from pyspark.sql.functions import col + + +# This file defines a sample transformation. +# Edit the sample below or add new transformations +# using "+ Add" in the file browser. + + +@dp.table +def sample_trips_my_default_python(): + return spark.read.table("samples.nyctaxi.trips") diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/transformations/sample_zones_my_default_python.py b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/transformations/sample_zones_my_default_python.py new file mode 100644 index 0000000000..a9282b4dcc --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/transformations/sample_zones_my_default_python.py @@ -0,0 +1,17 @@ +from pyspark import pipelines as dp +from pyspark.sql.functions import col, sum + + +# This file defines a sample transformation. +# Edit the sample below or add new transformations +# using "+ Add" in the file browser. + + +@dp.table +def sample_zones_my_default_python(): + # Read from the "sample_trips" table, then sum all the fares + return ( + spark.read.table(f"sample_trips_my_default_python") + .groupBy(col("pickup_zip")) + .agg(sum("fare_amount").alias("total_fare")) + ) diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python/main.py b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python/main.py index 04e8be4de0..1f9c88b322 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python/main.py +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python/main.py @@ -1,13 +1,21 @@ +import argparse from databricks.sdk.runtime import spark -from pyspark.sql import DataFrame +from my_default_python import taxis -def find_all_taxis() -> DataFrame: - return spark.read.table("samples.nyctaxi.trips") +def main(): + # Process command-line arguments + parser = argparse.ArgumentParser(description="Databricks job with catalog and schema parameters") + parser.add_argument("--catalog", required=True) + parser.add_argument("--schema", required=True) + args = parser.parse_args() + # Set the default catalog and schema + spark.sql(f"USE CATALOG {args.catalog}") + spark.sql(f"USE SCHEMA {args.schema}") -def main(): - find_all_taxis().show(5) + # Example: just find all taxis from a sample catalog + taxis.find_all_taxis().show(5) if __name__ == "__main__": diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python/taxis.py b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python/taxis.py new file mode 100644 index 0000000000..a7309cd4c5 --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python/taxis.py @@ -0,0 +1,7 @@ +from databricks.sdk.runtime import spark +from pyspark.sql import DataFrame + + +def find_all_taxis() -> DataFrame: + """Find all taxi data.""" + return spark.read.table("samples.nyctaxi.trips") diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/pipeline.ipynb b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/pipeline.ipynb deleted file mode 100644 index 21e8560105..0000000000 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/pipeline.ipynb +++ /dev/null @@ -1,90 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "source": [ - "# Lakeflow Declarative Pipeline\n", - "\n", - "This Lakeflow Declarative Pipeline definition is executed using a pipeline defined in resources/my_default_python.pipeline.yml." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "# Import DLT and src/my_default_python\n", - "import dlt\n", - "import sys\n", - "\n", - "sys.path.append(spark.conf.get(\"bundle.sourcePath\", \".\"))\n", - "from pyspark.sql.functions import expr\n", - "from my_default_python import main" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "@dlt.view\n", - "def taxi_raw():\n", - " return main.find_all_taxis()\n", - "\n", - "\n", - "@dlt.table\n", - "def filtered_taxis():\n", - " return dlt.read(\"taxi_raw\").filter(expr(\"fare_amount < 30\"))" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "pipeline", - "widgets": {} - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb new file mode 100644 index 0000000000..9b14f9286c --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb @@ -0,0 +1,149 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "[UUID]", + "showTitle": false, + "title": "" + } + }, + "source": [ + "# Default notebook\n", + "\n", + "This default notebook is executed using a Lakeflow job as defined in resources/sample_job.job.yml." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "[UUID]", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Set default catalog and schema\n", + "catalog = dbutils.widgets.get(\"catalog\")\n", + "schema = dbutils.widgets.get(\"schema\")\n", + "spark.sql(f\"USE CATALOG {catalog}\")\n", + "spark.sql(f\"USE SCHEMA {schema}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "[UUID]", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "sys.path.append(\"../src\")\n", + "from my_default_python import taxis\n", + "\n", + "taxis.find_all_taxis().show(10)" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "environmentMetadata": { + "base_environment": "", + "dependencies": [ + "--editable .." + ], + "environment_version": "2" + }, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 2 + }, + "notebookName": "notebook", + "widgets": { + "catalog": { + "currentValue": "hive_metastore", + "nuid": "c4t4l0g-w1dg-3t12-3456-[NUMID]", + "typedWidgetInfo": { + "autoCreated": false, + "defaultValue": "hive_metastore", + "label": "Catalog", + "name": "catalog", + "options": { + "widgetDisplayType": "Text", + "validationRegex": null + }, + "parameterDataType": "String" + }, + "widgetInfo": { + "widgetType": "text", + "defaultValue": "hive_metastore", + "label": "Catalog", + "name": "catalog", + "options": { + "widgetType": "text", + "autoCreated": false, + "validationRegex": null + } + } + }, + "schema": { + "currentValue": "[USERNAME]", + "nuid": "5ch3m4-w1dg-3t98-7654-[NUMID]", + "typedWidgetInfo": { + "autoCreated": false, + "defaultValue": "default", + "label": "Schema", + "name": "schema", + "options": { + "widgetDisplayType": "Text", + "validationRegex": null + }, + "parameterDataType": "String" + }, + "widgetInfo": { + "widgetType": "text", + "defaultValue": "default", + "label": "Schema", + "name": "schema", + "options": { + "widgetType": "text", + "autoCreated": false, + "validationRegex": null + } + } + } + } + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/tests/conftest.py b/acceptance/bundle/templates/default-python/classic/output/my_default_python/tests/conftest.py index f80cb4395e..4df274fd43 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/tests/conftest.py +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/tests/conftest.py @@ -1,4 +1,8 @@ -"""This file configures pytest.""" +"""This file configures pytest. + +This file is in the root since it can be used for tests in any place in this +project, including tests under resources/. +""" import os, sys, pathlib from contextlib import contextmanager @@ -9,13 +13,54 @@ from databricks.sdk import WorkspaceClient from pyspark.sql import SparkSession import pytest + import json + import csv + import os except ImportError: raise ImportError( "Test dependencies not found.\n\nRun tests using 'uv run pytest'. See http://docs.astral.sh/uv to learn more about uv." ) -def enable_fallback_compute(): +@pytest.fixture() +def spark() -> SparkSession: + """Provide a SparkSession fixture for tests. + + Minimal example: + def test_uses_spark(spark): + df = spark.createDataFrame([(1,)], ["x"]) + assert df.count() == 1 + """ + return DatabricksSession.builder.getOrCreate() + + +@pytest.fixture() +def load_fixture(spark: SparkSession): + """Provide a callable to load JSON or CSV from fixtures/ directory. + + Example usage: + + def test_using_fixture(load_fixture): + data = load_fixture("my_data.json") + assert data.count() >= 1 + """ + + def _loader(filename: str): + path = pathlib.Path(__file__).parent.parent / "fixtures" / filename + suffix = path.suffix.lower() + if suffix == ".json": + rows = json.loads(path.read_text()) + return spark.createDataFrame(rows) + if suffix == ".csv": + with path.open(newline="") as f: + rows = list(csv.DictReader(f)) + return spark.createDataFrame(rows) + raise ValueError(f"Unsupported fixture type for: {filename}") + + return _loader + + +def _enable_fallback_compute(): """Enable serverless compute if no compute is specified.""" conf = WorkspaceClient().config if conf.serverless_compute_id or conf.cluster_id or os.environ.get("SPARK_REMOTE"): @@ -23,13 +68,13 @@ def enable_fallback_compute(): url = "https://docs.databricks.com/dev-tools/databricks-connect/cluster-config" print("☁️ no compute specified, falling back to serverless compute", file=sys.stderr) - print(f" see {url} for manual configuration", file=sys.stderr) + print(f" see {url} for manual configuration", file=sys.stdout) os.environ["DATABRICKS_SERVERLESS_COMPUTE_ID"] = "auto" @contextmanager -def allow_stderr_output(config: pytest.Config): +def _allow_stderr_output(config: pytest.Config): """Temporarily disable pytest output capture.""" capman = config.pluginmanager.get_plugin("capturemanager") if capman: @@ -41,8 +86,8 @@ def allow_stderr_output(config: pytest.Config): def pytest_configure(config: pytest.Config): """Configure pytest session.""" - with allow_stderr_output(config): - enable_fallback_compute() + with _allow_stderr_output(config): + _enable_fallback_compute() # Initialize Spark session eagerly, so it is available even when # SparkSession.builder.getOrCreate() is used. For DB Connect 15+, @@ -51,9 +96,3 @@ def pytest_configure(config: pytest.Config): DatabricksSession.builder.validateSession().getOrCreate() else: DatabricksSession.builder.getOrCreate() - - -@pytest.fixture(scope="session") -def spark() -> SparkSession: - """Provide a SparkSession fixture for tests.""" - return DatabricksSession.builder.getOrCreate() diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/tests/main_test.py b/acceptance/bundle/templates/default-python/classic/output/my_default_python/tests/main_test.py deleted file mode 100644 index 4bfd5e1550..0000000000 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/tests/main_test.py +++ /dev/null @@ -1,6 +0,0 @@ -from my_default_python import main - - -def test_find_all_taxis(): - taxis = main.find_all_taxis() - assert taxis.count() > 5 diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/tests/sample_taxis_test.py b/acceptance/bundle/templates/default-python/classic/output/my_default_python/tests/sample_taxis_test.py new file mode 100644 index 0000000000..e503dd2b2c --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/tests/sample_taxis_test.py @@ -0,0 +1,8 @@ +from databricks.sdk.runtime import spark +from pyspark.sql import DataFrame +from my_default_python import taxis + + +def test_find_all_taxis(): + results = taxis.find_all_taxis() + assert results.count() > 5 diff --git a/acceptance/bundle/templates/default-python/combinations/classic/output.txt b/acceptance/bundle/templates/default-python/combinations/classic/output.txt index f1a50c0b49..f192544aa5 100644 --- a/acceptance/bundle/templates/default-python/combinations/classic/output.txt +++ b/acceptance/bundle/templates/default-python/combinations/classic/output.txt @@ -1,8 +1,12 @@ >>> [CLI] bundle init default-python --config-file ./input.json - Welcome to the default Python template for Databricks Asset Bundles! -Workspace to use (auto-detected, edit in 'X[UNIQUE_NAME]/databricks.yml'): [DATABRICKS_URL] + +Please answer the below to tailor your project to your preferences. +You can always change your mind and change your configuration in the databricks.yml file later. + +Note that [DATABRICKS_URL] is used for initialization +(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). ✨ Your new project has been created in the 'X[UNIQUE_NAME]' directory! diff --git a/acceptance/bundle/templates/default-python/combinations/serverless/output.txt b/acceptance/bundle/templates/default-python/combinations/serverless/output.txt index fcc3c931ad..ee551a5b16 100644 --- a/acceptance/bundle/templates/default-python/combinations/serverless/output.txt +++ b/acceptance/bundle/templates/default-python/combinations/serverless/output.txt @@ -1,8 +1,12 @@ >>> [CLI] bundle init default-python --config-file ./input.json - Welcome to the default Python template for Databricks Asset Bundles! -Workspace to use (auto-detected, edit in 'X[UNIQUE_NAME]/databricks.yml'): [DATABRICKS_URL] + +Please answer the below to tailor your project to your preferences. +You can always change your mind and change your configuration in the databricks.yml file later. + +Note that [DATABRICKS_URL] is used for initialization +(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). ✨ Your new project has been created in the 'X[UNIQUE_NAME]' directory! diff --git a/acceptance/bundle/templates/default-python/fail-missing-uv/output.txt b/acceptance/bundle/templates/default-python/fail-missing-uv/output.txt index d9c70acfe6..0631cd6a2f 100644 --- a/acceptance/bundle/templates/default-python/fail-missing-uv/output.txt +++ b/acceptance/bundle/templates/default-python/fail-missing-uv/output.txt @@ -1,8 +1,12 @@ >>> [CLI] bundle init default-python --config-file ./input.json --output-dir output - Welcome to the default Python template for Databricks Asset Bundles! -Workspace to use (auto-detected, edit in 'fail_missing_uv/databricks.yml'): [DATABRICKS_URL] + +Please answer the below to tailor your project to your preferences. +You can always change your mind and change your configuration in the databricks.yml file later. + +Note that [DATABRICKS_URL] is used for initialization +(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). ✨ Your new project has been created in the 'fail_missing_uv' directory! diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json b/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json index 80c0c4dc53..be1f493d64 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json +++ b/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json @@ -28,8 +28,8 @@ "uuid": "[UUID]" }, "include": [ - "resources/project_name_[UNIQUE_NAME].job.yml", - "resources/project_name_[UNIQUE_NAME].pipeline.yml" + "resources/project_name_[UNIQUE_NAME]_etl.pipeline.yml", + "resources/sample_job.job.yml" ], "presets": { "artifacts_dynamic_version": true, @@ -43,7 +43,7 @@ }, "resources": { "jobs": { - "project_name_[UNIQUE_NAME]_job": { + "sample_job": { "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json" @@ -61,12 +61,22 @@ "data_security_mode": "SINGLE_USER", "node_type_id": "[NODE_TYPE_ID]", "num_workers": 0, - "spark_version": "15.4.x-scala2.12" + "spark_version": "16.4.x-scala2.12" } } ], "max_concurrent_runs": 4, - "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_job", + "name": "[dev [USERNAME]] sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], "queue": { "enabled": true }, @@ -74,10 +84,22 @@ "dev": "[USERNAME]" }, "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "dist/*.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/sample_notebook" + }, + "task_key": "notebook_task" + }, { "depends_on": [ { - "task_key": "refresh_pipeline" + "task_key": "notebook_task" } ], "job_cluster_key": "job_cluster", @@ -88,16 +110,15 @@ ], "python_wheel_task": { "entry_point": "main", - "package_name": "project_name_[UNIQUE_NAME]" - }, - "task_key": "main_task" - }, - { - "job_cluster_key": "job_cluster", - "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook" + "package_name": "project_name_[UNIQUE_NAME]", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] }, - "task_key": "notebook_task" + "task_key": "python_wheel_task" }, { "depends_on": [ @@ -106,7 +127,7 @@ } ], "pipeline_task": { - "pipeline_id": "${resources.pipelines.project_name_[UNIQUE_NAME]_pipeline.id}" + "pipeline_id": "${resources.pipelines.project_name_[UNIQUE_NAME]_etl.id}" }, "task_key": "refresh_pipeline" } @@ -121,26 +142,29 @@ } }, "pipelines": { - "project_name_[UNIQUE_NAME]_pipeline": { + "project_name_[UNIQUE_NAME]_etl": { "channel": "CURRENT", - "configuration": { - "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src" - }, "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json" }, "development": true, "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files" + ] + }, "libraries": [ { - "notebook": { - "path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/pipeline" + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/project_name_[UNIQUE_NAME]_etl/transformations/**" } } ], - "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_pipeline", - "schema": "project_name_[UNIQUE_NAME]_dev", + "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/project_name_[UNIQUE_NAME]_etl", + "schema": "[USERNAME]", "tags": { "dev": "[USERNAME]" } @@ -152,6 +176,18 @@ "." ] }, + "variables": { + "catalog": { + "default": "hive_metastore", + "description": "The catalog to use", + "value": "hive_metastore" + }, + "schema": { + "default": "[USERNAME]", + "description": "The schema to use", + "value": "[USERNAME]" + } + }, "workspace": { "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts", "file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files", diff --git a/acceptance/bundle/templates/default-python/integration_classic/output.txt b/acceptance/bundle/templates/default-python/integration_classic/output.txt index cd95a2415a..f255d1c7ad 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/output.txt +++ b/acceptance/bundle/templates/default-python/integration_classic/output.txt @@ -3,9 +3,13 @@ [UV_PYTHON] >>> [CLI] bundle init default-python --config-file ./input.json --output-dir . - Welcome to the default Python template for Databricks Asset Bundles! -Workspace to use (auto-detected, edit in 'project_name_[UNIQUE_NAME]/databricks.yml'): [DATABRICKS_URL] + +Please answer the below to tailor your project to your preferences. +You can always change your mind and change your configuration in the databricks.yml file later. + +Note that [DATABRICKS_URL] is used for initialization +(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). ✨ Your new project has been created in the 'project_name_[UNIQUE_NAME]' directory! @@ -41,12 +45,12 @@ Workspace: Path: /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev Resources: Jobs: - project_name_[UNIQUE_NAME]_job: - Name: [dev [USERNAME]] project_name_[UNIQUE_NAME]_job + sample_job: + Name: [dev [USERNAME]] sample_job URL: [DATABRICKS_URL]/jobs/[NUMID] Pipelines: - project_name_[UNIQUE_NAME]_pipeline: - Name: [dev [USERNAME]] project_name_[UNIQUE_NAME]_pipeline + project_name_[UNIQUE_NAME]_etl: + Name: [dev [USERNAME]] project_name_[UNIQUE_NAME]_etl URL: [DATABRICKS_URL]/pipelines/[UUID] >>> [CLI] bundle summary -t dev -o json @@ -60,7 +64,7 @@ Resources: + "id": "[NUMID]", "job_clusters": [ { -@@ -118,5 +119,6 @@ +@@ -139,5 +140,6 @@ "unit": "DAYS" } - } @@ -68,13 +72,13 @@ Resources: + "url": "[DATABRICKS_URL]/jobs/[NUMID]" } }, -@@ -133,4 +135,5 @@ - "development": true, - "edition": "ADVANCED", +@@ -156,4 +158,5 @@ + ] + }, + "id": "[UUID]", "libraries": [ { -@@ -144,5 +147,6 @@ +@@ -168,5 +171,6 @@ "tags": { "dev": "[USERNAME]" - } @@ -85,12 +89,12 @@ Resources: >>> [CLI] bundle destroy -t dev --auto-approve The following resources will be deleted: - delete job project_name_[UNIQUE_NAME]_job - delete pipeline project_name_[UNIQUE_NAME]_pipeline + delete job sample_job + delete pipeline project_name_[UNIQUE_NAME]_etl This action will result in the deletion of the following Lakeflow Declarative Pipelines along with the Streaming Tables (STs) and Materialized Views (MVs) managed by them: - delete pipeline project_name_[UNIQUE_NAME]_pipeline + delete pipeline project_name_[UNIQUE_NAME]_etl All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev @@ -139,7 +143,7 @@ Validation OK! "uuid": "[UUID]" }, @@ -32,14 +26,10 @@ - "resources/project_name_[UNIQUE_NAME].pipeline.yml" + "resources/sample_job.job.yml" ], - "presets": { - "artifacts_dynamic_version": true, @@ -166,13 +170,23 @@ Validation OK! + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/state/metadata.json" }, "edit_mode": "UI_LOCKED", -@@ -66,11 +56,9 @@ +@@ -66,6 +56,6 @@ } ], - "max_concurrent_runs": 4, -- "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_job", +- "name": "[dev [USERNAME]] sample_job", + "max_concurrent_runs": 1, -+ "name": "project_name_[UNIQUE_NAME]_job", ++ "name": "sample_job", + "parameters": [ + { +@@ -74,13 +64,11 @@ + }, + { +- "default": "[USERNAME]", ++ "default": "prod", + "name": "schema" + } + ], + "permissions": [], "queue": { "enabled": true @@ -181,26 +195,28 @@ Validation OK! - "dev": "[USERNAME]" }, "tasks": [ -@@ -96,5 +84,5 @@ - "job_cluster_key": "job_cluster", +@@ -93,5 +81,5 @@ + ], "notebook_task": { -- "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook" -+ "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/notebook" +- "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/sample_notebook" ++ "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/sample_notebook" }, "task_key": "notebook_task" -@@ -113,5 +101,5 @@ +@@ -116,5 +104,5 @@ + "hive_metastore", + "--schema", +- "[USERNAME]" ++ "prod" + ] + }, +@@ -134,5 +122,5 @@ ], "trigger": { - "pause_status": "PAUSED", + "pause_status": "UNPAUSED", "periodic": { "interval": 1, -@@ -125,24 +113,21 @@ - "channel": "CURRENT", - "configuration": { -- "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src" -+ "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src" - }, +@@ -147,11 +135,10 @@ "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json" @@ -208,25 +224,41 @@ Validation OK! }, - "development": true, "edition": "ADVANCED", - "libraries": [ + "environment": { + "dependencies": [ +- "--editable /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files" ++ "--editable /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files" + ] + }, +@@ -159,14 +146,12 @@ { - "notebook": { -- "path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/pipeline" -+ "path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/pipeline" + "glob": { +- "include": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/project_name_[UNIQUE_NAME]_etl/transformations/**" ++ "include": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/project_name_[UNIQUE_NAME]_etl/transformations/**" } } ], -- "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_pipeline", -- "schema": "project_name_[UNIQUE_NAME]_dev", +- "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_etl", +- "root_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/project_name_[UNIQUE_NAME]_etl", +- "schema": "[USERNAME]", - "tags": { - "dev": "[USERNAME]" - } -+ "name": "project_name_[UNIQUE_NAME]_pipeline", ++ "name": "project_name_[UNIQUE_NAME]_etl", + "permissions": [], -+ "schema": "project_name_[UNIQUE_NAME]_prod" ++ "root_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/project_name_[UNIQUE_NAME]_etl", ++ "schema": "prod" } } -@@ -154,10 +139,10 @@ +@@ -184,16 +169,16 @@ + }, + "schema": { +- "default": "[USERNAME]", ++ "default": "prod", + "description": "The schema to use", +- "value": "[USERNAME]" ++ "value": "prod" + } }, "workspace": { - "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts", @@ -260,12 +292,12 @@ Workspace: Path: /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod Resources: Jobs: - project_name_[UNIQUE_NAME]_job: - Name: project_name_[UNIQUE_NAME]_job + sample_job: + Name: sample_job URL: [DATABRICKS_URL]/jobs/[NUMID] Pipelines: - project_name_[UNIQUE_NAME]_pipeline: - Name: project_name_[UNIQUE_NAME]_pipeline + project_name_[UNIQUE_NAME]_etl: + Name: project_name_[UNIQUE_NAME]_etl URL: [DATABRICKS_URL]/pipelines/[UUID] >>> [CLI] bundle summary -t prod -o json @@ -300,7 +332,7 @@ Resources: "uuid": "[UUID]" }, @@ -32,14 +26,10 @@ - "resources/project_name_[UNIQUE_NAME].pipeline.yml" + "resources/sample_job.job.yml" ], - "presets": { - "artifacts_dynamic_version": true, @@ -327,13 +359,23 @@ Resources: + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/state/metadata.json" }, "edit_mode": "UI_LOCKED", -@@ -67,11 +57,9 @@ +@@ -67,6 +57,6 @@ } ], - "max_concurrent_runs": 4, -- "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_job", +- "name": "[dev [USERNAME]] sample_job", + "max_concurrent_runs": 1, -+ "name": "project_name_[UNIQUE_NAME]_job", ++ "name": "sample_job", + "parameters": [ + { +@@ -75,13 +65,11 @@ + }, + { +- "default": "[USERNAME]", ++ "default": "prod", + "name": "schema" + } + ], + "permissions": [], "queue": { "enabled": true @@ -342,26 +384,28 @@ Resources: - "dev": "[USERNAME]" }, "tasks": [ -@@ -97,5 +85,5 @@ - "job_cluster_key": "job_cluster", +@@ -94,5 +82,5 @@ + ], "notebook_task": { -- "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook" -+ "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/notebook" +- "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/sample_notebook" ++ "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/sample_notebook" }, "task_key": "notebook_task" -@@ -114,5 +102,5 @@ +@@ -117,5 +105,5 @@ + "hive_metastore", + "--schema", +- "[USERNAME]" ++ "prod" + ] + }, +@@ -135,5 +123,5 @@ ], "trigger": { - "pause_status": "PAUSED", + "pause_status": "UNPAUSED", "periodic": { "interval": 1, -@@ -127,11 +115,10 @@ - "channel": "CURRENT", - "configuration": { -- "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src" -+ "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src" - }, +@@ -149,11 +137,10 @@ "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json" @@ -369,26 +413,41 @@ Resources: }, - "development": true, "edition": "ADVANCED", - "id": "[UUID]", -@@ -139,13 +126,11 @@ + "environment": { + "dependencies": [ +- "--editable /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files" ++ "--editable /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files" + ] + }, +@@ -162,14 +149,12 @@ { - "notebook": { -- "path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/pipeline" -+ "path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/pipeline" + "glob": { +- "include": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/project_name_[UNIQUE_NAME]_etl/transformations/**" ++ "include": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/project_name_[UNIQUE_NAME]_etl/transformations/**" } } ], -- "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_pipeline", -- "schema": "project_name_[UNIQUE_NAME]_dev", +- "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_etl", +- "root_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/project_name_[UNIQUE_NAME]_etl", +- "schema": "[USERNAME]", - "tags": { - "dev": "[USERNAME]" - }, -+ "name": "project_name_[UNIQUE_NAME]_pipeline", ++ "name": "project_name_[UNIQUE_NAME]_etl", + "permissions": [], -+ "schema": "project_name_[UNIQUE_NAME]_prod", ++ "root_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/project_name_[UNIQUE_NAME]_etl", ++ "schema": "prod", "url": "[DATABRICKS_URL]/pipelines/[UUID]" } -@@ -158,10 +143,10 @@ +@@ -188,16 +173,16 @@ + }, + "schema": { +- "default": "[USERNAME]", ++ "default": "prod", + "description": "The schema to use", +- "value": "[USERNAME]" ++ "value": "prod" + } }, "workspace": { - "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts", @@ -407,12 +466,12 @@ Resources: >>> [CLI] bundle destroy -t prod --auto-approve The following resources will be deleted: - delete job project_name_[UNIQUE_NAME]_job - delete pipeline project_name_[UNIQUE_NAME]_pipeline + delete job sample_job + delete pipeline project_name_[UNIQUE_NAME]_etl This action will result in the deletion of the following Lakeflow Declarative Pipelines along with the Streaming Tables (STs) and Materialized Views (MVs) managed by them: - delete pipeline project_name_[UNIQUE_NAME]_pipeline + delete pipeline project_name_[UNIQUE_NAME]_etl All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod diff --git a/acceptance/bundle/templates/default-python/no-uc/output.txt b/acceptance/bundle/templates/default-python/no-uc/output.txt index 6abf52cf09..20e30bb6fb 100644 --- a/acceptance/bundle/templates/default-python/no-uc/output.txt +++ b/acceptance/bundle/templates/default-python/no-uc/output.txt @@ -1,8 +1,12 @@ >>> [CLI] bundle init default-python --config-file ./input.json --output-dir output - Welcome to the default Python template for Databricks Asset Bundles! -Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): [DATABRICKS_URL] + +Please answer the below to tailor your project to your preferences. +You can always change your mind and change your configuration in the databricks.yml file later. + +Note that [DATABRICKS_URL] is used for initialization +(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). ✨ Your new project has been created in the 'my_default_python' directory! diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt index a6a92dfd4e..c150171c97 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt @@ -1,8 +1,12 @@ >>> [CLI] bundle init default-python --config-file [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/input.json --output-dir output - Welcome to the default Python template for Databricks Asset Bundles! -Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): [DATABRICKS_URL] + +Please answer the below to tailor your project to your preferences. +You can always change your mind and change your configuration in the databricks.yml file later. + +Note that [DATABRICKS_URL] is used for initialization +(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). ✨ Your new project has been created in the 'my_default_python' directory! @@ -10,13 +14,57 @@ Please refer to the README.md file for "getting started" instructions. See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. >>> diff.py [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output output/ ---- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/resources/my_default_python.pipeline.yml -+++ output/my_default_python/resources/my_default_python.pipeline.yml -@@ -4,6 +4,5 @@ - my_default_python_pipeline: - name: my_default_python_pipeline -- ## Catalog is required for serverless compute -- catalog: main +--- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/databricks.yml ++++ output/my_default_python/databricks.yml +@@ -32,5 +32,5 @@ + host: [DATABRICKS_URL] + variables: +- catalog: hive_metastore ++ catalog: customcatalog + schema: ${workspace.current_user.short_name} + prod: +@@ -41,5 +41,5 @@ + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + variables: +- catalog: hive_metastore + catalog: customcatalog - schema: my_default_python_${bundle.target} - serverless: true + schema: prod + permissions: +--- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/resources/default_python_etl.pipeline.yml ++++ output/my_default_python/resources/default_python_etl.pipeline.yml +@@ -5,5 +5,4 @@ + default_python_etl: + name: default_python_etl +- ## Catalog is required for serverless compute + catalog: ${var.catalog} + schema: ${var.schema} +--- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/src/default_python_etl/explorations/sample_exploration.ipynb ++++ output/my_default_python/src/default_python_etl/explorations/sample_exploration.ipynb +@@ -38,5 +38,5 @@ + "# !!! Before performing any data analysis, make sure to run the pipeline to materialize the sample datasets. The tables referenced in this notebook depend on that step./n", + "/n", +- "display(spark.sql(/"SELECT * FROM hive_metastore.[USERNAME].sample_trips_my_default_python/"))" ++ "display(spark.sql(/"SELECT * FROM customcatalog.[USERNAME].sample_trips_my_default_python/"))" + ] + } +--- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/src/sample_notebook.ipynb ++++ output/my_default_python/src/sample_notebook.ipynb +@@ -82,9 +82,9 @@ + "widgets": { + "catalog": { +- "currentValue": "hive_metastore", ++ "currentValue": "customcatalog", + "nuid": "c4t4l0g-w1dg-3t12-3456-[NUMID]", + "typedWidgetInfo": { + "autoCreated": false, +- "defaultValue": "hive_metastore", ++ "defaultValue": "customcatalog", + "label": "Catalog", + "name": "catalog", +@@ -97,5 +97,5 @@ + "widgetInfo": { + "widgetType": "text", +- "defaultValue": "hive_metastore", ++ "defaultValue": "customcatalog", + "label": "Catalog", + "name": "catalog", diff --git a/acceptance/bundle/templates/default-python/serverless/output.txt b/acceptance/bundle/templates/default-python/serverless/output.txt index 930e756de7..980d6786c6 100644 --- a/acceptance/bundle/templates/default-python/serverless/output.txt +++ b/acceptance/bundle/templates/default-python/serverless/output.txt @@ -1,8 +1,12 @@ >>> [CLI] bundle init default-python --config-file ./input.json --output-dir output - Welcome to the default Python template for Databricks Asset Bundles! -Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): [DATABRICKS_URL] + +Please answer the below to tailor your project to your preferences. +You can always change your mind and change your configuration in the databricks.yml file later. + +Note that [DATABRICKS_URL] is used for initialization +(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). ✨ Your new project has been created in the 'my_default_python' directory! diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json index 5d15eba363..75a111a6a9 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ "databricks.databricks", - "ms-python.vscode-pylance", - "redhat.vscode-yaml" + "redhat.vscode-yaml", + "ms-python.black-formatter" ] } diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/settings.json b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/settings.json index 8ee87c30d4..c49593bc59 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/settings.json +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/settings.json @@ -1,16 +1,39 @@ { - "python.analysis.stubPath": ".vscode", "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", "python.testing.pytestArgs": [ "." ], - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, - "python.analysis.extraPaths": ["src"], "files.exclude": { "**/*.egg-info": true, "**/__pycache__": true, ".pytest_cache": true, + "dist": true, + }, + "files.associations": { + "**/.gitkeep": "markdown" + }, + + // Pylance settings (VS Code) + // Set typeCheckingMode to "basic" to enable type checking! + "python.analysis.typeCheckingMode": "off", + "python.analysis.extraPaths": ["src", "lib", "resources"], + "python.analysis.diagnosticMode": "workspace", + "python.analysis.stubPath": ".vscode", + + // Pyright settings (Cursor) + // Set typeCheckingMode to "basic" to enable type checking! + "cursorpyright.analysis.typeCheckingMode": "off", + "cursorpyright.analysis.extraPaths": ["src", "lib", "resources"], + "cursorpyright.analysis.diagnosticMode": "workspace", + "cursorpyright.analysis.stubPath": ".vscode", + + // General Python settings + "python.defaultInterpreterPath": "./.venv/bin/python", + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnSave": true, }, } diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md index e01be4259d..75a9fb55b7 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md @@ -1,9 +1,13 @@ # my_default_python -The 'my_default_python' project was generated by using the default-python template. +The 'my_default_python' project was generated by using the default template. + +* `src/`: Python source code for this project. + * `src/my_default_python/`: Shared Python code that can be used by jobs and pipelines. +* `resources/`: Resource configurations (jobs, pipelines, etc.) +* `tests/`: Unit tests for the shared Python code. +* `fixtures/`: Fixtures for data sets (primarily used for testing). -For documentation on the Databricks Asset Bundles format use for this project, -and for CI/CD configuration, see https://docs.databricks.com/aws/en/dev-tools/bundles. ## Getting started @@ -17,13 +21,13 @@ Choose how you want to work on this project: (c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html - -Dependencies for this project should be installed using uv: +If you're developing with an IDE, dependencies for this project should be installed using uv: * Make sure you have the UV package manager installed. It's an alternative to tools like pip: https://docs.astral.sh/uv/getting-started/installation/. * Run `uv sync --dev` to install the project's dependencies. + # Using this project using the CLI The Databricks workspace and IDE extensions provide a graphical interface for working @@ -42,17 +46,16 @@ with this project. It's also possible to interact with it directly using the CLI is optional here.) This deploys everything that's defined for this project. - For example, the default template would deploy a job called - `[dev yourname] my_default_python_job` to your workspace. - You can find that job by opening your workpace and clicking on **Jobs & Pipelines**. + For example, the default template would deploy a pipeline called + `[dev yourname] default_python_etl` to your workspace. + You can find that resource by opening your workpace and clicking on **Jobs & Pipelines**. 3. Similarly, to deploy a production copy, type: ``` $ databricks bundle deploy --target prod ``` - - Note that the default job from the template has a schedule that runs every day - (defined in resources/my_default_python.job.yml). The schedule + Note the default template has a includes a job that runs the pipeline every day + (defined in resources/sample_job.job.yml). The schedule is paused when deploying in development mode (see https://docs.databricks.com/dev-tools/bundles/deployment-modes.html). @@ -65,3 +68,4 @@ with this project. It's also possible to interact with it directly using the CLI ``` $ uv run pytest ``` + diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/databricks.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/databricks.yml index bdbe7080bc..23a8437a22 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/databricks.yml +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/databricks.yml @@ -4,14 +4,21 @@ bundle: name: my_default_python uuid: [UUID] +include: + - resources/*.yml + - resources/*/*.yml + artifacts: python_artifact: type: whl build: uv build --wheel -include: - - resources/*.yml - - resources/*/*.yml +# Variable declarations. These variables are assigned in the dev/prod targets below. +variables: + catalog: + description: The catalog to use + schema: + description: The schema to use targets: dev: @@ -23,13 +30,18 @@ targets: default: true workspace: host: [DATABRICKS_URL] - + variables: + catalog: hive_metastore + schema: ${workspace.current_user.short_name} prod: mode: production workspace: host: [DATABRICKS_URL] # We explicitly deploy to /Workspace/Users/[USERNAME] to make sure we only have a single copy. root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + variables: + catalog: hive_metastore + schema: prod permissions: - user_name: [USERNAME] level: CAN_MANAGE diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/fixtures/.gitkeep b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/fixtures/.gitkeep index fa25d2745e..77a906614c 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/fixtures/.gitkeep +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/fixtures/.gitkeep @@ -1,22 +1,9 @@ -# Fixtures +# Test fixtures directory -This folder is reserved for fixtures, such as CSV files. - -Below is an example of how to load fixtures as a data frame: +Add JSON or CSV files here. In tests, use them with `load_fixture()`: ``` -import pandas as pd -import os - -def get_absolute_path(*relative_parts): - if 'dbutils' in globals(): - base_dir = os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) # type: ignore - path = os.path.normpath(os.path.join(base_dir, *relative_parts)) - return path if path.startswith("/Workspace") else "/Workspace" + path - else: - return os.path.join(*relative_parts) - -csv_file = get_absolute_path("..", "fixtures", "mycsv.csv") -df = pd.read_csv(csv_file) -display(df) +def test_using_fixture(load_fixture): + data = load_fixture("my_data.json") + assert len(data) >= 1 ``` diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/out.gitignore b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/out.gitignore index 0dab7f4995..e566c51f74 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/out.gitignore +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/out.gitignore @@ -6,3 +6,5 @@ __pycache__/ .venv/ scratch/** !scratch/README.md +**/explorations/** +**/!explorations/README.md diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml index ef43b9429f..607ea88364 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml @@ -3,33 +3,27 @@ name = "my_default_python" version = "0.0.1" authors = [{ name = "[USERNAME]" }] requires-python = ">=3.10,<=3.13" +dependencies = [ + # Any dependencies for jobs and pipelines in this project can be added here + # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies + # + # LIMITATION: for pipelines, dependencies are cached during development; + # add dependencies to the 'environment' section of pipeline.yml file instead +] [dependency-groups] dev = [ "pytest", - - # Code completion support for Lakeflow Declarative Pipelines, also install databricks-connect "databricks-dlt", - - # databricks-connect can be used to run parts of this project locally. - # Note that for local development, you should use a version that is not newer - # than the remote cluster or serverless compute you connect to. - # See also https://docs.databricks.com/dev-tools/databricks-connect.html. "databricks-connect>=15.4,<15.5", ] -[tool.pytest.ini_options] -pythonpath = "src" -testpaths = [ - "tests", -] +[project.scripts] +main = "my_default_python.main:main" [build-system] requires = ["hatchling"] build-backend = "hatchling.build" -[tool.hatch.build.targets.wheel] -packages = ["src/my_default_python"] - -[project.scripts] -main = "my_default_python.main:main" +[tool.black] +line-length = 125 diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/default_python_etl.pipeline.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/default_python_etl.pipeline.yml new file mode 100644 index 0000000000..d5cbd51cf8 --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/default_python_etl.pipeline.yml @@ -0,0 +1,21 @@ +# The main pipeline for my_default_python + +resources: + pipelines: + default_python_etl: + name: default_python_etl + ## Catalog is required for serverless compute + catalog: ${var.catalog} + schema: ${var.schema} + serverless: true + root_path: "../src/default_python_etl" + + libraries: + - glob: + include: ../src/default_python_etl/transformations/** + + environment: + dependencies: + # We include every dependency defined by pyproject.toml by defining an editable environment + # that points to the folder where pyproject.toml is deployed. + - --editable ${workspace.file_path} diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.job.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.job.yml deleted file mode 100644 index f617f4adc7..0000000000 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.job.yml +++ /dev/null @@ -1,45 +0,0 @@ -# The main job for my_default_python. -resources: - jobs: - my_default_python_job: - name: my_default_python_job - - trigger: - # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger - periodic: - interval: 1 - unit: DAYS - - #email_notifications: - # on_failure: - # - your_email@example.com - - tasks: - - task_key: notebook_task - notebook_task: - notebook_path: ../src/notebook.ipynb - - - task_key: refresh_pipeline - depends_on: - - task_key: notebook_task - pipeline_task: - pipeline_id: ${resources.pipelines.my_default_python_pipeline.id} - - - task_key: main_task - depends_on: - - task_key: refresh_pipeline - environment_key: default - python_wheel_task: - package_name: my_default_python - entry_point: main - - # A list of task execution environment specifications that can be referenced by tasks of this job. - environments: - - environment_key: default - - # Full documentation of this spec can be found at: - # https://docs.databricks.com/api/workspace/jobs/create#environments-spec - spec: - environment_version: "2" - dependencies: - - ../dist/*.whl diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml deleted file mode 100644 index 545a5ce556..0000000000 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml +++ /dev/null @@ -1,15 +0,0 @@ -# The main pipeline for my_default_python -resources: - pipelines: - my_default_python_pipeline: - name: my_default_python_pipeline - ## Catalog is required for serverless compute - catalog: main - schema: my_default_python_${bundle.target} - serverless: true - libraries: - - notebook: - path: ../src/pipeline.ipynb - - configuration: - bundle.sourcePath: ${workspace.file_path}/src diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml new file mode 100644 index 0000000000..c47eb8655e --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml @@ -0,0 +1,55 @@ +# A sample job for my_default_python. + +resources: + jobs: + sample_job: + name: sample_job + + trigger: + # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger + periodic: + interval: 1 + unit: DAYS + + #email_notifications: + # on_failure: + # - your_email@example.com + + parameters: + - name: catalog + default: ${var.catalog} + - name: schema + default: ${var.schema} + + tasks: + - task_key: notebook_task + notebook_task: + notebook_path: ../src/sample_notebook.ipynb + environment_key: default + - task_key: python_wheel_task + depends_on: + - task_key: notebook_task + python_wheel_task: + package_name: my_default_python + entry_point: main + parameters: + - "--catalog" + - "${var.catalog}" + - "--schema" + - "${var.schema}" + environment_key: default + - task_key: refresh_pipeline + depends_on: + - task_key: notebook_task + pipeline_task: + pipeline_id: ${resources.pipelines.default_python_etl.id} + + environments: + - environment_key: default + spec: + environment_version: "2" + dependencies: + # By default we just include the .whl file generated for the my_default_python package. + # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html + # for more information on how to add other libraries. + - ../dist/*.whl diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/README.md b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/README.md deleted file mode 100644 index e6cfb81b46..0000000000 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# scratch - -This folder is reserved for personal, exploratory notebooks. -By default these are not committed to Git, as 'scratch' is listed in .gitignore. diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/exploration.ipynb b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/exploration.ipynb deleted file mode 100644 index 3f589fed74..0000000000 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/exploration.ipynb +++ /dev/null @@ -1,61 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "import sys\n", - "\n", - "sys.path.append(\"../src\")\n", - "from my_default_python import main\n", - "\n", - "main.get_taxis().show(10)" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "ipynb-notebook", - "widgets": {} - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/README.md b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/README.md new file mode 100644 index 0000000000..b59a51fffb --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/README.md @@ -0,0 +1,20 @@ +# my_default_python + +This folder defines all source code for the my_default_python pipeline: + +- `explorations/`: Ad-hoc notebooks used to explore the data processed by this pipeline. +- `transformations/`: All dataset definitions and transformations. +- `utilities/` (optional): Utility functions and Python modules used in this pipeline. +- `data_sources/` (optional): View definitions describing the source data for this pipeline. + +## Getting Started + +To get started, go to the `transformations` folder -- most of the relevant source code lives there: + +* By convention, every dataset under `transformations` is in a separate file. +* Take a look at the sample called "sample_trips_my_default_python.py" to get familiar with the syntax. + Read more about the syntax at https://docs.databricks.com/dlt/python-ref.html. +* If you're using the workspace UI, use `Run file` to run and preview a single transformation. +* If you're using the CLI, use `databricks bundle run default_python_etl --select sample_trips_my_default_python` to run a single transformation. + +For more tutorials and reference material, see https://docs.databricks.com/dlt. diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/notebook.ipynb b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/explorations/sample_exploration.ipynb similarity index 50% rename from acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/notebook.ipynb rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/explorations/sample_exploration.ipynb index 27c3f19e34..e326a35169 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/notebook.ipynb +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/explorations/sample_exploration.ipynb @@ -8,23 +8,16 @@ "inputWidgets": {}, "nuid": "[UUID]", "showTitle": false, + "tableResultSettingsMap": {}, "title": "" } }, "source": [ - "# Default notebook\n", + "### Example Exploratory Notebook\n", "\n", - "This default notebook is executed using Databricks Workflows as defined in resources/my_default_python.job.yml." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" + "Use this notebook to explore the data generated by the pipeline in your preferred programming language.\n", + "\n", + "**Note**: This notebook is not executed as part of the pipeline." ] }, { @@ -32,42 +25,37 @@ "execution_count": 0, "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, + "cellMetadata": {}, "inputWidgets": {}, "nuid": "[UUID]", "showTitle": false, + "tableResultSettingsMap": {}, "title": "" } }, "outputs": [], "source": [ - "from my_default_python import main\n", + "# !!! Before performing any data analysis, make sure to run the pipeline to materialize the sample datasets. The tables referenced in this notebook depend on that step.\n", "\n", - "main.find_all_taxis().show(10)" + "display(spark.sql(\"SELECT * FROM hive_metastore.[USERNAME].sample_trips_my_default_python\"))" ] } ], "metadata": { "application/vnd.databricks.v1+notebook": { + "computePreferences": null, "dashboards": [], + "environmentMetadata": null, + "inputWidgetPreferences": null, "language": "python", "notebookMetadata": { "pythonIndentUnit": 2 }, - "notebookName": "notebook", + "notebookName": "sample_exploration", "widgets": {} }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, "language_info": { - "name": "python", - "version": "3.11.4" + "name": "python" } }, "nbformat": 4, diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/transformations/sample_trips_my_default_python.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/transformations/sample_trips_my_default_python.py new file mode 100644 index 0000000000..eb18701f08 --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/transformations/sample_trips_my_default_python.py @@ -0,0 +1,12 @@ +from pyspark import pipelines as dp +from pyspark.sql.functions import col + + +# This file defines a sample transformation. +# Edit the sample below or add new transformations +# using "+ Add" in the file browser. + + +@dp.table +def sample_trips_my_default_python(): + return spark.read.table("samples.nyctaxi.trips") diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/transformations/sample_zones_my_default_python.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/transformations/sample_zones_my_default_python.py new file mode 100644 index 0000000000..a9282b4dcc --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/transformations/sample_zones_my_default_python.py @@ -0,0 +1,17 @@ +from pyspark import pipelines as dp +from pyspark.sql.functions import col, sum + + +# This file defines a sample transformation. +# Edit the sample below or add new transformations +# using "+ Add" in the file browser. + + +@dp.table +def sample_zones_my_default_python(): + # Read from the "sample_trips" table, then sum all the fares + return ( + spark.read.table(f"sample_trips_my_default_python") + .groupBy(col("pickup_zip")) + .agg(sum("fare_amount").alias("total_fare")) + ) diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/main.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/main.py index 04e8be4de0..1f9c88b322 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/main.py +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/main.py @@ -1,13 +1,21 @@ +import argparse from databricks.sdk.runtime import spark -from pyspark.sql import DataFrame +from my_default_python import taxis -def find_all_taxis() -> DataFrame: - return spark.read.table("samples.nyctaxi.trips") +def main(): + # Process command-line arguments + parser = argparse.ArgumentParser(description="Databricks job with catalog and schema parameters") + parser.add_argument("--catalog", required=True) + parser.add_argument("--schema", required=True) + args = parser.parse_args() + # Set the default catalog and schema + spark.sql(f"USE CATALOG {args.catalog}") + spark.sql(f"USE SCHEMA {args.schema}") -def main(): - find_all_taxis().show(5) + # Example: just find all taxis from a sample catalog + taxis.find_all_taxis().show(5) if __name__ == "__main__": diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/taxis.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/taxis.py new file mode 100644 index 0000000000..a7309cd4c5 --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/taxis.py @@ -0,0 +1,7 @@ +from databricks.sdk.runtime import spark +from pyspark.sql import DataFrame + + +def find_all_taxis() -> DataFrame: + """Find all taxi data.""" + return spark.read.table("samples.nyctaxi.trips") diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/pipeline.ipynb b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/pipeline.ipynb deleted file mode 100644 index 21e8560105..0000000000 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/pipeline.ipynb +++ /dev/null @@ -1,90 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "source": [ - "# Lakeflow Declarative Pipeline\n", - "\n", - "This Lakeflow Declarative Pipeline definition is executed using a pipeline defined in resources/my_default_python.pipeline.yml." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "# Import DLT and src/my_default_python\n", - "import dlt\n", - "import sys\n", - "\n", - "sys.path.append(spark.conf.get(\"bundle.sourcePath\", \".\"))\n", - "from pyspark.sql.functions import expr\n", - "from my_default_python import main" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "@dlt.view\n", - "def taxi_raw():\n", - " return main.find_all_taxis()\n", - "\n", - "\n", - "@dlt.table\n", - "def filtered_taxis():\n", - " return dlt.read(\"taxi_raw\").filter(expr(\"fare_amount < 30\"))" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "pipeline", - "widgets": {} - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb new file mode 100644 index 0000000000..9b14f9286c --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb @@ -0,0 +1,149 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "[UUID]", + "showTitle": false, + "title": "" + } + }, + "source": [ + "# Default notebook\n", + "\n", + "This default notebook is executed using a Lakeflow job as defined in resources/sample_job.job.yml." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "[UUID]", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Set default catalog and schema\n", + "catalog = dbutils.widgets.get(\"catalog\")\n", + "schema = dbutils.widgets.get(\"schema\")\n", + "spark.sql(f\"USE CATALOG {catalog}\")\n", + "spark.sql(f\"USE SCHEMA {schema}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "[UUID]", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "sys.path.append(\"../src\")\n", + "from my_default_python import taxis\n", + "\n", + "taxis.find_all_taxis().show(10)" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "environmentMetadata": { + "base_environment": "", + "dependencies": [ + "--editable .." + ], + "environment_version": "2" + }, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 2 + }, + "notebookName": "notebook", + "widgets": { + "catalog": { + "currentValue": "hive_metastore", + "nuid": "c4t4l0g-w1dg-3t12-3456-[NUMID]", + "typedWidgetInfo": { + "autoCreated": false, + "defaultValue": "hive_metastore", + "label": "Catalog", + "name": "catalog", + "options": { + "widgetDisplayType": "Text", + "validationRegex": null + }, + "parameterDataType": "String" + }, + "widgetInfo": { + "widgetType": "text", + "defaultValue": "hive_metastore", + "label": "Catalog", + "name": "catalog", + "options": { + "widgetType": "text", + "autoCreated": false, + "validationRegex": null + } + } + }, + "schema": { + "currentValue": "[USERNAME]", + "nuid": "5ch3m4-w1dg-3t98-7654-[NUMID]", + "typedWidgetInfo": { + "autoCreated": false, + "defaultValue": "default", + "label": "Schema", + "name": "schema", + "options": { + "widgetDisplayType": "Text", + "validationRegex": null + }, + "parameterDataType": "String" + }, + "widgetInfo": { + "widgetType": "text", + "defaultValue": "default", + "label": "Schema", + "name": "schema", + "options": { + "widgetType": "text", + "autoCreated": false, + "validationRegex": null + } + } + } + } + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/conftest.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/conftest.py index f80cb4395e..4df274fd43 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/conftest.py +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/conftest.py @@ -1,4 +1,8 @@ -"""This file configures pytest.""" +"""This file configures pytest. + +This file is in the root since it can be used for tests in any place in this +project, including tests under resources/. +""" import os, sys, pathlib from contextlib import contextmanager @@ -9,13 +13,54 @@ from databricks.sdk import WorkspaceClient from pyspark.sql import SparkSession import pytest + import json + import csv + import os except ImportError: raise ImportError( "Test dependencies not found.\n\nRun tests using 'uv run pytest'. See http://docs.astral.sh/uv to learn more about uv." ) -def enable_fallback_compute(): +@pytest.fixture() +def spark() -> SparkSession: + """Provide a SparkSession fixture for tests. + + Minimal example: + def test_uses_spark(spark): + df = spark.createDataFrame([(1,)], ["x"]) + assert df.count() == 1 + """ + return DatabricksSession.builder.getOrCreate() + + +@pytest.fixture() +def load_fixture(spark: SparkSession): + """Provide a callable to load JSON or CSV from fixtures/ directory. + + Example usage: + + def test_using_fixture(load_fixture): + data = load_fixture("my_data.json") + assert data.count() >= 1 + """ + + def _loader(filename: str): + path = pathlib.Path(__file__).parent.parent / "fixtures" / filename + suffix = path.suffix.lower() + if suffix == ".json": + rows = json.loads(path.read_text()) + return spark.createDataFrame(rows) + if suffix == ".csv": + with path.open(newline="") as f: + rows = list(csv.DictReader(f)) + return spark.createDataFrame(rows) + raise ValueError(f"Unsupported fixture type for: {filename}") + + return _loader + + +def _enable_fallback_compute(): """Enable serverless compute if no compute is specified.""" conf = WorkspaceClient().config if conf.serverless_compute_id or conf.cluster_id or os.environ.get("SPARK_REMOTE"): @@ -23,13 +68,13 @@ def enable_fallback_compute(): url = "https://docs.databricks.com/dev-tools/databricks-connect/cluster-config" print("☁️ no compute specified, falling back to serverless compute", file=sys.stderr) - print(f" see {url} for manual configuration", file=sys.stderr) + print(f" see {url} for manual configuration", file=sys.stdout) os.environ["DATABRICKS_SERVERLESS_COMPUTE_ID"] = "auto" @contextmanager -def allow_stderr_output(config: pytest.Config): +def _allow_stderr_output(config: pytest.Config): """Temporarily disable pytest output capture.""" capman = config.pluginmanager.get_plugin("capturemanager") if capman: @@ -41,8 +86,8 @@ def allow_stderr_output(config: pytest.Config): def pytest_configure(config: pytest.Config): """Configure pytest session.""" - with allow_stderr_output(config): - enable_fallback_compute() + with _allow_stderr_output(config): + _enable_fallback_compute() # Initialize Spark session eagerly, so it is available even when # SparkSession.builder.getOrCreate() is used. For DB Connect 15+, @@ -51,9 +96,3 @@ def pytest_configure(config: pytest.Config): DatabricksSession.builder.validateSession().getOrCreate() else: DatabricksSession.builder.getOrCreate() - - -@pytest.fixture(scope="session") -def spark() -> SparkSession: - """Provide a SparkSession fixture for tests.""" - return DatabricksSession.builder.getOrCreate() diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/main_test.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/main_test.py deleted file mode 100644 index 4bfd5e1550..0000000000 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/main_test.py +++ /dev/null @@ -1,6 +0,0 @@ -from my_default_python import main - - -def test_find_all_taxis(): - taxis = main.find_all_taxis() - assert taxis.count() > 5 diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/sample_taxis_test.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/sample_taxis_test.py new file mode 100644 index 0000000000..e503dd2b2c --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/sample_taxis_test.py @@ -0,0 +1,8 @@ +from databricks.sdk.runtime import spark +from pyspark.sql import DataFrame +from my_default_python import taxis + + +def test_find_all_taxis(): + results = taxis.find_all_taxis() + assert results.count() > 5 diff --git a/acceptance/bundle/templates/telemetry/default-python/out.databricks.yml b/acceptance/bundle/templates/telemetry/default-python/out.databricks.yml index 687383d471..62bd43d325 100644 --- a/acceptance/bundle/templates/telemetry/default-python/out.databricks.yml +++ b/acceptance/bundle/templates/telemetry/default-python/out.databricks.yml @@ -4,14 +4,21 @@ bundle: name: my_default_python uuid: [BUNDLE-UUID] +include: + - resources/*.yml + - resources/*/*.yml + artifacts: python_artifact: type: whl build: uv build --wheel -include: - - resources/*.yml - - resources/*/*.yml +# Variable declarations. These variables are assigned in the dev/prod targets below. +variables: + catalog: + description: The catalog to use + schema: + description: The schema to use targets: dev: @@ -23,20 +30,20 @@ targets: default: true workspace: host: [DATABRICKS_URL] - + variables: + catalog: hive_metastore + schema: ${workspace.current_user.short_name} presets: - # Set dynamic_version: true on all artifacts of type "whl". - # This makes "bundle deploy" add a timestamp to wheel's version before uploading, - # new wheel takes over the previous installation even if actual wheel version is unchanged. - # See https://docs.databricks.com/aws/en/dev-tools/bundles/settings artifacts_dynamic_version: true - prod: mode: production workspace: host: [DATABRICKS_URL] # We explicitly deploy to /Workspace/Users/[USERNAME] to make sure we only have a single copy. root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + variables: + catalog: hive_metastore + schema: prod permissions: - user_name: [USERNAME] level: CAN_MANAGE diff --git a/acceptance/bundle/templates/telemetry/default-python/out.requests.txt b/acceptance/bundle/templates/telemetry/default-python/out.requests.txt index f36603307e..fef494494f 100644 --- a/acceptance/bundle/templates/telemetry/default-python/out.requests.txt +++ b/acceptance/bundle/templates/telemetry/default-python/out.requests.txt @@ -5,7 +5,7 @@ ] }, "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" + "path": "/api/2.1/unity-catalog/current-metastore-assignment" } { "headers": { @@ -14,7 +14,7 @@ ] }, "method": "GET", - "path": "/api/2.1/unity-catalog/current-metastore-assignment" + "path": "/api/2.0/preview/scim/v2/Me" } { "headers": { @@ -28,7 +28,7 @@ "uploadTime": [UNIX_TIME_MILLIS], "items": [], "protoLogs": [ - "{\"frontend_log_event_id\":\"[UUID]\",\"entry\":{\"databricks_cli_log\":{\"execution_context\":{\"cmd_exec_id\":\"[CMD-EXEC-ID]\",\"version\":\"[DEV_VERSION]\",\"command\":\"bundle_init\",\"operating_system\":\"[OS]\",\"execution_time_ms\":\"SMALL_INT\",\"exit_code\":0},\"bundle_init_event\":{\"bundle_uuid\":\"[BUNDLE-UUID]\",\"template_name\":\"default-python\",\"template_enum_args\":[{\"key\":\"include_dlt\",\"value\":\"no\"},{\"key\":\"include_notebook\",\"value\":\"yes\"},{\"key\":\"include_python\",\"value\":\"yes\"},{\"key\":\"serverless\",\"value\":\"no\"}]}}}}" + "{\"frontend_log_event_id\":\"[UUID]\",\"entry\":{\"databricks_cli_log\":{\"execution_context\":{\"cmd_exec_id\":\"[CMD-EXEC-ID]\",\"version\":\"[DEV_VERSION]\",\"command\":\"bundle_init\",\"operating_system\":\"[OS]\",\"execution_time_ms\":\"SMALL_INT\",\"exit_code\":0},\"bundle_init_event\":{\"bundle_uuid\":\"[BUNDLE-UUID]\",\"template_name\":\"default-python\",\"template_enum_args\":[{\"key\":\"include_job\",\"value\":\"yes\"},{\"key\":\"include_pipeline\",\"value\":\"yes\"},{\"key\":\"include_python\",\"value\":\"yes\"},{\"key\":\"language\",\"value\":\"python\"},{\"key\":\"personal_schemas\",\"value\":\"yes\"},{\"key\":\"serverless\",\"value\":\"no\"}]}}}}" ] } } diff --git a/acceptance/bundle/templates/telemetry/default-python/output.txt b/acceptance/bundle/templates/telemetry/default-python/output.txt index 6124901bf9..d2882e3eb8 100644 --- a/acceptance/bundle/templates/telemetry/default-python/output.txt +++ b/acceptance/bundle/templates/telemetry/default-python/output.txt @@ -1,6 +1,10 @@ - Welcome to the default Python template for Databricks Asset Bundles! -Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): [DATABRICKS_URL] + +Please answer the below to tailor your project to your preferences. +You can always change your mind and change your configuration in the databricks.yml file later. + +Note that [DATABRICKS_URL] is used for initialization +(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). ✨ Your new project has been created in the 'my_default_python' directory! @@ -25,17 +29,25 @@ See also the documentation at https://docs.databricks.com/dev-tools/bundles/inde "template_name": "default-python", "template_enum_args": [ { - "key": "include_dlt", - "value": "no" + "key": "include_job", + "value": "yes" }, { - "key": "include_notebook", + "key": "include_pipeline", "value": "yes" }, { "key": "include_python", "value": "yes" }, + { + "key": "language", + "value": "python" + }, + { + "key": "personal_schemas", + "value": "yes" + }, { "key": "serverless", "value": "no" From e63b542bc0d87bc69c0bfcb694759cbd42bb4641 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Tue, 14 Oct 2025 19:30:45 +0200 Subject: [PATCH 03/24] Minor tweak, update tests --- .../classic/output/my_default_python/pyproject.toml | 2 +- .../serverless/output/my_default_python/pyproject.toml | 2 +- .../python/output/my_lakeflow_pipelines/pyproject.toml | 2 +- .../sql/output/my_lakeflow_pipelines/pyproject.toml | 2 +- .../default/template/{{.project_name}}/pyproject.toml.tmpl | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml index 607ea88364..12114d057f 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [ # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies # # LIMITATION: for pipelines, dependencies are cached during development; - # add dependencies to the 'environment' section of pipeline.yml file instead + # add dependencies to the 'environment' section of your pipeline.yml file instead ] [dependency-groups] diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml index 607ea88364..12114d057f 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [ # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies # # LIMITATION: for pipelines, dependencies are cached during development; - # add dependencies to the 'environment' section of pipeline.yml file instead + # add dependencies to the 'environment' section of your pipeline.yml file instead ] [dependency-groups] diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml index 45f619c9b4..c5cf343c0b 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [ # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies # # LIMITATION: for pipelines, dependencies are cached during development; - # add dependencies to the 'environment' section of pipeline.yml file instead + # add dependencies to the 'environment' section of your pipeline.yml file instead ] [dependency-groups] diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/pyproject.toml b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/pyproject.toml index 45f619c9b4..c5cf343c0b 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/pyproject.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [ # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies # # LIMITATION: for pipelines, dependencies are cached during development; - # add dependencies to the 'environment' section of pipeline.yml file instead + # add dependencies to the 'environment' section of your pipeline.yml file instead ] [dependency-groups] diff --git a/libs/template/templates/default/template/{{.project_name}}/pyproject.toml.tmpl b/libs/template/templates/default/template/{{.project_name}}/pyproject.toml.tmpl index 71972a952b..cab493cb87 100644 --- a/libs/template/templates/default/template/{{.project_name}}/pyproject.toml.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/pyproject.toml.tmpl @@ -8,7 +8,7 @@ dependencies = [ # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies # # LIMITATION: for pipelines, dependencies are cached during development; - # add dependencies to the 'environment' section of pipeline.yml file instead + # add dependencies to the 'environment' section of your pipeline.yml file instead ] [dependency-groups] From 6321f5c4d95baaf17c1cb0c0aa8b48c463681e60 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Tue, 14 Oct 2025 20:46:23 +0200 Subject: [PATCH 04/24] Formatting --- .../src/sample_notebook.ipynb | 26 +++++++++---------- .../serverless-customcatalog/output.txt | 4 +-- .../src/sample_notebook.ipynb | 26 +++++++++---------- .../src/sample_notebook.ipynb.tmpl | 26 +++++++++---------- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb index 9b14f9286c..226cb2d064 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb @@ -35,7 +35,7 @@ "catalog = dbutils.widgets.get(\"catalog\")\n", "schema = dbutils.widgets.get(\"schema\")\n", "spark.sql(f\"USE CATALOG {catalog}\")\n", - "spark.sql(f\"USE SCHEMA {schema}\")\n" + "spark.sql(f\"USE SCHEMA {schema}\")" ] }, { @@ -89,21 +89,21 @@ "label": "Catalog", "name": "catalog", "options": { - "widgetDisplayType": "Text", - "validationRegex": null + "validationRegex": null, + "widgetDisplayType": "Text" }, "parameterDataType": "String" }, "widgetInfo": { - "widgetType": "text", "defaultValue": "hive_metastore", "label": "Catalog", "name": "catalog", "options": { - "widgetType": "text", "autoCreated": false, - "validationRegex": null - } + "validationRegex": null, + "widgetType": "text" + }, + "widgetType": "text" } }, "schema": { @@ -115,21 +115,21 @@ "label": "Schema", "name": "schema", "options": { - "widgetDisplayType": "Text", - "validationRegex": null + "validationRegex": null, + "widgetDisplayType": "Text" }, "parameterDataType": "String" }, "widgetInfo": { - "widgetType": "text", "defaultValue": "default", "label": "Schema", "name": "schema", "options": { - "widgetType": "text", "autoCreated": false, - "validationRegex": null - } + "validationRegex": null, + "widgetType": "text" + }, + "widgetType": "text" } } } diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt index c150171c97..039b7fd049 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt @@ -61,9 +61,9 @@ See also the documentation at https://docs.databricks.com/dev-tools/bundles/inde + "defaultValue": "customcatalog", "label": "Catalog", "name": "catalog", -@@ -97,5 +97,5 @@ +@@ -96,5 +96,5 @@ + }, "widgetInfo": { - "widgetType": "text", - "defaultValue": "hive_metastore", + "defaultValue": "customcatalog", "label": "Catalog", diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb index 9b14f9286c..226cb2d064 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb @@ -35,7 +35,7 @@ "catalog = dbutils.widgets.get(\"catalog\")\n", "schema = dbutils.widgets.get(\"schema\")\n", "spark.sql(f\"USE CATALOG {catalog}\")\n", - "spark.sql(f\"USE SCHEMA {schema}\")\n" + "spark.sql(f\"USE SCHEMA {schema}\")" ] }, { @@ -89,21 +89,21 @@ "label": "Catalog", "name": "catalog", "options": { - "widgetDisplayType": "Text", - "validationRegex": null + "validationRegex": null, + "widgetDisplayType": "Text" }, "parameterDataType": "String" }, "widgetInfo": { - "widgetType": "text", "defaultValue": "hive_metastore", "label": "Catalog", "name": "catalog", "options": { - "widgetType": "text", "autoCreated": false, - "validationRegex": null - } + "validationRegex": null, + "widgetType": "text" + }, + "widgetType": "text" } }, "schema": { @@ -115,21 +115,21 @@ "label": "Schema", "name": "schema", "options": { - "widgetDisplayType": "Text", - "validationRegex": null + "validationRegex": null, + "widgetDisplayType": "Text" }, "parameterDataType": "String" }, "widgetInfo": { - "widgetType": "text", "defaultValue": "default", "label": "Schema", "name": "schema", "options": { - "widgetType": "text", "autoCreated": false, - "validationRegex": null - } + "validationRegex": null, + "widgetType": "text" + }, + "widgetType": "text" } } } diff --git a/libs/template/templates/default/template/{{.project_name}}/src/sample_notebook.ipynb.tmpl b/libs/template/templates/default/template/{{.project_name}}/src/sample_notebook.ipynb.tmpl index 67430c2218..23d2003538 100644 --- a/libs/template/templates/default/template/{{.project_name}}/src/sample_notebook.ipynb.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/src/sample_notebook.ipynb.tmpl @@ -35,7 +35,7 @@ "catalog = dbutils.widgets.get(\"catalog\")\n", "schema = dbutils.widgets.get(\"schema\")\n", "spark.sql(f\"USE CATALOG {catalog}\")\n", - "spark.sql(f\"USE SCHEMA {schema}\")\n" + "spark.sql(f\"USE SCHEMA {schema}\")" ] }, { @@ -94,21 +94,21 @@ "label": "Catalog", "name": "catalog", "options": { - "widgetDisplayType": "Text", - "validationRegex": null + "validationRegex": null, + "widgetDisplayType": "Text" }, "parameterDataType": "String" }, "widgetInfo": { - "widgetType": "text", "defaultValue": "{{.default_catalog}}", "label": "Catalog", "name": "catalog", "options": { - "widgetType": "text", "autoCreated": false, - "validationRegex": null - } + "validationRegex": null, + "widgetType": "text" + }, + "widgetType": "text" } }, "schema": { @@ -120,21 +120,21 @@ "label": "Schema", "name": "schema", "options": { - "widgetDisplayType": "Text", - "validationRegex": null + "validationRegex": null, + "widgetDisplayType": "Text" }, "parameterDataType": "String" }, "widgetInfo": { - "widgetType": "text", "defaultValue": "default", "label": "Schema", "name": "schema", "options": { - "widgetType": "text", "autoCreated": false, - "validationRegex": null - } + "validationRegex": null, + "widgetType": "text" + }, + "widgetType": "text" } } } From 17d685585cbe462e03972bf09c477d245b19784a Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Tue, 14 Oct 2025 21:40:41 +0200 Subject: [PATCH 05/24] Fix whitespace in template acceptance test outputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regenerate acceptance test outputs for default-python template to remove extra trailing newline in README files. The template source was already correct; the acceptance test outputs just needed to be regenerated using the -update flag. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../default-python/classic/out.compare-vs-serverless.diff | 6 ++++++ .../classic/output/my_default_python/README.md | 1 - .../serverless/output/my_default_python/README.md | 1 - 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff index d3850c8fc0..a5257783df 100644 --- a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff +++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff @@ -1,3 +1,9 @@ +--- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/README.md ++++ output/my_default_python/README.md +@@ -69,3 +69,2 @@ + $ uv run pytest + ``` +- --- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/databricks.yml +++ output/my_default_python/databricks.yml @@ -34,4 +34,6 @@ diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md index 75a9fb55b7..b07e72a585 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md @@ -68,4 +68,3 @@ with this project. It's also possible to interact with it directly using the CLI ``` $ uv run pytest ``` - diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md index 75a9fb55b7..b07e72a585 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md @@ -68,4 +68,3 @@ with this project. It's also possible to interact with it directly using the CLI ``` $ uv run pytest ``` - From 37df48c714966f9000883e0e47b9bf0d8301aa8c Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Tue, 14 Oct 2025 21:56:04 +0200 Subject: [PATCH 06/24] Fix template whitespace: use {{- end}} to strip trailing newline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README template was generating an extra newline at the end of files. Fixed by using {{- end}} instead of {{end}} to strip the newline that comes after the closing code fence. This ensures generated README files end with exactly one newline as required by the whitespace linter. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../default-python/classic/out.compare-vs-serverless.diff | 6 ------ .../default/template/{{.project_name}}/README.md.tmpl | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff index a5257783df..d3850c8fc0 100644 --- a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff +++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff @@ -1,9 +1,3 @@ ---- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/README.md -+++ output/my_default_python/README.md -@@ -69,3 +69,2 @@ - $ uv run pytest - ``` -- --- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/databricks.yml +++ output/my_default_python/databricks.yml @@ -34,4 +34,6 @@ diff --git a/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl index f82bb9094f..9b0d6b2736 100644 --- a/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl @@ -92,4 +92,4 @@ with this project. It's also possible to interact with it directly using the CLI ``` $ uv run pytest ``` -{{end}} +{{- end}} From af524bb993eaffe059d65f93854d544a162fc6ef Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Wed, 15 Oct 2025 09:23:36 +0200 Subject: [PATCH 07/24] Update NEXT_CHANGELOG.md --- NEXT_CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 714bfe74a4..8eb9caee71 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -11,7 +11,7 @@ ### Dependency updates ### Bundles -* Updated the default-python template with improved structure and functionality. +* Updated the default-python template to follow the Lakeflow conventions: pipelines as source files, pyproject.toml ([#3712](https://github.com/databricks/cli/pull/3712)). * Updated the internal lakeflow-pipelines template to use an "src" layout ([#3671](https://github.com/databricks/cli/pull/3671)). ### API Changes From 5ed406752eab47f175b34b857c5570c409275564 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Thu, 16 Oct 2025 09:00:00 +0200 Subject: [PATCH 08/24] Update libs/template/templates/default-python/databricks_template_schema.json Co-authored-by: Julia Crawford (Databricks) --- .../templates/default-python/databricks_template_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/template/templates/default-python/databricks_template_schema.json b/libs/template/templates/default-python/databricks_template_schema.json index a323e4360b..392e6da568 100644 --- a/libs/template/templates/default-python/databricks_template_schema.json +++ b/libs/template/templates/default-python/databricks_template_schema.json @@ -1,6 +1,6 @@ { "template_dir": "../default", - "welcome_message": "Welcome to the default Python template for Databricks Asset Bundles!\n\nPlease answer the below to tailor your project to your preferences.\nYou can always change your mind and change your configuration in the databricks.yml file later.\n\nNote that {{workspace_host}} is used for initialization\n(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile).", + "welcome_message": "Welcome to the default Python template for Databricks Asset Bundles!\n\nAnswer the following questions to customize your project.\nYou can always change your configuration in the databricks.yml file later.\n\nNote that {{workspace_host}} is used for initialization.\n(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.)", "properties": { "project_name": { "type": "string", From bb33f6e106a385cb853e8e5a041a5d7f37a019d9 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Thu, 16 Oct 2025 09:00:54 +0200 Subject: [PATCH 09/24] Update libs/template/templates/default-python/databricks_template_schema.json --- .../templates/default-python/databricks_template_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/template/templates/default-python/databricks_template_schema.json b/libs/template/templates/default-python/databricks_template_schema.json index 392e6da568..6c0d23dcc9 100644 --- a/libs/template/templates/default-python/databricks_template_schema.json +++ b/libs/template/templates/default-python/databricks_template_schema.json @@ -29,7 +29,7 @@ "type": "string", "default": "yes", "enum": ["yes", "no"], - "description": "Include a Lakeflow ETL pipeline", + "description": "Include an ETL pipeline", "order": 4 }, "include_python": { From 7890ddb02836fecad1919e4a7e261dfbf3ba0109 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Thu, 16 Oct 2025 09:01:01 +0200 Subject: [PATCH 10/24] Update libs/template/templates/default-python/databricks_template_schema.json Co-authored-by: Julia Crawford (Databricks) --- .../templates/default-python/databricks_template_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/template/templates/default-python/databricks_template_schema.json b/libs/template/templates/default-python/databricks_template_schema.json index 6c0d23dcc9..7d83af8bb5 100644 --- a/libs/template/templates/default-python/databricks_template_schema.json +++ b/libs/template/templates/default-python/databricks_template_schema.json @@ -56,7 +56,7 @@ }, "personal_schemas": { "type": "string", - "description": "Use a personal schema for each user working on this project\n(this is recommended, your personal schema will be '{{.default_catalog}}.{{short_name}}')", + "description": "Use a personal schema for each user working on this project.\n(This is recommended. Your personal schema will be '{{.default_catalog}}.{{short_name}}'.)", "default": "yes", "enum": [ "yes", From d588ebec183564f9e96ce5b88067b2e854c1ee29 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Thu, 16 Oct 2025 09:01:21 +0200 Subject: [PATCH 11/24] Update libs/template/templates/default-python/databricks_template_schema.json Co-authored-by: Julia Crawford (Databricks) --- .../templates/default-python/databricks_template_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/template/templates/default-python/databricks_template_schema.json b/libs/template/templates/default-python/databricks_template_schema.json index 7d83af8bb5..8f994925f9 100644 --- a/libs/template/templates/default-python/databricks_template_schema.json +++ b/libs/template/templates/default-python/databricks_template_schema.json @@ -36,7 +36,7 @@ "type": "string", "default": "yes", "enum": ["yes", "no"], - "description": "Include a sample Python package that is built to a wheel file", + "description": "Include a sample Python package that builds into a wheel file", "order": 5 }, "serverless": { From a3673b8422fdc421a24b89288d312efe597bd990 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Thu, 16 Oct 2025 09:01:34 +0200 Subject: [PATCH 12/24] Update libs/template/templates/default-python/databricks_template_schema.json Co-authored-by: Julia Crawford (Databricks) --- .../templates/default-python/databricks_template_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/template/templates/default-python/databricks_template_schema.json b/libs/template/templates/default-python/databricks_template_schema.json index 8f994925f9..1823b38903 100644 --- a/libs/template/templates/default-python/databricks_template_schema.json +++ b/libs/template/templates/default-python/databricks_template_schema.json @@ -60,7 +60,7 @@ "default": "yes", "enum": [ "yes", - "no (advanced: I will customize the schema configuration later in databricks.yml)" + "no, I will customize the schema configuration later in databricks.yml" ], "order": 8 }, From e1d4b499fc3b34283d10a525d574f25a8ca61e7c Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Thu, 16 Oct 2025 09:01:49 +0200 Subject: [PATCH 13/24] Update libs/template/templates/default-python/databricks_template_schema.json Co-authored-by: Julia Crawford (Databricks) --- .../templates/default-python/databricks_template_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/template/templates/default-python/databricks_template_schema.json b/libs/template/templates/default-python/databricks_template_schema.json index 1823b38903..6f16a59723 100644 --- a/libs/template/templates/default-python/databricks_template_schema.json +++ b/libs/template/templates/default-python/databricks_template_schema.json @@ -85,5 +85,5 @@ "order": 10 } }, - "success_message": "\n✨ Your new project has been created in the '{{.project_name}}' directory!\n\nPlease refer to the README.md file for \"getting started\" instructions.\nSee also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html." + "success_message": "\n✨ Your new project has been created in the '{{.project_name}}' directory!\n\nTo get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html." } From 03c28534d955a233b67bb4d7e6a98f2dfc8a0dc9 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Thu, 16 Oct 2025 09:07:25 +0200 Subject: [PATCH 14/24] Update acceptance tests for default-python template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update output expectations to match the new welcome and success messages in the default-python template schema. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../templates/default-python/classic/output.txt | 11 +++++------ .../default-python/fail-missing-uv/output.txt | 11 +++++------ .../default-python/integration_classic/output.txt | 11 +++++------ .../bundle/templates/default-python/no-uc/output.txt | 11 +++++------ .../serverless-customcatalog/output.txt | 11 +++++------ .../templates/default-python/serverless/output.txt | 11 +++++------ 6 files changed, 30 insertions(+), 36 deletions(-) diff --git a/acceptance/bundle/templates/default-python/classic/output.txt b/acceptance/bundle/templates/default-python/classic/output.txt index e78cc4b8c2..5c8fea374e 100644 --- a/acceptance/bundle/templates/default-python/classic/output.txt +++ b/acceptance/bundle/templates/default-python/classic/output.txt @@ -2,16 +2,15 @@ >>> [CLI] bundle init default-python --config-file ./input.json --output-dir output Welcome to the default Python template for Databricks Asset Bundles! -Please answer the below to tailor your project to your preferences. -You can always change your mind and change your configuration in the databricks.yml file later. +Answer the following questions to customize your project. +You can always change your configuration in the databricks.yml file later. -Note that [DATABRICKS_URL] is used for initialization -(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). +Note that [DATABRICKS_URL] is used for initialization. +(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.) ✨ Your new project has been created in the 'my_default_python' directory! -Please refer to the README.md file for "getting started" instructions. -See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. >>> [CLI] bundle validate -t dev Name: my_default_python diff --git a/acceptance/bundle/templates/default-python/fail-missing-uv/output.txt b/acceptance/bundle/templates/default-python/fail-missing-uv/output.txt index 0631cd6a2f..f96a96c1c5 100644 --- a/acceptance/bundle/templates/default-python/fail-missing-uv/output.txt +++ b/acceptance/bundle/templates/default-python/fail-missing-uv/output.txt @@ -2,16 +2,15 @@ >>> [CLI] bundle init default-python --config-file ./input.json --output-dir output Welcome to the default Python template for Databricks Asset Bundles! -Please answer the below to tailor your project to your preferences. -You can always change your mind and change your configuration in the databricks.yml file later. +Answer the following questions to customize your project. +You can always change your configuration in the databricks.yml file later. -Note that [DATABRICKS_URL] is used for initialization -(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). +Note that [DATABRICKS_URL] is used for initialization. +(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.) ✨ Your new project has been created in the 'fail_missing_uv' directory! -Please refer to the README.md file for "getting started" instructions. -See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. >>> [CLI] bundle validate Name: fail_missing_uv diff --git a/acceptance/bundle/templates/default-python/integration_classic/output.txt b/acceptance/bundle/templates/default-python/integration_classic/output.txt index f255d1c7ad..a519959642 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/output.txt +++ b/acceptance/bundle/templates/default-python/integration_classic/output.txt @@ -5,16 +5,15 @@ >>> [CLI] bundle init default-python --config-file ./input.json --output-dir . Welcome to the default Python template for Databricks Asset Bundles! -Please answer the below to tailor your project to your preferences. -You can always change your mind and change your configuration in the databricks.yml file later. +Answer the following questions to customize your project. +You can always change your configuration in the databricks.yml file later. -Note that [DATABRICKS_URL] is used for initialization -(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). +Note that [DATABRICKS_URL] is used for initialization. +(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.) ✨ Your new project has been created in the 'project_name_[UNIQUE_NAME]' directory! -Please refer to the README.md file for "getting started" instructions. -See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. >>> [CLI] bundle validate -t dev Name: project_name_[UNIQUE_NAME] diff --git a/acceptance/bundle/templates/default-python/no-uc/output.txt b/acceptance/bundle/templates/default-python/no-uc/output.txt index 20e30bb6fb..4c34df71ef 100644 --- a/acceptance/bundle/templates/default-python/no-uc/output.txt +++ b/acceptance/bundle/templates/default-python/no-uc/output.txt @@ -2,13 +2,12 @@ >>> [CLI] bundle init default-python --config-file ./input.json --output-dir output Welcome to the default Python template for Databricks Asset Bundles! -Please answer the below to tailor your project to your preferences. -You can always change your mind and change your configuration in the databricks.yml file later. +Answer the following questions to customize your project. +You can always change your configuration in the databricks.yml file later. -Note that [DATABRICKS_URL] is used for initialization -(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). +Note that [DATABRICKS_URL] is used for initialization. +(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.) ✨ Your new project has been created in the 'my_default_python' directory! -Please refer to the README.md file for "getting started" instructions. -See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt index 039b7fd049..9ae640d43a 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt @@ -2,16 +2,15 @@ >>> [CLI] bundle init default-python --config-file [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/input.json --output-dir output Welcome to the default Python template for Databricks Asset Bundles! -Please answer the below to tailor your project to your preferences. -You can always change your mind and change your configuration in the databricks.yml file later. +Answer the following questions to customize your project. +You can always change your configuration in the databricks.yml file later. -Note that [DATABRICKS_URL] is used for initialization -(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). +Note that [DATABRICKS_URL] is used for initialization. +(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.) ✨ Your new project has been created in the 'my_default_python' directory! -Please refer to the README.md file for "getting started" instructions. -See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. >>> diff.py [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output output/ --- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/databricks.yml diff --git a/acceptance/bundle/templates/default-python/serverless/output.txt b/acceptance/bundle/templates/default-python/serverless/output.txt index 980d6786c6..d3b26ddd5b 100644 --- a/acceptance/bundle/templates/default-python/serverless/output.txt +++ b/acceptance/bundle/templates/default-python/serverless/output.txt @@ -2,16 +2,15 @@ >>> [CLI] bundle init default-python --config-file ./input.json --output-dir output Welcome to the default Python template for Databricks Asset Bundles! -Please answer the below to tailor your project to your preferences. -You can always change your mind and change your configuration in the databricks.yml file later. +Answer the following questions to customize your project. +You can always change your configuration in the databricks.yml file later. -Note that [DATABRICKS_URL] is used for initialization -(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). +Note that [DATABRICKS_URL] is used for initialization. +(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.) ✨ Your new project has been created in the 'my_default_python' directory! -Please refer to the README.md file for "getting started" instructions. -See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. >>> [CLI] bundle validate -t dev Name: my_default_python From 0910a805a9eeca380360a2e1a1c9cb1937842651 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Thu, 16 Oct 2025 10:31:02 +0200 Subject: [PATCH 15/24] Fix URL --- .../default-python/classic/output/my_default_python/README.md | 2 +- .../serverless/output/my_default_python/README.md | 2 +- .../python/output/my_lakeflow_pipelines/README.md | 2 +- .../sql/output/my_lakeflow_pipelines/README.md | 2 +- .../default-python/template/{{.project_name}}/README.md.tmpl | 2 +- .../templates/default/template/{{.project_name}}/README.md.tmpl | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md index b07e72a585..907deacb90 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md @@ -17,7 +17,7 @@ Choose how you want to work on this project: https://docs.databricks.com/dev-tools/bundles/workspace. (b) Locally with an IDE like Cursor or VS Code, see - https://docs.databricks.com/vscode-ext. + https://docs.databricks.com/dev-tools/vscode-ext.html. (c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md index b07e72a585..907deacb90 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md @@ -17,7 +17,7 @@ Choose how you want to work on this project: https://docs.databricks.com/dev-tools/bundles/workspace. (b) Locally with an IDE like Cursor or VS Code, see - https://docs.databricks.com/vscode-ext. + https://docs.databricks.com/dev-tools/vscode-ext.html. (c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/README.md b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/README.md index 7bb321e833..f9c9e3c5b7 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/README.md +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/README.md @@ -13,7 +13,7 @@ Choose how you want to work on this project: https://docs.databricks.com/dev-tools/bundles/workspace. (b) Locally with an IDE like Cursor or VS Code, see - https://docs.databricks.com/vscode-ext. + https://docs.databricks.com/dev-tools/vscode-ext.html. (c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/README.md b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/README.md index 89233d4f04..b073fc5038 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/README.md +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/README.md @@ -13,7 +13,7 @@ Choose how you want to work on this project: https://docs.databricks.com/dev-tools/bundles/workspace. (b) Locally with an IDE like Cursor or VS Code, see - https://docs.databricks.com/vscode-ext. + https://docs.databricks.com/dev-tools/vscode-ext.html. (c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html diff --git a/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl index 02da531477..9b9e365272 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl @@ -13,7 +13,7 @@ Choose how you want to work on this project: https://docs.databricks.com/dev-tools/bundles/workspace. (b) Locally with an IDE like Cursor or VS Code, see - https://docs.databricks.com/vscode-ext. + https://docs.databricks.com/dev-tools/vscode-ext.html. (c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html diff --git a/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl index 9b0d6b2736..3287d5ed6b 100644 --- a/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl @@ -24,7 +24,7 @@ Choose how you want to work on this project: https://docs.databricks.com/dev-tools/bundles/workspace. (b) Locally with an IDE like Cursor or VS Code, see - https://docs.databricks.com/vscode-ext. + https://docs.databricks.com/dev-tools/vscode-ext.html. (c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html From 5e7f05e5e5a37dac6f55a9ad3cbc477974209580 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Thu, 16 Oct 2025 10:51:30 +0200 Subject: [PATCH 16/24] Remove the project_name_short special casing --- .../classic/out.compare-vs-serverless.diff | 12 +++++----- .../classic/out.plan_dev.direct-exp.json | 16 ++++++------- .../classic/out.plan_dev.terraform.json | 2 +- .../classic/out.plan_prod.direct-exp.json | 16 ++++++------- .../classic/out.plan_prod.terraform.json | 2 +- .../classic/out.requests.dev.direct-exp.txt | 10 ++++---- .../classic/out.requests.dev.terraform.txt | 10 ++++---- .../classic/out.requests.prod.direct-exp.txt | 10 ++++---- .../classic/out.requests.prod.terraform.txt | 10 ++++---- .../default-python/classic/output.txt | 4 ++-- .../output/my_default_python/README.md | 2 +- ...yml => my_default_python_etl.pipeline.yml} | 8 +++---- .../resources/sample_job.job.yml | 2 +- .../README.md | 2 +- .../explorations/sample_exploration.ipynb | 0 .../sample_trips_my_default_python.py | 0 .../sample_zones_my_default_python.py | 0 .../combinations/classic/output.txt | 11 ++++----- .../combinations/serverless/output.txt | 11 ++++----- .../serverless-customcatalog/output.txt | 12 +++++----- .../output/my_default_python/README.md | 2 +- ...yml => my_default_python_etl.pipeline.yml} | 8 +++---- .../resources/sample_job.job.yml | 2 +- .../README.md | 2 +- .../explorations/sample_exploration.ipynb | 0 .../sample_trips_my_default_python.py | 0 .../sample_zones_my_default_python.py | 0 .../output/my_lakeflow_pipelines/README.md | 2 +- ...=> my_lakeflow_pipelines_etl.pipeline.yml} | 8 +++---- .../resources/sample_job.job.yml | 2 +- .../README.md | 2 +- .../explorations/sample_exploration.ipynb | 0 .../sample_trips_my_lakeflow_pipelines.py | 0 .../sample_zones_my_lakeflow_pipelines.py | 0 .../output/my_lakeflow_pipelines/README.md | 2 +- ...=> my_lakeflow_pipelines_etl.pipeline.yml} | 8 +++---- .../resources/sample_job.job.yml | 2 +- .../README.md | 2 +- .../explorations/sample_exploration.ipynb | 0 .../sample_trips_my_lakeflow_pipelines.sql | 0 .../sample_zones_my_lakeflow_pipelines.sql | 0 .../databricks_template_schema.json | 24 +++++++------------ .../default/template/__preamble.tmpl | 8 +++---- .../template/{{.project_name}}/README.md.tmpl | 4 ++-- .../resources/sample_job.job.yml.tmpl | 2 +- ...> {{.project_name}}_etl.pipeline.yml.tmpl} | 8 +++---- .../README.md.tmpl | 2 +- .../sample_exploration.ipynb.tmpl | 0 .../sample_trips_{{.project_name}}.py.tmpl | 0 .../sample_trips_{{.project_name}}.sql.tmpl | 0 .../sample_zones_{{.project_name}}.py.tmpl | 0 .../sample_zones_{{.project_name}}.sql.tmpl | 0 .../databricks_template_schema.json | 22 ++++++----------- .../databricks_template_schema.json | 22 ++++++----------- 54 files changed, 124 insertions(+), 150 deletions(-) rename acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/{default_python_etl.pipeline.yml => my_default_python_etl.pipeline.yml} (74%) rename acceptance/bundle/templates/default-python/classic/output/my_default_python/src/{default_python_etl => my_default_python_etl}/README.md (86%) rename acceptance/bundle/templates/default-python/classic/output/my_default_python/src/{default_python_etl => my_default_python_etl}/explorations/sample_exploration.ipynb (100%) rename acceptance/bundle/templates/default-python/classic/output/my_default_python/src/{default_python_etl => my_default_python_etl}/transformations/sample_trips_my_default_python.py (100%) rename acceptance/bundle/templates/default-python/classic/output/my_default_python/src/{default_python_etl => my_default_python_etl}/transformations/sample_zones_my_default_python.py (100%) rename acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/{default_python_etl.pipeline.yml => my_default_python_etl.pipeline.yml} (73%) rename acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/{default_python_etl => my_default_python_etl}/README.md (86%) rename acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/{default_python_etl => my_default_python_etl}/explorations/sample_exploration.ipynb (100%) rename acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/{default_python_etl => my_default_python_etl}/transformations/sample_trips_my_default_python.py (100%) rename acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/{default_python_etl => my_default_python_etl}/transformations/sample_zones_my_default_python.py (100%) rename acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/{lakeflow_pipelines_etl.pipeline.yml => my_lakeflow_pipelines_etl.pipeline.yml} (72%) rename acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/{lakeflow_pipelines_etl => my_lakeflow_pipelines_etl}/README.md (86%) rename acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/{lakeflow_pipelines_etl => my_lakeflow_pipelines_etl}/explorations/sample_exploration.ipynb (100%) rename acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/{lakeflow_pipelines_etl => my_lakeflow_pipelines_etl}/transformations/sample_trips_my_lakeflow_pipelines.py (100%) rename acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/{lakeflow_pipelines_etl => my_lakeflow_pipelines_etl}/transformations/sample_zones_my_lakeflow_pipelines.py (100%) rename acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/{lakeflow_pipelines_etl.pipeline.yml => my_lakeflow_pipelines_etl.pipeline.yml} (72%) rename acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/{lakeflow_pipelines_etl => my_lakeflow_pipelines_etl}/README.md (86%) rename acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/{lakeflow_pipelines_etl => my_lakeflow_pipelines_etl}/explorations/sample_exploration.ipynb (100%) rename acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/{lakeflow_pipelines_etl => my_lakeflow_pipelines_etl}/transformations/sample_trips_my_lakeflow_pipelines.sql (100%) rename acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/{lakeflow_pipelines_etl => my_lakeflow_pipelines_etl}/transformations/sample_zones_my_lakeflow_pipelines.sql (100%) rename libs/template/templates/default/template/{{.project_name}}/resources/{{{.project_name_short}}_etl.pipeline.yml.tmpl => {{.project_name}}_etl.pipeline.yml.tmpl} (84%) rename libs/template/templates/default/template/{{.project_name}}/src/{{{.project_name_short}}_etl => {{.project_name}}_etl}/README.md.tmpl (92%) rename libs/template/templates/default/template/{{.project_name}}/src/{{{.project_name_short}}_etl => {{.project_name}}_etl}/explorations/sample_exploration.ipynb.tmpl (100%) rename libs/template/templates/default/template/{{.project_name}}/src/{{{.project_name_short}}_etl => {{.project_name}}_etl}/transformations/sample_trips_{{.project_name}}.py.tmpl (100%) rename libs/template/templates/default/template/{{.project_name}}/src/{{{.project_name_short}}_etl => {{.project_name}}_etl}/transformations/sample_trips_{{.project_name}}.sql.tmpl (100%) rename libs/template/templates/default/template/{{.project_name}}/src/{{{.project_name_short}}_etl => {{.project_name}}_etl}/transformations/sample_zones_{{.project_name}}.py.tmpl (100%) rename libs/template/templates/default/template/{{.project_name}}/src/{{{.project_name_short}}_etl => {{.project_name}}_etl}/transformations/sample_zones_{{.project_name}}.sql.tmpl (100%) diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff index d3850c8fc0..e9c80598dd 100644 --- a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff +++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff @@ -7,18 +7,18 @@ + artifacts_dynamic_version: true prod: mode: production ---- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/resources/default_python_etl.pipeline.yml -+++ output/my_default_python/resources/default_python_etl.pipeline.yml +--- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml ++++ output/my_default_python/resources/my_default_python_etl.pipeline.yml @@ -5,8 +5,7 @@ - default_python_etl: - name: default_python_etl + my_default_python_etl: + name: my_default_python_etl - ## Catalog is required for serverless compute - catalog: ${var.catalog} + ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: + # catalog: ${var.catalog} schema: ${var.schema} - serverless: true - root_path: "../src/default_python_etl" + root_path: "../src/my_default_python_etl" --- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/resources/sample_job.job.yml +++ output/my_default_python/resources/sample_job.job.yml @@ -47,7 +47,7 @@ - task_key: refresh_pipeline depends_on: @@ -45,11 +55,11 @@ - pipeline_id: ${resources.pipelines.default_python_etl.id} + pipeline_id: ${resources.pipelines.my_default_python_etl.id} - environments: - - environment_key: default diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct-exp.json b/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct-exp.json index a51ef092ea..a41ab36888 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct-exp.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct-exp.json @@ -3,8 +3,8 @@ "resources.jobs.sample_job": { "depends_on": [ { - "node": "resources.pipelines.default_python_etl", - "label": "${resources.pipelines.default_python_etl.id}" + "node": "resources.pipelines.my_default_python_etl", + "label": "${resources.pipelines.my_default_python_etl.id}" } ], "action": "create", @@ -93,7 +93,7 @@ } ], "pipeline_task": { - "pipeline_id": "${resources.pipelines.default_python_etl.id}" + "pipeline_id": "${resources.pipelines.my_default_python_etl.id}" }, "task_key": "refresh_pipeline" } @@ -107,11 +107,11 @@ } }, "vars": { - "tasks[2].pipeline_task.pipeline_id": "${resources.pipelines.default_python_etl.id}" + "tasks[2].pipeline_task.pipeline_id": "${resources.pipelines.my_default_python_etl.id}" } } }, - "resources.pipelines.default_python_etl": { + "resources.pipelines.my_default_python_etl": { "action": "create", "new_state": { "config": { @@ -130,12 +130,12 @@ "libraries": [ { "glob": { - "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/default_python_etl/transformations/**" + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations/**" } } ], - "name": "[dev [USERNAME]] default_python_etl", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/default_python_etl", + "name": "[dev [USERNAME]] my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl", "schema": "[USERNAME]", "tags": { "dev": "[USERNAME]" diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_dev.terraform.json b/acceptance/bundle/templates/default-python/classic/out.plan_dev.terraform.json index 56f43e5adb..876a72a786 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_dev.terraform.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_dev.terraform.json @@ -3,7 +3,7 @@ "resources.jobs.sample_job": { "action": "create" }, - "resources.pipelines.default_python_etl": { + "resources.pipelines.my_default_python_etl": { "action": "create" } } diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct-exp.json b/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct-exp.json index e9d40f366b..6bc50d8d0a 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct-exp.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct-exp.json @@ -3,8 +3,8 @@ "resources.jobs.sample_job": { "depends_on": [ { - "node": "resources.pipelines.default_python_etl", - "label": "${resources.pipelines.default_python_etl.id}" + "node": "resources.pipelines.my_default_python_etl", + "label": "${resources.pipelines.my_default_python_etl.id}" } ], "action": "create", @@ -90,7 +90,7 @@ } ], "pipeline_task": { - "pipeline_id": "${resources.pipelines.default_python_etl.id}" + "pipeline_id": "${resources.pipelines.my_default_python_etl.id}" }, "task_key": "refresh_pipeline" } @@ -104,11 +104,11 @@ } }, "vars": { - "tasks[2].pipeline_task.pipeline_id": "${resources.pipelines.default_python_etl.id}" + "tasks[2].pipeline_task.pipeline_id": "${resources.pipelines.my_default_python_etl.id}" } } }, - "resources.pipelines.default_python_etl": { + "resources.pipelines.my_default_python_etl": { "action": "create", "new_state": { "config": { @@ -126,12 +126,12 @@ "libraries": [ { "glob": { - "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/default_python_etl/transformations/**" + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations/**" } } ], - "name": "default_python_etl", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/default_python_etl", + "name": "my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl", "schema": "prod" } } diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_prod.terraform.json b/acceptance/bundle/templates/default-python/classic/out.plan_prod.terraform.json index 56f43e5adb..876a72a786 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_prod.terraform.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_prod.terraform.json @@ -3,7 +3,7 @@ "resources.jobs.sample_job": { "action": "create" }, - "resources.pipelines.default_python_etl": { + "resources.pipelines.my_default_python_etl": { "action": "create" } } diff --git a/acceptance/bundle/templates/default-python/classic/out.requests.dev.direct-exp.txt b/acceptance/bundle/templates/default-python/classic/out.requests.dev.direct-exp.txt index 0f89b32629..436c881470 100644 --- a/acceptance/bundle/templates/default-python/classic/out.requests.dev.direct-exp.txt +++ b/acceptance/bundle/templates/default-python/classic/out.requests.dev.direct-exp.txt @@ -17,12 +17,12 @@ "libraries": [ { "glob": { - "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/default_python_etl/transformations/**" + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations/**" } } ], - "name": "[dev [USERNAME]] default_python_etl", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/default_python_etl", + "name": "[dev [USERNAME]] my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl", "schema": "[USERNAME]", "tags": { "dev": "[USERNAME]" @@ -69,14 +69,14 @@ "method": "POST", "path": "/api/2.0/workspace/mkdirs", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/default_python_etl/transformations" + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python" } } { "method": "POST", "path": "/api/2.0/workspace/mkdirs", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python" + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations" } } { diff --git a/acceptance/bundle/templates/default-python/classic/out.requests.dev.terraform.txt b/acceptance/bundle/templates/default-python/classic/out.requests.dev.terraform.txt index 91e9828451..6bac5cd373 100644 --- a/acceptance/bundle/templates/default-python/classic/out.requests.dev.terraform.txt +++ b/acceptance/bundle/templates/default-python/classic/out.requests.dev.terraform.txt @@ -17,12 +17,12 @@ "libraries": [ { "glob": { - "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/default_python_etl/transformations/**" + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations/**" } } ], - "name": "[dev [USERNAME]] default_python_etl", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/default_python_etl", + "name": "[dev [USERNAME]] my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl", "schema": "[USERNAME]", "tags": { "dev": "[USERNAME]" @@ -69,14 +69,14 @@ "method": "POST", "path": "/api/2.0/workspace/mkdirs", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/default_python_etl/transformations" + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python" } } { "method": "POST", "path": "/api/2.0/workspace/mkdirs", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python" + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations" } } { diff --git a/acceptance/bundle/templates/default-python/classic/out.requests.prod.direct-exp.txt b/acceptance/bundle/templates/default-python/classic/out.requests.prod.direct-exp.txt index 55311f4093..c72ffaf194 100644 --- a/acceptance/bundle/templates/default-python/classic/out.requests.prod.direct-exp.txt +++ b/acceptance/bundle/templates/default-python/classic/out.requests.prod.direct-exp.txt @@ -16,12 +16,12 @@ "libraries": [ { "glob": { - "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/default_python_etl/transformations/**" + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations/**" } } ], - "name": "default_python_etl", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/default_python_etl", + "name": "my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl", "schema": "prod" } } @@ -72,14 +72,14 @@ "method": "POST", "path": "/api/2.0/workspace/mkdirs", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/default_python_etl/transformations" + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python" } } { "method": "POST", "path": "/api/2.0/workspace/mkdirs", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python" + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations" } } { diff --git a/acceptance/bundle/templates/default-python/classic/out.requests.prod.terraform.txt b/acceptance/bundle/templates/default-python/classic/out.requests.prod.terraform.txt index cf913ee737..f09f9278f4 100644 --- a/acceptance/bundle/templates/default-python/classic/out.requests.prod.terraform.txt +++ b/acceptance/bundle/templates/default-python/classic/out.requests.prod.terraform.txt @@ -16,12 +16,12 @@ "libraries": [ { "glob": { - "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/default_python_etl/transformations/**" + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations/**" } } ], - "name": "default_python_etl", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/default_python_etl", + "name": "my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl", "schema": "prod" } } @@ -72,14 +72,14 @@ "method": "POST", "path": "/api/2.0/workspace/mkdirs", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/default_python_etl/transformations" + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python" } } { "method": "POST", "path": "/api/2.0/workspace/mkdirs", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python" + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations" } } { diff --git a/acceptance/bundle/templates/default-python/classic/output.txt b/acceptance/bundle/templates/default-python/classic/output.txt index 5c8fea374e..78119b61fa 100644 --- a/acceptance/bundle/templates/default-python/classic/output.txt +++ b/acceptance/bundle/templates/default-python/classic/output.txt @@ -35,14 +35,14 @@ Validation OK! >>> [CLI] bundle plan -t dev Building python_artifact... create jobs.sample_job -create pipelines.default_python_etl +create pipelines.my_default_python_etl Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged >>> [CLI] bundle plan -t prod Building python_artifact... create jobs.sample_job -create pipelines.default_python_etl +create pipelines.my_default_python_etl Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged Building python_artifact... diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md index 907deacb90..4707c76a8f 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md @@ -47,7 +47,7 @@ with this project. It's also possible to interact with it directly using the CLI This deploys everything that's defined for this project. For example, the default template would deploy a pipeline called - `[dev yourname] default_python_etl` to your workspace. + `[dev yourname] my_default_python_etl` to your workspace. You can find that resource by opening your workpace and clicking on **Jobs & Pipelines**. 3. Similarly, to deploy a production copy, type: diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/default_python_etl.pipeline.yml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python_etl.pipeline.yml similarity index 74% rename from acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/default_python_etl.pipeline.yml rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python_etl.pipeline.yml index 82df765028..896b271a0d 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/default_python_etl.pipeline.yml +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python_etl.pipeline.yml @@ -2,16 +2,16 @@ resources: pipelines: - default_python_etl: - name: default_python_etl + my_default_python_etl: + name: my_default_python_etl ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: # catalog: ${var.catalog} schema: ${var.schema} - root_path: "../src/default_python_etl" + root_path: "../src/my_default_python_etl" libraries: - glob: - include: ../src/default_python_etl/transformations/** + include: ../src/my_default_python_etl/transformations/** environment: dependencies: diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/sample_job.job.yml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/sample_job.job.yml index b91da70d42..1c62e20bc4 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/sample_job.job.yml +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/sample_job.job.yml @@ -52,7 +52,7 @@ resources: depends_on: - task_key: notebook_task pipeline_task: - pipeline_id: ${resources.pipelines.default_python_etl.id} + pipeline_id: ${resources.pipelines.my_default_python_etl.id} job_clusters: - job_cluster_key: job_cluster diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/README.md b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python_etl/README.md similarity index 86% rename from acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/README.md rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python_etl/README.md index b59a51fffb..17f7212f88 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/README.md +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python_etl/README.md @@ -15,6 +15,6 @@ To get started, go to the `transformations` folder -- most of the relevant sourc * Take a look at the sample called "sample_trips_my_default_python.py" to get familiar with the syntax. Read more about the syntax at https://docs.databricks.com/dlt/python-ref.html. * If you're using the workspace UI, use `Run file` to run and preview a single transformation. -* If you're using the CLI, use `databricks bundle run default_python_etl --select sample_trips_my_default_python` to run a single transformation. +* If you're using the CLI, use `databricks bundle run my_default_python_etl --select sample_trips_my_default_python` to run a single transformation. For more tutorials and reference material, see https://docs.databricks.com/dlt. diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/explorations/sample_exploration.ipynb b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python_etl/explorations/sample_exploration.ipynb similarity index 100% rename from acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/explorations/sample_exploration.ipynb rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python_etl/explorations/sample_exploration.ipynb diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/transformations/sample_trips_my_default_python.py b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python_etl/transformations/sample_trips_my_default_python.py similarity index 100% rename from acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/transformations/sample_trips_my_default_python.py rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python_etl/transformations/sample_trips_my_default_python.py diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/transformations/sample_zones_my_default_python.py b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python_etl/transformations/sample_zones_my_default_python.py similarity index 100% rename from acceptance/bundle/templates/default-python/classic/output/my_default_python/src/default_python_etl/transformations/sample_zones_my_default_python.py rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python_etl/transformations/sample_zones_my_default_python.py diff --git a/acceptance/bundle/templates/default-python/combinations/classic/output.txt b/acceptance/bundle/templates/default-python/combinations/classic/output.txt index f192544aa5..c2b00c6bb3 100644 --- a/acceptance/bundle/templates/default-python/combinations/classic/output.txt +++ b/acceptance/bundle/templates/default-python/combinations/classic/output.txt @@ -2,16 +2,15 @@ >>> [CLI] bundle init default-python --config-file ./input.json Welcome to the default Python template for Databricks Asset Bundles! -Please answer the below to tailor your project to your preferences. -You can always change your mind and change your configuration in the databricks.yml file later. +Answer the following questions to customize your project. +You can always change your configuration in the databricks.yml file later. -Note that [DATABRICKS_URL] is used for initialization -(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). +Note that [DATABRICKS_URL] is used for initialization. +(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.) ✨ Your new project has been created in the 'X[UNIQUE_NAME]' directory! -Please refer to the README.md file for "getting started" instructions. -See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. >>> yamlcheck.py diff --git a/acceptance/bundle/templates/default-python/combinations/serverless/output.txt b/acceptance/bundle/templates/default-python/combinations/serverless/output.txt index ee551a5b16..ef53968285 100644 --- a/acceptance/bundle/templates/default-python/combinations/serverless/output.txt +++ b/acceptance/bundle/templates/default-python/combinations/serverless/output.txt @@ -2,16 +2,15 @@ >>> [CLI] bundle init default-python --config-file ./input.json Welcome to the default Python template for Databricks Asset Bundles! -Please answer the below to tailor your project to your preferences. -You can always change your mind and change your configuration in the databricks.yml file later. +Answer the following questions to customize your project. +You can always change your configuration in the databricks.yml file later. -Note that [DATABRICKS_URL] is used for initialization -(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). +Note that [DATABRICKS_URL] is used for initialization. +(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.) ✨ Your new project has been created in the 'X[UNIQUE_NAME]' directory! -Please refer to the README.md file for "getting started" instructions. -See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. >>> yamlcheck.py diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt index 9ae640d43a..8257e8d97f 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt @@ -29,16 +29,16 @@ To get started, refer to the project README.md file and the documentation at htt + catalog: customcatalog schema: prod permissions: ---- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/resources/default_python_etl.pipeline.yml -+++ output/my_default_python/resources/default_python_etl.pipeline.yml +--- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml ++++ output/my_default_python/resources/my_default_python_etl.pipeline.yml @@ -5,5 +5,4 @@ - default_python_etl: - name: default_python_etl + my_default_python_etl: + name: my_default_python_etl - ## Catalog is required for serverless compute catalog: ${var.catalog} schema: ${var.schema} ---- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/src/default_python_etl/explorations/sample_exploration.ipynb -+++ output/my_default_python/src/default_python_etl/explorations/sample_exploration.ipynb +--- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/src/my_default_python_etl/explorations/sample_exploration.ipynb ++++ output/my_default_python/src/my_default_python_etl/explorations/sample_exploration.ipynb @@ -38,5 +38,5 @@ "# !!! Before performing any data analysis, make sure to run the pipeline to materialize the sample datasets. The tables referenced in this notebook depend on that step./n", "/n", diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md index 907deacb90..4707c76a8f 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md @@ -47,7 +47,7 @@ with this project. It's also possible to interact with it directly using the CLI This deploys everything that's defined for this project. For example, the default template would deploy a pipeline called - `[dev yourname] default_python_etl` to your workspace. + `[dev yourname] my_default_python_etl` to your workspace. You can find that resource by opening your workpace and clicking on **Jobs & Pipelines**. 3. Similarly, to deploy a production copy, type: diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/default_python_etl.pipeline.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml similarity index 73% rename from acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/default_python_etl.pipeline.yml rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml index d5cbd51cf8..eed63f83df 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/default_python_etl.pipeline.yml +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml @@ -2,17 +2,17 @@ resources: pipelines: - default_python_etl: - name: default_python_etl + my_default_python_etl: + name: my_default_python_etl ## Catalog is required for serverless compute catalog: ${var.catalog} schema: ${var.schema} serverless: true - root_path: "../src/default_python_etl" + root_path: "../src/my_default_python_etl" libraries: - glob: - include: ../src/default_python_etl/transformations/** + include: ../src/my_default_python_etl/transformations/** environment: dependencies: diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml index c47eb8655e..b0d0c7640c 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml @@ -42,7 +42,7 @@ resources: depends_on: - task_key: notebook_task pipeline_task: - pipeline_id: ${resources.pipelines.default_python_etl.id} + pipeline_id: ${resources.pipelines.my_default_python_etl.id} environments: - environment_key: default diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/README.md b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python_etl/README.md similarity index 86% rename from acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/README.md rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python_etl/README.md index b59a51fffb..17f7212f88 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/README.md +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python_etl/README.md @@ -15,6 +15,6 @@ To get started, go to the `transformations` folder -- most of the relevant sourc * Take a look at the sample called "sample_trips_my_default_python.py" to get familiar with the syntax. Read more about the syntax at https://docs.databricks.com/dlt/python-ref.html. * If you're using the workspace UI, use `Run file` to run and preview a single transformation. -* If you're using the CLI, use `databricks bundle run default_python_etl --select sample_trips_my_default_python` to run a single transformation. +* If you're using the CLI, use `databricks bundle run my_default_python_etl --select sample_trips_my_default_python` to run a single transformation. For more tutorials and reference material, see https://docs.databricks.com/dlt. diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/explorations/sample_exploration.ipynb b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python_etl/explorations/sample_exploration.ipynb similarity index 100% rename from acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/explorations/sample_exploration.ipynb rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python_etl/explorations/sample_exploration.ipynb diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/transformations/sample_trips_my_default_python.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python_etl/transformations/sample_trips_my_default_python.py similarity index 100% rename from acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/transformations/sample_trips_my_default_python.py rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python_etl/transformations/sample_trips_my_default_python.py diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/transformations/sample_zones_my_default_python.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python_etl/transformations/sample_zones_my_default_python.py similarity index 100% rename from acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/default_python_etl/transformations/sample_zones_my_default_python.py rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python_etl/transformations/sample_zones_my_default_python.py diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/README.md b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/README.md index f9c9e3c5b7..af884ca91a 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/README.md +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/README.md @@ -36,7 +36,7 @@ with this project. It's also possible to interact with it directly using the CLI This deploys everything that's defined for this project. For example, the default template would deploy a pipeline called - `[dev yourname] lakeflow_pipelines_etl` to your workspace. + `[dev yourname] my_lakeflow_pipelines_etl` to your workspace. You can find that resource by opening your workpace and clicking on **Jobs & Pipelines**. 3. Similarly, to deploy a production copy, type: diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/lakeflow_pipelines_etl.pipeline.yml b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/my_lakeflow_pipelines_etl.pipeline.yml similarity index 72% rename from acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/lakeflow_pipelines_etl.pipeline.yml rename to acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/my_lakeflow_pipelines_etl.pipeline.yml index 76b8ac2bde..0a5bf1e7ee 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/lakeflow_pipelines_etl.pipeline.yml +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/my_lakeflow_pipelines_etl.pipeline.yml @@ -2,17 +2,17 @@ resources: pipelines: - lakeflow_pipelines_etl: - name: lakeflow_pipelines_etl + my_lakeflow_pipelines_etl: + name: my_lakeflow_pipelines_etl ## Catalog is required for serverless compute catalog: ${var.catalog} schema: ${var.schema} serverless: true - root_path: "../src/lakeflow_pipelines_etl" + root_path: "../src/my_lakeflow_pipelines_etl" libraries: - glob: - include: ../src/lakeflow_pipelines_etl/transformations/** + include: ../src/my_lakeflow_pipelines_etl/transformations/** environment: dependencies: diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml index c55849f2ff..a72aa871eb 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml @@ -24,7 +24,7 @@ resources: tasks: - task_key: refresh_pipeline pipeline_task: - pipeline_id: ${resources.pipelines.lakeflow_pipelines_etl.id} + pipeline_id: ${resources.pipelines.my_lakeflow_pipelines_etl.id} environments: - environment_key: default diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/README.md b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/README.md similarity index 86% rename from acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/README.md rename to acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/README.md index 470115709d..c675a73a2c 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/README.md +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/README.md @@ -15,6 +15,6 @@ To get started, go to the `transformations` folder -- most of the relevant sourc * Take a look at the sample called "sample_trips_my_lakeflow_pipelines.py" to get familiar with the syntax. Read more about the syntax at https://docs.databricks.com/dlt/python-ref.html. * If you're using the workspace UI, use `Run file` to run and preview a single transformation. -* If you're using the CLI, use `databricks bundle run lakeflow_pipelines_etl --select sample_trips_my_lakeflow_pipelines` to run a single transformation. +* If you're using the CLI, use `databricks bundle run my_lakeflow_pipelines_etl --select sample_trips_my_lakeflow_pipelines` to run a single transformation. For more tutorials and reference material, see https://docs.databricks.com/dlt. diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/explorations/sample_exploration.ipynb b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/explorations/sample_exploration.ipynb similarity index 100% rename from acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/explorations/sample_exploration.ipynb rename to acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/explorations/sample_exploration.ipynb diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/transformations/sample_trips_my_lakeflow_pipelines.py b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/transformations/sample_trips_my_lakeflow_pipelines.py similarity index 100% rename from acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/transformations/sample_trips_my_lakeflow_pipelines.py rename to acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/transformations/sample_trips_my_lakeflow_pipelines.py diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/transformations/sample_zones_my_lakeflow_pipelines.py b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/transformations/sample_zones_my_lakeflow_pipelines.py similarity index 100% rename from acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/transformations/sample_zones_my_lakeflow_pipelines.py rename to acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/transformations/sample_zones_my_lakeflow_pipelines.py diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/README.md b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/README.md index b073fc5038..d715967218 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/README.md +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/README.md @@ -36,7 +36,7 @@ with this project. It's also possible to interact with it directly using the CLI This deploys everything that's defined for this project. For example, the default template would deploy a pipeline called - `[dev yourname] lakeflow_pipelines_etl` to your workspace. + `[dev yourname] my_lakeflow_pipelines_etl` to your workspace. You can find that resource by opening your workpace and clicking on **Jobs & Pipelines**. 3. Similarly, to deploy a production copy, type: diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/lakeflow_pipelines_etl.pipeline.yml b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/my_lakeflow_pipelines_etl.pipeline.yml similarity index 72% rename from acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/lakeflow_pipelines_etl.pipeline.yml rename to acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/my_lakeflow_pipelines_etl.pipeline.yml index 76b8ac2bde..0a5bf1e7ee 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/lakeflow_pipelines_etl.pipeline.yml +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/my_lakeflow_pipelines_etl.pipeline.yml @@ -2,17 +2,17 @@ resources: pipelines: - lakeflow_pipelines_etl: - name: lakeflow_pipelines_etl + my_lakeflow_pipelines_etl: + name: my_lakeflow_pipelines_etl ## Catalog is required for serverless compute catalog: ${var.catalog} schema: ${var.schema} serverless: true - root_path: "../src/lakeflow_pipelines_etl" + root_path: "../src/my_lakeflow_pipelines_etl" libraries: - glob: - include: ../src/lakeflow_pipelines_etl/transformations/** + include: ../src/my_lakeflow_pipelines_etl/transformations/** environment: dependencies: diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml index c55849f2ff..a72aa871eb 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml @@ -24,7 +24,7 @@ resources: tasks: - task_key: refresh_pipeline pipeline_task: - pipeline_id: ${resources.pipelines.lakeflow_pipelines_etl.id} + pipeline_id: ${resources.pipelines.my_lakeflow_pipelines_etl.id} environments: - environment_key: default diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/README.md b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/README.md similarity index 86% rename from acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/README.md rename to acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/README.md index 470115709d..c675a73a2c 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/README.md +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/README.md @@ -15,6 +15,6 @@ To get started, go to the `transformations` folder -- most of the relevant sourc * Take a look at the sample called "sample_trips_my_lakeflow_pipelines.py" to get familiar with the syntax. Read more about the syntax at https://docs.databricks.com/dlt/python-ref.html. * If you're using the workspace UI, use `Run file` to run and preview a single transformation. -* If you're using the CLI, use `databricks bundle run lakeflow_pipelines_etl --select sample_trips_my_lakeflow_pipelines` to run a single transformation. +* If you're using the CLI, use `databricks bundle run my_lakeflow_pipelines_etl --select sample_trips_my_lakeflow_pipelines` to run a single transformation. For more tutorials and reference material, see https://docs.databricks.com/dlt. diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/explorations/sample_exploration.ipynb b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/explorations/sample_exploration.ipynb similarity index 100% rename from acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/explorations/sample_exploration.ipynb rename to acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/explorations/sample_exploration.ipynb diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/transformations/sample_trips_my_lakeflow_pipelines.sql b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/transformations/sample_trips_my_lakeflow_pipelines.sql similarity index 100% rename from acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/transformations/sample_trips_my_lakeflow_pipelines.sql rename to acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/transformations/sample_trips_my_lakeflow_pipelines.sql diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/transformations/sample_zones_my_lakeflow_pipelines.sql b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/transformations/sample_zones_my_lakeflow_pipelines.sql similarity index 100% rename from acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/lakeflow_pipelines_etl/transformations/sample_zones_my_lakeflow_pipelines.sql rename to acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines_etl/transformations/sample_zones_my_lakeflow_pipelines.sql diff --git a/libs/template/templates/default-python/databricks_template_schema.json b/libs/template/templates/default-python/databricks_template_schema.json index 6f16a59723..40e686cfc1 100644 --- a/libs/template/templates/default-python/databricks_template_schema.json +++ b/libs/template/templates/default-python/databricks_template_schema.json @@ -10,41 +10,33 @@ "pattern": "^[A-Za-z0-9_]+$", "pattern_match_failure_message": "Name must consist of letters, numbers, and underscores." }, - "project_name_short": { - "//": "This is a derived property based on project_name (it replaces my_project with sample and strips _project|_app|_service)", - "skip_prompt_if": {}, - "type": "string", - "default": "{{if eq .project_name \"my_project\"}}sample{{else}}{{with (regexp \"^(my_)?(.*)(_project|_app|_service)?$\").FindStringSubmatch .project_name}}{{index . 2}}{{else}}{{.project_name}}{{end}}{{end}}", - "description": "Short name for the project", - "order": 2 - }, "include_job": { "type": "string", "default": "yes", "enum": ["yes", "no"], "description": "Include a Lakeflow job that runs a notebook", - "order": 3 + "order": 2 }, "include_pipeline": { "type": "string", "default": "yes", "enum": ["yes", "no"], "description": "Include an ETL pipeline", - "order": 4 + "order": 3 }, "include_python": { "type": "string", "default": "yes", "enum": ["yes", "no"], "description": "Include a sample Python package that builds into a wheel file", - "order": 5 + "order": 4 }, "serverless": { "type": "string", "default": "yes", "enum": ["yes", "no"], "description": "Use serverless compute", - "order": 6 + "order": 5 }, "default_catalog": { "type": "string", @@ -52,7 +44,7 @@ "pattern": "^\\w*$", "pattern_match_failure_message": "Invalid catalog name.", "description": "Default catalog for any tables created by this project{{if eq (default_catalog) \"\"}} (leave blank when not using Unity Catalog){{end}}", - "order": 7 + "order": 6 }, "personal_schemas": { "type": "string", @@ -62,7 +54,7 @@ "yes", "no, I will customize the schema configuration later in databricks.yml" ], - "order": 8 + "order": 7 }, "language": { "//": "This property is always set to 'python'", @@ -74,7 +66,7 @@ "python", "sql" ], - "order": 9 + "order": 8 }, "lakeflow_only": { "//": "This property is always set to 'no' for default-python", @@ -82,7 +74,7 @@ "type": "string", "default": "no", "description": "Internal flag for lakeflow-only templates", - "order": 10 + "order": 9 } }, "success_message": "\n✨ Your new project has been created in the '{{.project_name}}' directory!\n\nTo get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html." diff --git a/libs/template/templates/default/template/__preamble.tmpl b/libs/template/templates/default/template/__preamble.tmpl index 0d3e26e0a8..1248b59850 100644 --- a/libs/template/templates/default/template/__preamble.tmpl +++ b/libs/template/templates/default/template/__preamble.tmpl @@ -29,12 +29,12 @@ This file only contains template directives; it is skipped for the actual output {{end}} {{if not $pipeline}} - {{skip "{{.project_name}}/resources/{{.project_name_short}}_etl.pipeline.yml"}} - {{skip "{{.project_name}}/src/{{.project_name_short}}_etl"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_etl.pipeline.yml"}} + {{skip "{{.project_name}}/src/{{.project_name}}_etl"}} {{else if $sql_language}} - {{skip "{{.project_name}}/src/{{.project_name_short}}_etl/transformations/*.py"}} + {{skip "{{.project_name}}/src/{{.project_name}}_etl/transformations/*.py"}} {{else}} - {{skip "{{.project_name}}/src/{{.project_name_short}}_etl/transformations/*.sql"}} + {{skip "{{.project_name}}/src/{{.project_name}}_etl/transformations/*.sql"}} {{end}} {{if or $sql_language $lakeflow_only}} diff --git a/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl index 3287d5ed6b..445e5f8290 100644 --- a/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl @@ -57,11 +57,11 @@ with this project. It's also possible to interact with it directly using the CLI This deploys everything that's defined for this project. {{- if eq .include_pipeline "yes"}} For example, the default template would deploy a pipeline called - `[dev yourname] {{.project_name_short}}_etl` to your workspace. + `[dev yourname] {{.project_name}}_etl` to your workspace. You can find that resource by opening your workpace and clicking on **Jobs & Pipelines**. {{- else if eq .include_job "yes"}} For example, the default template would deploy a job called - `[dev yourname] {{.project_name_short}}_job` to your workspace. + `[dev yourname] {{.project_name}}_job` to your workspace. You can find that resource by opening your workpace and clicking on **Jobs & Pipelines**. {{- end}} diff --git a/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.job.yml.tmpl b/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.job.yml.tmpl index 236711e6d9..0695196067 100644 --- a/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.job.yml.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.job.yml.tmpl @@ -85,7 +85,7 @@ resources: - task_key: python_wheel_task {{- end}} pipeline_task: - pipeline_id: ${resources.pipelines.{{.project_name_short}}_etl.id} + pipeline_id: ${resources.pipelines.{{.project_name}}_etl.id} {{- end}} {{- if $serverless}} diff --git a/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name_short}}_etl.pipeline.yml.tmpl b/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl.pipeline.yml.tmpl similarity index 84% rename from libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name_short}}_etl.pipeline.yml.tmpl rename to libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl.pipeline.yml.tmpl index 2ab868e4f2..9bfde2257d 100644 --- a/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name_short}}_etl.pipeline.yml.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl.pipeline.yml.tmpl @@ -4,11 +4,11 @@ resources: pipelines: - {{.project_name_short}}_etl: + {{.project_name}}_etl: {{- /* Note that pipeline names must be unique in a worskspace, * so we use the project name as part as the name. */}} - name: {{.project_name_short}}_etl + name: {{.project_name}}_etl {{- if or (eq default_catalog "") (eq default_catalog "hive_metastore")}} {{- if $serverless }} ## Catalog is required for serverless compute @@ -24,11 +24,11 @@ resources: {{- if $serverless }} serverless: true {{- end}} - root_path: "../src/{{.project_name_short}}_etl" + root_path: "../src/{{.project_name}}_etl" libraries: - glob: - include: ../src/{{.project_name_short}}_etl/transformations/** + include: ../src/{{.project_name}}_etl/transformations/** environment: dependencies: diff --git a/libs/template/templates/default/template/{{.project_name}}/src/{{.project_name_short}}_etl/README.md.tmpl b/libs/template/templates/default/template/{{.project_name}}/src/{{.project_name}}_etl/README.md.tmpl similarity index 92% rename from libs/template/templates/default/template/{{.project_name}}/src/{{.project_name_short}}_etl/README.md.tmpl rename to libs/template/templates/default/template/{{.project_name}}/src/{{.project_name}}_etl/README.md.tmpl index a8e96f5046..dce5866887 100644 --- a/libs/template/templates/default/template/{{.project_name}}/src/{{.project_name_short}}_etl/README.md.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/src/{{.project_name}}_etl/README.md.tmpl @@ -15,6 +15,6 @@ To get started, go to the `transformations` folder -- most of the relevant sourc * Take a look at the sample called "sample_trips_{{.project_name}}.py" to get familiar with the syntax. Read more about the syntax at https://docs.databricks.com/dlt/python-ref.html. * If you're using the workspace UI, use `Run file` to run and preview a single transformation. -* If you're using the CLI, use `databricks bundle run {{.project_name_short}}_etl --select sample_trips_{{.project_name}}` to run a single transformation. +* If you're using the CLI, use `databricks bundle run {{.project_name}}_etl --select sample_trips_{{.project_name}}` to run a single transformation. For more tutorials and reference material, see https://docs.databricks.com/dlt. diff --git a/libs/template/templates/default/template/{{.project_name}}/src/{{.project_name_short}}_etl/explorations/sample_exploration.ipynb.tmpl b/libs/template/templates/default/template/{{.project_name}}/src/{{.project_name}}_etl/explorations/sample_exploration.ipynb.tmpl similarity index 100% rename from libs/template/templates/default/template/{{.project_name}}/src/{{.project_name_short}}_etl/explorations/sample_exploration.ipynb.tmpl rename to libs/template/templates/default/template/{{.project_name}}/src/{{.project_name}}_etl/explorations/sample_exploration.ipynb.tmpl diff --git a/libs/template/templates/default/template/{{.project_name}}/src/{{.project_name_short}}_etl/transformations/sample_trips_{{.project_name}}.py.tmpl b/libs/template/templates/default/template/{{.project_name}}/src/{{.project_name}}_etl/transformations/sample_trips_{{.project_name}}.py.tmpl similarity index 100% rename from libs/template/templates/default/template/{{.project_name}}/src/{{.project_name_short}}_etl/transformations/sample_trips_{{.project_name}}.py.tmpl rename to libs/template/templates/default/template/{{.project_name}}/src/{{.project_name}}_etl/transformations/sample_trips_{{.project_name}}.py.tmpl diff --git a/libs/template/templates/default/template/{{.project_name}}/src/{{.project_name_short}}_etl/transformations/sample_trips_{{.project_name}}.sql.tmpl b/libs/template/templates/default/template/{{.project_name}}/src/{{.project_name}}_etl/transformations/sample_trips_{{.project_name}}.sql.tmpl similarity index 100% rename from libs/template/templates/default/template/{{.project_name}}/src/{{.project_name_short}}_etl/transformations/sample_trips_{{.project_name}}.sql.tmpl rename to libs/template/templates/default/template/{{.project_name}}/src/{{.project_name}}_etl/transformations/sample_trips_{{.project_name}}.sql.tmpl diff --git a/libs/template/templates/default/template/{{.project_name}}/src/{{.project_name_short}}_etl/transformations/sample_zones_{{.project_name}}.py.tmpl b/libs/template/templates/default/template/{{.project_name}}/src/{{.project_name}}_etl/transformations/sample_zones_{{.project_name}}.py.tmpl similarity index 100% rename from libs/template/templates/default/template/{{.project_name}}/src/{{.project_name_short}}_etl/transformations/sample_zones_{{.project_name}}.py.tmpl rename to libs/template/templates/default/template/{{.project_name}}/src/{{.project_name}}_etl/transformations/sample_zones_{{.project_name}}.py.tmpl diff --git a/libs/template/templates/default/template/{{.project_name}}/src/{{.project_name_short}}_etl/transformations/sample_zones_{{.project_name}}.sql.tmpl b/libs/template/templates/default/template/{{.project_name}}/src/{{.project_name}}_etl/transformations/sample_zones_{{.project_name}}.sql.tmpl similarity index 100% rename from libs/template/templates/default/template/{{.project_name}}/src/{{.project_name_short}}_etl/transformations/sample_zones_{{.project_name}}.sql.tmpl rename to libs/template/templates/default/template/{{.project_name}}/src/{{.project_name}}_etl/transformations/sample_zones_{{.project_name}}.sql.tmpl diff --git a/libs/template/templates/experimental-default-python-vnext/databricks_template_schema.json b/libs/template/templates/experimental-default-python-vnext/databricks_template_schema.json index 1566462147..9b66b28884 100644 --- a/libs/template/templates/experimental-default-python-vnext/databricks_template_schema.json +++ b/libs/template/templates/experimental-default-python-vnext/databricks_template_schema.json @@ -19,41 +19,33 @@ "pattern": "^[A-Za-z0-9_]+$", "pattern_match_failure_message": "Name must consist of letters, numbers, and underscores." }, - "project_name_short": { - "//": "This is a derived property based on project_name (it replaces my_project with sample and strips _project|_app|_service)", - "skip_prompt_if": {}, - "type": "string", - "default": "{{if eq .project_name \"my_project\"}}sample{{else}}{{with (regexp \"^(my_)?(.*)(_project|_app|_service)?$\").FindStringSubmatch .project_name}}{{index . 2}}{{else}}{{.project_name}}{{end}}{{end}}", - "description": "Short name for the project", - "order": 2 - }, "include_job": { "type": "string", "default": "yes", "enum": ["yes", "no"], "description": "Include a Lakeflow job that runs a notebook", - "order": 3 + "order": 2 }, "include_pipeline": { "type": "string", "default": "yes", "enum": ["yes", "no"], "description": "Include a Lakeflow ETL pipeline", - "order": 4 + "order": 3 }, "include_python": { "type": "string", "default": "yes", "enum": ["yes", "no"], "description": "Include a sample Python package that is built to a wheel file", - "order": 5 + "order": 4 }, "serverless": { "type": "string", "default": "yes", "enum": ["yes", "no"], "description": "Use serverless compute", - "order": 6 + "order": 5 }, "default_catalog": { "type": "string", @@ -61,7 +53,7 @@ "pattern": "^\\w*$", "pattern_match_failure_message": "Invalid catalog name.", "description": "Default catalog for any tables created by this project{{if eq (default_catalog) \"\"}} (leave blank when not using Unity Catalog){{end}}", - "order": 7 + "order": 6 }, "personal_schemas": { "type": "string", @@ -71,7 +63,7 @@ "yes", "no (advanced: I will customize the schema configuration later in databricks.yml)" ], - "order": 8 + "order": 7 }, "language": { "//": "This property is always set to 'python'", @@ -83,7 +75,7 @@ "python", "sql" ], - "order": 9 + "order": 8 } }, "success_message": "\n✨ Your new project has been created in the '{{.project_name}}' directory!\n\nPlease refer to the README.md file for \"getting started\" instructions.\nSee also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html." diff --git a/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json b/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json index f64dcf54ff..0ad8443982 100644 --- a/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json +++ b/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json @@ -19,14 +19,6 @@ "pattern": "^[a-z0-9_]+$", "pattern_match_failure_message": "Name must consist of letters, numbers, and underscores." }, - "project_name_short": { - "//": "This is a derived property based on project_name (it replaces lakeflow_project with lakeflow and strips _project|_app|_service)", - "skip_prompt_if": {}, - "type": "string", - "default": "{{if or (eq .project_name \"lakeflow_project\") (eq .project_name \"my_project\")}}lakeflow{{else}}{{with (regexp \"^(my_|lakeflow_)?(.*)(_project|_app|_service)?$\").FindStringSubmatch .project_name}}{{index . 2}}{{else}}{{.project_name}}{{end}}{{end}}", - "description": "Short name for the project", - "order": 2 - }, "include_job": { "//": "For the present template, the answer here is always 'no'", "skip_prompt_if": {}, @@ -34,7 +26,7 @@ "type": "string", "enum": ["yes", "no"], "description": "Include a Lakeflow job that runs a notebook", - "order": 3 + "order": 2 }, "include_pipeline": { "//": "For the present template, the answer here is always 'yes'", @@ -43,7 +35,7 @@ "type": "string", "enum": ["yes", "no"], "description": "Include a Lakeflow ETL pipeline", - "order": 4 + "order": 3 }, "include_python": { "//": "For the present template, the answer here is always 'no'", @@ -52,7 +44,7 @@ "default": "no", "enum": ["yes", "no"], "description": "Include a sample Python package that is built to a wheel file", - "order": 5 + "order": 4 }, "serverless": { "//": "For the present template, the answer here is always 'yes', since it can be easily changed", @@ -61,7 +53,7 @@ "default": "yes", "enum": ["yes", "no"], "description": "Use serverless compute?", - "order": 6 + "order": 5 }, "default_catalog": { "type": "string", @@ -69,7 +61,7 @@ "pattern": "^\\w*$", "pattern_match_failure_message": "Invalid catalog name.", "description": "Default catalog for any tables created by this project{{if eq (default_catalog) \"\"}} (leave blank when not using Unity Catalog){{end}}", - "order": 7 + "order": 6 }, "personal_schemas": { "type": "string", @@ -79,7 +71,7 @@ "yes", "no (advanced: I will customize the schema configuration later in databricks.yml)" ], - "order": 8 + "order": 7 }, "language": { "type": "string", @@ -89,7 +81,7 @@ "python", "sql" ], - "order": 9 + "order": 8 } }, "success_message": "\n✨ Your new project has been created in the '{{.project_name}}' directory!\n\nPlease refer to the README.md file for \"getting started\" instructions." From 1e34b95ab8b6bd6359786df70027d135bbe5d26b Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Wed, 22 Oct 2025 10:03:20 +0200 Subject: [PATCH 17/24] Avoid using environemnts in notebook task (PrPr feature) --- .../classic/out.compare-vs-serverless.diff | 7 +++---- .../my_default_python/resources/sample_job.job.yml | 1 - .../templates/default-python/template/__preamble.tmpl | 10 +++++----- .../resources/sample_job.job.yml.tmpl | 7 ++++--- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff index e9c80598dd..3e2eed02e7 100644 --- a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff +++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff @@ -22,10 +22,9 @@ --- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/resources/sample_job.job.yml +++ output/my_default_python/resources/sample_job.job.yml -@@ -26,5 +26,10 @@ +@@ -26,4 +26,10 @@ notebook_task: notebook_path: ../src/sample_notebook.ipynb -- environment_key: default + job_cluster_key: job_cluster + libraries: + # By default we just include the .whl file generated for the my_default_python package. @@ -34,7 +33,7 @@ + - whl: ../dist/*.whl - task_key: python_wheel_task depends_on: -@@ -38,5 +43,10 @@ +@@ -37,5 +43,10 @@ - "--schema" - "${var.schema}" - environment_key: default @@ -46,7 +45,7 @@ + - whl: ../dist/*.whl - task_key: refresh_pipeline depends_on: -@@ -45,11 +55,11 @@ +@@ -44,11 +55,11 @@ pipeline_id: ${resources.pipelines.my_default_python_etl.id} - environments: diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml index b0d0c7640c..7fd0530bd4 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml @@ -25,7 +25,6 @@ resources: - task_key: notebook_task notebook_task: notebook_path: ../src/sample_notebook.ipynb - environment_key: default - task_key: python_wheel_task depends_on: - task_key: notebook_task diff --git a/libs/template/templates/default-python/template/__preamble.tmpl b/libs/template/templates/default-python/template/__preamble.tmpl index e579c34fb0..d3eaa303bd 100644 --- a/libs/template/templates/default-python/template/__preamble.tmpl +++ b/libs/template/templates/default-python/template/__preamble.tmpl @@ -4,8 +4,8 @@ This file only template directives; it is skipped for the actual output. {{skip "__preamble"}} -{{$notDLT := not (eq .include_dlt "yes")}} -{{$notNotebook := not (eq .include_notebook "yes")}} +{{$notPipeline := not (eq .include_pipeline "yes")}} +{{$notJob := not (eq .include_job "yes")}} {{$notPython := not (eq .include_python "yes")}} {{if $notPython}} @@ -16,16 +16,16 @@ This file only template directives; it is skipped for the actual output. {{skip "{{.project_name}}/requirements-dev.txt"}} {{end}} -{{if $notDLT}} +{{if $notPipeline}} {{skip "{{.project_name}}/src/pipeline.ipynb"}} {{skip "{{.project_name}}/resources/{{.project_name}}.pipeline.yml"}} {{end}} -{{if $notNotebook}} +{{if $notJob}} {{skip "{{.project_name}}/src/notebook.ipynb"}} {{end}} -{{if (and $notDLT $notNotebook $notPython)}} +{{if (and $notPipeline $notJob $notPython)}} {{skip "{{.project_name}}/resources/{{.project_name}}.job.yml"}} {{else}} {{skip "{{.project_name}}/resources/.gitkeep"}} diff --git a/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.job.yml.tmpl b/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.job.yml.tmpl index 0695196067..00c921ef44 100644 --- a/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.job.yml.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.job.yml.tmpl @@ -33,10 +33,11 @@ resources: notebook_task: notebook_path: ../src/sample_notebook.ipynb {{- if $serverless}} - {{- /* Environments for notebooks are still in private preview */}} - {{- if $python_package}} + {{- /* Environments for notebooks are still in private preview + - if $python_package environment_key: default - {{- end}} + - end + */}} {{- else}} job_cluster_key: job_cluster {{- if $python_package}} From 52c67eb553a661f152da01fec00fe2e6a368ce4a Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Wed, 22 Oct 2025 10:10:43 +0200 Subject: [PATCH 18/24] Cleanup Remove unused files (default-python now depends on ../default!) --- .../default-python/library/versions.tmpl | 24 ---- .../default-python/template/__preamble.tmpl | 32 ------ .../template/{{.project_name}}/.gitignore | 8 -- .../.vscode/__builtins__.pyi | 3 - .../{{.project_name}}/.vscode/extensions.json | 7 -- .../{{.project_name}}/.vscode/settings.json | 16 --- .../template/{{.project_name}}/README.md.tmpl | 68 ------------ .../{{.project_name}}/databricks.yml.tmpl | 44 -------- .../{{.project_name}}/fixtures/.gitkeep.tmpl | 27 ----- .../{{.project_name}}/pyproject.toml.tmpl | 35 ------ .../{{.project_name}}/resources/.gitkeep | 1 - .../resources/{{.project_name}}.job.yml.tmpl | 90 --------------- .../{{.project_name}}.pipeline.yml.tmpl | 25 ----- .../{{.project_name}}/scratch/README.md | 4 - .../scratch/exploration.ipynb.tmpl | 65 ----------- .../{{.project_name}}/src/notebook.ipynb.tmpl | 79 ------------- .../{{.project_name}}/src/pipeline.ipynb.tmpl | 104 ------------------ .../src/{{.project_name}}/__init__.py | 0 .../src/{{.project_name}}/main.py.tmpl | 14 --- .../{{.project_name}}/tests/conftest.py.tmpl | 59 ---------- .../{{.project_name}}/tests/main_test.py.tmpl | 6 - .../README.md | 5 - .../databricks_template_schema.json | 82 -------------- 23 files changed, 798 deletions(-) delete mode 100644 libs/template/templates/default-python/library/versions.tmpl delete mode 100644 libs/template/templates/default-python/template/__preamble.tmpl delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/.gitignore delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/.vscode/__builtins__.pyi delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/.vscode/extensions.json delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/.vscode/settings.json delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/fixtures/.gitkeep.tmpl delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/pyproject.toml.tmpl delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/resources/.gitkeep delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/scratch/README.md delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/scratch/exploration.ipynb.tmpl delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/src/pipeline.ipynb.tmpl delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/src/{{.project_name}}/__init__.py delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/src/{{.project_name}}/main.py.tmpl delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/tests/conftest.py.tmpl delete mode 100644 libs/template/templates/default-python/template/{{.project_name}}/tests/main_test.py.tmpl delete mode 100644 libs/template/templates/experimental-default-python-vnext/README.md delete mode 100644 libs/template/templates/experimental-default-python-vnext/databricks_template_schema.json diff --git a/libs/template/templates/default-python/library/versions.tmpl b/libs/template/templates/default-python/library/versions.tmpl deleted file mode 100644 index 79c3955c99..0000000000 --- a/libs/template/templates/default-python/library/versions.tmpl +++ /dev/null @@ -1,24 +0,0 @@ -{{/* The latest LTS DBR version; this should be updated a few months after each LTS. - */}} -{{define "latest_lts_dbr_version" -}} - 15.4.x-scala2.12 -{{- end}} - -{{/* A safe version of DB Connect that is compatible with at least half the - * clusters running in production. - * - * We need to be very conservative in updating this, since a newer version can - * only connect to compute of that same version and higher. If the version is - * deemed too old, customers can update the version themselves after initializing - * the template. - */}} -{{define "conservative_db_connect_version_spec" -}} - >=15.4,<15.5 -{{- end}} - -{{/* DB Connect 15 only supports versions up to 3.13 because of a limitation in - * pyarrow: https://arrow.apache.org/docs/python/install.html#python-compatibility - */}} -{{define "conservative_db_connect_python_version_spec" -}} - >=3.10,<=3.13 -{{- end}} diff --git a/libs/template/templates/default-python/template/__preamble.tmpl b/libs/template/templates/default-python/template/__preamble.tmpl deleted file mode 100644 index d3eaa303bd..0000000000 --- a/libs/template/templates/default-python/template/__preamble.tmpl +++ /dev/null @@ -1,32 +0,0 @@ -# Preamble - -This file only template directives; it is skipped for the actual output. - -{{skip "__preamble"}} - -{{$notPipeline := not (eq .include_pipeline "yes")}} -{{$notJob := not (eq .include_job "yes")}} -{{$notPython := not (eq .include_python "yes")}} - -{{if $notPython}} - {{skip "{{.project_name}}/src/{{.project_name}}"}} - {{skip "{{.project_name}}/tests/main_test.py"}} - {{skip "{{.project_name}}/setup.py"}} - {{skip "{{.project_name}}/pytest.ini"}} - {{skip "{{.project_name}}/requirements-dev.txt"}} -{{end}} - -{{if $notPipeline}} - {{skip "{{.project_name}}/src/pipeline.ipynb"}} - {{skip "{{.project_name}}/resources/{{.project_name}}.pipeline.yml"}} -{{end}} - -{{if $notJob}} - {{skip "{{.project_name}}/src/notebook.ipynb"}} -{{end}} - -{{if (and $notPipeline $notJob $notPython)}} - {{skip "{{.project_name}}/resources/{{.project_name}}.job.yml"}} -{{else}} - {{skip "{{.project_name}}/resources/.gitkeep"}} -{{end}} diff --git a/libs/template/templates/default-python/template/{{.project_name}}/.gitignore b/libs/template/templates/default-python/template/{{.project_name}}/.gitignore deleted file mode 100644 index 0dab7f4995..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -.databricks/ -build/ -dist/ -__pycache__/ -*.egg-info -.venv/ -scratch/** -!scratch/README.md diff --git a/libs/template/templates/default-python/template/{{.project_name}}/.vscode/__builtins__.pyi b/libs/template/templates/default-python/template/{{.project_name}}/.vscode/__builtins__.pyi deleted file mode 100644 index 0edd5181bc..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/.vscode/__builtins__.pyi +++ /dev/null @@ -1,3 +0,0 @@ -# Typings for Pylance in Visual Studio Code -# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md -from databricks.sdk.runtime import * diff --git a/libs/template/templates/default-python/template/{{.project_name}}/.vscode/extensions.json b/libs/template/templates/default-python/template/{{.project_name}}/.vscode/extensions.json deleted file mode 100644 index 5d15eba363..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/.vscode/extensions.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "recommendations": [ - "databricks.databricks", - "ms-python.vscode-pylance", - "redhat.vscode-yaml" - ] -} diff --git a/libs/template/templates/default-python/template/{{.project_name}}/.vscode/settings.json b/libs/template/templates/default-python/template/{{.project_name}}/.vscode/settings.json deleted file mode 100644 index 8ee87c30d4..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/.vscode/settings.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "python.analysis.stubPath": ".vscode", - "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", - "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", - "python.testing.pytestArgs": [ - "." - ], - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, - "python.analysis.extraPaths": ["src"], - "files.exclude": { - "**/*.egg-info": true, - "**/__pycache__": true, - ".pytest_cache": true, - }, -} diff --git a/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl deleted file mode 100644 index 9b9e365272..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl +++ /dev/null @@ -1,68 +0,0 @@ -# {{.project_name}} - -The '{{.project_name}}' project was generated by using the default-python template. - -For documentation on the Databricks Asset Bundles format use for this project, -and for CI/CD configuration, see https://docs.databricks.com/aws/en/dev-tools/bundles. - -## Getting started - -Choose how you want to work on this project: - -(a) Directly in your Databricks workspace, see - https://docs.databricks.com/dev-tools/bundles/workspace. - -(b) Locally with an IDE like Cursor or VS Code, see - https://docs.databricks.com/dev-tools/vscode-ext.html. - -(c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html - -{{if (eq .include_python "yes") }} -Dependencies for this project should be installed using uv: - -* Make sure you have the UV package manager installed. - It's an alternative to tools like pip: https://docs.astral.sh/uv/getting-started/installation/. -* Run `uv sync --dev` to install the project's dependencies. -{{- end}} - -# Using this project using the CLI - -The Databricks workspace and IDE extensions provide a graphical interface for working -with this project. It's also possible to interact with it directly using the CLI: - -1. Authenticate to your Databricks workspace, if you have not done so already: - ``` - $ databricks configure - ``` - -2. To deploy a development copy of this project, type: - ``` - $ databricks bundle deploy --target dev - ``` - (Note that "dev" is the default target, so the `--target` parameter - is optional here.) - - This deploys everything that's defined for this project. - For example, the default template would deploy a job called - `[dev yourname] {{.project_name}}_job` to your workspace. - You can find that job by opening your workpace and clicking on **Jobs & Pipelines**. - -3. Similarly, to deploy a production copy, type: - ``` - $ databricks bundle deploy --target prod - ``` - - Note that the default job from the template has a schedule that runs every day - (defined in resources/{{.project_name}}.job.yml). The schedule - is paused when deploying in development mode (see - https://docs.databricks.com/dev-tools/bundles/deployment-modes.html). - -4. To run a job or pipeline, use the "run" command: - ``` - $ databricks bundle run - ``` - -5. Finally, to run tests locally, use `pytest`: - ``` - $ uv run pytest - ``` diff --git a/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl deleted file mode 100644 index 99e9d3b7c2..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl +++ /dev/null @@ -1,44 +0,0 @@ -{{$with_classic := (ne .serverless "yes") -}} -{{$with_python := (eq .include_python "yes") -}} -# This is a Databricks asset bundle definition for {{.project_name}}. -# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. -bundle: - name: {{.project_name}} - uuid: {{bundle_uuid}} -{{ if $with_python }} -artifacts: - python_artifact: - type: whl - build: uv build --wheel -{{ end }} -include: - - resources/*.yml - - resources/*/*.yml - -targets: - dev: - # The default target uses 'mode: development' to create a development copy. - # - Deployed resources get prefixed with '[dev my_user_name]' - # - Any job schedules and triggers are paused by default. - # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. - mode: development - default: true - workspace: - host: {{workspace_host}} -{{ if ($with_classic) }} - presets: - # Set dynamic_version: true on all artifacts of type "whl". - # This makes "bundle deploy" add a timestamp to wheel's version before uploading, - # new wheel takes over the previous installation even if actual wheel version is unchanged. - # See https://docs.databricks.com/aws/en/dev-tools/bundles/settings - artifacts_dynamic_version: true -{{ end }} - prod: - mode: production - workspace: - host: {{workspace_host}} - # We explicitly deploy to /Workspace/Users/{{user_name}} to make sure we only have a single copy. - root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} - permissions: - - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} - level: CAN_MANAGE diff --git a/libs/template/templates/default-python/template/{{.project_name}}/fixtures/.gitkeep.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/fixtures/.gitkeep.tmpl deleted file mode 100644 index ee95703028..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/fixtures/.gitkeep.tmpl +++ /dev/null @@ -1,27 +0,0 @@ -# Fixtures -{{- /* -We don't want to have too many README.md files, since they -stand out so much. But we do need to have a file here to make -sure the folder is added to Git. -*/}} - -This folder is reserved for fixtures, such as CSV files. - -Below is an example of how to load fixtures as a data frame: - -``` -import pandas as pd -import os - -def get_absolute_path(*relative_parts): - if 'dbutils' in globals(): - base_dir = os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) # type: ignore - path = os.path.normpath(os.path.join(base_dir, *relative_parts)) - return path if path.startswith("/Workspace") else "/Workspace" + path - else: - return os.path.join(*relative_parts) - -csv_file = get_absolute_path("..", "fixtures", "mycsv.csv") -df = pd.read_csv(csv_file) -display(df) -``` diff --git a/libs/template/templates/default-python/template/{{.project_name}}/pyproject.toml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/pyproject.toml.tmpl deleted file mode 100644 index 3d83de0077..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/pyproject.toml.tmpl +++ /dev/null @@ -1,35 +0,0 @@ -[project] -name = "{{.project_name}}" -version = "0.0.1" -authors = [{ name = "{{user_name}}" }] -requires-python = "{{template "conservative_db_connect_python_version_spec"}}" - -[dependency-groups] -dev = [ - "pytest", - - # Code completion support for Lakeflow Declarative Pipelines, also install databricks-connect - "databricks-dlt", - - # databricks-connect can be used to run parts of this project locally. - # Note that for local development, you should use a version that is not newer - # than the remote cluster or serverless compute you connect to. - # See also https://docs.databricks.com/dev-tools/databricks-connect.html. - "databricks-connect{{template "conservative_db_connect_version_spec"}}", -] - -[tool.pytest.ini_options] -pythonpath = "src" -testpaths = [ - "tests", -] - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/{{.project_name}}"] - -[project.scripts] -main = "{{.project_name}}.main:main" diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/.gitkeep b/libs/template/templates/default-python/template/{{.project_name}}/resources/.gitkeep deleted file mode 100644 index 3e09c14c18..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -This folder is reserved for Databricks Asset Bundles resource definitions. diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl deleted file mode 100644 index 2ad0f2f5fa..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl +++ /dev/null @@ -1,90 +0,0 @@ -# The main job for {{.project_name}}. - -{{- /* Clarify what this job is for Lakeflow Declarative Pipelines only users. */}} -{{if and (eq .include_dlt "yes") (and (eq .include_notebook "no") (eq .include_python "no")) -}} -# This job runs {{.project_name}}_pipeline on a schedule. -{{end -}} -{{$with_serverless := (eq .serverless "yes") -}} - -resources: - jobs: - {{.project_name}}_job: - name: {{.project_name}}_job - - trigger: - # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger - periodic: - interval: 1 - unit: DAYS - - #email_notifications: - # on_failure: - # - your_email@example.com - - tasks: - {{- if eq .include_notebook "yes" }} - - task_key: notebook_task - {{- if not $with_serverless}} - job_cluster_key: job_cluster{{end}} - notebook_task: - notebook_path: ../src/notebook.ipynb - {{- end -}} - {{- if (eq .include_dlt "yes") }} - - - task_key: refresh_pipeline - {{- if (eq .include_notebook "yes" )}} - depends_on: - - task_key: notebook_task - {{- end}} - pipeline_task: - {{- /* TODO: we should find a way that doesn't use magics for the below, like ./{{project_name}}.pipeline.yml */}} - pipeline_id: ${resources.pipelines.{{.project_name}}_pipeline.id} - {{- end -}} - {{- if (eq .include_python "yes") }} - - - task_key: main_task - {{- if (eq .include_dlt "yes") }} - depends_on: - - task_key: refresh_pipeline - {{- else if (eq .include_notebook "yes" )}} - depends_on: - - task_key: notebook_task -{{end}} - {{- if $with_serverless }} - environment_key: default - {{- else }} - job_cluster_key: job_cluster{{end}} - python_wheel_task: - package_name: {{.project_name}} - entry_point: main - {{- if not $with_serverless }} - libraries: - # By default we just include the .whl file generated for the {{.project_name}} package. - # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html - # for more information on how to add other libraries. - - whl: ../dist/*.whl -{{- end -}} -{{else}} -{{- end}} -{{if $with_serverless}}{{if (eq .include_python "yes")}} - # A list of task execution environment specifications that can be referenced by tasks of this job. - environments: - - environment_key: default - - # Full documentation of this spec can be found at: - # https://docs.databricks.com/api/workspace/jobs/create#environments-spec - spec: - environment_version: "2" - dependencies: - - ../dist/*.whl -{{end}}{{ else }} - job_clusters: - - job_cluster_key: job_cluster - new_cluster: - spark_version: {{template "latest_lts_dbr_version"}} - node_type_id: {{smallest_node_type}} - data_security_mode: SINGLE_USER - autoscale: - min_workers: 1 - max_workers: 4 -{{end -}} diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl deleted file mode 100644 index 093b087a01..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl +++ /dev/null @@ -1,25 +0,0 @@ -{{$with_serverless := (eq .serverless "yes") -}} -# The main pipeline for {{.project_name}} -resources: - pipelines: - {{.project_name}}_pipeline: - name: {{.project_name}}_pipeline - {{- if or (eq default_catalog "") (eq default_catalog "hive_metastore")}} - {{- if $with_serverless }} - ## Catalog is required for serverless compute - catalog: main{{else}} - ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: - # catalog: catalog_name{{end}} - {{- else}} - catalog: {{default_catalog}} - {{- end}} - schema: {{.project_name}}_${bundle.target} - {{- if $with_serverless }} - serverless: true - {{- end}} - libraries: - - notebook: - path: ../src/pipeline.ipynb - - configuration: - bundle.sourcePath: ${workspace.file_path}/src diff --git a/libs/template/templates/default-python/template/{{.project_name}}/scratch/README.md b/libs/template/templates/default-python/template/{{.project_name}}/scratch/README.md deleted file mode 100644 index e6cfb81b46..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/scratch/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# scratch - -This folder is reserved for personal, exploratory notebooks. -By default these are not committed to Git, as 'scratch' is listed in .gitignore. diff --git a/libs/template/templates/default-python/template/{{.project_name}}/scratch/exploration.ipynb.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/scratch/exploration.ipynb.tmpl deleted file mode 100644 index d5c05798ac..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/scratch/exploration.ipynb.tmpl +++ /dev/null @@ -1,65 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "6bca260b-13d1-448f-8082-30b60a85c9ae", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - {{- if (eq .include_python "yes") }} - "import sys\n", - "\n", - "sys.path.append(\"../src\")\n", - "from {{.project_name}} import main\n", - "\n", - "main.get_taxis().show(10)" - {{else}} - "spark.read.table(\"samples.nyctaxi.trips\")" - {{end -}} - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "ipynb-notebook", - "widgets": {} - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl deleted file mode 100644 index 53cb3040c6..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl +++ /dev/null @@ -1,79 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "ee353e42-ff58-4955-9608-12865bd0950e", - "showTitle": false, - "title": "" - } - }, - "source": [ - "# Default notebook\n", - "\n", - "This default notebook is executed using Databricks Workflows as defined in resources/{{.project_name}}.job.yml." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "6bca260b-13d1-448f-8082-30b60a85c9ae", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - {{- if (eq .include_python "yes") }} - "from {{.project_name}} import main\n", - "\n", - "main.find_all_taxis().show(10)" - {{else}} - "display(spark.read.table(\"samples.nyctaxi.trips\"))" - {{end -}} - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "notebook", - "widgets": {} - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/pipeline.ipynb.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/src/pipeline.ipynb.tmpl deleted file mode 100644 index 342fafcf6f..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/src/pipeline.ipynb.tmpl +++ /dev/null @@ -1,104 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "9a626959-61c8-4bba-84d2-2a4ecab1f7ec", - "showTitle": false, - "title": "" - } - }, - "source": [ - "# Lakeflow Declarative Pipeline\n", - "\n", - "This Lakeflow Declarative Pipeline definition is executed using a pipeline defined in resources/{{.project_name}}.pipeline.yml." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "9198e987-5606-403d-9f6d-8f14e6a4017f", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - {{- if (eq .include_python "yes") }} - "# Import DLT and src/{{.project_name}}\n", - "import dlt\n", - "import sys\n", - "\n", - "sys.path.append(spark.conf.get(\"bundle.sourcePath\", \".\"))\n", - "from pyspark.sql.functions import expr\n", - "from {{.project_name}} import main" - {{else}} - "import dlt\n", - "from pyspark.sql.functions import expr\n", - "from pyspark.sql import SparkSession\n", - "spark = SparkSession.builder.getOrCreate()" - {{end -}} - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "3fc19dba-61fd-4a89-8f8c-24fee63bfb14", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - {{- if (eq .include_python "yes") }} - "@dlt.view\n", - "def taxi_raw():\n", - " return main.find_all_taxis()\n", - {{else}} - "\n", - "@dlt.view\n", - "def taxi_raw():\n", - " return spark.read.format(\"json\").load(\"/databricks-datasets/nyctaxi/sample/json/\")\n", - {{end -}} - "\n", - "\n", - "@dlt.table\n", - "def filtered_taxis():\n", - " return dlt.read(\"taxi_raw\").filter(expr(\"fare_amount < 30\"))" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "pipeline", - "widgets": {} - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/{{.project_name}}/__init__.py b/libs/template/templates/default-python/template/{{.project_name}}/src/{{.project_name}}/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/{{.project_name}}/main.py.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/src/{{.project_name}}/main.py.tmpl deleted file mode 100644 index 04e8be4de0..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/src/{{.project_name}}/main.py.tmpl +++ /dev/null @@ -1,14 +0,0 @@ -from databricks.sdk.runtime import spark -from pyspark.sql import DataFrame - - -def find_all_taxis() -> DataFrame: - return spark.read.table("samples.nyctaxi.trips") - - -def main(): - find_all_taxis().show(5) - - -if __name__ == "__main__": - main() diff --git a/libs/template/templates/default-python/template/{{.project_name}}/tests/conftest.py.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/tests/conftest.py.tmpl deleted file mode 100644 index f80cb4395e..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/tests/conftest.py.tmpl +++ /dev/null @@ -1,59 +0,0 @@ -"""This file configures pytest.""" - -import os, sys, pathlib -from contextlib import contextmanager - - -try: - from databricks.connect import DatabricksSession - from databricks.sdk import WorkspaceClient - from pyspark.sql import SparkSession - import pytest -except ImportError: - raise ImportError( - "Test dependencies not found.\n\nRun tests using 'uv run pytest'. See http://docs.astral.sh/uv to learn more about uv." - ) - - -def enable_fallback_compute(): - """Enable serverless compute if no compute is specified.""" - conf = WorkspaceClient().config - if conf.serverless_compute_id or conf.cluster_id or os.environ.get("SPARK_REMOTE"): - return - - url = "https://docs.databricks.com/dev-tools/databricks-connect/cluster-config" - print("☁️ no compute specified, falling back to serverless compute", file=sys.stderr) - print(f" see {url} for manual configuration", file=sys.stderr) - - os.environ["DATABRICKS_SERVERLESS_COMPUTE_ID"] = "auto" - - -@contextmanager -def allow_stderr_output(config: pytest.Config): - """Temporarily disable pytest output capture.""" - capman = config.pluginmanager.get_plugin("capturemanager") - if capman: - with capman.global_and_fixture_disabled(): - yield - else: - yield - - -def pytest_configure(config: pytest.Config): - """Configure pytest session.""" - with allow_stderr_output(config): - enable_fallback_compute() - - # Initialize Spark session eagerly, so it is available even when - # SparkSession.builder.getOrCreate() is used. For DB Connect 15+, - # we validate version compatibility with the remote cluster. - if hasattr(DatabricksSession.builder, "validateSession"): - DatabricksSession.builder.validateSession().getOrCreate() - else: - DatabricksSession.builder.getOrCreate() - - -@pytest.fixture(scope="session") -def spark() -> SparkSession: - """Provide a SparkSession fixture for tests.""" - return DatabricksSession.builder.getOrCreate() diff --git a/libs/template/templates/default-python/template/{{.project_name}}/tests/main_test.py.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/tests/main_test.py.tmpl deleted file mode 100644 index 084454eb3e..0000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/tests/main_test.py.tmpl +++ /dev/null @@ -1,6 +0,0 @@ -from {{.project_name}} import main - - -def test_find_all_taxis(): - taxis = main.find_all_taxis() - assert taxis.count() > 5 diff --git a/libs/template/templates/experimental-default-python-vnext/README.md b/libs/template/templates/experimental-default-python-vnext/README.md deleted file mode 100644 index e5d65586db..0000000000 --- a/libs/template/templates/experimental-default-python-vnext/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# experimental-default-python-vnext - -This experimental internal template is a specialized form of the 'default' template. -It is available as a hidden template in the CLI for testing purposes. -It is designed to replace the 'default-python' template in an upcoming version. diff --git a/libs/template/templates/experimental-default-python-vnext/databricks_template_schema.json b/libs/template/templates/experimental-default-python-vnext/databricks_template_schema.json deleted file mode 100644 index 9b66b28884..0000000000 --- a/libs/template/templates/experimental-default-python-vnext/databricks_template_schema.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "template_dir": "../default", - "welcome_message": "Welcome to the experimental \"vNext\" default Python template for Databricks Asset Bundles!\n\n⚠️ EXPERIMENTAL: This template is experimental and may be removed or changed without notice.\n\nPlease answer the below to tailor your project to your preferences.\nYou can always change your mind and change your configuration in the databricks.yml file later.\n\nNote that {{workspace_host}} is used for initialization\n(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile).", - "properties": { - "lakeflow_only": { - "//": "This template isn't specifically customized for Lakeflow Pipelines", - "skip_prompt_if": {}, - "default": "no", - "type": "string", - "enum": ["yes", "no"], - "description": "Adjust template for Lakeflow Pipelines", - "order": 0 - }, - "project_name": { - "type": "string", - "default": "my_project", - "description": "\nUnique name for this project", - "order": 1, - "pattern": "^[A-Za-z0-9_]+$", - "pattern_match_failure_message": "Name must consist of letters, numbers, and underscores." - }, - "include_job": { - "type": "string", - "default": "yes", - "enum": ["yes", "no"], - "description": "Include a Lakeflow job that runs a notebook", - "order": 2 - }, - "include_pipeline": { - "type": "string", - "default": "yes", - "enum": ["yes", "no"], - "description": "Include a Lakeflow ETL pipeline", - "order": 3 - }, - "include_python": { - "type": "string", - "default": "yes", - "enum": ["yes", "no"], - "description": "Include a sample Python package that is built to a wheel file", - "order": 4 - }, - "serverless": { - "type": "string", - "default": "yes", - "enum": ["yes", "no"], - "description": "Use serverless compute", - "order": 5 - }, - "default_catalog": { - "type": "string", - "default": "{{default_catalog}}", - "pattern": "^\\w*$", - "pattern_match_failure_message": "Invalid catalog name.", - "description": "Default catalog for any tables created by this project{{if eq (default_catalog) \"\"}} (leave blank when not using Unity Catalog){{end}}", - "order": 6 - }, - "personal_schemas": { - "type": "string", - "description": "Use a personal schema for each user working on this project\n(this is recommended, your personal schema will be '{{.default_catalog}}.{{short_name}}')", - "default": "yes", - "enum": [ - "yes", - "no (advanced: I will customize the schema configuration later in databricks.yml)" - ], - "order": 7 - }, - "language": { - "//": "This property is always set to 'python'", - "skip_prompt_if": {}, - "type": "string", - "default": "python", - "description": "Initial language for this project", - "enum": [ - "python", - "sql" - ], - "order": 8 - } - }, - "success_message": "\n✨ Your new project has been created in the '{{.project_name}}' directory!\n\nPlease refer to the README.md file for \"getting started\" instructions.\nSee also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html." -} From 24b914bb5c72625e9a3999e4f89bcb1e705fb169 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Sun, 2 Nov 2025 21:39:23 +0100 Subject: [PATCH 19/24] Update acceptance tests for current state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- acceptance/auth/bundle_and_profile/output.txt | 2 +- .../bundle/templates-machinery/wrong-url/output.txt | 2 +- .../templates/telemetry/default-python/output.txt | 11 +++++------ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/acceptance/auth/bundle_and_profile/output.txt b/acceptance/auth/bundle_and_profile/output.txt index e05169b680..a4953c7c87 100644 --- a/acceptance/auth/bundle_and_profile/output.txt +++ b/acceptance/auth/bundle_and_profile/output.txt @@ -13,7 +13,7 @@ === Inside the bundle, profile flag not matching bundle host. Should use profile from the flag and not the bundle. >>> errcode [CLI] current-user me -p profile_name -Error: Get "https://non-existing-subdomain.databricks.com/api/2.0/preview/scim/v2/Me": (redacted) +Error: Credential was not sent or was of an unsupported type for this API. [ReqId: [UUID]] Exit code: 1 diff --git a/acceptance/bundle/templates-machinery/wrong-url/output.txt b/acceptance/bundle/templates-machinery/wrong-url/output.txt index 3edc026879..07c7c38f2f 100644 --- a/acceptance/bundle/templates-machinery/wrong-url/output.txt +++ b/acceptance/bundle/templates-machinery/wrong-url/output.txt @@ -1,5 +1,5 @@ Error: git clone failed: git clone https://invalid-domain-123.databricks.com/hello/world (redacted) --no-tags --depth=1: exit status 128. Cloning into '(redacted)'... -fatal: unable to access 'https://invalid-domain-123.databricks.com/hello/world/': Could not resolve host: invalid-domain-123.databricks.com +fatal: repository 'https://invalid-domain-123.databricks.com/hello/world/' not found Exit code: 1 diff --git a/acceptance/bundle/templates/telemetry/default-python/output.txt b/acceptance/bundle/templates/telemetry/default-python/output.txt index d2882e3eb8..0ae361c932 100644 --- a/acceptance/bundle/templates/telemetry/default-python/output.txt +++ b/acceptance/bundle/templates/telemetry/default-python/output.txt @@ -1,15 +1,14 @@ Welcome to the default Python template for Databricks Asset Bundles! -Please answer the below to tailor your project to your preferences. -You can always change your mind and change your configuration in the databricks.yml file later. +Answer the following questions to customize your project. +You can always change your configuration in the databricks.yml file later. -Note that [DATABRICKS_URL] is used for initialization -(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile). +Note that [DATABRICKS_URL] is used for initialization. +(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.) ✨ Your new project has been created in the 'my_default_python' directory! -Please refer to the README.md file for "getting started" instructions. -See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. >>> cat out.requests.txt { From 27f48d9392f940f97b8dab843995110b7c89b043 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Mon, 3 Nov 2025 11:14:38 +0100 Subject: [PATCH 20/24] Apply fixes for non-UC support, catalog references, and PyDABs compatibility This commit applies several critical fixes to the template system: 1. Non-UC workspace support (deeeb38): Set default_catalog to "hive_metastore" when no UC metastore is available, ensuring templates work on non-UC workspaces. Applied to default-python and lakeflow-pipelines templates. 2. Fix catalog property references (63758b4): Changed template conditionals from using the helper function `default_catalog` to the property `.default_catalog` in pipeline templates. This ensures proper evaluation of the user-provided value. 3. Serverless catalog field (0088b28): Always emit the catalog field for serverless pipelines, changing the comment to clarify that "Serverless compute requires Unity Catalog". Applied to both YAML and Python pipeline templates. 4. PyDABs compatibility: Added missing enable_pydabs property to default-python template schema to support the merged PyDABs infrastructure from main. Acceptance test outputs have been updated to reflect these changes. --- acceptance/auth/bundle_and_profile/output.txt | 6 +- .../classic/out.compare-vs-serverless.diff | 2 +- .../classic/out.plan_prod.direct.json | 46 ++++++++++ .../out.plan_dev.direct.json | 74 +++++++++------ .../out.plan_dev.terraform.json | 4 +- .../out.plan_prod.direct.json | 90 ++++++++++++------- .../out.plan_prod.terraform.json | 4 +- .../integration_classic/output.txt | 58 ++++++++---- .../serverless-customcatalog/output.txt | 2 +- .../my_default_python_etl.pipeline.yml | 2 +- .../my_lakeflow_pipelines_etl.pipeline.yml | 1 - .../my_lakeflow_pipelines_etl.pipeline.yml | 1 - .../output/my_pydabs/pyproject.toml | 2 +- .../databricks_template_schema.json | 10 ++- .../{{.project_name}}_etl.pipeline.yml.tmpl | 6 +- .../{{.project_name}}_etl_pipeline.py.tmpl | 6 +- .../databricks_template_schema.json | 2 +- 17 files changed, 219 insertions(+), 97 deletions(-) diff --git a/acceptance/auth/bundle_and_profile/output.txt b/acceptance/auth/bundle_and_profile/output.txt index a4953c7c87..5501ab5a6e 100644 --- a/acceptance/auth/bundle_and_profile/output.txt +++ b/acceptance/auth/bundle_and_profile/output.txt @@ -13,13 +13,13 @@ === Inside the bundle, profile flag not matching bundle host. Should use profile from the flag and not the bundle. >>> errcode [CLI] current-user me -p profile_name -Error: Credential was not sent or was of an unsupported type for this API. [ReqId: [UUID]] +Error: Get "https://non.existing.subdomain.databricks.com/api/2.0/preview/scim/v2/Me": (redacted) Exit code: 1 === Inside the bundle, target and not matching profile >>> errcode [CLI] current-user me -t dev -p profile_name -Error: cannot resolve bundle auth configuration: the host in the profile (https://non-existing-subdomain.databricks.com) doesn’t match the host configured in the bundle ([DATABRICKS_TARGET]). The profile "DEFAULT" has host="[DATABRICKS_TARGET]" that matches host in the bundle. To select it, pass "-p DEFAULT" +Error: cannot resolve bundle auth configuration: the host in the profile (https://non.existing.subdomain.databricks.com) doesn’t match the host configured in the bundle ([DATABRICKS_TARGET]). The profile "DEFAULT" has host="[DATABRICKS_TARGET]" that matches host in the bundle. To select it, pass "-p DEFAULT" Exit code: 1 @@ -48,7 +48,7 @@ Validation OK! === Bundle commands load bundle configuration with -p flag, validation not OK (profile host don't match bundle host) >>> errcode [CLI] bundle validate -p profile_name -Error: cannot resolve bundle auth configuration: the host in the profile (https://non-existing-subdomain.databricks.com) doesn’t match the host configured in the bundle ([DATABRICKS_TARGET]). The profile "DEFAULT" has host="[DATABRICKS_TARGET]" that matches host in the bundle. To select it, pass "-p DEFAULT" +Error: cannot resolve bundle auth configuration: the host in the profile (https://non.existing.subdomain.databricks.com) doesn’t match the host configured in the bundle ([DATABRICKS_TARGET]). The profile "DEFAULT" has host="[DATABRICKS_TARGET]" that matches host in the bundle. To select it, pass "-p DEFAULT" Name: test-auth Target: dev diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff index 3e2eed02e7..d923dfd998 100644 --- a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff +++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff @@ -12,7 +12,7 @@ @@ -5,8 +5,7 @@ my_default_python_etl: name: my_default_python_etl -- ## Catalog is required for serverless compute +- # Serverless compute requires Unity Catalog - catalog: ${var.catalog} + ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: + # catalog: ${var.catalog} diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json index 6bc50d8d0a..519c93ea8e 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json @@ -108,6 +108,29 @@ } } }, + "resources.jobs.sample_job.permissions": { + "depends_on": [ + { + "node": "resources.jobs.sample_job", + "label": "${resources.jobs.sample_job.id}" + } + ], + "action": "create", + "new_state": { + "config": { + "object_id": "", + "permissions": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + }, + "vars": { + "object_id": "/jobs/${resources.jobs.sample_job.id}" + } + } + }, "resources.pipelines.my_default_python_etl": { "action": "create", "new_state": { @@ -135,6 +158,29 @@ "schema": "prod" } } + }, + "resources.pipelines.my_default_python_etl.permissions": { + "depends_on": [ + { + "node": "resources.pipelines.my_default_python_etl", + "label": "${resources.pipelines.my_default_python_etl.id}" + } + ], + "action": "create", + "new_state": { + "config": { + "object_id": "", + "permissions": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + }, + "vars": { + "object_id": "/pipelines/${resources.pipelines.my_default_python_etl.id}" + } + } } } } diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json b/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json index 9d76455f1b..9893dc0401 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json +++ b/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json @@ -1,10 +1,10 @@ { "plan": { - "resources.jobs.project_name_[UNIQUE_NAME]_job": { + "resources.jobs.sample_job": { "depends_on": [ { - "node": "resources.pipelines.project_name_[UNIQUE_NAME]_pipeline", - "label": "${resources.pipelines.project_name_[UNIQUE_NAME]_pipeline.id}" + "node": "resources.pipelines.project_name_[UNIQUE_NAME]_etl", + "label": "${resources.pipelines.project_name_[UNIQUE_NAME]_etl.id}" } ], "action": "create", @@ -27,12 +27,22 @@ "data_security_mode": "SINGLE_USER", "node_type_id": "[NODE_TYPE_ID]", "num_workers": 0, - "spark_version": "15.4.x-scala2.12" + "spark_version": "16.4.x-scala2.12" } } ], "max_concurrent_runs": 4, - "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_job", + "name": "[dev [USERNAME]] sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], "queue": { "enabled": true }, @@ -40,10 +50,22 @@ "dev": "[USERNAME]" }, "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts/.internal/project_name_[UNIQUE_NAME]-0.0.1+[NUMID]-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/sample_notebook" + }, + "task_key": "notebook_task" + }, { "depends_on": [ { - "task_key": "refresh_pipeline" + "task_key": "notebook_task" } ], "job_cluster_key": "job_cluster", @@ -54,16 +76,15 @@ ], "python_wheel_task": { "entry_point": "main", - "package_name": "project_name_[UNIQUE_NAME]" - }, - "task_key": "main_task" - }, - { - "job_cluster_key": "job_cluster", - "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook" + "package_name": "project_name_[UNIQUE_NAME]", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] }, - "task_key": "notebook_task" + "task_key": "python_wheel_task" }, { "depends_on": [ @@ -72,7 +93,7 @@ } ], "pipeline_task": { - "pipeline_id": "${resources.pipelines.project_name_[UNIQUE_NAME]_pipeline.id}" + "pipeline_id": "${resources.pipelines.project_name_[UNIQUE_NAME]_etl.id}" }, "task_key": "refresh_pipeline" } @@ -86,33 +107,36 @@ } }, "vars": { - "tasks[2].pipeline_task.pipeline_id": "${resources.pipelines.project_name_[UNIQUE_NAME]_pipeline.id}" + "tasks[2].pipeline_task.pipeline_id": "${resources.pipelines.project_name_[UNIQUE_NAME]_etl.id}" } } }, - "resources.pipelines.project_name_[UNIQUE_NAME]_pipeline": { + "resources.pipelines.project_name_[UNIQUE_NAME]_etl": { "action": "create", "new_state": { "config": { "channel": "CURRENT", - "configuration": { - "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src" - }, "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json" }, "development": true, "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files" + ] + }, "libraries": [ { - "notebook": { - "path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/pipeline" + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/project_name_[UNIQUE_NAME]_etl/transformations/**" } } ], - "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_pipeline", - "schema": "project_name_[UNIQUE_NAME]_dev", + "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/project_name_[UNIQUE_NAME]_etl", + "schema": "[USERNAME]", "tags": { "dev": "[USERNAME]" } diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.terraform.json b/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.terraform.json index fe5ba8af20..be4a28ca7f 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.terraform.json +++ b/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.terraform.json @@ -1,9 +1,9 @@ { "plan": { - "resources.jobs.project_name_[UNIQUE_NAME]_job": { + "resources.jobs.sample_job": { "action": "create" }, - "resources.pipelines.project_name_[UNIQUE_NAME]_pipeline": { + "resources.pipelines.project_name_[UNIQUE_NAME]_etl": { "action": "create" } } diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json b/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json index 4cfd71d3ac..599487f46d 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json +++ b/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json @@ -1,10 +1,10 @@ { "plan": { - "resources.jobs.project_name_[UNIQUE_NAME]_job": { + "resources.jobs.sample_job": { "depends_on": [ { - "node": "resources.pipelines.project_name_[UNIQUE_NAME]_pipeline", - "label": "${resources.pipelines.project_name_[UNIQUE_NAME]_pipeline.id}" + "node": "resources.pipelines.project_name_[UNIQUE_NAME]_etl", + "label": "${resources.pipelines.project_name_[UNIQUE_NAME]_etl.id}" } ], "action": "create", @@ -27,20 +27,42 @@ "data_security_mode": "SINGLE_USER", "node_type_id": "[NODE_TYPE_ID]", "num_workers": 0, - "spark_version": "15.4.x-scala2.12" + "spark_version": "16.4.x-scala2.12" } } ], "max_concurrent_runs": 1, - "name": "project_name_[UNIQUE_NAME]_job", + "name": "sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "prod", + "name": "schema" + } + ], "queue": { "enabled": true }, "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/artifacts/.internal/project_name_[UNIQUE_NAME]-0.0.1-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/sample_notebook" + }, + "task_key": "notebook_task" + }, { "depends_on": [ { - "task_key": "refresh_pipeline" + "task_key": "notebook_task" } ], "job_cluster_key": "job_cluster", @@ -51,16 +73,15 @@ ], "python_wheel_task": { "entry_point": "main", - "package_name": "project_name_[UNIQUE_NAME]" - }, - "task_key": "main_task" - }, - { - "job_cluster_key": "job_cluster", - "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/notebook" + "package_name": "project_name_[UNIQUE_NAME]", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "prod" + ] }, - "task_key": "notebook_task" + "task_key": "python_wheel_task" }, { "depends_on": [ @@ -69,7 +90,7 @@ } ], "pipeline_task": { - "pipeline_id": "${resources.pipelines.project_name_[UNIQUE_NAME]_pipeline.id}" + "pipeline_id": "${resources.pipelines.project_name_[UNIQUE_NAME]_etl.id}" }, "task_key": "refresh_pipeline" } @@ -83,15 +104,15 @@ } }, "vars": { - "tasks[2].pipeline_task.pipeline_id": "${resources.pipelines.project_name_[UNIQUE_NAME]_pipeline.id}" + "tasks[2].pipeline_task.pipeline_id": "${resources.pipelines.project_name_[UNIQUE_NAME]_etl.id}" } } }, - "resources.jobs.project_name_[UNIQUE_NAME]_job.permissions": { + "resources.jobs.sample_job.permissions": { "depends_on": [ { - "node": "resources.jobs.project_name_[UNIQUE_NAME]_job", - "label": "${resources.jobs.project_name_[UNIQUE_NAME]_job.id}" + "node": "resources.jobs.sample_job", + "label": "${resources.jobs.sample_job.id}" } ], "action": "create", @@ -106,40 +127,43 @@ ] }, "vars": { - "object_id": "/jobs/${resources.jobs.project_name_[UNIQUE_NAME]_job.id}" + "object_id": "/jobs/${resources.jobs.sample_job.id}" } } }, - "resources.pipelines.project_name_[UNIQUE_NAME]_pipeline": { + "resources.pipelines.project_name_[UNIQUE_NAME]_etl": { "action": "create", "new_state": { "config": { "channel": "CURRENT", - "configuration": { - "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src" - }, "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/state/metadata.json" }, "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files" + ] + }, "libraries": [ { - "notebook": { - "path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/pipeline" + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/project_name_[UNIQUE_NAME]_etl/transformations/**" } } ], - "name": "project_name_[UNIQUE_NAME]_pipeline", - "schema": "project_name_[UNIQUE_NAME]_prod" + "name": "project_name_[UNIQUE_NAME]_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/project_name_[UNIQUE_NAME]_etl", + "schema": "prod" } } }, - "resources.pipelines.project_name_[UNIQUE_NAME]_pipeline.permissions": { + "resources.pipelines.project_name_[UNIQUE_NAME]_etl.permissions": { "depends_on": [ { - "node": "resources.pipelines.project_name_[UNIQUE_NAME]_pipeline", - "label": "${resources.pipelines.project_name_[UNIQUE_NAME]_pipeline.id}" + "node": "resources.pipelines.project_name_[UNIQUE_NAME]_etl", + "label": "${resources.pipelines.project_name_[UNIQUE_NAME]_etl.id}" } ], "action": "create", @@ -154,7 +178,7 @@ ] }, "vars": { - "object_id": "/pipelines/${resources.pipelines.project_name_[UNIQUE_NAME]_pipeline.id}" + "object_id": "/pipelines/${resources.pipelines.project_name_[UNIQUE_NAME]_etl.id}" } } } diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.terraform.json b/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.terraform.json index fe5ba8af20..be4a28ca7f 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.terraform.json +++ b/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.terraform.json @@ -1,9 +1,9 @@ { "plan": { - "resources.jobs.project_name_[UNIQUE_NAME]_job": { + "resources.jobs.sample_job": { "action": "create" }, - "resources.pipelines.project_name_[UNIQUE_NAME]_pipeline": { + "resources.pipelines.project_name_[UNIQUE_NAME]_etl": { "action": "create" } } diff --git a/acceptance/bundle/templates/default-python/integration_classic/output.txt b/acceptance/bundle/templates/default-python/integration_classic/output.txt index a519959642..b7ace0b92b 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/output.txt +++ b/acceptance/bundle/templates/default-python/integration_classic/output.txt @@ -26,6 +26,7 @@ Workspace: Validation OK! >>> [CLI] bundle validate -t dev -o json +Building python_artifact... >>> [CLI] bundle deploy -t dev Building python_artifact... @@ -178,15 +179,20 @@ Validation OK! + "name": "sample_job", "parameters": [ { -@@ -74,13 +64,11 @@ +@@ -74,13 +64,16 @@ }, { - "default": "[USERNAME]", + "default": "prod", "name": "schema" ++ } ++ ], ++ "permissions": [ ++ { ++ "level": "IS_OWNER", ++ "service_principal_name": "[USERNAME]" } ], -+ "permissions": [], "queue": { "enabled": true - }, @@ -194,28 +200,28 @@ Validation OK! - "dev": "[USERNAME]" }, "tasks": [ -@@ -93,5 +81,5 @@ +@@ -93,5 +86,5 @@ ], "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/sample_notebook" + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/sample_notebook" }, "task_key": "notebook_task" -@@ -116,5 +104,5 @@ +@@ -116,5 +109,5 @@ "hive_metastore", "--schema", - "[USERNAME]" + "prod" ] }, -@@ -134,5 +122,5 @@ +@@ -134,5 +127,5 @@ ], "trigger": { - "pause_status": "PAUSED", + "pause_status": "UNPAUSED", "periodic": { "interval": 1, -@@ -147,11 +135,10 @@ +@@ -147,11 +140,10 @@ "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json" @@ -229,7 +235,7 @@ Validation OK! + "--editable /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files" ] }, -@@ -159,14 +146,12 @@ +@@ -159,14 +151,17 @@ { "glob": { - "include": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/project_name_[UNIQUE_NAME]_etl/transformations/**" @@ -244,12 +250,17 @@ Validation OK! - "dev": "[USERNAME]" - } + "name": "project_name_[UNIQUE_NAME]_etl", -+ "permissions": [], ++ "permissions": [ ++ { ++ "level": "IS_OWNER", ++ "service_principal_name": "[USERNAME]" ++ } ++ ], + "root_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/project_name_[UNIQUE_NAME]_etl", + "schema": "prod" } } -@@ -184,16 +169,16 @@ +@@ -184,16 +179,16 @@ }, "schema": { - "default": "[USERNAME]", @@ -273,6 +284,7 @@ Validation OK! + "state_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/state" } } +Building python_artifact... >>> [CLI] bundle deploy -t prod Building python_artifact... @@ -367,15 +379,20 @@ Resources: + "name": "sample_job", "parameters": [ { -@@ -75,13 +65,11 @@ +@@ -75,13 +65,16 @@ }, { - "default": "[USERNAME]", + "default": "prod", "name": "schema" ++ } ++ ], ++ "permissions": [ ++ { ++ "level": "IS_OWNER", ++ "service_principal_name": "[USERNAME]" } ], -+ "permissions": [], "queue": { "enabled": true - }, @@ -383,28 +400,28 @@ Resources: - "dev": "[USERNAME]" }, "tasks": [ -@@ -94,5 +82,5 @@ +@@ -94,5 +87,5 @@ ], "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/sample_notebook" + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/sample_notebook" }, "task_key": "notebook_task" -@@ -117,5 +105,5 @@ +@@ -117,5 +110,5 @@ "hive_metastore", "--schema", - "[USERNAME]" + "prod" ] }, -@@ -135,5 +123,5 @@ +@@ -135,5 +128,5 @@ ], "trigger": { - "pause_status": "PAUSED", + "pause_status": "UNPAUSED", "periodic": { "interval": 1, -@@ -149,11 +137,10 @@ +@@ -149,11 +142,10 @@ "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json" @@ -418,7 +435,7 @@ Resources: + "--editable /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files" ] }, -@@ -162,14 +149,12 @@ +@@ -162,14 +154,17 @@ { "glob": { - "include": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/project_name_[UNIQUE_NAME]_etl/transformations/**" @@ -433,12 +450,17 @@ Resources: - "dev": "[USERNAME]" - }, + "name": "project_name_[UNIQUE_NAME]_etl", -+ "permissions": [], ++ "permissions": [ ++ { ++ "level": "IS_OWNER", ++ "service_principal_name": "[USERNAME]" ++ } ++ ], + "root_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/project_name_[UNIQUE_NAME]_etl", + "schema": "prod", "url": "[DATABRICKS_URL]/pipelines/[UUID]" } -@@ -188,16 +173,16 @@ +@@ -188,16 +183,16 @@ }, "schema": { - "default": "[USERNAME]", diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt index 8257e8d97f..b50932e80e 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt @@ -34,7 +34,7 @@ To get started, refer to the project README.md file and the documentation at htt @@ -5,5 +5,4 @@ my_default_python_etl: name: my_default_python_etl -- ## Catalog is required for serverless compute +- # Serverless compute requires Unity Catalog catalog: ${var.catalog} schema: ${var.schema} --- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/src/my_default_python_etl/explorations/sample_exploration.ipynb diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml index eed63f83df..bea3ab82ac 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml @@ -4,7 +4,7 @@ resources: pipelines: my_default_python_etl: name: my_default_python_etl - ## Catalog is required for serverless compute + # Serverless compute requires Unity Catalog catalog: ${var.catalog} schema: ${var.schema} serverless: true diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/my_lakeflow_pipelines_etl.pipeline.yml b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/my_lakeflow_pipelines_etl.pipeline.yml index 0a5bf1e7ee..9563c82ef0 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/my_lakeflow_pipelines_etl.pipeline.yml +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/my_lakeflow_pipelines_etl.pipeline.yml @@ -4,7 +4,6 @@ resources: pipelines: my_lakeflow_pipelines_etl: name: my_lakeflow_pipelines_etl - ## Catalog is required for serverless compute catalog: ${var.catalog} schema: ${var.schema} serverless: true diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/my_lakeflow_pipelines_etl.pipeline.yml b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/my_lakeflow_pipelines_etl.pipeline.yml index 0a5bf1e7ee..9563c82ef0 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/my_lakeflow_pipelines_etl.pipeline.yml +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/my_lakeflow_pipelines_etl.pipeline.yml @@ -4,7 +4,6 @@ resources: pipelines: my_lakeflow_pipelines_etl: name: my_lakeflow_pipelines_etl - ## Catalog is required for serverless compute catalog: ${var.catalog} schema: ${var.schema} serverless: true diff --git a/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml b/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml index 35cc70b58b..1dba700078 100644 --- a/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml +++ b/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [ # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies # # LIMITATION: for pipelines, dependencies are cached during development; - # add dependencies to the 'environment' section of pipeline.yml file instead + # add dependencies to the 'environment' section of your pipeline.yml file instead ] [dependency-groups] diff --git a/libs/template/templates/default-python/databricks_template_schema.json b/libs/template/templates/default-python/databricks_template_schema.json index 40e686cfc1..5a54255f61 100644 --- a/libs/template/templates/default-python/databricks_template_schema.json +++ b/libs/template/templates/default-python/databricks_template_schema.json @@ -40,7 +40,7 @@ }, "default_catalog": { "type": "string", - "default": "{{default_catalog}}", + "default": "{{if eq (default_catalog) \"\"}}hive_metastore{{else}}{{default_catalog}}{{end}}", "pattern": "^\\w*$", "pattern_match_failure_message": "Invalid catalog name.", "description": "Default catalog for any tables created by this project{{if eq (default_catalog) \"\"}} (leave blank when not using Unity Catalog){{end}}", @@ -75,6 +75,14 @@ "default": "no", "description": "Internal flag for lakeflow-only templates", "order": 9 + }, + "enable_pydabs": { + "//": "This property is always set to 'no' for default-python", + "skip_prompt_if": {}, + "type": "string", + "default": "no", + "description": "Use Python instead of YAML for resource definitions", + "order": 10 } }, "success_message": "\n✨ Your new project has been created in the '{{.project_name}}' directory!\n\nTo get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html." diff --git a/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl.pipeline.yml.tmpl b/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl.pipeline.yml.tmpl index 9bfde2257d..5f65b2b299 100644 --- a/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl.pipeline.yml.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl.pipeline.yml.tmpl @@ -9,9 +9,9 @@ resources: * so we use the project name as part as the name. */}} name: {{.project_name}}_etl -{{- if or (eq default_catalog "") (eq default_catalog "hive_metastore")}} - {{- if $serverless }} - ## Catalog is required for serverless compute +{{- if or (eq .default_catalog "") (eq .default_catalog "hive_metastore")}} + {{- if $serverless}} + # Serverless compute requires Unity Catalog catalog: ${var.catalog} {{- else}} ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: diff --git a/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl_pipeline.py.tmpl b/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl_pipeline.py.tmpl index cb6af37f81..c43c7ec95d 100644 --- a/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl_pipeline.py.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl_pipeline.py.tmpl @@ -10,9 +10,9 @@ The main pipeline for {{.project_name}} { {{- /* Note that pipeline names must be unique in a worskspace, so we use the project name as part as the name. */}} "name": "{{.project_name}}_etl", -{{- if or (eq default_catalog "") (eq default_catalog "hive_metastore")}} - {{- if $serverless }} - ## Catalog is required for serverless compute +{{- if or (eq .default_catalog "") (eq .default_catalog "hive_metastore")}} + {{- if $serverless}} + # Serverless compute requires Unity Catalog "catalog": "${var.catalog}", {{- else}} ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: diff --git a/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json b/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json index bf918f3d72..951adebddd 100644 --- a/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json +++ b/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json @@ -65,7 +65,7 @@ }, "default_catalog": { "type": "string", - "default": "{{default_catalog}}", + "default": "{{if eq (default_catalog) \"\"}}hive_metastore{{else}}{{default_catalog}}{{end}}", "pattern": "^\\w*$", "pattern_match_failure_message": "Invalid catalog name.", "description": "Default catalog for any tables created by this project{{if eq (default_catalog) \"\"}} (leave blank when not using Unity Catalog){{end}}", From fe7fca4db233f220151218f7d2618a9ba00093c5 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Mon, 3 Nov 2025 11:45:00 +0100 Subject: [PATCH 21/24] Add terraform state files to .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index b002446fca..834ed4d8fc 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,7 @@ tools/golangci-lint # Release artifacts dist/ pr-* + +# Terraform state files +bundle/deploy/terraform/terraform.tfstate +bundle/deploy/terraform/terraform.tfstate.backup From b5598b473d3842835d3c9ea844b656169bf9b005 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Mon, 3 Nov 2025 19:03:12 +0100 Subject: [PATCH 22/24] Cleanup .gitignore, add development entries --- .gitignore | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 834ed4d8fc..b1ac989e33 100644 --- a/.gitignore +++ b/.gitignore @@ -42,8 +42,7 @@ tools/golangci-lint # Release artifacts dist/ -pr-* -# Terraform state files -bundle/deploy/terraform/terraform.tfstate -bundle/deploy/terraform/terraform.tfstate.backup +# Local development notes, tmp +/pr-* +/tmp/ From 2d6df7d244b5e0efc359769796ba717deb576016 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Tue, 4 Nov 2025 09:22:55 +0100 Subject: [PATCH 23/24] Fix serverless DLT catalog for workspaces without default catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integration tests were failing because when a UC workspace has no metastore default catalog, the template would use "hive_metastore" for serverless pipelines, which the API rejects. Hardcode 'catalog: main' instead of '${var.catalog}' for this case. This provides a clear error if "main" doesn't exist and matches the approach used in origin/main. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../classic/out.compare-vs-serverless.diff | 4 ++-- .../default-python/serverless-customcatalog/output.txt | 8 +++++--- .../resources/my_default_python_etl.pipeline.yml | 4 ++-- .../resources/{{.project_name}}_etl.pipeline.yml.tmpl | 9 +++++++-- .../resources/{{.project_name}}_etl_pipeline.py.tmpl | 9 +++++++-- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff index d923dfd998..f4929ebb1b 100644 --- a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff +++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff @@ -12,8 +12,8 @@ @@ -5,8 +5,7 @@ my_default_python_etl: name: my_default_python_etl -- # Serverless compute requires Unity Catalog -- catalog: ${var.catalog} +- # Catalog is required for serverless compute +- catalog: main + ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: + # catalog: ${var.catalog} schema: ${var.schema} diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt index b50932e80e..daa2aba833 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt @@ -31,12 +31,14 @@ To get started, refer to the project README.md file and the documentation at htt permissions: --- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml +++ output/my_default_python/resources/my_default_python_etl.pipeline.yml -@@ -5,5 +5,4 @@ +@@ -5,6 +5,5 @@ my_default_python_etl: name: my_default_python_etl -- # Serverless compute requires Unity Catalog - catalog: ${var.catalog} +- # Catalog is required for serverless compute +- catalog: main ++ catalog: ${var.catalog} schema: ${var.schema} + serverless: true --- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/src/my_default_python_etl/explorations/sample_exploration.ipynb +++ output/my_default_python/src/my_default_python_etl/explorations/sample_exploration.ipynb @@ -38,5 +38,5 @@ diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml index bea3ab82ac..3d6ad105c7 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml @@ -4,8 +4,8 @@ resources: pipelines: my_default_python_etl: name: my_default_python_etl - # Serverless compute requires Unity Catalog - catalog: ${var.catalog} + # Catalog is required for serverless compute + catalog: main schema: ${var.schema} serverless: true root_path: "../src/my_default_python_etl" diff --git a/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl.pipeline.yml.tmpl b/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl.pipeline.yml.tmpl index 5f65b2b299..f71058bdad 100644 --- a/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl.pipeline.yml.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl.pipeline.yml.tmpl @@ -11,8 +11,13 @@ resources: name: {{.project_name}}_etl {{- if or (eq .default_catalog "") (eq .default_catalog "hive_metastore")}} {{- if $serverless}} - # Serverless compute requires Unity Catalog - catalog: ${var.catalog} + {{- /* This is an unsupported configuration: Unity Catalog disabled, serverless enabled. + * To make tests pass and to offer some user guidance we hardcode 'main' here, since + * because ${var.catalog} would resolve to 'hive_metastore', which is rejected by the API. + * Setting 'main' also leads to a clear "catalog not found" error for users. + */}} + # Catalog is required for serverless compute + catalog: main {{- else}} ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: # catalog: ${var.catalog} diff --git a/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl_pipeline.py.tmpl b/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl_pipeline.py.tmpl index c43c7ec95d..4cb2e803f3 100644 --- a/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl_pipeline.py.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/resources/{{.project_name}}_etl_pipeline.py.tmpl @@ -12,8 +12,13 @@ The main pipeline for {{.project_name}} "name": "{{.project_name}}_etl", {{- if or (eq .default_catalog "") (eq .default_catalog "hive_metastore")}} {{- if $serverless}} - # Serverless compute requires Unity Catalog - "catalog": "${var.catalog}", + {{- /* This is an unsupported configuration: Unity Catalog disabled, serverless enabled. + * To make tests pass and to offer some user guidance we hardcode 'main' here, since + * because ${var.catalog} would resolve to 'hive_metastore', which is rejected by the API. + * Setting 'main' also leads to a clear "catalog not found" error for users. + */}} + # Catalog is required for serverless compute + "catalog": "main", {{- else}} ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: # "catalog": "${var.catalog}", From e7daf19df7bd7102803bbacf8bd86cbea497d11b Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Tue, 4 Nov 2025 10:49:58 +0100 Subject: [PATCH 24/24] QA --- .gitignore | 4 ---- NEXT_CHANGELOG.md | 1 - .../templates/default-python/databricks_template_schema.json | 2 +- libs/template/templates/default/README.md | 2 +- .../lakeflow-pipelines/databricks_template_schema.json | 2 +- .../template/templates/pydabs/databricks_template_schema.json | 2 +- 6 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index b1ac989e33..6bbb4c86f6 100644 --- a/.gitignore +++ b/.gitignore @@ -42,7 +42,3 @@ tools/golangci-lint # Release artifacts dist/ - -# Local development notes, tmp -/pr-* -/tmp/ diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index ff4292f436..22d61e9020 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -10,7 +10,6 @@ ### Bundles * Updated the default-python template to follow the Lakeflow conventions: pipelines as source files, pyproject.toml ([#3712](https://github.com/databricks/cli/pull/3712)). -* Updated the internal lakeflow-pipelines template to use an "src" layout ([#3671](https://github.com/databricks/cli/pull/3671)). * Fix a permissions bug adding second IS\_OWNER and causing "The job must have exactly one owner." error. Introduced in 0.274.0. ([#3850](https://github.com/databricks/cli/pull/3850)) ### API Changes diff --git a/libs/template/templates/default-python/databricks_template_schema.json b/libs/template/templates/default-python/databricks_template_schema.json index 5a54255f61..218578de28 100644 --- a/libs/template/templates/default-python/databricks_template_schema.json +++ b/libs/template/templates/default-python/databricks_template_schema.json @@ -14,7 +14,7 @@ "type": "string", "default": "yes", "enum": ["yes", "no"], - "description": "Include a Lakeflow job that runs a notebook", + "description": "Include a job that runs a notebook", "order": 2 }, "include_pipeline": { diff --git a/libs/template/templates/default/README.md b/libs/template/templates/default/README.md index 87d4e0b656..0b964180d9 100644 --- a/libs/template/templates/default/README.md +++ b/libs/template/templates/default/README.md @@ -1,4 +1,4 @@ # Default template This template directory acts as a default template that is referenced -from the lakeflow-pipelines and default-python templates. +from the lakeflow-pipelines, default-python, and pydabs templates. diff --git a/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json b/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json index 951adebddd..153145390e 100644 --- a/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json +++ b/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json @@ -33,7 +33,7 @@ "default": "no", "type": "string", "enum": ["yes", "no"], - "description": "Include a Lakeflow job that runs a notebook", + "description": "Include a job that runs a notebook", "order": 3 }, "include_pipeline": { diff --git a/libs/template/templates/pydabs/databricks_template_schema.json b/libs/template/templates/pydabs/databricks_template_schema.json index 75a764453e..f3b67e2737 100644 --- a/libs/template/templates/pydabs/databricks_template_schema.json +++ b/libs/template/templates/pydabs/databricks_template_schema.json @@ -14,7 +14,7 @@ "type": "string", "default": "yes", "enum": ["yes", "no"], - "description": "Include a Lakeflow job that runs a notebook", + "description": "Include a job that runs a notebook", "order": 2 }, "include_pipeline": {