From a0e49a403380557776cc6cf9c0b073497be391f4 Mon Sep 17 00:00:00 2001
From: "Precious Onu (Arnold)" <50643194+poarnold@users.noreply.github.com>
Date: Tue, 20 May 2025 14:17:15 +0100
Subject: [PATCH 1/2] Use databricks cli for data_pipeline_ci_cd.yml

---
 azure-data-pipeline/data_pipeline_ci_cd.yml | 34 +++++++++++++--------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/azure-data-pipeline/data_pipeline_ci_cd.yml b/azure-data-pipeline/data_pipeline_ci_cd.yml
index 8bdbfb9..994a1f7 100644
--- a/azure-data-pipeline/data_pipeline_ci_cd.yml
+++ b/azure-data-pipeline/data_pipeline_ci_cd.yml
@@ -2,12 +2,12 @@ name: CICD
 pr:
   branches:
     include:
-    - master
+    - main
     - adf_publish
 trigger:
   branches:
     include:
-    - master
+    - main
   paths:
     include:
     - scripts/
@@ -53,18 +53,26 @@ stages:
                 addToPath: true
                 architecture: 'x64'
               displayName: 'Use Python3'
-            # Need to install DevOps for Azure Databricks extension
-            - task: configuredatabricks@0
-              inputs:
-                url: '$(DATABRICKS_URL)'
-                token: '$(databricks-token)'
-              displayName: 'Configure Databricks CLI'    
+          
+            # Install Databricks CLI
+            - script: |
+                curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh
+                echo "databricks version" $(databricks --version)
+              displayName: 'Install Databricks CLI'
+            
+            - script: |
+                mkdir -p ~/.databricks
+                echo "[DEFAULT]" > ~/.databrickscfg
+                echo "host = $DATABRICKS_URL" >> ~/.databrickscfg
+                echo "token = $(databricks-token)" >> ~/.databrickscfg
+                databricks auth profiles
+              displayName: "Configure Databricks CLI"
+            
+            - script: |
+                echo "Uploading notebooks from $(Pipeline.Workspace)/notebooks to /Shared..."
+                databricks workspace import-dir "$(Pipeline.Workspace)/notebooks" /Shared
+              displayName: "Upload Notebooks to /Shared in the Databricks cluster"
 
-            - task: deploynotebooks@0
-              inputs:
-                notebooksFolderPath: '$(Pipeline.Workspace)/notebooks'
-                workspaceFolder: '/Shared'
-              displayName: 'Deploy (copy) data processing notebook to the Databricks cluster'
   - deployment: "Deploy_to_ADF"
     displayName: 'Deploy to ADF'
     timeoutInMinutes: 0

From 412ee7618b5b46904eaa656b60fc5796496a5bba Mon Sep 17 00:00:00 2001
From: "Precious Onu (Arnold)" <50643194+poarnold@users.noreply.github.com>
Date: Tue, 3 Jun 2025 13:30:47 +0100
Subject: [PATCH 2/2] update branch trigger for data_pipeline_test.yml

---
 azure-data-pipeline/data_pipeline_test.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/azure-data-pipeline/data_pipeline_test.yml b/azure-data-pipeline/data_pipeline_test.yml
index b2a85bf..8333980 100644
--- a/azure-data-pipeline/data_pipeline_test.yml
+++ b/azure-data-pipeline/data_pipeline_test.yml
@@ -4,6 +4,7 @@ trigger:
   branches:
     include:
     - master
+    - main
   paths:
     include:
     - scripts/
@@ -31,4 +32,4 @@ stages:
             azureSubscription: $(AZURE_RM_CONNECTION)
             ScriptPath: '$(Build.SourcesDirectory)/azure-data-pipeline/scripts/Invoke-ADFPipeline.ps1'
             ScriptArguments: '-ResourceGroupName $(RESOURCE_GROUP) -DataFactoryName $(DATA_FACTORY_TEST_NAME) -PipelineName $(ADF_PIPELINE_NAME)'
-            azurePowerShellVersion: LatestVersion
\ No newline at end of file
+            azurePowerShellVersion: LatestVersion