Split Weekly Generation Pipeline

.azure-pipelines/common-templates/download-openapi-docs.yml

@@ -73,12 +73,16 @@ steps:
           $diff = git diff --name-only
           $ModulesWithChanges = @{}
           $diff | %{
-            if (($_ -match 'openApiDocs\/(v1.0|beta)\/(.*).yml') -and !$ModulesWithChanges.ContainsKey($matches.2))
-            {
-              $ModulesWithChanges.Add($matches.2, $matches.1)
+            if ($_ -match 'openApiDocs\/(v1.0|beta)\/(.*)\.yml') {
+              $version = if ($matches[1] -eq 'v1.0') { 'v1.0' } else { 'beta' }
+              $moduleName = "$($matches[2])_$version"
+              if (!$ModulesWithChanges.ContainsKey($moduleName)) {
+                $ModulesWithChanges.Add($moduleName, $matches[1])
+              }
             }
           }
           $ModuleNames = $ModulesWithChanges.Keys
+          Write-Host "Modules with changes: $ModuleNames"
         }
         Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames"
 

.azure-pipelines/generation-templates/individualized-workload-modules.yml

@@ -0,0 +1,56 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+
+parameters:
+  - name: Test
+    type: boolean
+    default: true
+  - name: Pack
+    type: boolean
+    default: true
+  - name: Sign
+    type: boolean
+    default: true
+  - name: ModuleName
+    type: string
+    default: ""    
+  - name: ModuleVersion
+    type: string
+    default: ""
+
+steps:
+  - task: PowerShell@2
+    displayName: Generate Workload Modules
+    inputs:
+      targetType: inline
+      pwsh: true
+      script: |
+        . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -EnableSigning:$${{ parameters.Sign }} -Build -ExcludeExampleTemplates -ExcludeNotesSection -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }}
+  - template: ../common-templates/guardian-analyzer.yml
+
+  - task: PowerShell@2
+    displayName: Test Workload Modules
+    enabled: false
+    inputs:
+      targetType: inline
+      pwsh: true
+      script: |
+        . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -SkipGeneration -Test -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }}
+
+  - task: PowerShell@2
+    displayName: Find Duplicate Commands
+    inputs:
+      targetType: inline
+      pwsh: true
+      script: |
+        . $(System.DefaultWorkingDirectory)/tools/PostGeneration/FindDuplicateCommand.ps1 -SourcePath "$(System.DefaultWorkingDirectory)/src/"
+  
+  - ${{ if eq(parameters.Pack, true) }}:
+      - task: PowerShell@2
+        displayName: Pack Workload Modules
+        inputs:
+          targetType: inline
+          pwsh: true
+          script: |
+            . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -SkipGeneration -Pack -ArtifactsLocation $(Build.ArtifactStagingDirectory)

.azure-pipelines/module-metadata-generation.yml

@@ -0,0 +1,154 @@
+# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool.
+# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected.
+# This pipeline will be extended to the OneESPT template
+# If you are not using the E+D shared hosted pool with windows-2022, replace the pool section with your hosted pool, os, and image name. If you are using a Linux image, you must specify an additional windows image for SDL: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/sdlanalysis/overview#how-to-specify-a-windows-pool-for-the-sdl-source-analysis-stage
+# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish Module Artifacts' in the templateContext section.
+# The Task 'NuGetCommand@2' has been converted to an output named 'Publish NuGet to feed' in the templateContext section.
+# Output added to job "MsGraphPsSdkWeeklyGeneration" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness.
+# Output added to job "MsGraphPsSdkWeeklyGeneration" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness.
+name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r)
+parameters:
+- name: BuildAgent
+  displayName: Build Agent
+  default: 1es-windows-ps-compute-m
+- name: BaseBranch
+  displayName: Base Branch
+  default: dev
+- name: Test
+  type: boolean
+  default: true
+- name: Pack
+  type: boolean
+  default: true
+- name: Sign
+  type: boolean
+  default: true
+- name: CreatePullRequest
+  type: boolean
+  default: true
+
+variables:
+  BuildAgent: ${{ parameters.BuildAgent }}
+  Branch: "ModuleMetadataRefresh"
+  BaseBranch: ${{ parameters.BaseBranch }}
+
+trigger:
+  branches:
+    include:
+    - dev
+
+resources:
+  repositories:
+  - repository: 1ESPipelineTemplates
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+extends:
+  template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates
+  parameters:
+    pool: $(BuildAgent)
+    sdl:
+      binskim:
+        enabled: false
+        justificationForDisabling: "Binskim keeps on crashing and failing the weekly build pipeline. Disabling it for now because we are unable to publish the artifacts to internal feeds."
+      credscan:
+        suppressionsFile: $(Build.SourcesDirectory)/.azure-pipelines/config/credscan/credscan-suppressions.json
+      policheck:
+        exclusionFile: $(Build.SourcesDirectory)/.azure-pipelines/config/policheck/policheck-exclusions.xml
+    customBuildTags:
+    - ES365AIMigrationTooling
+    stages:
+    - stage: stage
+      jobs:
+      - job: CreateMetadataRefreshBranch
+        displayName: Create Metadata Refresh Branch
+        timeoutInMinutes: 1200
+        steps:
+        - template: .azure-pipelines/common-templates/checkout.yml@self
+          parameters:
+            TargetBranch: ${{ parameters.BaseBranch }}
+        - task: PowerShell@2
+          name: "ComputeBranch"
+          displayName: "Compute weekly branch name"
+          inputs:
+            targetType: inline
+            script: |
+              $branch = "{0}/{1}" -f "$(Branch)", (Get-Date -Format yyyyMMddHHmm)
+              Write-Host "##vso[task.setvariable variable=WeeklyBranch;isOutput=true]$branch" 
+        - task: Bash@3
+          displayName: "Create weekly branch"
+          inputs:
+            targetType: inline
+            script: |
+              git status
+              git branch $(ComputeBranch.WeeklyBranch)
+              git checkout $(ComputeBranch.WeeklyBranch)
+              git status  
+        - task: PowerShell@2
+          name: BuildOpenApiMetadataDetectionTool
+          displayName: Build tool for detecting metadata changes
+          inputs:
+            pwsh: true
+            targetType: inline
+            script: dotnet build --configuration Release
+            workingDirectory: "$(System.DefaultWorkingDirectory)/tools/OpenApiInfoGenerator/OpenApiInfoGenerator"             
+
+
+
+      - job: MsGraphPsSdkWeeklyGeneration
+        dependsOn: CreateMetadataRefreshBranch
+        displayName: Microsoft Graph PowerShell SDK Generation
+        condition: succeeded()
+        timeoutInMinutes: 840
+        variables:
+          WeeklyBranch: $[ dependencies.CreateMetadataRefreshBranch.outputs['ComputeBranch.WeeklyBranch'] ]
+        templateContext:
+          outputs:
+          - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}:
+            - output: pipelineArtifact
+              displayName: 'Publish Module Artifacts'
+              targetPath: "$(Build.ArtifactStagingDirectory)"
+              artifactName: "drop"
+              publishLocation: "Container"
+          - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}:
+            - output: nuget
+              displayName: 'Publish NuGet to feed'
+              packageParentPath: '$(Build.ArtifactStagingDirectory)'
+              packagesToPush: $(Build.ArtifactStagingDirectory)/**/Microsoft.Graph.*.nupkg
+              publishVstsFeed: $(PROJECT_NAME)/$(FEED_NAME)
+              allowPackageConflicts: true
+        steps:
+        - template: .azure-pipelines/common-templates/checkout.yml@self
+          parameters:
+            TargetBranch: $(WeeklyBranch)
+        - template: .azure-pipelines/common-templates/install-tools.yml@self
+        - template: .azure-pipelines/common-templates/security-pre-checks.yml@self
+        - template: .azure-pipelines/generation-templates/authentication-module.yml@self
+          parameters:
+            Test: ${{ parameters.Test }}
+            Pack: ${{ parameters.Pack }}
+            Sign: ${{ parameters.Sign }}
+        - template: .azure-pipelines/generation-templates/workload-modules.yml@self
+          parameters:
+            Test: ${{ parameters.Test }}
+            Pack: ${{ parameters.Pack }}
+            Sign: ${{ parameters.Sign }}
+        - template: .azure-pipelines/generation-templates/meta-module.yml@self
+          parameters:
+            Test: ${{ parameters.Test }}
+            Pack: ${{ parameters.Pack }}
+            Sign: ${{ parameters.Sign }}
+        - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}:
+          - template: .azure-pipelines/common-templates/esrp/codesign-nuget.yml@self
+            parameters:
+              FolderPath: "$(Build.ArtifactStagingDirectory)"
+              Pattern: "Microsoft.Graph*.nupkg"
+        - template: .azure-pipelines/generation-templates/generate-command-metadata.yml@self
+        - template: .azure-pipelines/common-templates/security-post-checks.yml@self
+        - ${{ if eq(parameters.CreatePullRequest, true) }}:
+          - template: .azure-pipelines/common-templates/create-pr.yml@self
+            parameters:
+              BaseBranch: $(BaseBranch)
+              TargetBranch: $(WeeklyBranch)
+              Title: "[v2] Weekly OpenApiDocs Refresh"
+              Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR."

.azure-pipelines/weekly-generation.yml

@@ -34,6 +34,15 @@ parameters:
   displayName: Skip OpenAPI Docs Download
   default: false
   type: boolean
+- name: SpecifyModules
+  displayName: Specify Modules to Generate
+  default: false
+  type: boolean
+- name: ModulesToGenerate
+  displayName: Modules To Generate
+  default: Insert Module Names Here separated by space, e.g., Users_v1.0 Users_beta  
+  type: string
+
 variables:
   Branch: "WeeklyApiRefresh"
   BaseBranch: ${{ parameters.BaseBranch }}
@@ -82,28 +91,80 @@ extends:
             BuildAgent: $(BuildAgent)
             SkipForceRefresh: $(SkipForceRefresh)
             SkipOpenAPIDocsDownload: ${{ parameters.SkipOpenAPIDocsDownload }}
-      - job: MsGraphPsSdkWeeklyGeneration
+
+      - job: SetUpModuleMatrix
+        displayName: Setup Module JSON for Matrix
         dependsOn: RefreshOpenAPIDocuments
+        condition: and(succeeded(), ne(dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'], ''))
+        timeoutInMinutes: 840
+        variables:
+          WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ]
+          ModulesToUpdate: $[ dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'] ]
+        steps:
+        - template: .azure-pipelines/common-templates/checkout.yml@self
+          parameters:
+            TargetBranch: $(WeeklyBranch)
+
+        - task: PowerShell@2
+          name: CreateMatrixJson
+          displayName: Create Matrix Json 
+          inputs:
+            targetType: inline
+            pwsh: true
+            script: |
+              if ($${{ parameters.SpecifyModules }}) {
+                $modules = "${{ parameters.ModulesToGenerate }}"
+              } else {
+                $modules = $env:ModulesToUpdate
+              }
+
+              Write-Host "ModulesToUpdate: $modules"
+              $testDataArray = $modules -split ' '
+              $jsonOutput = @{}
+
+              foreach ($item in $testDataArray) {
+                if ($item -notmatch '_') {
+                  # If '_' is not present, create two versions: v1.0 and beta
+                  $jsonOutput["${item}_v1.0"] = @{
+                    moduleVersion = 'v1.0'
+                    moduleName    = $item
+                  }
+                  $jsonOutput["${item}_beta"] = @{
+                    moduleVersion = 'beta'
+                    moduleName    = $item
+                  }
+                } else {
+                  $name, $version = $item -split '_'
+                  $jsonOutput[$item] = @{
+                    moduleVersion = $version
+                    moduleName    = $name
+                  }
+                }
+              }
+
+              $result = $jsonOutput | ConvertTo-Json -Compress
+              Write-Host "##vso[task.setvariable variable=matrixJson;isOutput=true]$result"
+
+      - job: MsGraphPsSdkWeeklyGeneration
+        dependsOn:
+          - SetUpModuleMatrix
+          - RefreshOpenAPIDocuments
         displayName: Microsoft Graph PowerShell SDK Generation
         condition: and(succeeded(), ne(dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'], ''))
         timeoutInMinutes: 840
         variables:
           WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ]
+          ModulesMatrix: $[ dependencies.SetUpModuleMatrix.outputs['CreateMatrixJson.matrixJson'] ]
         templateContext:
           outputs:
           - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}:
             - output: pipelineArtifact
               displayName: 'Publish Module Artifacts'
               targetPath: "$(Build.ArtifactStagingDirectory)"
-              artifactName: "drop"
+              artifactName: "$(moduleName)_$(moduleVersion)_drop"
               publishLocation: "Container"
-          - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}:
-            - output: nuget
-              displayName: 'Publish NuGet to feed'
-              packageParentPath: '$(Build.ArtifactStagingDirectory)'
-              packagesToPush: $(Build.ArtifactStagingDirectory)/**/Microsoft.Graph.*.nupkg
-              publishVstsFeed: $(PROJECT_NAME)/$(FEED_NAME)
-              allowPackageConflicts: true
+        strategy:
+          matrix: $[ dependencies.SetUpModuleMatrix.outputs['CreateMatrixJson.matrixJson'] ]
         steps:
         - template: .azure-pipelines/common-templates/checkout.yml@self
           parameters:
@@ -115,27 +176,30 @@ extends:
             Test: ${{ parameters.Test }}
             Pack: ${{ parameters.Pack }}
             Sign: ${{ parameters.Sign }}
-        - template: .azure-pipelines/generation-templates/workload-modules.yml@self
+        - template: .azure-pipelines/generation-templates/individualized-workload-modules.yml@self
           parameters:
             Test: ${{ parameters.Test }}
             Pack: ${{ parameters.Pack }}
             Sign: ${{ parameters.Sign }}
-        - template: .azure-pipelines/generation-templates/meta-module.yml@self
-          parameters:
-            Test: ${{ parameters.Test }}
-            Pack: ${{ parameters.Pack }}
-            Sign: ${{ parameters.Sign }}
-        - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}:
-          - template: .azure-pipelines/common-templates/esrp/codesign-nuget.yml@self
-            parameters:
-              FolderPath: "$(Build.ArtifactStagingDirectory)"
-              Pattern: "Microsoft.Graph*.nupkg"
-        - template: .azure-pipelines/generation-templates/generate-command-metadata.yml@self
+            ModuleName: $(moduleName)
+            ModuleVersion: $(moduleVersion)
         - template: .azure-pipelines/common-templates/security-post-checks.yml@self
-        - ${{ if eq(parameters.CreatePullRequest, true) }}:
-          - template: .azure-pipelines/common-templates/create-pr.yml@self
-            parameters:
-              BaseBranch: $(BaseBranch)
-              TargetBranch: $(WeeklyBranch)
-              Title: "[v2] Weekly OpenApiDocs Refresh"
-              Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR."
+
+      - job: CreatePullRequest
+        displayName: Create Pull Request for all changes
+        dependsOn:
+          - MsGraphPsSdkWeeklyGeneration
+          - RefreshOpenAPIDocuments
+        condition: and(succeeded(), eq(dependencies.MsGraphPsSdkWeeklyGeneration.result, 'Succeeded'), eq(${{ parameters.CreatePullRequest }}, true))
+        variables:
+          WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ]
+        steps:
+        - template: .azure-pipelines/common-templates/checkout.yml@self
+          parameters:
+            TargetBranch: $(WeeklyBranch)
+        - template: .azure-pipelines/common-templates/create-pr.yml@self
+          parameters:
+            BaseBranch: $(BaseBranch)
+            TargetBranch: $(WeeklyBranch)
+            Title: "[v2] Weekly OpenApiDocs Refresh"
+            Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR."