From fb839668cd024e1257d0b460e64cf368d015651c Mon Sep 17 00:00:00 2001 From: ramsessanchez <63934382+ramsessanchez@users.noreply.github.com> Date: Wed, 8 Oct 2025 15:49:31 -0700 Subject: [PATCH 1/5] Split Weekly Generation Pipeline Split Weekly generation pipeline into two separate pipelines, one to retrieve openApi updates and individually build the modules. A second pipeline to take the current api docs and generate the commandMetadata when we approach a release date. --- .../download-openapi-docs.yml | 11 +- .../individualized-workload-modules.yml | 58 +++++++ .../module-metadata-generation.yml | 154 ++++++++++++++++++ .azure-pipelines/weekly-generation.yml | 120 ++++++++++---- 4 files changed, 312 insertions(+), 31 deletions(-) create mode 100644 .azure-pipelines/generation-templates/individualized-workload-modules.yml create mode 100644 .azure-pipelines/module-metadata-generation.yml diff --git a/.azure-pipelines/common-templates/download-openapi-docs.yml b/.azure-pipelines/common-templates/download-openapi-docs.yml index 9a0a1c58053..91c25b65b4b 100644 --- a/.azure-pipelines/common-templates/download-openapi-docs.yml +++ b/.azure-pipelines/common-templates/download-openapi-docs.yml @@ -73,12 +73,17 @@ steps: $diff = git diff --name-only $ModulesWithChanges = @{} $diff | %{ - if (($_ -match 'openApiDocs\/(v1.0|beta)\/(.*).yml') -and !$ModulesWithChanges.ContainsKey($matches.2)) - { - $ModulesWithChanges.Add($matches.2, $matches.1) + if ($_ -match 'openApiDocs\/(v1.0|beta)\/(.*)\.yml') { + $version = if ($matches[1] -eq 'v1.0') { 'v1.0' } else { 'beta' } + $moduleName = "$($matches[2])_$version" + if (!$ModulesWithChanges.ContainsKey($moduleName)) { + $ModulesWithChanges.Add($moduleName, $matches[1]) + } } } $ModuleNames = $ModulesWithChanges.Keys + Write-Host "Modules with changes: $ModuleNames" + Write-Host "Testing Line" } Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames" diff --git a/.azure-pipelines/generation-templates/individualized-workload-modules.yml b/.azure-pipelines/generation-templates/individualized-workload-modules.yml new file mode 100644 index 00000000000..74bf9c9cdda --- /dev/null +++ b/.azure-pipelines/generation-templates/individualized-workload-modules.yml @@ -0,0 +1,58 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +parameters: + - name: Test + type: boolean + default: true + - name: Pack + type: boolean + default: true + - name: Sign + type: boolean + default: true + - name: ModuleName + type: string + default: "" + - name: ModuleVersion + type: string + default: "" + +steps: + - task: PowerShell@2 + displayName: Generate Workload Modules + inputs: + targetType: inline + pwsh: true + script: | + . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -EnableSigning:$${{ parameters.Sign }} -Build -ExcludeExampleTemplates -ExcludeNotesSection -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }} + - template: ../common-templates/guardian-analyzer.yml + + - task: PowerShell@2 + displayName: Test Workload Modules + enabled: false + inputs: + targetType: inline + pwsh: true + script: | + . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -SkipGeneration -Test -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }} + + - task: PowerShell@2 + displayName: Find Duplicate Commands + inputs: + targetType: inline + pwsh: true + script: | + . $(System.DefaultWorkingDirectory)/tools/PostGeneration/FindDuplicateCommand.ps1 -SourcePath "$(System.DefaultWorkingDirectory)/src/" + + - ${{ if eq(parameters.Pack, true) }}: + - task: PowerShell@2 + displayName: Pack Workload Modules + inputs: + targetType: inline + pwsh: true + script: | + . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -SkipGeneration -Pack -ArtifactsLocation $(Build.ArtifactStagingDirectory) \ No newline at end of file diff --git a/.azure-pipelines/module-metadata-generation.yml b/.azure-pipelines/module-metadata-generation.yml new file mode 100644 index 00000000000..690612accae --- /dev/null +++ b/.azure-pipelines/module-metadata-generation.yml @@ -0,0 +1,154 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. +# This pipeline will be extended to the OneESPT template +# If you are not using the E+D shared hosted pool with windows-2022, replace the pool section with your hosted pool, os, and image name. If you are using a Linux image, you must specify an additional windows image for SDL: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/sdlanalysis/overview#how-to-specify-a-windows-pool-for-the-sdl-source-analysis-stage +# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish Module Artifacts' in the templateContext section. +# The Task 'NuGetCommand@2' has been converted to an output named 'Publish NuGet to feed' in the templateContext section. +# Output added to job "MsGraphPsSdkWeeklyGeneration" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness. +# Output added to job "MsGraphPsSdkWeeklyGeneration" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness. +name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) +parameters: +- name: BuildAgent + displayName: Build Agent + default: 1es-windows-ps-compute-m +- name: BaseBranch + displayName: Base Branch + default: dev +- name: Test + type: boolean + default: true +- name: Pack + type: boolean + default: true +- name: Sign + type: boolean + default: true +- name: CreatePullRequest + type: boolean + default: true + +variables: + BuildAgent: ${{ parameters.BuildAgent }} + Branch: "ModuleMetadataRefresh" + BaseBranch: ${{ parameters.BaseBranch }} + +trigger: + branches: + include: + - dev + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: $(BuildAgent) + sdl: + binskim: + enabled: false + justificationForDisabling: "Binskim keeps on crushing and failing the weekly build pipeline. Disabling it for now because we are unable to publish the artifacts to internal feeds." + credscan: + suppressionsFile: $(Build.SourcesDirectory)/.azure-pipelines/config/credscan/credscan-suppressions.json + policheck: + exclusionFile: $(Build.SourcesDirectory)/.azure-pipelines/config/policheck/policheck-exclusions.xml + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: stage + jobs: + - job: CreateMetadataRefreshBranch + displayName: Create Metadata Refresh Branch + timeoutInMinutes: 1200 + steps: + - template: ./checkout.yml + parameters: + TargetBranch: ${{ parameters.BaseBranch }} + - task: PowerShell@2 + name: "ComputeBranch" + displayName: "Compute weekly branch name" + inputs: + targetType: inline + script: | + $branch = "{0}/{1}" -f "$(Branch)", (Get-Date -Format yyyyMMddHHmm) + Write-Host "##vso[task.setvariable variable=WeeklyBranch;isOutput=true]$branch" + - task: Bash@3 + displayName: "Create weekly branch" + inputs: + targetType: inline + script: | + git status + git branch $(ComputeBranch.WeeklyBranch) + git checkout $(ComputeBranch.WeeklyBranch) + git status + - task: PowerShell@2 + name: BuildOpenApiMetadataDetectionTool + displayName: Build tool for detecting metadata changes + inputs: + pwsh: true + targetType: inline + script: dotnet build --configuration Release + workingDirectory: "$(System.DefaultWorkingDirectory)/tools/OpenApiInfoGenerator/OpenApiInfoGenerator" + + + + - job: MsGraphPsSdkWeeklyGeneration + dependsOn: CreateMetadataRefreshBranch + displayName: Microsoft Graph PowerShell SDK Generation + condition: succeeded() + timeoutInMinutes: 840 + variables: + WeeklyBranch: $[ dependencies.CreateMetadataRefreshBranch.outputs['ComputeBranch.WeeklyBranch'] ] + templateContext: + outputs: + - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}: + - output: pipelineArtifact + displayName: 'Publish Module Artifacts' + targetPath: "$(Build.ArtifactStagingDirectory)" + artifactName: "drop" + publishLocation: "Container" + - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}: + - output: nuget + displayName: 'Publish NuGet to feed' + packageParentPath: '$(Build.ArtifactStagingDirectory)' + packagesToPush: $(Build.ArtifactStagingDirectory)/**/Microsoft.Graph.*.nupkg + publishVstsFeed: $(PROJECT_NAME)/$(FEED_NAME) + allowPackageConflicts: true + steps: + - template: .azure-pipelines/common-templates/checkout.yml@self + parameters: + TargetBranch: $(WeeklyBranch) + - template: .azure-pipelines/common-templates/install-tools.yml@self + - template: .azure-pipelines/common-templates/security-pre-checks.yml@self + - template: .azure-pipelines/generation-templates/authentication-module.yml@self + parameters: + Test: ${{ parameters.Test }} + Pack: ${{ parameters.Pack }} + Sign: ${{ parameters.Sign }} + - template: .azure-pipelines/generation-templates/workload-modules.yml@self + parameters: + Test: ${{ parameters.Test }} + Pack: ${{ parameters.Pack }} + Sign: ${{ parameters.Sign }} + - template: .azure-pipelines/generation-templates/meta-module.yml@self + parameters: + Test: ${{ parameters.Test }} + Pack: ${{ parameters.Pack }} + Sign: ${{ parameters.Sign }} + - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}: + - template: .azure-pipelines/common-templates/esrp/codesign-nuget.yml@self + parameters: + FolderPath: "$(Build.ArtifactStagingDirectory)" + Pattern: "Microsoft.Graph*.nupkg" + - template: .azure-pipelines/generation-templates/generate-command-metadata.yml@self + - template: .azure-pipelines/common-templates/security-post-checks.yml@self + - ${{ if eq(parameters.CreatePullRequest, true) }}: + - template: .azure-pipelines/common-templates/create-pr.yml@self + parameters: + BaseBranch: $(BaseBranch) + TargetBranch: $(WeeklyBranch) + Title: "[v2] Weekly OpenApiDocs Refresh" + Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR." \ No newline at end of file diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index 0ef0a27a210..ac366850a8e 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -34,6 +34,15 @@ parameters: displayName: Skip OpenAPI Docs Download default: false type: boolean +- name: SpecifyModules + displayName: Specify Modules to Generate + default: false + type: boolean +- name: ModulesToGenerate + displayName: Modules To Generate + default: Insert Module Names Here separated by space, e.g., Users_v1.0 Users_beta + type: string + variables: Branch: "WeeklyApiRefresh" BaseBranch: ${{ parameters.BaseBranch }} @@ -82,28 +91,80 @@ extends: BuildAgent: $(BuildAgent) SkipForceRefresh: $(SkipForceRefresh) SkipOpenAPIDocsDownload: ${{ parameters.SkipOpenAPIDocsDownload }} - - job: MsGraphPsSdkWeeklyGeneration + + - job: SetUpModuleMatrix + displayName: Setup Module JSON for Matrix dependsOn: RefreshOpenAPIDocuments + condition: and(succeeded(), ne(dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'], '')) + timeoutInMinutes: 840 + variables: + WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] + ModulesToUpdate: $[ dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'] ] + steps: + - template: .azure-pipelines/common-templates/checkout.yml@self + parameters: + TargetBranch: $(WeeklyBranch) + + - task: PowerShell@2 + name: CreateMatrixJson + displayName: Create Matrix Json + inputs: + targetType: inline + pwsh: true + script: | + if ($${{ parameters.SpecifyModules }}) { + $modules = "${{ parameters.ModulesToGenerate }}" + } else { + $modules = $env:ModulesToUpdate + } + + Write-Host "ModulesToUpdate: $modules" + $testDataArray = $modules -split ' ' + $jsonOutput = @{} + + foreach ($item in $testDataArray) { + if ($item -notmatch '_') { + # If '_' is not present, create two versions: v1.0 and beta + $jsonOutput["${item}_v1.0"] = @{ + moduleVersion = 'v1.0' + moduleName = $item + } + $jsonOutput["${item}_beta"] = @{ + moduleVersion = 'beta' + moduleName = $item + } + } else { + $name, $version = $item -split '_' + $jsonOutput[$item] = @{ + moduleVersion = $version + moduleName = $name + } + } + } + + $result = $jsonOutput | ConvertTo-Json -Compress + Write-Host "##vso[task.setvariable variable=matrixJson;isOutput=true]$result" + + - job: MsGraphPsSdkWeeklyGeneration + dependsOn: + - SetUpModuleMatrix + - RefreshOpenAPIDocuments displayName: Microsoft Graph PowerShell SDK Generation condition: and(succeeded(), ne(dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'], '')) timeoutInMinutes: 840 variables: WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] + ModulesMatrix: $[ dependencies.SetUpModuleMatrix.outputs['CreateMatrixJson.matrixJson'] ] templateContext: outputs: - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}: - output: pipelineArtifact displayName: 'Publish Module Artifacts' targetPath: "$(Build.ArtifactStagingDirectory)" - artifactName: "drop" + artifactName: "$(moduleName)_$(moduleVersion)_drop" publishLocation: "Container" - - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}: - - output: nuget - displayName: 'Publish NuGet to feed' - packageParentPath: '$(Build.ArtifactStagingDirectory)' - packagesToPush: $(Build.ArtifactStagingDirectory)/**/Microsoft.Graph.*.nupkg - publishVstsFeed: $(PROJECT_NAME)/$(FEED_NAME) - allowPackageConflicts: true + strategy: + matrix: $[ dependencies.SetUpModuleMatrix.outputs['CreateMatrixJson.matrixJson'] ] steps: - template: .azure-pipelines/common-templates/checkout.yml@self parameters: @@ -115,27 +176,30 @@ extends: Test: ${{ parameters.Test }} Pack: ${{ parameters.Pack }} Sign: ${{ parameters.Sign }} - - template: .azure-pipelines/generation-templates/workload-modules.yml@self + - template: .azure-pipelines/generation-templates/individualized-workload-modules.yml@self parameters: Test: ${{ parameters.Test }} Pack: ${{ parameters.Pack }} Sign: ${{ parameters.Sign }} - - template: .azure-pipelines/generation-templates/meta-module.yml@self - parameters: - Test: ${{ parameters.Test }} - Pack: ${{ parameters.Pack }} - Sign: ${{ parameters.Sign }} - - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}: - - template: .azure-pipelines/common-templates/esrp/codesign-nuget.yml@self - parameters: - FolderPath: "$(Build.ArtifactStagingDirectory)" - Pattern: "Microsoft.Graph*.nupkg" - - template: .azure-pipelines/generation-templates/generate-command-metadata.yml@self + ModuleName: $(moduleName) + ModuleVersion: $(moduleVersion) - template: .azure-pipelines/common-templates/security-post-checks.yml@self - - ${{ if eq(parameters.CreatePullRequest, true) }}: - - template: .azure-pipelines/common-templates/create-pr.yml@self - parameters: - BaseBranch: $(BaseBranch) - TargetBranch: $(WeeklyBranch) - Title: "[v2] Weekly OpenApiDocs Refresh" - Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR." \ No newline at end of file + + - job: CreatePullRequest + displayName: Create Pull Request for all changes + dependsOn: + - MsGraphPsSdkWeeklyGeneration + - RefreshOpenAPIDocuments + condition: and(succeeded(), eq(dependencies.MsGraphPsSdkWeeklyGeneration.result, 'Succeeded'), eq(${{ parameters.CreatePullRequest }}, true)) + variables: + WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] + steps: + - template: .azure-pipelines/common-templates/checkout.yml@self + parameters: + TargetBranch: $(WeeklyBranch) + - template: .azure-pipelines/common-templates/create-pr.yml@self + parameters: + BaseBranch: $(BaseBranch) + TargetBranch: $(WeeklyBranch) + Title: "[v2] Weekly OpenApiDocs Refresh" + Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR." \ No newline at end of file From 088d3f2aa4ad7d575f27439202af96240a1453ee Mon Sep 17 00:00:00 2001 From: Ramses Sanchez-Hernandez <63934382+ramsessanchez@users.noreply.github.com> Date: Wed, 22 Oct 2025 09:32:00 -0700 Subject: [PATCH 2/5] remove testing line Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .azure-pipelines/common-templates/download-openapi-docs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.azure-pipelines/common-templates/download-openapi-docs.yml b/.azure-pipelines/common-templates/download-openapi-docs.yml index 91c25b65b4b..092a88042c9 100644 --- a/.azure-pipelines/common-templates/download-openapi-docs.yml +++ b/.azure-pipelines/common-templates/download-openapi-docs.yml @@ -83,7 +83,6 @@ steps: } $ModuleNames = $ModulesWithChanges.Keys Write-Host "Modules with changes: $ModuleNames" - Write-Host "Testing Line" } Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames" From 68a7e8cec35098f21d7dde32fad9d8d587eab051 Mon Sep 17 00:00:00 2001 From: Ramses Sanchez-Hernandez <63934382+ramsessanchez@users.noreply.github.com> Date: Wed, 22 Oct 2025 09:32:38 -0700 Subject: [PATCH 3/5] remove duplicate lines Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../generation-templates/individualized-workload-modules.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.azure-pipelines/generation-templates/individualized-workload-modules.yml b/.azure-pipelines/generation-templates/individualized-workload-modules.yml index 74bf9c9cdda..69319d77c89 100644 --- a/.azure-pipelines/generation-templates/individualized-workload-modules.yml +++ b/.azure-pipelines/generation-templates/individualized-workload-modules.yml @@ -1,8 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. parameters: - name: Test From c1ea55d386be1432d65cbc3d222616adb7293ab0 Mon Sep 17 00:00:00 2001 From: Ramses Sanchez-Hernandez <63934382+ramsessanchez@users.noreply.github.com> Date: Wed, 22 Oct 2025 09:41:46 -0700 Subject: [PATCH 4/5] update checkout step ref Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .azure-pipelines/module-metadata-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/module-metadata-generation.yml b/.azure-pipelines/module-metadata-generation.yml index 690612accae..0a645899b2c 100644 --- a/.azure-pipelines/module-metadata-generation.yml +++ b/.azure-pipelines/module-metadata-generation.yml @@ -64,7 +64,7 @@ extends: displayName: Create Metadata Refresh Branch timeoutInMinutes: 1200 steps: - - template: ./checkout.yml + - template: .azure-pipelines/common-templates/checkout.yml@self parameters: TargetBranch: ${{ parameters.BaseBranch }} - task: PowerShell@2 From 6add689439c67d60a7261e396a8cd0c3df6bbed0 Mon Sep 17 00:00:00 2001 From: Ramses Sanchez-Hernandez <63934382+ramsessanchez@users.noreply.github.com> Date: Wed, 22 Oct 2025 09:42:01 -0700 Subject: [PATCH 5/5] Update .azure-pipelines/module-metadata-generation.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .azure-pipelines/module-metadata-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/module-metadata-generation.yml b/.azure-pipelines/module-metadata-generation.yml index 0a645899b2c..744ea0c150b 100644 --- a/.azure-pipelines/module-metadata-generation.yml +++ b/.azure-pipelines/module-metadata-generation.yml @@ -50,7 +50,7 @@ extends: sdl: binskim: enabled: false - justificationForDisabling: "Binskim keeps on crushing and failing the weekly build pipeline. Disabling it for now because we are unable to publish the artifacts to internal feeds." + justificationForDisabling: "Binskim keeps on crashing and failing the weekly build pipeline. Disabling it for now because we are unable to publish the artifacts to internal feeds." credscan: suppressionsFile: $(Build.SourcesDirectory)/.azure-pipelines/config/credscan/credscan-suppressions.json policheck: