diff --git a/.azure-pipelines/common-templates/download-openapi-docs.yml b/.azure-pipelines/common-templates/download-openapi-docs.yml index 9a0a1c58053..092a88042c9 100644 --- a/.azure-pipelines/common-templates/download-openapi-docs.yml +++ b/.azure-pipelines/common-templates/download-openapi-docs.yml @@ -73,12 +73,16 @@ steps: $diff = git diff --name-only $ModulesWithChanges = @{} $diff | %{ - if (($_ -match 'openApiDocs\/(v1.0|beta)\/(.*).yml') -and !$ModulesWithChanges.ContainsKey($matches.2)) - { - $ModulesWithChanges.Add($matches.2, $matches.1) + if ($_ -match 'openApiDocs\/(v1.0|beta)\/(.*)\.yml') { + $version = if ($matches[1] -eq 'v1.0') { 'v1.0' } else { 'beta' } + $moduleName = "$($matches[2])_$version" + if (!$ModulesWithChanges.ContainsKey($moduleName)) { + $ModulesWithChanges.Add($moduleName, $matches[1]) + } } } $ModuleNames = $ModulesWithChanges.Keys + Write-Host "Modules with changes: $ModuleNames" } Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames" diff --git a/.azure-pipelines/generation-templates/individualized-workload-modules.yml b/.azure-pipelines/generation-templates/individualized-workload-modules.yml new file mode 100644 index 00000000000..69319d77c89 --- /dev/null +++ b/.azure-pipelines/generation-templates/individualized-workload-modules.yml @@ -0,0 +1,56 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + + +parameters: + - name: Test + type: boolean + default: true + - name: Pack + type: boolean + default: true + - name: Sign + type: boolean + default: true + - name: ModuleName + type: string + default: "" + - name: ModuleVersion + type: string + default: "" + +steps: + - task: PowerShell@2 + displayName: Generate Workload Modules + inputs: + targetType: inline + pwsh: true + script: | + . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -EnableSigning:$${{ parameters.Sign }} -Build -ExcludeExampleTemplates -ExcludeNotesSection -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }} + - template: ../common-templates/guardian-analyzer.yml + + - task: PowerShell@2 + displayName: Test Workload Modules + enabled: false + inputs: + targetType: inline + pwsh: true + script: | + . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -SkipGeneration -Test -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }} + + - task: PowerShell@2 + displayName: Find Duplicate Commands + inputs: + targetType: inline + pwsh: true + script: | + . $(System.DefaultWorkingDirectory)/tools/PostGeneration/FindDuplicateCommand.ps1 -SourcePath "$(System.DefaultWorkingDirectory)/src/" + + - ${{ if eq(parameters.Pack, true) }}: + - task: PowerShell@2 + displayName: Pack Workload Modules + inputs: + targetType: inline + pwsh: true + script: | + . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -SkipGeneration -Pack -ArtifactsLocation $(Build.ArtifactStagingDirectory) \ No newline at end of file diff --git a/.azure-pipelines/module-metadata-generation.yml b/.azure-pipelines/module-metadata-generation.yml new file mode 100644 index 00000000000..744ea0c150b --- /dev/null +++ b/.azure-pipelines/module-metadata-generation.yml @@ -0,0 +1,154 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. +# This pipeline will be extended to the OneESPT template +# If you are not using the E+D shared hosted pool with windows-2022, replace the pool section with your hosted pool, os, and image name. If you are using a Linux image, you must specify an additional windows image for SDL: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/sdlanalysis/overview#how-to-specify-a-windows-pool-for-the-sdl-source-analysis-stage +# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish Module Artifacts' in the templateContext section. +# The Task 'NuGetCommand@2' has been converted to an output named 'Publish NuGet to feed' in the templateContext section. +# Output added to job "MsGraphPsSdkWeeklyGeneration" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness. +# Output added to job "MsGraphPsSdkWeeklyGeneration" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness. +name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) +parameters: +- name: BuildAgent + displayName: Build Agent + default: 1es-windows-ps-compute-m +- name: BaseBranch + displayName: Base Branch + default: dev +- name: Test + type: boolean + default: true +- name: Pack + type: boolean + default: true +- name: Sign + type: boolean + default: true +- name: CreatePullRequest + type: boolean + default: true + +variables: + BuildAgent: ${{ parameters.BuildAgent }} + Branch: "ModuleMetadataRefresh" + BaseBranch: ${{ parameters.BaseBranch }} + +trigger: + branches: + include: + - dev + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: $(BuildAgent) + sdl: + binskim: + enabled: false + justificationForDisabling: "Binskim keeps on crashing and failing the weekly build pipeline. Disabling it for now because we are unable to publish the artifacts to internal feeds." + credscan: + suppressionsFile: $(Build.SourcesDirectory)/.azure-pipelines/config/credscan/credscan-suppressions.json + policheck: + exclusionFile: $(Build.SourcesDirectory)/.azure-pipelines/config/policheck/policheck-exclusions.xml + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: stage + jobs: + - job: CreateMetadataRefreshBranch + displayName: Create Metadata Refresh Branch + timeoutInMinutes: 1200 + steps: + - template: .azure-pipelines/common-templates/checkout.yml@self + parameters: + TargetBranch: ${{ parameters.BaseBranch }} + - task: PowerShell@2 + name: "ComputeBranch" + displayName: "Compute weekly branch name" + inputs: + targetType: inline + script: | + $branch = "{0}/{1}" -f "$(Branch)", (Get-Date -Format yyyyMMddHHmm) + Write-Host "##vso[task.setvariable variable=WeeklyBranch;isOutput=true]$branch" + - task: Bash@3 + displayName: "Create weekly branch" + inputs: + targetType: inline + script: | + git status + git branch $(ComputeBranch.WeeklyBranch) + git checkout $(ComputeBranch.WeeklyBranch) + git status + - task: PowerShell@2 + name: BuildOpenApiMetadataDetectionTool + displayName: Build tool for detecting metadata changes + inputs: + pwsh: true + targetType: inline + script: dotnet build --configuration Release + workingDirectory: "$(System.DefaultWorkingDirectory)/tools/OpenApiInfoGenerator/OpenApiInfoGenerator" + + + + - job: MsGraphPsSdkWeeklyGeneration + dependsOn: CreateMetadataRefreshBranch + displayName: Microsoft Graph PowerShell SDK Generation + condition: succeeded() + timeoutInMinutes: 840 + variables: + WeeklyBranch: $[ dependencies.CreateMetadataRefreshBranch.outputs['ComputeBranch.WeeklyBranch'] ] + templateContext: + outputs: + - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}: + - output: pipelineArtifact + displayName: 'Publish Module Artifacts' + targetPath: "$(Build.ArtifactStagingDirectory)" + artifactName: "drop" + publishLocation: "Container" + - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}: + - output: nuget + displayName: 'Publish NuGet to feed' + packageParentPath: '$(Build.ArtifactStagingDirectory)' + packagesToPush: $(Build.ArtifactStagingDirectory)/**/Microsoft.Graph.*.nupkg + publishVstsFeed: $(PROJECT_NAME)/$(FEED_NAME) + allowPackageConflicts: true + steps: + - template: .azure-pipelines/common-templates/checkout.yml@self + parameters: + TargetBranch: $(WeeklyBranch) + - template: .azure-pipelines/common-templates/install-tools.yml@self + - template: .azure-pipelines/common-templates/security-pre-checks.yml@self + - template: .azure-pipelines/generation-templates/authentication-module.yml@self + parameters: + Test: ${{ parameters.Test }} + Pack: ${{ parameters.Pack }} + Sign: ${{ parameters.Sign }} + - template: .azure-pipelines/generation-templates/workload-modules.yml@self + parameters: + Test: ${{ parameters.Test }} + Pack: ${{ parameters.Pack }} + Sign: ${{ parameters.Sign }} + - template: .azure-pipelines/generation-templates/meta-module.yml@self + parameters: + Test: ${{ parameters.Test }} + Pack: ${{ parameters.Pack }} + Sign: ${{ parameters.Sign }} + - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}: + - template: .azure-pipelines/common-templates/esrp/codesign-nuget.yml@self + parameters: + FolderPath: "$(Build.ArtifactStagingDirectory)" + Pattern: "Microsoft.Graph*.nupkg" + - template: .azure-pipelines/generation-templates/generate-command-metadata.yml@self + - template: .azure-pipelines/common-templates/security-post-checks.yml@self + - ${{ if eq(parameters.CreatePullRequest, true) }}: + - template: .azure-pipelines/common-templates/create-pr.yml@self + parameters: + BaseBranch: $(BaseBranch) + TargetBranch: $(WeeklyBranch) + Title: "[v2] Weekly OpenApiDocs Refresh" + Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR." \ No newline at end of file diff --git a/.azure-pipelines/weekly-generation.yml b/.azure-pipelines/weekly-generation.yml index 0ef0a27a210..ac366850a8e 100644 --- a/.azure-pipelines/weekly-generation.yml +++ b/.azure-pipelines/weekly-generation.yml @@ -34,6 +34,15 @@ parameters: displayName: Skip OpenAPI Docs Download default: false type: boolean +- name: SpecifyModules + displayName: Specify Modules to Generate + default: false + type: boolean +- name: ModulesToGenerate + displayName: Modules To Generate + default: Insert Module Names Here separated by space, e.g., Users_v1.0 Users_beta + type: string + variables: Branch: "WeeklyApiRefresh" BaseBranch: ${{ parameters.BaseBranch }} @@ -82,28 +91,80 @@ extends: BuildAgent: $(BuildAgent) SkipForceRefresh: $(SkipForceRefresh) SkipOpenAPIDocsDownload: ${{ parameters.SkipOpenAPIDocsDownload }} - - job: MsGraphPsSdkWeeklyGeneration + + - job: SetUpModuleMatrix + displayName: Setup Module JSON for Matrix dependsOn: RefreshOpenAPIDocuments + condition: and(succeeded(), ne(dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'], '')) + timeoutInMinutes: 840 + variables: + WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] + ModulesToUpdate: $[ dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'] ] + steps: + - template: .azure-pipelines/common-templates/checkout.yml@self + parameters: + TargetBranch: $(WeeklyBranch) + + - task: PowerShell@2 + name: CreateMatrixJson + displayName: Create Matrix Json + inputs: + targetType: inline + pwsh: true + script: | + if ($${{ parameters.SpecifyModules }}) { + $modules = "${{ parameters.ModulesToGenerate }}" + } else { + $modules = $env:ModulesToUpdate + } + + Write-Host "ModulesToUpdate: $modules" + $testDataArray = $modules -split ' ' + $jsonOutput = @{} + + foreach ($item in $testDataArray) { + if ($item -notmatch '_') { + # If '_' is not present, create two versions: v1.0 and beta + $jsonOutput["${item}_v1.0"] = @{ + moduleVersion = 'v1.0' + moduleName = $item + } + $jsonOutput["${item}_beta"] = @{ + moduleVersion = 'beta' + moduleName = $item + } + } else { + $name, $version = $item -split '_' + $jsonOutput[$item] = @{ + moduleVersion = $version + moduleName = $name + } + } + } + + $result = $jsonOutput | ConvertTo-Json -Compress + Write-Host "##vso[task.setvariable variable=matrixJson;isOutput=true]$result" + + - job: MsGraphPsSdkWeeklyGeneration + dependsOn: + - SetUpModuleMatrix + - RefreshOpenAPIDocuments displayName: Microsoft Graph PowerShell SDK Generation condition: and(succeeded(), ne(dependencies.RefreshOpenAPIDocuments.outputs['OpenAPIDocDiff.ModulesWithChanges'], '')) timeoutInMinutes: 840 variables: WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] + ModulesMatrix: $[ dependencies.SetUpModuleMatrix.outputs['CreateMatrixJson.matrixJson'] ] templateContext: outputs: - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}: - output: pipelineArtifact displayName: 'Publish Module Artifacts' targetPath: "$(Build.ArtifactStagingDirectory)" - artifactName: "drop" + artifactName: "$(moduleName)_$(moduleVersion)_drop" publishLocation: "Container" - - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}: - - output: nuget - displayName: 'Publish NuGet to feed' - packageParentPath: '$(Build.ArtifactStagingDirectory)' - packagesToPush: $(Build.ArtifactStagingDirectory)/**/Microsoft.Graph.*.nupkg - publishVstsFeed: $(PROJECT_NAME)/$(FEED_NAME) - allowPackageConflicts: true + strategy: + matrix: $[ dependencies.SetUpModuleMatrix.outputs['CreateMatrixJson.matrixJson'] ] steps: - template: .azure-pipelines/common-templates/checkout.yml@self parameters: @@ -115,27 +176,30 @@ extends: Test: ${{ parameters.Test }} Pack: ${{ parameters.Pack }} Sign: ${{ parameters.Sign }} - - template: .azure-pipelines/generation-templates/workload-modules.yml@self + - template: .azure-pipelines/generation-templates/individualized-workload-modules.yml@self parameters: Test: ${{ parameters.Test }} Pack: ${{ parameters.Pack }} Sign: ${{ parameters.Sign }} - - template: .azure-pipelines/generation-templates/meta-module.yml@self - parameters: - Test: ${{ parameters.Test }} - Pack: ${{ parameters.Pack }} - Sign: ${{ parameters.Sign }} - - ${{ if and(eq(parameters.Pack, true), eq(parameters.Sign, true)) }}: - - template: .azure-pipelines/common-templates/esrp/codesign-nuget.yml@self - parameters: - FolderPath: "$(Build.ArtifactStagingDirectory)" - Pattern: "Microsoft.Graph*.nupkg" - - template: .azure-pipelines/generation-templates/generate-command-metadata.yml@self + ModuleName: $(moduleName) + ModuleVersion: $(moduleVersion) - template: .azure-pipelines/common-templates/security-post-checks.yml@self - - ${{ if eq(parameters.CreatePullRequest, true) }}: - - template: .azure-pipelines/common-templates/create-pr.yml@self - parameters: - BaseBranch: $(BaseBranch) - TargetBranch: $(WeeklyBranch) - Title: "[v2] Weekly OpenApiDocs Refresh" - Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR." \ No newline at end of file + + - job: CreatePullRequest + displayName: Create Pull Request for all changes + dependsOn: + - MsGraphPsSdkWeeklyGeneration + - RefreshOpenAPIDocuments + condition: and(succeeded(), eq(dependencies.MsGraphPsSdkWeeklyGeneration.result, 'Succeeded'), eq(${{ parameters.CreatePullRequest }}, true)) + variables: + WeeklyBranch: $[ dependencies.RefreshOpenAPIDocuments.outputs['ComputeBranch.WeeklyBranch'] ] + steps: + - template: .azure-pipelines/common-templates/checkout.yml@self + parameters: + TargetBranch: $(WeeklyBranch) + - template: .azure-pipelines/common-templates/create-pr.yml@self + parameters: + BaseBranch: $(BaseBranch) + TargetBranch: $(WeeklyBranch) + Title: "[v2] Weekly OpenApiDocs Refresh" + Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR." \ No newline at end of file