From 24b3c0ef4459367b9ae4d81bf2a9ffdffb875f7d Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 21 Mar 2024 19:25:37 -0700 Subject: [PATCH 1/5] Remove various single-scenario YAML parameters in our build scripts and shift implementation responsibility to those scenarios. Uses Arcade defaults to remove richCodeNavigation property. Inline CodeQL injection to the CodeQL pipeline Use Arcade's Source-Build jobs now that we aren't using any features that require the steps support. --- eng/pipelines/common/global-build-job.yml | 45 ++------------- eng/pipelines/common/platform-matrix.yml | 67 ----------------------- eng/pipelines/global-build.yml | 23 ++++---- eng/pipelines/runtime-codeql.yml | 14 ++++- eng/pipelines/runtime-official.yml | 16 ++---- eng/pipelines/runtime.yml | 58 +++++++++----------- src/installer/prepare-artifacts.proj | 8 +-- 7 files changed, 60 insertions(+), 171 deletions(-) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index eb83415112e7b7..bc2826eaf06dc2 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -18,8 +18,6 @@ parameters: useContinueOnErrorDuringBuild: false shouldContinueOnError: false isOfficialBuild: false - isSourceBuild: false - isNonPortableSourceBuild: false runtimeFlavor: 'coreclr' runtimeVariant: '' helixQueues: '' @@ -27,11 +25,7 @@ parameters: testResultsFormat: '' postBuildSteps: [] extraVariablesTemplates: [] - isManualCodeQLBuild: false preBuildSteps: [] - enableRichCodeNavigation: false - richCodeNavigationLanguage: 'csharp' - disableComponentGovernance: '' jobs: - template: /eng/common/templates/job/job.yml @@ -49,9 +43,6 @@ jobs: enablePublishTestResults: ${{ parameters.enablePublishTestResults }} testResultsFormat: ${{ parameters.testResultsFormat }} - enableRichCodeNavigation: ${{ parameters.enableRichCodeNavigation }} - richCodeNavigationLanguage: ${{ parameters.richCodeNavigationLanguage }} - # Component governance does not work on musl machines ${{ if eq(parameters.osSubGroup, '_musl') }}: disableComponentGovernance: true @@ -133,10 +124,7 @@ jobs: - checkout: self clean: true - # If running in source build mode, a git stash will be used for the inner clone. Avoid setting a fetch depth, - # as a stash of a shallow cloned repo is not currently supported. - ${{ if ne(parameters.isSourceBuild, true) }}: - fetchDepth: $(checkoutFetchDepth) + fetchDepth: $(checkoutFetchDepth) - ${{ if and(eq(parameters.isOfficialBuild, true), notin(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator')) }}: - template: /eng/pipelines/common/restore-internal-tools.yml @@ -166,16 +154,6 @@ jobs: artifact: Mono_Offsets_${{monoCrossAOTTargetOS}} path: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles' - - ${{ if eq(parameters.isSourceBuild, true) }}: - - template: /eng/common/templates/steps/source-build.yml - parameters: - platform: - baseOS: ${{ parameters.baseOS }} - buildScript: $(Build.SourcesDirectory)$(dir)build$(scriptExt) - nonPortable: ${{ parameters.isNonPortableSourceBuild }} - targetRID: ${{ parameters.targetRid }} - name: ${{ parameters.platform }} - - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }} displayName: Install Build Dependencies @@ -207,7 +185,6 @@ jobs: targetRid: ${{ parameters.targetRid }} nameSuffix: ${{ parameters.nameSuffix }} platform: ${{ parameters.platform }} - pgoType: ${{ parameters.pgoType }} shouldContinueOnError: ${{ parameters.shouldContinueOnError }} ${{ if ne(preBuildStep.forwardedParameters, '') }}: ${{ each parameter in preBuildStep.forwardedParameters }}: @@ -218,21 +195,11 @@ jobs: - ${{ preBuildStep }} # Build - - ${{ if eq(parameters.isSourceBuild, false) }}: - - ${{ if eq(parameters.isManualCodeQLBuild, true) }}: - - task: CodeQL3000Init@0 - displayName: Initialize CodeQL (manually-injected) - - - template: /eng/pipelines/common/templates/global-build-step.yml - parameters: - buildArgs: ${{ parameters.buildArgs }} - useContinueOnErrorDuringBuild: ${{ parameters.useContinueOnErrorDuringBuild }} - shouldContinueOnError: ${{ parameters.shouldContinueOnError }} - - - ${{ if eq(parameters.isManualCodeQLBuild, true) }}: - - task: CodeQL3000Finalize@0 - displayName: Finalize CodeQL (manually-injected) - #endif isSourceBuild + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: ${{ parameters.buildArgs }} + useContinueOnErrorDuringBuild: ${{ parameters.useContinueOnErrorDuringBuild }} + shouldContinueOnError: ${{ parameters.shouldContinueOnError }} - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos', 'android') }}: - script: | diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 7245e521680464..cb3c8b2d0a65e2 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -294,73 +294,6 @@ jobs: helixQueueGroup: ${{ parameters.helixQueueGroup }} ${{ insert }}: ${{ parameters.jobParameters }} -# Centos 8 Stream x64 Source Build - -- ${{ if containsValue(parameters.platforms, 'SourceBuild_centos8_x64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: linux - archType: x64 - targetRid: centos.8-x64 - platform: centos8_linux_x64 - shouldContinueOnError: ${{ parameters.shouldContinueOnError }} - container: SourceBuild_centos_x64 - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - buildConfig: ${{ parameters.buildConfig }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - isSourceBuild: true - isNonPortableSourceBuild: true - ${{ insert }}: ${{ parameters.jobParameters }} - -# Non-existent RID Source Build - -- ${{ if containsValue(parameters.platforms, 'SourceBuild_banana24_x64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: linux - archType: x64 - targetRid: banana.24-x64 - platform: banana24_linux_x64 - shouldContinueOnError: ${{ parameters.shouldContinueOnError }} - container: SourceBuild_centos_x64 # Run the unknown-rid build on a platform with a known RID so our RID graph tooling can automatically add it to the RID graph. - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - baseOS: linux - buildConfig: ${{ parameters.buildConfig }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - isSourceBuild: true - isNonPortableSourceBuild: true - ${{ insert }}: ${{ parameters.jobParameters }} - -# Portable Linux x64 Source Build - -- ${{ if containsValue(parameters.platforms, 'SourceBuild_linux_x64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: linux - archType: x64 - targetRid: linux-x64 - platform: linux_x64 - shouldContinueOnError: ${{ parameters.shouldContinueOnError }} - container: SourceBuild_linux_x64 - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - buildConfig: ${{ parameters.buildConfig }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - isSourceBuild: true - isNonPortableSourceBuild: false - ${{ insert }}: ${{ parameters.jobParameters }} - # GCC Linux x64 Build - ${{ if containsValue(parameters.platforms, 'gcc_linux_x64') }}: diff --git a/eng/pipelines/global-build.yml b/eng/pipelines/global-build.yml index 62f3cc067fe5b8..e7cff308898a92 100644 --- a/eng/pipelines/global-build.yml +++ b/eng/pipelines/global-build.yml @@ -161,17 +161,16 @@ extends: eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) - # - # SourceBuild Build - # - - template: /eng/pipelines/common/platform-matrix.yml + - stage: Source Build Validation + dependsOn: + - EvaluatePaths + condition: + eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true) + jobs: + - template: /eng/common/templates/jobs/source-build.yml parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: Release platforms: - - SourceBuild_linux_x64 - jobParameters: - nameSuffix: PortableSourceBuild - timeoutInMinutes: 95 - condition: - eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true) + - name: Linux_x64 + targetRid: linux-x64 + container: SourceBuild_linux_x64 + diff --git a/eng/pipelines/runtime-codeql.yml b/eng/pipelines/runtime-codeql.yml index 9eff98fef5ab9d..599622f88f62a0 100644 --- a/eng/pipelines/runtime-codeql.yml +++ b/eng/pipelines/runtime-codeql.yml @@ -47,7 +47,12 @@ extends: nameSuffix: AllSubsets_CoreCLR isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 360 - isManualCodeQLBuild: true + preBuildSteps: + - task: CodeQL3000Init@0 + displayName: Initialize CodeQL (manually-injected) + postBuildSteps: + - task: CodeQL3000Finalize@0 + displayName: Finalize CodeQL (manually-injected) # # Build Mono runtime packs @@ -65,4 +70,9 @@ extends: nameSuffix: AllSubsets_Mono isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 360 - isManualCodeQLBuild: true + preBuildSteps: + - task: CodeQL3000Init@0 + displayName: Initialize CodeQL (manually-injected) + postBuildSteps: + - task: CodeQL3000Finalize@0 + displayName: Finalize CodeQL (manually-injected) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 0cc897455e2e06..0d3a2f40ea9554 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -580,20 +580,12 @@ extends: # # Build SourceBuild packages # - - template: /eng/pipelines/common/platform-matrix.yml + - template: /eng/common/templates/jobs/source-build.yml parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: Release - helixQueueGroup: ci platforms: - - SourceBuild_linux_x64 - jobParameters: - nameSuffix: PortableSourceBuild - postBuildSteps: - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: SourceBuildPackages - timeoutInMinutes: 95 + - name: Linux_x64 + targetRid: linux-x64 + container: SourceBuild_linux_x64 # # Build PGO Instrumented CoreCLR Release diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 3cce0b5a17499f..778ac192dfd86a 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -1819,36 +1819,28 @@ extends: eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) - # - # Sourcebuild legs - # We have 3 important legs for source-build: - # - Centos.8 (ensures that known non-portable RID is working) - # - Linux-x64 portable (used for dependency flow and downstream PR verification) - # - Banana.24 - Non-existent RID to ensure we don't break RIDs we don't know about. - # - # Running all of these everywhere is wasteful. Run Banana.24 and CentOS.8 in rolling CI, - # Run Linux-x64 in PR. - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: Release - helixQueueGroup: pr - platforms: - - SourceBuild_centos8_x64 - jobParameters: - nameSuffix: centos8SourceBuild - timeoutInMinutes: 95 - condition: eq(variables['isRollingBuild'], true) - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: Release - helixQueueGroup: pr - platforms: - - SourceBuild_banana24_x64 - jobParameters: - nameSuffix: banana24SourceBuild - timeoutInMinutes: 95 - condition: eq(variables['isRollingBuild'], true) + - stage: Source Build Validation + dependsOn: [] + condition: eq(variables['isRollingBuild'], true) + jobs: + # + # Sourcebuild legs + # We have 3 important legs for source-build: + # - Centos.8 (ensures that known non-portable RID is working) + # - Linux-x64 portable (used for dependency flow and downstream PR verification) + # - Banana.24 - Non-existent RID to ensure we don't break RIDs we don't know about. + # + # Running all of these everywhere is wasteful. Run Banana.24 and CentOS.8 in rolling CI, + # Run Linux-x64 in PR. + - template: /eng/common/templates/jobs/source-build.yml + parameters: + platforms: + - name: CentOS.8 + targetRid: centos.8-x64 + nonPortable: true + container: SourceBuild_centos_x64 + - name: Nonexistent RID + baseOS: linux + targetRid: banana.24-x64 + nonPortable: true + container: SourceBuild_centos_x64 diff --git a/src/installer/prepare-artifacts.proj b/src/installer/prepare-artifacts.proj index f00a65b86896b3..393b50eafe206a 100644 --- a/src/installer/prepare-artifacts.proj +++ b/src/installer/prepare-artifacts.proj @@ -58,8 +58,8 @@ - @@ -145,10 +145,6 @@ $(InstallersRelativePath)workloads/%(Filename)%(Extension) true - - - - From 41d94f0b8fa93eb2018b044f5a9a129c6da8c8e1 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 21 Mar 2024 19:47:00 -0700 Subject: [PATCH 2/5] Fix stage names --- eng/pipelines/global-build.yml | 3 ++- eng/pipelines/runtime.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/global-build.yml b/eng/pipelines/global-build.yml index e7cff308898a92..957c0277efb443 100644 --- a/eng/pipelines/global-build.yml +++ b/eng/pipelines/global-build.yml @@ -161,7 +161,8 @@ extends: eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) - - stage: Source Build Validation + - stage: SourceBuild + displayName: Source Build Validation dependsOn: - EvaluatePaths condition: diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 778ac192dfd86a..9ce7ed589a0eea 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -1819,7 +1819,8 @@ extends: eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) - - stage: Source Build Validation + - stage: SourceBuild + displayName: Source Build Validation dependsOn: [] condition: eq(variables['isRollingBuild'], true) jobs: From 1c28abd7add76a58fd85bd281db8c95aa3b3e9f9 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 21 Mar 2024 19:48:47 -0700 Subject: [PATCH 3/5] Fix job names --- eng/pipelines/runtime.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 9ce7ed589a0eea..3d3fc6b47f4bdd 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -1836,11 +1836,11 @@ extends: - template: /eng/common/templates/jobs/source-build.yml parameters: platforms: - - name: CentOS.8 + - name: CentOS8 targetRid: centos.8-x64 nonPortable: true container: SourceBuild_centos_x64 - - name: Nonexistent RID + - name: NonexistentRID baseOS: linux targetRid: banana.24-x64 nonPortable: true From 1c44c8e6f375bc2c71b6fe25c21631a1afe22fac Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 21 Mar 2024 19:56:16 -0700 Subject: [PATCH 4/5] Use local variable from dependency in condition --- eng/pipelines/global-build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/global-build.yml b/eng/pipelines/global-build.yml index 957c0277efb443..eecadfa1a5283f 100644 --- a/eng/pipelines/global-build.yml +++ b/eng/pipelines/global-build.yml @@ -165,8 +165,9 @@ extends: displayName: Source Build Validation dependsOn: - EvaluatePaths - condition: - eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true) + variables: + runSourceBuild: $[ stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'] ] + condition: eq(variables['runSourceBuild'], true) jobs: - template: /eng/common/templates/jobs/source-build.yml parameters: From b2207ac770c67f6e95cc9a9468d0f36f5d5dd6d0 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 21 Mar 2024 20:01:12 -0700 Subject: [PATCH 5/5] Just always run the source-build validation (stage-level variables wasn't working) --- eng/pipelines/global-build.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/global-build.yml b/eng/pipelines/global-build.yml index eecadfa1a5283f..a1e83c778aeb3d 100644 --- a/eng/pipelines/global-build.yml +++ b/eng/pipelines/global-build.yml @@ -161,18 +161,12 @@ extends: eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) - - stage: SourceBuild - displayName: Source Build Validation - dependsOn: - - EvaluatePaths - variables: - runSourceBuild: $[ stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'] ] - condition: eq(variables['runSourceBuild'], true) - jobs: + # + # Build Portable SourceBuild + # - template: /eng/common/templates/jobs/source-build.yml parameters: platforms: - name: Linux_x64 targetRid: linux-x64 container: SourceBuild_linux_x64 -