From aa443b66330b4dd9f27cd795c26b31a459b80a5a Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 6 Sep 2023 17:06:50 -0700 Subject: [PATCH 1/2] Move CoreCLR community platforms to build with the global build template (#91646) --- eng/build.sh | 2 +- eng/pipelines/common/platform-matrix.yml | 2 +- eng/pipelines/coreclr/templates/build-job.yml | 21 ++------- eng/pipelines/runtime.yml | 44 ++++++++++++++++++- 4 files changed, 48 insertions(+), 21 deletions(-) diff --git a/eng/build.sh b/eng/build.sh index 772d7ac8be82cb..4d0d17a220447d 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -32,7 +32,7 @@ usage() echo " [Default: Debug]" echo " --os Target operating system: windows, linux, freebsd, osx, maccatalyst, tvos," echo " tvossimulator, ios, iossimulator, android, browser, wasi, netbsd, illumos, solaris" - echo " linux-musl, linux-bionic or haiku." + echo " linux-musl, linux-bionic, tizen, or haiku." echo " [Default: Your machine's OS.]" echo " --outputrid Optional argument that overrides the target rid name." echo " --projects Project or solution file(s) to build." diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 8399c82ecd7695..c2f754d6ee2385 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -883,7 +883,7 @@ jobs: jobTemplate: ${{ parameters.jobTemplate }} helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} variables: ${{ parameters.variables }} - osGroup: tizen + osGroup: linux # Our build scripts don't support Tizen and have always used Linux as the OS parameter. archType: armel targetRid: tizen-armel platform: tizen_armel diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index 2e128ea54e1287..d2d9a1ecb8d4cb 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -95,9 +95,6 @@ jobs: - ${{ if ne(parameters.testGroup, 'innerloop') }}: - name: clrRuntimeComponentsBuildArg value: '-component runtime -component alljits -component paltests -component nativeaot -component spmi ' - - ${{ if and(eq(parameters.osGroup, 'linux'), eq(parameters.archType, 'x86')) }}: - - name: clrRuntimeComponentsBuildArg - value: '-component runtime -component jit -component iltools -component spmi ' - name: pgoInstrumentArg value: '' @@ -108,12 +105,6 @@ jobs: - name: SignType value: $[ coalesce(variables.OfficialSignType, 'real') ] - - name: clrRuntimePortableBuildArg - value: '' - - ${{ if eq(parameters.osGroup, 'tizen') }}: - - name: clrRuntimePortableBuildArg - value: '-portablebuild=false' - # Set a default empty argument for the pgo path. # This will be set during the 'native prerequisites' step if PGO optimization is enabled. - name: CoreClrPgoDataArg @@ -185,14 +176,14 @@ jobs: # Build CoreCLR Runtime - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrRuntimeComponentsBuildArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(clrRuntimePortableBuildArg) $(CoreClrPgoDataArg) $(nativeSymbols) + - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrRuntimeComponentsBuildArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) $(nativeSymbols) displayName: Build CoreCLR Runtime - ${{ if eq(parameters.osGroup, 'windows') }}: - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) displayName: Build CoreCLR Runtime - ${{ if or(eq(parameters.crossBuild, 'true'), ne(parameters.archType, 'x64')) }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -hostarch x64 $(osArg) -ci $(compilerArg) -component crosscomponents -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" $(officialBuildIdArg) $(clrRuntimePortableBuildArg) + - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -hostarch x64 $(osArg) -ci $(compilerArg) -component crosscomponents -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" $(officialBuildIdArg) displayName: Build CoreCLR Cross-Arch Tools (Tools that run on x64 targeting x86) - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: @@ -202,12 +193,8 @@ jobs: displayName: Disk Usage after Build # Build CoreCLR Managed Components - - ${{ if or(ne(parameters.osGroup, 'linux'), ne(parameters.archType, 'x86')) }}: - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+clr.paltestlist $(crossArg) $(compilerArg) -arch $(archType) $(osArg) -c $(buildConfig) $(pgoInstrumentArg) $(officialBuildIdArg) -ci - displayName: Build managed product components and packages - - ${{ if and(eq(parameters.osGroup, 'linux'), eq(parameters.archType, 'x86')) }}: - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(pgoInstrumentArg) $(officialBuildIdArg) -ci - displayName: Build managed product components and packages + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+clr.paltestlist $(crossArg) $(compilerArg) -arch $(archType) $(osArg) -c $(buildConfig) $(pgoInstrumentArg) $(officialBuildIdArg) -ci + displayName: Build managed product components and packages # Build native test components - ${{ if and(ne(parameters.isOfficialBuild, true), ne(parameters.disableClrTest, true)) }}: diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 3aa0b6504819a7..f2657982aee552 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -70,7 +70,6 @@ extends: jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml buildConfig: checked platforms: - - linux_x86 - linux_x64 - linux_arm - linux_arm64 @@ -79,7 +78,6 @@ extends: - linux_musl_arm64 - linux_musl_x64 - osx_arm64 - - tizen_armel - windows_x86 - windows_x64 - windows_arm64 @@ -188,6 +186,48 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr_jit.containsChange'], true), eq(variables['isRollingBuild'], true))) + # + # Build CoreCLR with no R2R + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: checked + runtimeFlavor: coreclr + platforms: + - linux_x86 + jobParameters: + testScope: innerloop + nameSuffix: CoreCLR_NoR2R + buildArgs: -s clr.runtime+clr.jit+clr.iltools+clr.spmi+clr.corelib -c $(_BuildConfig) + timeoutInMinutes: 120 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + # + # Build CoreCLR as a non-portable build + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: checked + runtimeFlavor: coreclr + platforms: + - tizen_armel + jobParameters: + testScope: innerloop + nameSuffix: CoreCLR_NonPortable + buildArgs: -s clr.native+clr.tools+clr.corelib+clr.nativecorelib+clr.aot+clr.packages -c $(_BuildConfig) /p:PortableBuild=false + timeoutInMinutes: 120 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + # # CoreCLR NativeAOT debug build and smoke tests # Only when CoreCLR is changed From cc403bd81adef8d2a23b877ff1614a47c5393ec7 Mon Sep 17 00:00:00 2001 From: Juan Hoyos <19413848+hoyosjs@users.noreply.github.com> Date: Thu, 7 Sep 2023 00:28:10 -0700 Subject: [PATCH 2/2] Try to disable VSIX update service and collect VS update logs (#91708) * Tweak invocations, disable service updates, detect vswhere missing * Try disabling update early in build * Only collect logs on official build on exit --- eng/collect_vsinfo.ps1 | 65 +++++++++++++++++++ eng/disable_vsupdate.ps1 | 23 +++++++ eng/native/init-vs-env.cmd | 6 ++ eng/pipelines/common/global-build-job.yml | 5 ++ eng/pipelines/coreclr/templates/build-job.yml | 7 +- eng/pipelines/installer/jobs/build-job.yml | 6 ++ eng/pipelines/mono/templates/build-job.yml | 4 ++ 7 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 eng/collect_vsinfo.ps1 create mode 100644 eng/disable_vsupdate.ps1 diff --git a/eng/collect_vsinfo.ps1 b/eng/collect_vsinfo.ps1 new file mode 100644 index 00000000000000..e2178fe4b24f8f --- /dev/null +++ b/eng/collect_vsinfo.ps1 @@ -0,0 +1,65 @@ +<# +.PARAMETER ArchiveRunName +Name of the run for vs logs + +.NOTES +Returns 0 if succeeds, 1 otherwise +#> +[CmdletBinding(PositionalBinding=$false)] +Param ( + [Parameter(Mandatory=$True)] + [string] $ArchiveRunName +) + +. $PSScriptRoot/common/tools.ps1 + +$ProgressPreference = "SilentlyContinue" +$LogDir = Join-Path $LogDir $ArchiveRunName +mkdir $LogDir + +$vscollect_uri="http://aka.ms/vscollect.exe" +$vscollect="$env:TEMP\vscollect.exe" + +if (-not (Test-Path $vscollect)) { + Retry({ + Write-Host "GET $vscollect_uri" + Invoke-WebRequest $vscollect_uri -OutFile $vscollect -UseBasicParsing + }) + + if (-not (Test-Path $vscollect)) { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download vscollect." + exit 1 + } +} + +&"$vscollect" +Move-Item $env:TEMP\vslogs.zip "$LogDir" + +$vswhere = "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" +if (-not (Test-Path -Path "$vswhere" -PathType Leaf)) +{ + Write-Error "Couldn't locate vswhere at $vswhere" + exit 1 +} + +&"$vswhere" -all -prerelease -products * | Tee-Object -FilePath "$LogDir\vs_where.log" + +$vsdir = &"$vswhere" -latest -prerelease -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + +if (-not (Test-Path $vsdir)) +{ + $procDumpDir = Join-Path $ToolsDir "procdump" + $procDumpToolPath = Join-Path $procDumpDir "procdump.exe" + $procdump_uri = "https://download.sysinternals.com/files/Procdump.zip" + + if (-not (Test-Path $procDumpToolPath)) { + Retry({ + Write-Host "GET $procdump_uri" + Invoke-WebRequest $procdump_uri -OutFile "$TempDir\Procdump.zip" -UseBasicParsing + }) + + Expand-Archive -Path "$TempDir\Procdump.zip" $procDumpDir + } + + &"$procDumpToolPath" -ma -accepteula VSIXAutoUpdate.exe "$LogDir" +} diff --git a/eng/disable_vsupdate.ps1 b/eng/disable_vsupdate.ps1 new file mode 100644 index 00000000000000..d8d365f0577103 --- /dev/null +++ b/eng/disable_vsupdate.ps1 @@ -0,0 +1,23 @@ +schtasks /change /tn "\Microsoft\VisualStudio\VSIX Auto Update" /disable + +$vswhere = "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" +if (-not (Test-Path -Path "$vswhere" -PathType Leaf)) +{ + Write-Error "Couldn't locate vswhere at $vswhere" + exit 1 +} + +$vsdir = &"$vswhere" -latest -prerelease -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath +$vsregedit = "$vsdir\Common7\IDE\VsRegEdit.exe" + +if (-not (Test-Path -Path "$vsregedit" )) +{ + Write-Error "VSWhere returned path: $vsdir, but regedit $vsregedit doesn't exist." + exit 1 +} + +Write-Output "VSWhere returned path: $vsdir, using regedit $vsregedit" +Write-Output "Disabling updates through VS Registry:" + +&"$vsdir\Common7\IDE\VsRegEdit.exe" set local HKCU ExtensionManager AutomaticallyCheckForUpdates2Override dword 0 +&"$vsdir\Common7\IDE\VsRegEdit.exe" read local HKCU ExtensionManager AutomaticallyCheckForUpdates2Override dword diff --git a/eng/native/init-vs-env.cmd b/eng/native/init-vs-env.cmd index 273f49b3392c2d..befe98ab407f72 100644 --- a/eng/native/init-vs-env.cmd +++ b/eng/native/init-vs-env.cmd @@ -26,6 +26,8 @@ if defined VisualStudioVersion goto :VSDetected set "__VSWhere=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" set "__VSCOMNTOOLS=" +if not exist "%__VSWhere%" goto :VSWhereMissing + if exist "%__VSWhere%" ( for /f "tokens=*" %%p in ( '"%__VSWhere%" -latest -prerelease -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath' @@ -56,6 +58,10 @@ echo %__MsgPrefix%Error: Visual Studio 2022 with C++ tools required. ^ Please see https://github.com/dotnet/runtime/blob/main/docs/workflow/requirements/windows-requirements.md for build requirements. exit /b 1 +:VSWhereMissing +echo %__MsgPrefix%Error: vswhere couldn not be found in Visual Studio Installer directory at "%__VSWhere%" +exit /b 1 + :SetVCEnvironment if "%__VCBuildArch%"=="" exit /b 0 diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 39e7d9b5c53ced..0d2ceac932d721 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -254,6 +254,11 @@ jobs: shouldContinueOnError: ${{ parameters.shouldContinueOnError }} ${{ insert }}: ${{ parameters.extraStepsParameters }} + - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: + - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log + displayName: Collect vslogs on exit + condition: always() + - task: PublishBuildArtifacts@1 displayName: Publish Logs inputs: diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index d2d9a1ecb8d4cb..01d12e6452b4cf 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -119,7 +119,6 @@ jobs: - ${{ parameters.variables }} steps: - # Install native dependencies # Linux builds use docker images with dependencies preinstalled, # and FreeBSD builds use a build agent with dependencies @@ -282,6 +281,12 @@ jobs: parameters: name: ${{ parameters.platform }} + - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: + - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log + displayName: Collect vslogs on exit + condition: always() + + # Publish Logs - task: PublishPipelineArtifact@1 displayName: Publish Logs diff --git a/eng/pipelines/installer/jobs/build-job.yml b/eng/pipelines/installer/jobs/build-job.yml index 1d89cfad70eb20..3e18fb3d96392a 100644 --- a/eng/pipelines/installer/jobs/build-job.yml +++ b/eng/pipelines/installer/jobs/build-job.yml @@ -370,6 +370,12 @@ jobs: displayName: Build and Package continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }} + - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: + - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log + displayName: Collect vslogs on exit + condition: always() + + - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: - script: | du -sh $(Build.SourcesDirectory)/* diff --git a/eng/pipelines/mono/templates/build-job.yml b/eng/pipelines/mono/templates/build-job.yml index 845078804fbc7f..0c823e8d573710 100644 --- a/eng/pipelines/mono/templates/build-job.yml +++ b/eng/pipelines/mono/templates/build-job.yml @@ -171,6 +171,10 @@ jobs: - script: build$(scriptExt) -subset mono+clr.hosts -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(aotCrossParameter) $(llvmParameter) -pack $(OutputRidArg) displayName: Build nupkg + - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: + - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log + displayName: Collect vslogs on exit + condition: always() # Publish Logs - task: PublishPipelineArtifact@1 displayName: Publish Logs