diff --git a/Directory.Build.props b/Directory.Build.props index 7e5c028ecfa209..084308a0dd1fc1 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -192,6 +192,8 @@ false false + + false true + true true diff --git a/build.sh b/build.sh index befbb3bb08957f..9df7f38ad7163f 100755 --- a/build.sh +++ b/build.sh @@ -26,7 +26,7 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" if is_cygwin_or_mingw; then # if bash shell running on Windows (not WSL), # pass control to batch build script. - "$scriptroot/build.cmd" $@ + "$scriptroot/build.cmd" "$@" else - "$scriptroot/eng/build.sh" $@ + "$scriptroot/eng/build.sh" "$@" fi diff --git a/eng/Publishing.props b/eng/Publishing.props index 8050b1279ee3c8..beba39632cf7ef 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -15,7 +15,7 @@ We can't use NETCoreSdkRuntimeIdentifier here as the Arcade SDK projects don't import the .NET SDK. Instead, just make sure we include the assets targeting "not the output rid", which will catch the host assets. --> - + <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.NETCore.ILAsm.*.nupkg" Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.NETCore.ILAsm.*.nupkg" /> @@ -51,7 +51,7 @@ This ensures that we don't produce these files in the "Repo source build" builds, but we do produce them in both the VMR and the runtime official build. --> - + true true @@ -89,12 +89,12 @@ DependsOnTargets="GetNonStableProductVersion" BeforeTargets="PublishToAzureDevOpsArtifacts" AfterTargets="GenerateChecksumsFromArtifacts"> - + - + $(OutputRID) - true + true true @@ -44,7 +44,7 @@ - + diff --git a/eng/Subsets.props b/eng/Subsets.props index 96c5cf476590ee..4aa019b3c8499b 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -83,7 +83,7 @@ - + true diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 95d081cac00ffc..9ae818cc373938 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,5 +1,5 @@ - + https://github.com/dotnet/icu @@ -41,91 +41,91 @@ https://github.com/dotnet/llvm-project da5dd054a531e6fea65643b7e754285b73eab433 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 https://github.com/dotnet/runtime-assets @@ -263,33 +263,33 @@ https://github.com/dotnet/llvm-project da5dd054a531e6fea65643b7e754285b73eab433 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 https://github.com/dotnet/xharness @@ -303,9 +303,9 @@ https://github.com/dotnet/xharness e85bb14e85357ab678c2bcb0b6f2bac634fdd49b - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 https://dev.azure.com/dnceng/internal/_git/dotnet-optimization @@ -331,29 +331,29 @@ https://github.com/dotnet/runtime-assets 1cfc6ba21d0377b51f17eac4fdc2557f7b1e8693 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 - + https://github.com/dotnet/dotnet - 85778473549347b3e4bad3ea009e9438df7b11bb + 62e9d324a3c6678a0415b12af92354b6b03677c8 https://dev.azure.com/dnceng/internal/_git/dotnet-optimization diff --git a/eng/Versions.props b/eng/Versions.props index 2e7f6a32b19171..5a39a15b6987be 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -36,17 +36,17 @@ - 10.0.0-preview.25260.104 + 10.0.0-preview.25304.104 - 5.0.0-1.25260.104 + 5.0.0-1.25304.104 - 5.0.0-1.25260.104 - 5.0.0-1.25260.104 - 5.0.0-1.25260.104 + 5.0.0-1.25304.104 + 5.0.0-1.25304.104 + 5.0.0-1.25304.104 - 10.0.100-preview.5.25260.104 + 10.0.100-preview.6.25304.104 - 10.0.0-beta.25260.104 - 10.0.0-beta.25260.104 - 10.0.0-beta.25260.104 - 10.0.0-beta.25260.104 - 2.9.2-beta.25260.104 - 10.0.0-beta.25260.104 - 2.9.2-beta.25260.104 - 10.0.0-beta.25260.104 - 10.0.0-beta.25260.104 - 10.0.0-beta.25260.104 - 10.0.0-beta.25260.104 - 10.0.0-beta.25260.104 - 10.0.0-beta.25260.104 - 10.0.0-beta.25260.104 - 10.0.0-beta.25260.104 - 10.0.0-beta.25260.104 + 10.0.0-beta.25304.104 + 10.0.0-beta.25304.104 + 10.0.0-beta.25304.104 + 10.0.0-beta.25304.104 + 2.9.2-beta.25304.104 + 10.0.0-beta.25304.104 + 2.9.2-beta.25304.104 + 10.0.0-beta.25304.104 + 10.0.0-beta.25304.104 + 10.0.0-beta.25304.104 + 10.0.0-beta.25304.104 + 10.0.0-beta.25304.104 + 10.0.0-beta.25304.104 + 10.0.0-beta.25304.104 + 10.0.0-beta.25304.104 + 10.0.0-beta.25304.104 1.4.0 6.0.0-preview.1.102 - 10.0.0-preview.5.25260.104 + 10.0.0-preview.6.25304.104 6.0.0 - 10.0.0-preview.5.25260.104 - 10.0.0-preview.5.25260.104 + 10.0.0-preview.6.25304.104 + 10.0.0-preview.6.25304.104 6.0.0 4.6.1 @@ -128,16 +128,16 @@ 8.0.0 8.0.1 5.0.0 - 10.0.0-preview.5.25260.104 - 10.0.0-preview.5.25260.104 + 10.0.0-preview.6.25304.104 + 10.0.0-preview.6.25304.104 6.0.0 5.0.0 5.0.0 5.0.0 7.0.0 - 10.0.0-preview.5.25260.104 + 10.0.0-preview.6.25304.104 7.0.0 - 10.0.0-preview.5.25260.104 + 10.0.0-preview.6.25304.104 8.0.0 4.5.1 @@ -180,7 +180,7 @@ 2.0.0 17.10.0-beta1.24272.1 - 2.0.0-beta5.25279.2 + 2.0.0-beta5.25304.104 3.1.16 2.1.0 2.0.3 @@ -226,7 +226,7 @@ 9.0.0-preview-20241010.1 - 0.11.5-alpha.25260.104 + 0.11.5-alpha.25304.104 10.0.0-preview.5.25261.1 @@ -258,7 +258,7 @@ Note: when the name is updated, make sure to update dependency name in eng/pipelines/common/xplat-setup.yml like - DarcDependenciesChanged.Microsoft_NET_Workload_Emscripten_Current_Manifest-10_0_100_Transport --> - 10.0.100-preview.5.25260.104 + 10.0.100-preview.6.25304.104 $(MicrosoftNETWorkloadEmscriptenCurrentManifest100100TransportVersion) 1.1.87-gba258badda diff --git a/eng/build.sh b/eng/build.sh index 6a82dc2cbb917a..f777325b184594 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -153,9 +153,9 @@ showSubsetHelp() "$scriptroot/common/build.sh" "-restore" "-build" "/p:Subset=help" "/clp:nosummary /tl:false" } -arguments='' +arguments=() cmakeargs='' -extraargs='' +extraargs=() crossBuild=0 portableBuild=1 bootstrap=0 @@ -178,7 +178,7 @@ while [[ $# > 0 ]]; do exit 0 fi - arguments="$arguments /p:Subset=$1" + arguments+=("/p:Subset=$1") shift 1 continue fi @@ -201,7 +201,7 @@ while [[ $# > 0 ]]; do showSubsetHelp exit 0 fi - arguments="$arguments /p:Subset=$2" + arguments+=("/p:Subset=$2") shift 2 fi ;; @@ -241,7 +241,7 @@ while [[ $# > 0 ]]; do exit 1 ;; esac - arguments="$arguments -configuration $val" + arguments+=("-configuration" "$val") shift 2 ;; @@ -251,7 +251,7 @@ while [[ $# > 0 ]]; do exit 1 fi val="$(echo "$2" | tr "[:upper:]" "[:lower:]")" - arguments="$arguments /p:BuildTargetFramework=$val" + arguments+=("/p:BuildTargetFramework=$val") shift 2 ;; @@ -306,12 +306,12 @@ while [[ $# > 0 ]]; do exit 1 ;; esac - arguments="$arguments /p:TargetOS=$os" + arguments+=("/p:TargetOS=$os") shift 2 ;; -pack) - arguments="$arguments --pack /p:BuildAllConfigurations=true" + arguments+=("--pack" "/p:BuildAllConfigurations=true") shift 1 ;; @@ -320,17 +320,17 @@ while [[ $# > 0 ]]; do echo "No test scope supplied. See help (--help) for supported test scope values." 1>&2 exit 1 fi - arguments="$arguments /p:TestScope=$2" + arguments+=("/p:TestScope=$2") shift 2 ;; -testnobuild) - arguments="$arguments /p:TestNoBuild=true" + arguments+=("/p:TestNoBuild=true") shift 1 ;; -coverage) - arguments="$arguments /p:Coverage=true" + arguments+=("/p:Coverage=true") shift 1 ;; @@ -350,7 +350,7 @@ while [[ $# > 0 ]]; do exit 1 ;; esac - arguments="$arguments /p:RuntimeConfiguration=$val" + arguments+=("/p:RuntimeConfiguration=$val") shift 2 ;; @@ -370,12 +370,12 @@ while [[ $# > 0 ]]; do exit 1 ;; esac - arguments="$arguments /p:RuntimeFlavor=$val" + arguments+=("/p:RuntimeFlavor=$val") shift 2 ;; -usemonoruntime) - arguments="$arguments /p:PrimaryRuntimeFlavor=Mono" + arguments+=("/p:PrimaryRuntimeFlavor=Mono") shift 1 ;; @@ -395,7 +395,7 @@ while [[ $# > 0 ]]; do exit 1 ;; esac - arguments="$arguments /p:LibrariesConfiguration=$val" + arguments+=("/p:LibrariesConfiguration=$val") shift 2 ;; @@ -415,25 +415,25 @@ while [[ $# > 0 ]]; do exit 1 ;; esac - arguments="$arguments /p:HostConfiguration=$val" + arguments+=("/p:HostConfiguration=$val") shift 2 ;; -cross) crossBuild=1 - arguments="$arguments /p:CrossBuild=True" + arguments+=("/p:CrossBuild=True") shift 1 ;; *crossbuild=true*) crossBuild=1 - extraargs="$extraargs $1" + extraargs+=("$1") shift 1 ;; -clang*) compiler="${opt/#-/}" # -clang-9 => clang-9 or clang-9 => (unchanged) - arguments="$arguments /p:Compiler=$compiler /p:CppCompilerAndLinker=$compiler" + arguments+=("/p:Compiler=$compiler" "/p:CppCompilerAndLinker=$compiler") shift 1 ;; @@ -448,7 +448,7 @@ while [[ $# > 0 ]]; do -gcc*) compiler="${opt/#-/}" # -gcc-9 => gcc-9 or gcc-9 => (unchanged) - arguments="$arguments /p:Compiler=$compiler /p:CppCompilerAndLinker=$compiler" + arguments+=("/p:Compiler=$compiler" "/p:CppCompilerAndLinker=$compiler") shift 1 ;; @@ -457,7 +457,7 @@ while [[ $# > 0 ]]; do echo "No value for outputrid is supplied. See help (--help) for supported values." 1>&2 exit 1 fi - arguments="$arguments /p:OutputRID=$(echo "$2" | tr "[:upper:]" "[:lower:]")" + arguments+=("/p:OutputRID=$(echo "$2" | tr "[:upper:]" "[:lower:]")") shift 2 ;; @@ -469,7 +469,7 @@ while [[ $# > 0 ]]; do passedPortable="$(echo "$2" | tr "[:upper:]" "[:lower:]")" if [ "$passedPortable" = false ]; then portableBuild=0 - arguments="$arguments /p:PortableBuild=false" + arguments+=("/p:PortableBuild=false") fi shift 2 ;; @@ -481,7 +481,7 @@ while [[ $# > 0 ]]; do fi passedKeepNativeSymbols="$(echo "$2" | tr "[:upper:]" "[:lower:]")" if [ "$passedKeepNativeSymbols" = true ]; then - arguments="$arguments /p:KeepNativeSymbols=true" + arguments+=("/p:KeepNativeSymbols=true") fi shift 2 ;; @@ -489,30 +489,30 @@ while [[ $# > 0 ]]; do -ninja) if [ -z ${2+x} ]; then - arguments="$arguments /p:Ninja=true" + arguments+=("/p:Ninja=true") shift 1 else ninja="$(echo "$2" | tr "[:upper:]" "[:lower:]")" if [ "$ninja" = true ]; then - arguments="$arguments /p:Ninja=true" + arguments+=("/p:Ninja=true") shift 2 elif [ "$ninja" = false ]; then - arguments="$arguments /p:Ninja=false" + arguments+=("/p:Ninja=false") shift 2 else - arguments="$arguments /p:Ninja=true" + arguments+=("/p:Ninja=true") shift 1 fi fi ;; -pgoinstrument) - arguments="$arguments /p:PgoInstrument=true" + arguments+=("/p:PgoInstrument=true") shift 1 ;; -use-bootstrap) - arguments="$arguments /p:UseBootstrap=true" + arguments+=("/p:UseBootstrap=true") shift 1 ;; @@ -526,30 +526,30 @@ while [[ $# > 0 ]]; do echo "No value for -fsanitize is supplied. See help (--help) for supported values." 1>&2 exit 1 fi - arguments="$arguments /p:EnableNativeSanitizers=$2" + arguments+=("/p:EnableNativeSanitizers=$2") shift 2 ;; -fsanitize=*) sanitizers="${opt/#-fsanitize=/}" # -fsanitize=address => address - arguments="$arguments /p:EnableNativeSanitizers=$sanitizers" + arguments+=("/p:EnableNativeSanitizers=$sanitizers") shift 2 ;; -verbose) - arguments="$arguments /p:CoreclrVerbose=true" + arguments+=("/p:CoreclrVerbose=true") shift 1 ;; *) - extraargs="$extraargs $1" + extraargs+=("$1") shift 1 ;; esac done if [ ${#actInt[@]} -eq 0 ]; then - arguments="-restore -build $arguments" + arguments=("-restore" "-build" "${arguments[@]}") fi if [[ "$os" == "browser" ]]; then @@ -566,11 +566,11 @@ if [[ "$os" == "wasi" ]]; then fi if [[ "${TreatWarningsAsErrors:-}" == "false" ]]; then - arguments="$arguments -warnAsError false" + arguments+=("-warnAsError" "false") fi # disable terminal logger for now: https://github.com/dotnet/runtime/issues/97211 -arguments="$arguments -tl:false" +arguments+=("-tl:false") initDistroRid "$os" "$arch" "$crossBuild" @@ -581,23 +581,31 @@ export DOTNETSDK_ALLOW_TARGETING_PACK_CACHING=0 # URL-encode space (%20) to avoid quoting issues until the msbuild call in /eng/common/tools.sh. # In *proj files (XML docs), URL-encoded string are rendered in their decoded form. cmakeargs="${cmakeargs// /%20}" -arguments="$arguments /p:TargetArchitecture=$arch /p:BuildArchitecture=$hostArch" -arguments="$arguments /p:CMakeArgs=\"$cmakeargs\" $extraargs" +arguments+=("/p:TargetArchitecture=$arch" "/p:BuildArchitecture=$hostArch") +arguments+=("/p:CMakeArgs=\"$cmakeargs\"" ${extraargs[@]+"${extraargs[@]}"}) if [[ "$bootstrap" == "1" ]]; then # Strip build actions other than -restore and -build from the arguments for the bootstrap build. - bootstrapArguments="$arguments" - for flag in --sign --publish --pack --test -sign -publish -pack -test; do - bootstrapArguments="${bootstrapArguments//$flag/}" + bootstrapArguments=() + for argument in "${arguments[@]}"; do + add=1 + for flag in --sign --publish --pack --test -sign -publish -pack -test; do + if [[ "$argument" == "$flag" ]]; then + add=0 + fi + done + if [[ $add == 1 ]]; then + bootstrapArguments+=("$argument") + fi done - "$scriptroot/common/build.sh" $bootstrapArguments /p:Subset=bootstrap -bl:$scriptroot/../artifacts/log/bootstrap.binlog + "$scriptroot/common/build.sh" ${bootstrapArguments[@]+"${bootstrapArguments[@]}"} /p:Subset=bootstrap -bl:$scriptroot/../artifacts/log/bootstrap.binlog # Remove artifacts from the bootstrap build so the product build is a "clean" build. echo "Cleaning up artifacts from bootstrap build..." rm -r "$scriptroot/../artifacts/bin" # Remove all directories in obj except for the source-built-upstream-cache directory to avoid breaking SourceBuild. find "$scriptroot/../artifacts/obj" -mindepth 1 -maxdepth 1 ! -name 'source-built-upstream-cache' -exec rm -rf {} + - arguments="$arguments /p:UseBootstrap=true" + arguments+=("/p:UseBootstrap=true") fi -"$scriptroot/common/build.sh" $arguments +"$scriptroot/common/build.sh" ${arguments[@]+"${arguments[@]}"} diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index e8acaef91b456a..8cfee107e7a35e 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -21,6 +21,7 @@ Param( [switch] $publish, [switch] $clean, [switch][Alias('pb')]$productBuild, + [switch]$fromVMR, [switch][Alias('bl')]$binaryLog, [switch][Alias('nobl')]$excludeCIBinarylog, [switch] $ci, @@ -64,7 +65,7 @@ function Print-Usage() { Write-Host "" Write-Host "Advanced settings:" - Write-Host " -projects Semi-colon delimited list of slnx/proj's to build. Globbing is supported (*.slnx)" + Write-Host " -projects Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)" Write-Host " -ci Set when running on CI server" Write-Host " -excludeCIBinarylog Don't output binary log (short: -nobl)" Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" @@ -74,6 +75,7 @@ function Print-Usage() { Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" Write-Host " -nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" Write-Host " -buildCheck Sets /check msbuild parameter" + Write-Host " -fromVMR Set when building from within the VMR" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." @@ -127,7 +129,8 @@ function Build { /p:Deploy=$deploy ` /p:Test=$test ` /p:Pack=$pack ` - /p:DotNetBuildRepo=$productBuild ` + /p:DotNetBuild=$productBuild ` + /p:DotNetBuildFromVMR=$fromVMR ` /p:IntegrationTest=$integrationTest ` /p:PerformanceTest=$performanceTest ` /p:Sign=$sign ` diff --git a/eng/common/build.sh b/eng/common/build.sh index 36fba82a37930d..9767bb411a4f0c 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -43,6 +43,7 @@ usage() echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" echo " --buildCheck Sets /check msbuild parameter" + echo " --fromVMR Set when building from within the VMR" echo "" echo "Command line arguments not listed above are passed thru to msbuild." echo "Arguments can also be passed in with a single hyphen." @@ -64,6 +65,7 @@ restore=false build=false source_build=false product_build=false +from_vmr=false rebuild=false test=false integration_test=false @@ -89,7 +91,7 @@ verbosity='minimal' runtime_source_feed='' runtime_source_feed_key='' -properties='' +properties=() while [[ $# > 0 ]]; do opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in @@ -129,19 +131,22 @@ while [[ $# > 0 ]]; do -pack) pack=true ;; - -sourcebuild|-sb) + -sourcebuild|-source-build|-sb) build=true source_build=true product_build=true restore=true pack=true ;; - -productBuild|-pb) + -productbuild|-product-build|-pb) build=true product_build=true restore=true pack=true ;; + -fromvmr|-from-vmr) + from_vmr=true + ;; -test|-t) test=true ;; @@ -187,7 +192,7 @@ while [[ $# > 0 ]]; do shift ;; *) - properties="$properties $1" + properties+=("$1") ;; esac @@ -221,7 +226,7 @@ function Build { InitializeCustomToolset if [[ ! -z "$projects" ]]; then - properties="$properties /p:Projects=$projects" + properties+=("/p:Projects=$projects") fi local bl="" @@ -241,8 +246,9 @@ function Build { /p:RepoRoot="$repo_root" \ /p:Restore=$restore \ /p:Build=$build \ - /p:DotNetBuildRepo=$product_build \ + /p:DotNetBuild=$product_build \ /p:DotNetBuildSourceOnly=$source_build \ + /p:DotNetBuildFromVMR=$from_vmr \ /p:Rebuild=$rebuild \ /p:Test=$test \ /p:Pack=$pack \ @@ -251,7 +257,7 @@ function Build { /p:Sign=$sign \ /p:Publish=$publish \ /p:RestoreStaticGraphEnableBinaryLogger=$binary_log \ - $properties + ${properties[@]+"${properties[@]}"} ExitWithExitCode 0 } diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml index 4f1dc42e02c557..d5303229c97e27 100644 --- a/eng/common/core-templates/job/publish-build-assets.yml +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -32,6 +32,12 @@ parameters: # Optional: 🌤️ or not the build has assets it wants to publish to BAR isAssetlessBuild: false + # Optional, publishing version + publishingVersion: 3 + + # Optional: A minimatch pattern for the asset manifests to publish to BAR + assetManifestsPattern: '*/manifests/**/*.xml' + jobs: - job: Asset_Registry_Publish @@ -77,13 +83,31 @@ jobs: clean: true - ${{ if eq(parameters.isAssetlessBuild, 'false') }}: - - task: DownloadPipelineArtifact@2 - displayName: Download Asset Manifests - inputs: - artifactName: AssetManifests - targetPath: '$(Build.StagingDirectory)/AssetManifests' - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} + - ${{ if eq(parameters.publishingVersion, 3) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Asset Manifests + inputs: + artifactName: AssetManifests + targetPath: '$(Build.StagingDirectory)/AssetManifests' + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + - ${{ if eq(parameters.publishingVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download V4 asset manifests + inputs: + itemPattern: '*/manifests/**/*.xml' + targetPath: '$(Build.StagingDirectory)/AllAssetManifests' + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + - task: CopyFiles@2 + displayName: Copy V4 asset manifests to AssetManifests + inputs: + SourceFolder: '$(Build.StagingDirectory)/AllAssetManifests' + Contents: ${{ parameters.assetManifestsPattern }} + TargetFolder: '$(Build.StagingDirectory)/AssetManifests' + flattenFolders: true + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} - task: NuGetAuthenticate@1 @@ -120,6 +144,17 @@ jobs: Copy-Item -Path $symbolExclusionfile -Destination "$(Build.StagingDirectory)/ReleaseConfigs" } + - ${{ if eq(parameters.publishingVersion, 4) }}: + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + targetPath: '$(Build.ArtifactStagingDirectory)/MergedManifest.xml' + artifactName: AssetManifests + displayName: 'Publish Merged Manifest' + retryCountOnTaskFailure: 10 # for any logs being locked + sbomEnabled: false # we don't need SBOM for logs + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml index 5757915edb3ea1..a151fd811e3e48 100644 --- a/eng/common/core-templates/post-build/post-build.yml +++ b/eng/common/core-templates/post-build/post-build.yml @@ -193,9 +193,6 @@ stages: buildId: $(AzDOBuildId) artifactName: PackageArtifacts checkDownloadedFiles: true - itemPattern: | - ** - !**/Microsoft.SourceBuild.Intermediate.*.nupkg # This is necessary whenever we want to publish/restore to an AzDO private feed # Since sdk-task.ps1 tries to restore packages we need to do this authentication here diff --git a/eng/common/core-templates/steps/install-microbuild.yml b/eng/common/core-templates/steps/install-microbuild.yml index 2bcf974ee15318..a3540ba00c7eb2 100644 --- a/eng/common/core-templates/steps/install-microbuild.yml +++ b/eng/common/core-templates/steps/install-microbuild.yml @@ -48,38 +48,3 @@ steps: eq(variables['_SignType'], 'real') ) )) - - # Workaround for ESRP CLI on Linux - https://github.com/dotnet/source-build/issues/4964 - - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, 'true') }}: - - task: UseDotNet@2 - displayName: Install .NET 9.0 SDK for ESRP CLI Workaround - inputs: - packageType: sdk - version: 9.0.x - installationPath: ${{ parameters.microBuildOutputFolder }}/.dotnet - workingDirectory: ${{ parameters.microBuildOutputFolder }} - condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) - - - task: PowerShell@2 - displayName: Workaround for ESRP CLI on Linux - inputs: - targetType: 'inline' - script: | - Write-Host "Copying Linux Path" - $MBSIGN_APPFOLDER = '$(MBSIGN_APPFOLDER)' - $MBSIGN_APPFOLDER = ($MBSIGN_APPFOLDER -replace '/build', '') - - $versionRegex = '\d+\.\d+\.\d+' - $package = Get-ChildItem -Path $MBSIGN_APPFOLDER -Directory | - Where-Object { $_.Name -match $versionRegex } - - if ($package.Count -ne 1) { - Write-Host "There should be exactly one matching subfolder, but found $($package.Count)." - exit 1 - } - - $MBSIGN_APPFOLDER = $package[0].FullName + '/build' - $MBSIGN_APPFOLDER | Write-Host - $SignConfigPath = $MBSIGN_APPFOLDER + '/signconfig.xml' - Copy-Item -Path "$(MBSIGN_APPFOLDER)/signconfig.xml" -Destination $SignConfigPath -Force - condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml index f2a0f347fdd67a..0dde553c3ebfbb 100644 --- a/eng/common/core-templates/steps/source-build.yml +++ b/eng/common/core-templates/steps/source-build.yml @@ -51,13 +51,12 @@ steps: ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ --configuration $buildConfig \ --restore --build --pack -bl \ + --source-build \ ${{ parameters.platform.buildArguments }} \ $internalRuntimeDownloadArgs \ $targetRidArgs \ $baseRidArgs \ $portableBuildArgs \ - /p:DotNetBuildSourceOnly=true \ - /p:DotNetBuildRepo=true \ displayName: Build - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index 36dbd45e1ce866..e889f439b8dc99 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -68,7 +68,7 @@ function InstallDarcCli { fi fi - local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" + local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" echo "Installing Darc CLI version $darcVersion..." echo "You may need to restart your command shell if this is the first dotnet tool you have installed." diff --git a/eng/common/templates/steps/vmr-sync.yml b/eng/common/templates/steps/vmr-sync.yml new file mode 100644 index 00000000000000..599afb6186b8fd --- /dev/null +++ b/eng/common/templates/steps/vmr-sync.yml @@ -0,0 +1,207 @@ +### These steps synchronize new code from product repositories into the VMR (https://github.com/dotnet/dotnet). +### They initialize the darc CLI and pull the new updates. +### Changes are applied locally onto the already cloned VMR (located in $vmrPath). + +parameters: +- name: targetRef + displayName: Target revision in dotnet/ to synchronize + type: string + default: $(Build.SourceVersion) + +- name: vmrPath + displayName: Path where the dotnet/dotnet is checked out to + type: string + default: $(Agent.BuildDirectory)/vmr + +- name: additionalSyncs + displayName: Optional list of package names whose repo's source will also be synchronized in the local VMR, e.g. NuGet.Protocol + type: object + default: [] + +steps: +- checkout: vmr + displayName: Clone dotnet/dotnet + path: vmr + clean: true + +- checkout: self + displayName: Clone $(Build.Repository.Name) + path: repo + fetchDepth: 0 + +# This step is needed so that when we get a detached HEAD / shallow clone, +# we still pull the commit into the temporary repo clone to use it during the sync. +# Also unshallow the clone so that forwardflow command would work. +- script: | + git branch repo-head + git rev-parse HEAD + displayName: Label PR commit + workingDirectory: $(Agent.BuildDirectory)/repo + +- script: | + vmr_sha=$(grep -oP '(?<=Sha=")[^"]*' $(Agent.BuildDirectory)/repo/eng/Version.Details.xml) + echo "##vso[task.setvariable variable=vmr_sha]$vmr_sha" + displayName: Obtain the vmr sha from Version.Details.xml (Unix) + condition: ne(variables['Agent.OS'], 'Windows_NT') + workingDirectory: $(Agent.BuildDirectory)/repo + +- powershell: | + [xml]$xml = Get-Content -Path $(Agent.BuildDirectory)/repo/eng/Version.Details.xml + $vmr_sha = $xml.SelectSingleNode("//Source").Sha + Write-Output "##vso[task.setvariable variable=vmr_sha]$vmr_sha" + displayName: Obtain the vmr sha from Version.Details.xml (Windows) + condition: eq(variables['Agent.OS'], 'Windows_NT') + workingDirectory: $(Agent.BuildDirectory)/repo + +- script: | + git fetch --all + git checkout $(vmr_sha) + displayName: Checkout VMR at correct sha for repo flow + workingDirectory: ${{ parameters.vmrPath }} + +- script: | + git config --global user.name "dotnet-maestro[bot]" + git config --global user.email "dotnet-maestro[bot]@users.noreply.github.com" + displayName: Set git author to dotnet-maestro[bot] + workingDirectory: ${{ parameters.vmrPath }} + +- script: | + ./eng/common/vmr-sync.sh \ + --vmr ${{ parameters.vmrPath }} \ + --tmp $(Agent.TempDirectory) \ + --azdev-pat '$(dn-bot-all-orgs-code-r)' \ + --ci \ + --debug + + if [ "$?" -ne 0 ]; then + echo "##vso[task.logissue type=error]Failed to synchronize the VMR" + exit 1 + fi + displayName: Sync repo into VMR (Unix) + condition: ne(variables['Agent.OS'], 'Windows_NT') + workingDirectory: $(Agent.BuildDirectory)/repo + +- script: | + git config --global diff.astextplain.textconv echo + git config --system core.longpaths true + displayName: Configure Windows git (longpaths, astextplain) + condition: eq(variables['Agent.OS'], 'Windows_NT') + +- powershell: | + ./eng/common/vmr-sync.ps1 ` + -vmr ${{ parameters.vmrPath }} ` + -tmp $(Agent.TempDirectory) ` + -azdevPat '$(dn-bot-all-orgs-code-r)' ` + -ci ` + -debugOutput + + if ($LASTEXITCODE -ne 0) { + echo "##vso[task.logissue type=error]Failed to synchronize the VMR" + exit 1 + } + displayName: Sync repo into VMR (Windows) + condition: eq(variables['Agent.OS'], 'Windows_NT') + workingDirectory: $(Agent.BuildDirectory)/repo + +- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - task: CopyFiles@2 + displayName: Collect failed patches + condition: failed() + inputs: + SourceFolder: '$(Agent.TempDirectory)' + Contents: '*.patch' + TargetFolder: '$(Build.ArtifactStagingDirectory)/FailedPatches' + + - publish: '$(Build.ArtifactStagingDirectory)/FailedPatches' + artifact: $(System.JobDisplayName)_FailedPatches + displayName: Upload failed patches + condition: failed() + +- ${{ each assetName in parameters.additionalSyncs }}: + # The vmr-sync script ends up staging files in the local VMR so we have to commit those + - script: + git commit --allow-empty -am "Forward-flow $(Build.Repository.Name)" + displayName: Commit local VMR changes + workingDirectory: ${{ parameters.vmrPath }} + + - script: | + set -ex + + echo "Searching for details of asset ${{ assetName }}..." + + # Use darc to get dependencies information + dependencies=$(./.dotnet/dotnet darc get-dependencies --name '${{ assetName }}' --ci) + + # Extract repository URL and commit hash + repository=$(echo "$dependencies" | grep 'Repo:' | sed 's/Repo:[[:space:]]*//' | head -1) + + if [ -z "$repository" ]; then + echo "##vso[task.logissue type=error]Asset ${{ assetName }} not found in the dependency list" + exit 1 + fi + + commit=$(echo "$dependencies" | grep 'Commit:' | sed 's/Commit:[[:space:]]*//' | head -1) + + echo "Updating the VMR from $repository / $commit..." + cd .. + git clone $repository ${{ assetName }} + cd ${{ assetName }} + git checkout $commit + git branch "sync/$commit" + + ./eng/common/vmr-sync.sh \ + --vmr ${{ parameters.vmrPath }} \ + --tmp $(Agent.TempDirectory) \ + --azdev-pat '$(dn-bot-all-orgs-code-r)' \ + --ci \ + --debug + + if [ "$?" -ne 0 ]; then + echo "##vso[task.logissue type=error]Failed to synchronize the VMR" + exit 1 + fi + displayName: Sync ${{ assetName }} into (Unix) + condition: ne(variables['Agent.OS'], 'Windows_NT') + workingDirectory: $(Agent.BuildDirectory)/repo + + - powershell: | + $ErrorActionPreference = 'Stop' + + Write-Host "Searching for details of asset ${{ assetName }}..." + + $dependencies = .\.dotnet\dotnet darc get-dependencies --name '${{ assetName }}' --ci + + $repository = $dependencies | Select-String -Pattern 'Repo:\s+([^\s]+)' | Select-Object -First 1 + $repository -match 'Repo:\s+([^\s]+)' | Out-Null + $repository = $matches[1] + + if ($repository -eq $null) { + Write-Error "Asset ${{ assetName }} not found in the dependency list" + exit 1 + } + + $commit = $dependencies | Select-String -Pattern 'Commit:\s+([^\s]+)' | Select-Object -First 1 + $commit -match 'Commit:\s+([^\s]+)' | Out-Null + $commit = $matches[1] + + Write-Host "Updating the VMR from $repository / $commit..." + cd .. + git clone $repository ${{ assetName }} + cd ${{ assetName }} + git checkout $commit + git branch "sync/$commit" + + .\eng\common\vmr-sync.ps1 ` + -vmr ${{ parameters.vmrPath }} ` + -tmp $(Agent.TempDirectory) ` + -azdevPat '$(dn-bot-all-orgs-code-r)' ` + -ci ` + -debugOutput + + if ($LASTEXITCODE -ne 0) { + echo "##vso[task.logissue type=error]Failed to synchronize the VMR" + exit 1 + } + displayName: Sync ${{ assetName }} into (Windows) + condition: ne(variables['Agent.OS'], 'Windows_NT') + workingDirectory: $(Agent.BuildDirectory)/repo diff --git a/eng/common/templates/vmr-build-pr.yml b/eng/common/templates/vmr-build-pr.yml new file mode 100644 index 00000000000000..670cf32c3bd1fa --- /dev/null +++ b/eng/common/templates/vmr-build-pr.yml @@ -0,0 +1,33 @@ +trigger: none +pr: + branches: + include: + - main + - release/* + paths: + exclude: + - documentation/* + - README.md + - CODEOWNERS + +variables: +- template: /eng/common/templates/variables/pool-providers.yml@self + +- name: skipComponentGovernanceDetection # we run CG on internal builds only + value: true + +- name: Codeql.Enabled # we run CodeQL on internal builds only + value: false + +resources: + repositories: + - repository: vmr + type: github + name: dotnet/dotnet + endpoint: dotnet + +stages: +- template: /eng/pipelines/templates/stages/vmr-build.yml@vmr + parameters: + isBuiltFromVmr: false + scope: lite diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 7373e5305465d7..c9e39595b586cb 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -65,10 +65,8 @@ $ErrorActionPreference = 'Stop' # Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed [string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null } -# True if the build is a product build -[bool]$productBuild = if (Test-Path variable:productBuild) { $productBuild } else { $false } - -[String[]]$properties = if (Test-Path variable:properties) { $properties } else { @() } +# True when the build is running within the VMR. +[bool]$fromVMR = if (Test-Path variable:fromVMR) { $fromVMR } else { $false } function Create-Directory ([string[]] $path) { New-Item -Path $path -Force -ItemType 'Directory' | Out-Null @@ -646,7 +644,6 @@ function GetNuGetPackageCachePath() { $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' } else { $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' - $env:RESTORENOHTTPCACHE = $true } } @@ -852,8 +849,8 @@ function MSBuild-Core() { } # When running on Azure Pipelines, override the returned exit code to avoid double logging. - # Skip this when the build is a child of the VMR orchestrator build. - if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$productBuild -and -not($properties -like "*DotNetBuildRepo=true*")) { + # Skip this when the build is a child of the VMR build. + if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$fromVMR) { Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error diff --git a/eng/common/tools.sh b/eng/common/tools.sh index cc007b1f15ad05..a7e84931510673 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -72,8 +72,8 @@ fi runtime_source_feed=${runtime_source_feed:-''} runtime_source_feed_key=${runtime_source_feed_key:-''} -# True if the build is a product build -product_build=${product_build:-false} +# True when the build is running within the VMR. +from_vmr=${from_vmr:-false} # Resolve any symlinks in the given path. function ResolvePath { @@ -345,14 +345,12 @@ function InitializeBuildTool { _InitializeBuildToolCommand="msbuild" } -# Set RestoreNoHttpCache as a workaround for https://github.com/NuGet/Home/issues/3116 function GetNuGetPackageCachePath { if [[ -z ${NUGET_PACKAGES:-} ]]; then if [[ "$use_global_nuget_cache" == true ]]; then export NUGET_PACKAGES="$HOME/.nuget/packages/" else export NUGET_PACKAGES="$repo_root/.packages/" - export RESTORENOHTTPCACHE=true fi fi @@ -506,8 +504,8 @@ function MSBuild-Core { echo "Build failed with exit code $exit_code. Check errors above." # When running on Azure Pipelines, override the returned exit code to avoid double logging. - # Skip this when the build is a child of the VMR orchestrator build. - if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$product_build" != true && "$properties" != *"DotNetBuildRepo=true"* ]]; then + # Skip this when the build is a child of the VMR build. + if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$from_vmr" != true ]]; then Write-PipelineSetResult -result "Failed" -message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error @@ -530,6 +528,7 @@ function GetDarc { fi "$eng_root/common/darc-init.sh" --toolpath "$darc_path" $version + darc_tool="$darc_path/darc" } # Returns a full path to an Arcade SDK task project file. diff --git a/eng/common/vmr-sync.ps1 b/eng/common/vmr-sync.ps1 new file mode 100755 index 00000000000000..97302f3205be9a --- /dev/null +++ b/eng/common/vmr-sync.ps1 @@ -0,0 +1,138 @@ +<# +.SYNOPSIS + +This script is used for synchronizing the current repository into a local VMR. +It pulls the current repository's code into the specified VMR directory for local testing or +Source-Build validation. + +.DESCRIPTION + +The tooling used for synchronization will clone the VMR repository into a temporary folder if +it does not already exist. These clones can be reused in future synchronizations, so it is +recommended to dedicate a folder for this to speed up re-runs. + +.EXAMPLE + Synchronize current repository into a local VMR: + ./vmr-sync.ps1 -vmrDir "$HOME/repos/dotnet" -tmpDir "$HOME/repos/tmp" + +.PARAMETER tmpDir +Required. Path to the temporary folder where repositories will be cloned + +.PARAMETER vmrBranch +Optional. Branch of the 'dotnet/dotnet' repo to synchronize. The VMR will be checked out to this branch + +.PARAMETER azdevPat +Optional. Azure DevOps PAT to use for cloning private repositories. + +.PARAMETER vmrDir +Optional. Path to the dotnet/dotnet repository. When null, gets cloned to the temporary folder + +.PARAMETER debugOutput +Optional. Enables debug logging in the darc vmr command. + +.PARAMETER ci +Optional. Denotes that the script is running in a CI environment. +#> +param ( + [Parameter(Mandatory=$true, HelpMessage="Path to the temporary folder where repositories will be cloned")] + [string][Alias('t', 'tmp')]$tmpDir, + [string][Alias('b', 'branch')]$vmrBranch, + [string]$remote, + [string]$azdevPat, + [string][Alias('v', 'vmr')]$vmrDir, + [switch]$ci, + [switch]$debugOutput +) + +function Fail { + Write-Host "> $($args[0])" -ForegroundColor 'Red' +} + +function Highlight { + Write-Host "> $($args[0])" -ForegroundColor 'Cyan' +} + +$verbosity = 'verbose' +if ($debugOutput) { + $verbosity = 'debug' +} +# Validation + +if (-not $tmpDir) { + Fail "Missing -tmpDir argument. Please specify the path to the temporary folder where the repositories will be cloned" + exit 1 +} + +# Sanitize the input + +if (-not $vmrDir) { + $vmrDir = Join-Path $tmpDir 'dotnet' +} + +if (-not (Test-Path -Path $tmpDir -PathType Container)) { + New-Item -ItemType Directory -Path $tmpDir | Out-Null +} + +# Prepare the VMR + +if (-not (Test-Path -Path $vmrDir -PathType Container)) { + Highlight "Cloning 'dotnet/dotnet' into $vmrDir.." + git clone https://github.com/dotnet/dotnet $vmrDir + + if ($vmrBranch) { + git -C $vmrDir switch -c $vmrBranch + } +} +else { + if ((git -C $vmrDir diff --quiet) -eq $false) { + Fail "There are changes in the working tree of $vmrDir. Please commit or stash your changes" + exit 1 + } + + if ($vmrBranch) { + Highlight "Preparing $vmrDir" + git -C $vmrDir checkout $vmrBranch + git -C $vmrDir pull + } +} + +Set-StrictMode -Version Latest + +# Prepare darc + +Highlight 'Installing .NET, preparing the tooling..' +. .\eng\common\tools.ps1 +$dotnetRoot = InitializeDotNetCli -install:$true +$darc = Get-Darc +$dotnet = "$dotnetRoot\dotnet.exe" + +Highlight "Starting the synchronization of VMR.." + +# Synchronize the VMR +$darcArgs = ( + "vmr", "forwardflow", + "--tmp", $tmpDir, + "--$verbosity", + $vmrDir +) + +if ($ci) { + $darcArgs += ("--ci") +} + +if ($azdevPat) { + $darcArgs += ("--azdev-pat", $azdevPat) +} + +& "$darc" $darcArgs + +if ($LASTEXITCODE -eq 0) { + Highlight "Synchronization succeeded" +} +else { + Fail "Synchronization of repo to VMR failed!" + Fail "'$vmrDir' is left in its last state (re-run of this script will reset it)." + Fail "Please inspect the logs which contain path to the failing patch file (use -debugOutput to get all the details)." + Fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script." + exit 1 +} diff --git a/eng/common/vmr-sync.sh b/eng/common/vmr-sync.sh new file mode 100755 index 00000000000000..44239e331c0cc0 --- /dev/null +++ b/eng/common/vmr-sync.sh @@ -0,0 +1,207 @@ +#!/bin/bash + +### This script is used for synchronizing the current repository into a local VMR. +### It pulls the current repository's code into the specified VMR directory for local testing or +### Source-Build validation. +### +### The tooling used for synchronization will clone the VMR repository into a temporary folder if +### it does not already exist. These clones can be reused in future synchronizations, so it is +### recommended to dedicate a folder for this to speed up re-runs. +### +### USAGE: +### Synchronize current repository into a local VMR: +### ./vmr-sync.sh --tmp "$HOME/repos/tmp" "$HOME/repos/dotnet" +### +### Options: +### -t, --tmp, --tmp-dir PATH +### Required. Path to the temporary folder where repositories will be cloned +### +### -b, --branch, --vmr-branch BRANCH_NAME +### Optional. Branch of the 'dotnet/dotnet' repo to synchronize. The VMR will be checked out to this branch +### +### --debug +### Optional. Turns on the most verbose logging for the VMR tooling +### +### --remote name:URI +### Optional. Additional remote to use during the synchronization +### This can be used to synchronize to a commit from a fork of the repository +### Example: 'runtime:https://github.com/yourfork/runtime' +### +### --azdev-pat +### Optional. Azure DevOps PAT to use for cloning private repositories. +### +### -v, --vmr, --vmr-dir PATH +### Optional. Path to the dotnet/dotnet repository. When null, gets cloned to the temporary folder + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +function print_help () { + sed -n '/^### /,/^$/p' "$source" | cut -b 5- +} + +COLOR_RED=$(tput setaf 1 2>/dev/null || true) +COLOR_CYAN=$(tput setaf 6 2>/dev/null || true) +COLOR_CLEAR=$(tput sgr0 2>/dev/null || true) +COLOR_RESET=uniquesearchablestring +FAILURE_PREFIX='> ' + +function fail () { + echo "${COLOR_RED}$FAILURE_PREFIX${1//${COLOR_RESET}/${COLOR_RED}}${COLOR_CLEAR}" >&2 +} + +function highlight () { + echo "${COLOR_CYAN}$FAILURE_PREFIX${1//${COLOR_RESET}/${COLOR_CYAN}}${COLOR_CLEAR}" +} + +tmp_dir='' +vmr_dir='' +vmr_branch='' +additional_remotes='' +verbosity=verbose +azdev_pat='' +ci=false + +while [[ $# -gt 0 ]]; do + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -t|--tmp|--tmp-dir) + tmp_dir=$2 + shift + ;; + -v|--vmr|--vmr-dir) + vmr_dir=$2 + shift + ;; + -b|--branch|--vmr-branch) + vmr_branch=$2 + shift + ;; + --remote) + additional_remotes="$additional_remotes $2" + shift + ;; + --azdev-pat) + azdev_pat=$2 + shift + ;; + --ci) + ci=true + ;; + -d|--debug) + verbosity=debug + ;; + -h|--help) + print_help + exit 0 + ;; + *) + fail "Invalid argument: $1" + print_help + exit 1 + ;; + esac + + shift +done + +# Validation + +if [[ -z "$tmp_dir" ]]; then + fail "Missing --tmp-dir argument. Please specify the path to the temporary folder where the repositories will be cloned" + exit 1 +fi + +# Sanitize the input + +if [[ -z "$vmr_dir" ]]; then + vmr_dir="$tmp_dir/dotnet" +fi + +if [[ ! -d "$tmp_dir" ]]; then + mkdir -p "$tmp_dir" +fi + +if [[ "$verbosity" == "debug" ]]; then + set -x +fi + +# Prepare the VMR + +if [[ ! -d "$vmr_dir" ]]; then + highlight "Cloning 'dotnet/dotnet' into $vmr_dir.." + git clone https://github.com/dotnet/dotnet "$vmr_dir" + + if [[ -n "$vmr_branch" ]]; then + git -C "$vmr_dir" switch -c "$vmr_branch" + fi +else + if ! git -C "$vmr_dir" diff --quiet; then + fail "There are changes in the working tree of $vmr_dir. Please commit or stash your changes" + exit 1 + fi + + if [[ -n "$vmr_branch" ]]; then + highlight "Preparing $vmr_dir" + git -C "$vmr_dir" checkout "$vmr_branch" + git -C "$vmr_dir" pull + fi +fi + +set -e + +# Prepare darc + +highlight 'Installing .NET, preparing the tooling..' +source "./eng/common/tools.sh" +InitializeDotNetCli true +GetDarc +dotnetDir=$( cd ./.dotnet/; pwd -P ) +dotnet=$dotnetDir/dotnet + +highlight "Starting the synchronization of VMR.." +set +e + +if [[ -n "$additional_remotes" ]]; then + additional_remotes="--additional-remotes $additional_remotes" +fi + +if [[ -n "$azdev_pat" ]]; then + azdev_pat="--azdev-pat $azdev_pat" +fi + +ci_arg='' +if [[ "$ci" == "true" ]]; then + ci_arg="--ci" +fi + +# Synchronize the VMR + +export DOTNET_ROOT="$dotnetDir" + +"$darc_tool" vmr forwardflow \ + --tmp "$tmp_dir" \ + $azdev_pat \ + --$verbosity \ + $ci_arg \ + $additional_remotes \ + "$vmr_dir" + +if [[ $? == 0 ]]; then + highlight "Synchronization succeeded" +else + fail "Synchronization of repo to VMR failed!" + fail "'$vmr_dir' is left in its last state (re-run of this script will reset it)." + fail "Please inspect the logs which contain path to the failing patch file (use --debug to get all the details)." + fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script." + exit 1 +fi diff --git a/eng/native/signing/auth.json b/eng/native/signing/auth.json index 6d0230d8b63f75..a467bd95b55536 100644 --- a/eng/native/signing/auth.json +++ b/eng/native/signing/auth.json @@ -1,23 +1,20 @@ { "Version" : "1.0.0", - "AuthenticationType" : "AAD_CERT", + "AuthenticationType" : "AAD_MSI_WIF", "TenantId" : "975f013f-7f24-47e8-a7d3-abc4752bf346", "ClientId" : "22346933-af99-4e94-97d5-7fa1dcf4bba6", - "AuthCert" : - { - "SubjectName" : "CN=22346933-af99-4e94-97d5-7fa1dcf4bba6.microsoft.com", - "StoreLocation" : "CurrentUser", - "StoreName": "My", - "SendX5c" : "true", - "WithAzureRegion": false - }, + "EsrpClientId": "22346933-af99-4e94-97d5-7fa1dcf4bba6", "RequestSigningCert" : { "GetCertFromKeyVault" : true, "KeyVaultName": "clrdiag-esrp-pme", "KeyVaultCertName": "dac-dnceng-esrpclient-cert", "SendX5c": false, - "WithAzureRegion": false + "WithAzureRegion": false, + "StoreLocation": null, + "StoreName": null, + "SubjectName": null }, - "OAuthToken": null + "OAuthToken": null, + "FederatedTokenData": {} } diff --git a/eng/pipelines/coreclr/templates/install-diagnostic-certs.yml b/eng/pipelines/coreclr/templates/install-diagnostic-certs.yml deleted file mode 100644 index 32fb99b3590ba3..00000000000000 --- a/eng/pipelines/coreclr/templates/install-diagnostic-certs.yml +++ /dev/null @@ -1,39 +0,0 @@ -parameters: - isOfficialBuild: false - certNames: [] - vaultName: '' - azureSubscription: '' - scriptRoot: '$(Build.SourcesDirectory)' - -steps: -- ${{ if and(eq(parameters.isOfficialBuild, true), ne(variables['Build.Reason'], 'PullRequest'), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/reltest/')), not(endsWith(variables['Build.SourceBranch'], '-staging'))) }}: - - task: AzureKeyVault@2 - inputs: - azureSubscription: ${{ parameters.azureSubscription }} - KeyVaultName: ${{ parameters.vaultName }} - SecretsFilter: ${{ join(',', parameters.certNames) }} - displayName: 'Download secrets: Diagnostic Certificates' - - - task: NuGetCommand@2 - displayName: Install ESRPClient Tool - inputs: - command: 'custom' - arguments: 'install microsoft.esrpclient - -Source https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - -DependencyVersion Highest -OutputDirectory $(Build.StagingDirectory)' - - - powershell: | - eng/pipelines/install-diagnostic-certs.ps1 "${{ join(',', parameters.certNames) }}" - $esrpclient = Get-ChildItem -Path '$(Build.StagingDirectory)\microsoft.esrpclient*' -Filter esrpclient.exe -Recurse | Select-Object -First 1 | select -ExpandProperty FullName - - if ($esrpclient -eq $null) { - throw "Failed to find esrpclient.exe in $(Build.StagingDirectory)" - } - - $signArgs = "/p:DotNetEsrpToolPath=$esrpclient" - echo "##vso[task.setvariable variable=_SignDiagnosticFilesArgs;]$signArgs" - displayName: 'Install diagnostic certificates' - workingDirectory: ${{ parameters.scriptRoot }} - env: - ${{ each cert in parameters.certNames }}: - ${{ cert }}: $(${{ cert }}) diff --git a/eng/pipelines/coreclr/templates/remove-diagnostic-certs.yml b/eng/pipelines/coreclr/templates/remove-diagnostic-certs.yml deleted file mode 100644 index c510ea8f177cf7..00000000000000 --- a/eng/pipelines/coreclr/templates/remove-diagnostic-certs.yml +++ /dev/null @@ -1,11 +0,0 @@ -parameters: - isOfficialBuild: false - scriptRoot: '$(Build.SourcesDirectory)' - -steps: -- ${{ if and(eq(parameters.isOfficialBuild, true), ne(variables['Build.Reason'], 'PullRequest'), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/reltest/')), not(endsWith(variables['Build.SourceBranch'], '-staging'))) }}: - - powershell: | - eng/pipelines/remove-diagnostic-certs.ps1 "$(DacCertificateThumbprints)" - workingDirectory: ${{ parameters.scriptRoot }} - displayName: 'Remove Diagnostic Certificates' - condition: always() diff --git a/eng/pipelines/install-diagnostic-certs.ps1 b/eng/pipelines/install-diagnostic-certs.ps1 deleted file mode 100644 index 74d3c43f75739a..00000000000000 --- a/eng/pipelines/install-diagnostic-certs.ps1 +++ /dev/null @@ -1,32 +0,0 @@ -[CmdletBinding()] -param( - [string] - [Parameter(Mandatory)] - $certList -) -# Required for the pipeline logging functions -$ci = $true -. $PSScriptRoot/../common/pipeline-logging-functions.ps1 - -$certs = $certList -split ',' -$thumbprints = @() -$certCollection = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2Collection -foreach ($cert in $certs) -{ - $certBytes = [System.Convert]::FromBase64String($(Get-Item "Env:$cert").Value) - $certCollection.Import($certBytes,$null, [System.Security.Cryptography.X509Certificates.X509KeyStorageFlags]::PersistKeySet) -} - -foreach ($cert in $certCollection) -{ - Write-Host "Installed certificate '$($cert.Thumbprint)' with subject: '$($cert.Subject)'" - $thumbprints += $cert.Thumbprint -} - -$store = Get-Item -Path Cert:\CurrentUser\My -$store.Open([System.Security.Cryptography.X509Certificates.OpenFlags]::ReadWrite) -$store.AddRange($certCollection) -$store.Close() - -Write-PipelineSetVariable -name "DacCertificateThumbprints" -Value "$($thumbprints -join ',')" -IsMultiJobVariable $false -Write-Host "Successfully installed diagnostic certificates" diff --git a/eng/pipelines/remove-diagnostic-certs.ps1 b/eng/pipelines/remove-diagnostic-certs.ps1 deleted file mode 100644 index 3ebea9f3e287ae..00000000000000 --- a/eng/pipelines/remove-diagnostic-certs.ps1 +++ /dev/null @@ -1,22 +0,0 @@ -[CmdletBinding()] -param( - [string] - [Parameter(Mandatory)] - $thumbprintList -) - -$thumbprints = $thumbprintList -split ',' -$store = Get-Item -Path Cert:\CurrentUser\My -$store.Open([System.Security.Cryptography.X509Certificates.OpenFlags]::ReadWrite) -foreach ($thumbprint in $thumbprints) -{ - $cert = $store.Certificates.Find([System.Security.Cryptography.X509Certificates.X509FindType]::FindByThumbprint, $thumbprint, $false) - if ($null -eq $cert) - { - Write-Host "Certificate with thumbprint '$thumbprint' not found in the user store." - } - $store.RemoveRange($cert) - Write-Host "Removed certificate '$thumbprint'" -} -$store.Close() -Write-Host "Successfully removed diagnostic certificates" diff --git a/global.json b/global.json index 8616c8a6541983..c8a292c031012a 100644 --- a/global.json +++ b/global.json @@ -1,18 +1,18 @@ { "sdk": { - "version": "10.0.100-preview.3.25201.16", + "version": "10.0.100-preview.6.25302.104", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "10.0.100-preview.3.25201.16" + "dotnet": "10.0.100-preview.6.25302.104" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.25260.104", - "Microsoft.DotNet.Helix.Sdk": "10.0.0-beta.25260.104", - "Microsoft.DotNet.SharedFramework.Sdk": "10.0.0-beta.25260.104", + "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.25304.104", + "Microsoft.DotNet.Helix.Sdk": "10.0.0-beta.25304.104", + "Microsoft.DotNet.SharedFramework.Sdk": "10.0.0-beta.25304.104", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.Build.Traversal": "3.4.0", - "Microsoft.NET.Sdk.IL": "10.0.0-preview.5.25260.104" + "Microsoft.NET.Sdk.IL": "10.0.0-preview.6.25304.104" } } diff --git a/src/coreclr/.nuget/Microsoft.CrossOsDiag.Private.CoreCLR/Microsoft.CrossOsDiag.Private.CoreCLR.proj b/src/coreclr/.nuget/Microsoft.CrossOsDiag.Private.CoreCLR/Microsoft.CrossOsDiag.Private.CoreCLR.proj index d4f80483746781..a9e84b88cfdb18 100644 --- a/src/coreclr/.nuget/Microsoft.CrossOsDiag.Private.CoreCLR/Microsoft.CrossOsDiag.Private.CoreCLR.proj +++ b/src/coreclr/.nuget/Microsoft.CrossOsDiag.Private.CoreCLR/Microsoft.CrossOsDiag.Private.CoreCLR.proj @@ -8,7 +8,7 @@ In dotnet/runtime's official build, we'll extract the runtime packs to a specific directory and pass that here. In the VMR, we'll restore the runtime packs as NuGet packages. --> - + $(NuGetPackageRoot) @@ -24,7 +24,7 @@ - + diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.Trim.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.Trim.cs index 4d654530d608ba..e19d7fd366d745 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.Trim.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.Trim.cs @@ -833,7 +833,7 @@ public static Span TrimEnd(this Span span) => span.Slice(0, ClampEnd(span, 0)); /// - /// Delimits all leading occurrences of whitespace charecters from the span. + /// Delimits all leading occurrences of whitespace characters from the span. /// /// The source span from which the characters are removed. private static int ClampStart(ReadOnlySpan span) @@ -852,7 +852,7 @@ private static int ClampStart(ReadOnlySpan span) } /// - /// Delimits all trailing occurrences of whitespace charecters from the span. + /// Delimits all trailing occurrences of whitespace characters from the span. /// /// The source span from which the characters are removed. /// The start index from which to being searching. diff --git a/src/libraries/oob-src.proj b/src/libraries/oob-src.proj index b1d9ddb7d5827b..9cef0fb3922a8d 100644 --- a/src/libraries/oob-src.proj +++ b/src/libraries/oob-src.proj @@ -8,7 +8,7 @@ true true + '$(DotNetBuildFromVMR)' == 'true'">true - + <_MonoCMakeArgs Include="-DENABLE_WERROR=1"/> diff --git a/src/tests/tracing/eventpipe/common/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj b/src/tests/tracing/eventpipe/common/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj index 064e21555e3d2a..fc7c42a9f4bdce 100644 --- a/src/tests/tracing/eventpipe/common/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj +++ b/src/tests/tracing/eventpipe/common/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj @@ -12,7 +12,7 @@ true true false - true + true false