From 543a821a7db51907b2b40de637a14f1b72e44207 Mon Sep 17 00:00:00 2001 From: Tomas Date: Sun, 15 Mar 2020 18:45:53 +0100 Subject: [PATCH 1/9] Crossgen2 / SuperIlc support for composite mode with shared framework This change adds SuperIlc support for the option --framework in --composite mode by first compiling the entire set of framework assemblies into one composite version bubble and then building the test on top of it as another composite version bubble. One practical upside of this mode is that it's naturally much faster than the "one-component" composite mode as that basically requires compiling the framework with each app. Thanks Tomas --- eng/pipelines/coreclr/crossgen2-composite.yml | 56 ++++++++ .../coreclr/templates/run-test-job.yml | 5 + .../coreclr/templates/send-to-helix-step.yml | 3 + src/coreclr/build-test.cmd | 37 +++++- src/coreclr/build-test.sh | 30 ++++- .../CompilationModuleGroup.ReadyToRun.cs | 6 + .../ReadyToRunCompilationModuleGroupBase.cs | 5 + .../Compiler/ReadyToRunLibraryRootProvider.cs | 4 +- ...RunSingleAssemblyCompilationModuleGroup.cs | 3 + .../SingleMethodCompilationModuleGroup.cs | 3 + .../src/tools/crossgen2/crossgen2/Program.cs | 15 ++- src/coreclr/src/vm/nativeimage.cpp | 8 +- src/coreclr/src/vm/readytoruninfo.cpp | 31 +++-- src/coreclr/tests/helixpublishwitharcade.proj | 3 + .../tests/src/CLRTest.CrossGen.targets | 124 +++++++++++++----- 15 files changed, 277 insertions(+), 56 deletions(-) create mode 100644 eng/pipelines/coreclr/crossgen2-composite.yml diff --git a/eng/pipelines/coreclr/crossgen2-composite.yml b/eng/pipelines/coreclr/crossgen2-composite.yml new file mode 100644 index 00000000000000..1fdeca4d9e25e8 --- /dev/null +++ b/eng/pipelines/coreclr/crossgen2-composite.yml @@ -0,0 +1,56 @@ +trigger: none + +pr: none + +schedules: +- cron: "0 6 * * *" + displayName: Mon through Sun at 10:00 PM (UTC-8:00) + branches: + include: + - master + always: true + +jobs: +# +# Checkout repository +# +- template: /eng/pipelines/common/checkout-job.yml + +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + buildConfig: checked + platforms: + - Linux_x64 + - OSX_x64 + - Windows_NT_x64 + jobParameters: + testGroup: innerloop + +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-test-job.yml + buildConfig: checked + platforms: + - OSX_x64 + - Windows_NT_x64 + jobParameters: + testGroup: innerloop + liveLibrariesBuildConfig: Release + +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + buildConfig: checked + platforms: + - Linux_x64 + - OSX_x64 + - Windows_NT_x64 + jobParameters: + testGroup: innerloop + readyToRun: true + crossgen2: true + compositeBuildMode: true + displayNameArgs: Composite + liveLibrariesBuildConfig: Release diff --git a/eng/pipelines/coreclr/templates/run-test-job.yml b/eng/pipelines/coreclr/templates/run-test-job.yml index 4bdb1d38891bf2..b1a8b29c2a83e0 100644 --- a/eng/pipelines/coreclr/templates/run-test-job.yml +++ b/eng/pipelines/coreclr/templates/run-test-job.yml @@ -11,6 +11,7 @@ parameters: readyToRun: false liveLibrariesBuildConfig: '' crossgen2: false + compositeBuildMode: false helixQueues: '' # If true, run the corefx tests instead of the coreclr ones corefxTests: false @@ -91,6 +92,9 @@ jobs: value: 'crossgen2' - name: LogNamePrefix value: TestRunLogs_R2R_CG2 + - ${{ if eq(parameters.compositeBuildMode, true) }}: + - name: crossgenArg + value: 'composite' # Set job timeouts # @@ -277,6 +281,7 @@ jobs: runCrossGen: ${{ and(eq(parameters.readyToRun, true), ne(parameters.crossgen2, true)) }} runCrossGen2: ${{ and(eq(parameters.readyToRun, true), eq(parameters.crossgen2, true)) }} + compositeBuildMode: ${{ parameters.compositeBuildMode }} runInUnloadableContext: ${{ parameters.runInUnloadableContext }} ${{ if eq(variables['System.TeamProject'], 'internal') }}: diff --git a/eng/pipelines/coreclr/templates/send-to-helix-step.yml b/eng/pipelines/coreclr/templates/send-to-helix-step.yml index 7209065b570b5e..0c9f46cbd75dc0 100644 --- a/eng/pipelines/coreclr/templates/send-to-helix-step.yml +++ b/eng/pipelines/coreclr/templates/send-to-helix-step.yml @@ -16,6 +16,7 @@ parameters: timeoutPerTestInMinutes: '' runCrossGen: '' runCrossGen2: '' + compositeBuildMode: false helixProjectArguments: '' runInUnloadableContext: '' longRunningGcTests: '' @@ -47,6 +48,7 @@ steps: _HelixType: ${{ parameters.helixType }} _RunCrossGen: ${{ parameters.runCrossGen }} _RunCrossGen2: ${{ parameters.runCrossGen2 }} + _CompositeBuildMode: ${{ parameters.compositeBuildMode }} _RunInUnloadableContext: ${{ parameters.runInUnloadableContext }} _LongRunningGcTests: ${{ parameters.longRunningGcTests }} _GcSimulatorTests: ${{ parameters.gcSimulatorTests }} @@ -86,6 +88,7 @@ steps: _HelixType: ${{ parameters.helixType }} _RunCrossGen: ${{ parameters.runCrossGen }} _RunCrossGen2: ${{ parameters.runCrossGen2 }} + _CompositeBuildMode: ${{ parameters.compositeBuildMode }} _RunInUnloadableContext: ${{ parameters.runInUnloadableContext }} _LongRunningGcTests: ${{ parameters.longRunningGcTests }} _GcSimulatorTests: ${{ parameters.gcSimulatorTests }} diff --git a/src/coreclr/build-test.cmd b/src/coreclr/build-test.cmd index 2dbb3b134ddaa6..63eed40b23f09b 100644 --- a/src/coreclr/build-test.cmd +++ b/src/coreclr/build-test.cmd @@ -59,6 +59,7 @@ set __RuntimeId= set __TargetsWindows=1 set __DoCrossgen= set __DoCrossgen2= +set __CompositeBuildMode= set __CopyNativeTestBinaries=0 set __CopyNativeProjectsAfterCombinedTestBuild=true set __SkipGenerateLayout=0 @@ -104,6 +105,7 @@ if /i "%1" == "buildagainstpackages" (echo error: Remove /BuildAgainstPackages if /i "%1" == "skiprestorepackages" (set __SkipRestorePackages=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "crossgen" (set __DoCrossgen=1&set __TestBuildMode=crossgen&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "crossgen2" (set __DoCrossgen2=1&set __TestBuildMode=crossgen2&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "composite" (set __CompositeBuildMode=1&set __DoCrossgen2=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "runtimeid" (set __RuntimeId=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) if /i "%1" == "targetsNonWindows" (set __TargetsWindows=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "Exclude" (set __Exclude=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) @@ -621,18 +623,45 @@ exit /b 1 set __TotalPrecompiled=0 set __FailedToPrecompile=0 set __FailedAssemblies= -for %%F in ("%CORE_ROOT%\System.*.dll";"%CORE_ROOT%\Microsoft.*.dll") do ( +set __CompositeOutputDir=%CORE_ROOT%\composite.out +set __CompositeResponseFile=%__CompositeOutputDir%\framework-r2r.dll.rsp + +if defined __CompositeBuildMode ( + mkdir !__CompositeOutputDir! + echo --composite>>!__CompositeResponseFile! + echo -O>>!__CompositeResponseFile! + echo --out^:%__CompositeOutputDir%\framework-r2r.dll>>!__CompositeResponseFile! +) + +for %%F in ("%CORE_ROOT%\System.*.dll";"%CORE_ROOT%\Microsoft.*.dll";%CORE_ROOT%\netstandard.dll;%CORE_ROOT%\mscorlib.dll) do ( if not "%%~nxF"=="Microsoft.CodeAnalysis.VisualBasic.dll" ( if not "%%~nxF"=="Microsoft.CodeAnalysis.CSharp.dll" ( if not "%%~nxF"=="Microsoft.CodeAnalysis.dll" ( if not "%%~nxF"=="System.Runtime.WindowsRuntime.dll" ( - call :PrecompileAssembly "%%F" %%~nxF __TotalPrecompiled __FailedToPrecompile __FailedAssemblies - echo Processed: !__TotalPrecompiled!, failed !__FailedToPrecompile! + if defined __CompositeBuildMode ( + echo %%F>>!__CompositeResponseFile! + ) else ( + call :PrecompileAssembly "%%F" %%~nxF __TotalPrecompiled __FailedToPrecompile __FailedAssemblies + echo Processed: !__TotalPrecompiled!, failed !__FailedToPrecompile! + ) ))))) ) +echo Composite response line^: %__CompositeResponseFile% +type "%__CompositeResponseFile%" + +if defined __CompositeBuildMode ( + set __CompositeCommandLine="%CORE_ROOT%\corerun" + set __CompositeCommandLine=!__CompositeCommandLine! "%CORE_ROOT%\crossgen2\crossgen2.dll" + set __CompositeCommandLine=!__CompositeCommandLine! "@%__CompositeResponseFile%" + echo Building composite R2R framework^: !__CompositeCommandLine! + !__CompositeCommandLine! + set __FailedToPrecompile=!ERRORLEVEL! + copy /Y "!__CompositeOutputDir!\*.*" "!CORE_ROOT!\" +) + if !__FailedToPrecompile! NEQ 0 ( - echo Failed assemblies: + @echo Failed assemblies: FOR %%G IN (!__FailedAssemblies!) do echo %%G ) diff --git a/src/coreclr/build-test.sh b/src/coreclr/build-test.sh index 24b7d3275b071f..5e11468bcc4f4c 100755 --- a/src/coreclr/build-test.sh +++ b/src/coreclr/build-test.sh @@ -183,15 +183,25 @@ precompile_coreroot_fx() local totalPrecompiled=0 local failedToPrecompile=0 + local compositeCommandLine="$overlayDir/corerun" + compositeCommandLine+=" $overlayDir/crossgen2/crossgen2.dll" + compositeCommandLine+=" --composite" + compositeCommandLine+=" -O" + compositeCommandLine+=" --out:$outputDir/framework-r2r.dll" declare -a failedAssemblies - filesToPrecompile=$(find -L "$overlayDir" -maxdepth 1 -iname Microsoft.\*.dll -o -iname System.\*.dll -type f) + filesToPrecompile=$(find -L "$overlayDir" -maxdepth 1 -iname Microsoft.\*.dll -o -iname System.\*.dll -o -iname netstandard.dll -o -iname mscorlib.dll -type f) for fileToPrecompile in ${filesToPrecompile}; do local filename="$fileToPrecompile" if is_skip_crossgen_test "$(basename $filename)"; then continue fi + if [[ "$__CompositeBuildMode" != 0 ]]; then + compositeCommandLine+=" $filename" + continue + fi + local commandLine="" if [[ "$__DoCrossgen" != 0 ]]; then @@ -224,6 +234,17 @@ precompile_coreroot_fx() echo "Processed: $totalPrecompiled, failed $failedToPrecompile" done + if [[ "$__CompositeBuildMode" != 0 ]]; then + # Compile the entire framework in composite build mode + echo "Compiling composite R2R framework: $compositeCommandLine" + $compositeCommandLine + local exitCode="$?" + if [[ "$exitCode" != 0 ]]; then + echo Unable to precompile composite framework, exit code is "$exitCode". + exit 1 + fi + fi + if [[ "$__DoCrossgen2" != 0 ]]; then # Copy the Crossgen-compiled assemblies back to CORE_ROOT mv -f "$outputDir"/* "$overlayDir"/ @@ -532,6 +553,12 @@ handle_arguments_local() { __TestBuildMode=crossgen2 ;; + composite|-composite) + __CompositeBuildMode=1 + __DoCrossgen2=1 + __TestBuildMode=crossgen2 + ;; + generatetesthostonly|-generatetesthostonly) __GenerateTestHostOnly=1 ;; @@ -592,6 +619,7 @@ __CrossBuild=0 __DistroRid="" __DoCrossgen=0 __DoCrossgen2=0 +__CompositeBuildMode=0 __DotNetCli="$__RepoRootDir/dotnet.sh" __GenerateLayoutOnly= __GenerateTestHostOnly= diff --git a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/CompilationModuleGroup.ReadyToRun.cs b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/CompilationModuleGroup.ReadyToRun.cs index acd8aa324e2a44..38ec589d85b583 100644 --- a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/CompilationModuleGroup.ReadyToRun.cs +++ b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/CompilationModuleGroup.ReadyToRun.cs @@ -82,6 +82,12 @@ public bool EnforceOwningType(EcmaModule module) /// public abstract bool IsInputBubble { get; } + /// Returns true when compiling in library output mode (e.g. when compiling the shared framework + /// or a 3rd party package into a composite R2R executable). When set to false, we're compiling a runnable + /// executable. + /// + public abstract bool IsLibraryBuildMode { get; } + /// /// List of input modules to use for the compilation. /// diff --git a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunCompilationModuleGroupBase.cs b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunCompilationModuleGroupBase.cs index 7d55bc55e32b47..ba1293ab328525 100644 --- a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunCompilationModuleGroupBase.cs +++ b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunCompilationModuleGroupBase.cs @@ -21,6 +21,7 @@ public abstract class ReadyToRunCompilationModuleGroupBase : CompilationModuleGr private readonly bool _compileGenericDependenciesFromVersionBubbleModuleSet; private readonly bool _isCompositeBuildMode; private readonly bool _isInputBubble; + private readonly bool _isLibraryBuildMode; private readonly ConcurrentDictionary _containsTypeLayoutCache = new ConcurrentDictionary(); private readonly ConcurrentDictionary _versionsWithTypeCache = new ConcurrentDictionary(); private readonly ConcurrentDictionary _versionsWithMethodCache = new ConcurrentDictionary(); @@ -29,6 +30,7 @@ public ReadyToRunCompilationModuleGroupBase( TypeSystemContext context, bool isCompositeBuildMode, bool isInputBubble, + bool isLibraryBuildMode, IEnumerable compilationModuleSet, IEnumerable versionBubbleModuleSet, bool compileGenericDependenciesFromVersionBubbleModuleSet) @@ -36,6 +38,7 @@ public ReadyToRunCompilationModuleGroupBase( _compilationModuleSet = new HashSet(compilationModuleSet); _isCompositeBuildMode = isCompositeBuildMode; _isInputBubble = isInputBubble; + _isLibraryBuildMode = isLibraryBuildMode; Debug.Assert(_isCompositeBuildMode || _compilationModuleSet.Count == 1); @@ -213,6 +216,8 @@ public sealed override bool TryGetModuleTokenForExternalType(TypeDesc type, out public sealed override bool IsInputBubble => _isInputBubble; + public sealed override bool IsLibraryBuildMode => _isLibraryBuildMode; + public sealed override IEnumerable CompilationModuleSet => _compilationModuleSet; private bool ComputeTypeVersionsWithCode(TypeDesc type) diff --git a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunLibraryRootProvider.cs b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunLibraryRootProvider.cs index 1870e823746876..332c5dc18d5a39 100644 --- a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunLibraryRootProvider.cs +++ b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunLibraryRootProvider.cs @@ -18,12 +18,14 @@ public class ReadyToRunRootProvider : ICompilationRootProvider private EcmaModule _module; private IEnumerable _profileData; private readonly bool _profileDrivenPartialNGen; + private readonly bool _isLibraryBuildMode; - public ReadyToRunRootProvider(EcmaModule module, ProfileDataManager profileDataManager, bool profileDrivenPartialNGen) + public ReadyToRunRootProvider(EcmaModule module, ProfileDataManager profileDataManager, bool profileDrivenPartialNGen, bool isLibraryBuildMode) { _module = module; _profileData = profileDataManager.GetMethodsForModuleDesc(module); _profileDrivenPartialNGen = profileDrivenPartialNGen; + _isLibraryBuildMode = isLibraryBuildMode; } public void AddCompilationRoots(IRootingServiceProvider rootProvider) diff --git a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunSingleAssemblyCompilationModuleGroup.cs b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunSingleAssemblyCompilationModuleGroup.cs index 01a6c1a18cbfdb..1a7c6818ba6b98 100644 --- a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunSingleAssemblyCompilationModuleGroup.cs +++ b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunSingleAssemblyCompilationModuleGroup.cs @@ -21,12 +21,15 @@ public ReadyToRunSingleAssemblyCompilationModuleGroup( TypeSystemContext context, bool isCompositeBuildMode, bool isInputBubble, + bool isLibraryBuildMode, IEnumerable compilationModuleSet, IEnumerable versionBubbleModuleSet, bool compileGenericDependenciesFromVersionBubbleModuleSet) : base(context, isCompositeBuildMode, isInputBubble, + isCompositeBuildMode: isCompositeBuildMode, + isLibraryBuildMode: isLibraryBuildMode, compilationModuleSet, versionBubbleModuleSet, compileGenericDependenciesFromVersionBubbleModuleSet) diff --git a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/SingleMethodCompilationModuleGroup.cs b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/SingleMethodCompilationModuleGroup.cs index f32c2295b55033..aa09bb90615b3a 100644 --- a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/SingleMethodCompilationModuleGroup.cs +++ b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/SingleMethodCompilationModuleGroup.cs @@ -21,6 +21,7 @@ public SingleMethodCompilationModuleGroup( TypeSystemContext context, bool isCompositeBuildMode, bool isInputBubble, + bool isLibraryBuildMode, IEnumerable compilationModuleSet, IEnumerable versionBubbleModuleSet, bool compileGenericDependenciesFromVersionBubbleModuleSet, @@ -28,6 +29,8 @@ public SingleMethodCompilationModuleGroup( base(context, isCompositeBuildMode, isInputBubble, + isCompositeBuildMode: isCompositeBuildMode, + isLibraryBuildMode: isLibraryBuildMode, compilationModuleSet, versionBubbleModuleSet, compileGenericDependenciesFromVersionBubbleModuleSet) diff --git a/src/coreclr/src/tools/crossgen2/crossgen2/Program.cs b/src/coreclr/src/tools/crossgen2/crossgen2/Program.cs index d6f98eb7d905b7..fd8bb55a87647e 100644 --- a/src/coreclr/src/tools/crossgen2/crossgen2/Program.cs +++ b/src/coreclr/src/tools/crossgen2/crossgen2/Program.cs @@ -313,6 +313,7 @@ private int Run() List inputModules = new List(); List rootingModules = new List(); HashSet versionBubbleModulesHash = new HashSet(); + bool isLibraryBuildMode = true; foreach (var inputFile in inputFilePaths) { @@ -320,6 +321,10 @@ private int Run() inputModules.Add(module); rootingModules.Add(module); versionBubbleModulesHash.Add(module); + if (module.EntryPoint != null) + { + isLibraryBuildMode = false; + } if (!_commandLineOptions.CompositeOrInputBubble) { @@ -332,6 +337,10 @@ private int Run() EcmaModule module = typeSystemContext.GetModuleFromPath(unrootedInputFile.Value); inputModules.Add(module); versionBubbleModulesHash.Add(module); + if (module.EntryPoint != null) + { + isLibraryBuildMode = false; + } } string systemModuleName = _commandLineOptions.SystemModule ?? DefaultSystemModule; @@ -394,6 +403,7 @@ private int Run() typeSystemContext, _commandLineOptions.Composite, _commandLineOptions.InputBubble, + isLibraryBuildMode, inputModules, versionBubbleModules, _commandLineOptions.CompileBubbleGenerics, @@ -407,6 +417,8 @@ private int Run() typeSystemContext, _commandLineOptions.Composite, _commandLineOptions.InputBubble, + isCompositeBuildMode: _commandLineOptions.Composite, + isLibraryBuildMode: isLibraryBuildMode, inputModules, versionBubbleModules, _commandLineOptions.CompileBubbleGenerics); @@ -436,7 +448,8 @@ private int Run() compilationRoots.Add(new ReadyToRunRootProvider( module, profileDataManager, - profileDrivenPartialNGen: _commandLineOptions.Partial)); + profileDrivenPartialNGen: _commandLineOptions.Partial, + isLibraryBuildMode: isLibraryBuildMode)); if (!_commandLineOptions.CompositeOrInputBubble) { diff --git a/src/coreclr/src/vm/nativeimage.cpp b/src/coreclr/src/vm/nativeimage.cpp index 7cda61993ddb95..81e22fb4e92e05 100644 --- a/src/coreclr/src/vm/nativeimage.cpp +++ b/src/coreclr/src/vm/nativeimage.cpp @@ -53,7 +53,7 @@ void NativeImage::Initialize(READYTORUN_HEADER *pHeader, LoaderAllocator *pLoade m_pReadyToRunInfo = new ReadyToRunInfo(/*pModule*/ NULL, m_pImageLayout, pHeader, /*compositeImage*/ NULL, pamTracker); m_pComponentAssemblies = m_pReadyToRunInfo->FindSection(ReadyToRunSectionType::ComponentAssemblies); - m_componentAssemblyCount = m_pComponentAssemblies->Size / sizeof(READYTORUN_COMPONENT_ASSEMBLIES_ENTRY); + m_componentAssemblyCount = (m_pComponentAssemblies != NULL ? m_pComponentAssemblies->Size / sizeof(READYTORUN_COMPONENT_ASSEMBLIES_ENTRY) : 1); // Check if the current module's image has native manifest metadata, otherwise the current->GetNativeAssemblyImport() asserts. m_pManifestMetadata = LoadManifestMetadata(); @@ -130,6 +130,11 @@ PTR_READYTORUN_CORE_HEADER NativeImage::GetComponentAssemblyHeader(LPCUTF8 simpl { STANDARD_VM_CONTRACT; + if (m_componentAssemblyCount == 1) + { + return dac_cast(&m_pReadyToRunInfo->GetReadyToRunHeader()->CoreHeader); + } + const AssemblyNameIndex *assemblyNameIndex = m_assemblySimpleNameToIndexMap.LookupPtr(simpleName); if (assemblyNameIndex != NULL) { @@ -138,6 +143,7 @@ PTR_READYTORUN_CORE_HEADER NativeImage::GetComponentAssemblyHeader(LPCUTF8 simpl (const READYTORUN_COMPONENT_ASSEMBLIES_ENTRY *)&pImageBase[m_pComponentAssemblies->VirtualAddress] + assemblyNameIndex->Index; return (PTR_READYTORUN_CORE_HEADER)&pImageBase[componentAssembly->ReadyToRunCoreHeader.VirtualAddress]; } + return NULL; } #endif diff --git a/src/coreclr/src/vm/readytoruninfo.cpp b/src/coreclr/src/vm/readytoruninfo.cpp index a47541991df1ee..18dafd7fa765d1 100644 --- a/src/coreclr/src/vm/readytoruninfo.cpp +++ b/src/coreclr/src/vm/readytoruninfo.cpp @@ -707,9 +707,9 @@ ReadyToRunInfo::ReadyToRunInfo(Module * pModule, PEImageLayout * pLayout, READYT } // For format version 4.1 and later, there is an optional inlining table - if (IsImageVersionAtLeast(4, 1)) + if (pModule != NULL && IsImageVersionAtLeast(4, 1)) { - IMAGE_DATA_DIRECTORY* pInlineTrackingInfoDir = m_pComposite->FindSection(ReadyToRunSectionType::InliningInfo2); + IMAGE_DATA_DIRECTORY* pInlineTrackingInfoDir = m_component.FindSection(ReadyToRunSectionType::InliningInfo2); if (pInlineTrackingInfoDir != NULL) { const BYTE* pInlineTrackingMapData = (const BYTE*)m_pComposite->GetImage()->GetDirectoryData(pInlineTrackingInfoDir); @@ -719,9 +719,9 @@ ReadyToRunInfo::ReadyToRunInfo(Module * pModule, PEImageLayout * pLayout, READYT } // For format version 2.1 and later, there is an optional inlining table - if (m_pPersistentInlineTrackingMap == nullptr && IsImageVersionAtLeast(2, 1)) + if (pModule != NULL && m_pPersistentInlineTrackingMap == nullptr && IsImageVersionAtLeast(2, 1)) { - IMAGE_DATA_DIRECTORY * pInlineTrackingInfoDir = m_pComposite->FindSection(ReadyToRunSectionType::InliningInfo); + IMAGE_DATA_DIRECTORY * pInlineTrackingInfoDir = m_component.FindSection(ReadyToRunSectionType::InliningInfo); if (pInlineTrackingInfoDir != NULL) { const BYTE* pInlineTrackingMapData = (const BYTE*)m_pComposite->GetImage()->GetDirectoryData(pInlineTrackingInfoDir); @@ -731,7 +731,7 @@ ReadyToRunInfo::ReadyToRunInfo(Module * pModule, PEImageLayout * pLayout, READYT } // For format version 2.2 and later, there is an optional profile-data section - if (IsImageVersionAtLeast(2, 2)) + if (pModule != NULL && IsImageVersionAtLeast(2, 2)) { IMAGE_DATA_DIRECTORY * pProfileDataInfoDir = m_pComposite->FindSection(ReadyToRunSectionType::ProfileDataInfo); if (pProfileDataInfoDir != NULL) @@ -744,16 +744,19 @@ ReadyToRunInfo::ReadyToRunInfo(Module * pModule, PEImageLayout * pLayout, READYT } // For format version 3.1 and later, there is an optional attributes section - IMAGE_DATA_DIRECTORY *attributesPresenceDataInfoDir = m_component.FindSection(ReadyToRunSectionType::AttributePresence); - if (attributesPresenceDataInfoDir != NULL) + if (pModule != NULL) { - NativeCuckooFilter newFilter( - (BYTE *)m_pComposite->GetLayout()->GetBase(), - m_pComposite->GetLayout()->GetVirtualSize(), - attributesPresenceDataInfoDir->VirtualAddress, - attributesPresenceDataInfoDir->Size); - - m_attributesPresence = newFilter; + IMAGE_DATA_DIRECTORY *attributesPresenceDataInfoDir = m_component.FindSection(ReadyToRunSectionType::AttributePresence); + if (attributesPresenceDataInfoDir != NULL) + { + NativeCuckooFilter newFilter( + (BYTE *)m_pComposite->GetLayout()->GetBase(), + m_pComposite->GetLayout()->GetVirtualSize(), + attributesPresenceDataInfoDir->VirtualAddress, + attributesPresenceDataInfoDir->Size); + + m_attributesPresence = newFilter; + } } } diff --git a/src/coreclr/tests/helixpublishwitharcade.proj b/src/coreclr/tests/helixpublishwitharcade.proj index 93276627286dea..dcf41938ca9ffb 100644 --- a/src/coreclr/tests/helixpublishwitharcade.proj +++ b/src/coreclr/tests/helixpublishwitharcade.proj @@ -27,6 +27,7 @@ PublishTestResults=$(_PublishTestResults); RunCrossGen=$(_RunCrossGen); RunCrossGen2=$(_RunCrossGen2); + CompositeBuildMode=$(_CompositeBuildMode); LongRunningGCTests=$(_LongRunningGCTests); GcSimulatorTests=$(_GcSimulatorTests); RunInUnloadableContext=$(_RunInUnloadableContext); @@ -194,6 +195,7 @@ + @@ -209,6 +211,7 @@ + diff --git a/src/coreclr/tests/src/CLRTest.CrossGen.targets b/src/coreclr/tests/src/CLRTest.CrossGen.targets index 258f3078b6f92b..f2da929774fb81 100644 --- a/src/coreclr/tests/src/CLRTest.CrossGen.targets +++ b/src/coreclr/tests/src/CLRTest.CrossGen.targets @@ -65,25 +65,56 @@ if [ ! -z ${RunCrossGen+x} ]%3B then fi # CrossGen2 Script if [ ! -z ${RunCrossGen2+x} ]%3B then - if [ ! -f $(MSBuildProjectName).org ]%3B then - TakeLock - if [ ! -f $(MSBuildProjectName).org ]%3B then - mkdir IL - cp $(MSBuildProjectName).dll IL/$(MSBuildProjectName).dll - mv $(MSBuildProjectName).dll $(MSBuildProjectName).org - __Command=$_DebuggerFullPath "$CORE_ROOT/corerun" "$CORE_ROOT/crossgen2/crossgen2.dll" -r:$CORE_ROOT/System.*.dll -r:$CORE_ROOT/Microsoft.*.dll -r:$CORE_ROOT/mscorlib.dll -r:$CORE_ROOT/netstandard.dll -r:$PWD/*.dll --targetarch=x64 -O --inputbubble $ExtraCrossGen2Args -o:$(scriptPath)$(MSBuildProjectName).dll $(scriptPath)$(MSBuildProjectName).org - echo $__Command - $__Command - __cg2ExitCode=$? - if [ $__cg2ExitCode -ne 0 ] - then - echo Crossgen2 failed with exitcode: $__cg2ExitCode - ReleaseLock - exit 1 - fi - fi - ReleaseLock + TakeLock + if [ ! -d IL ]%3B then + mkdir IL + if [ ! -z ${CompositeBuildMode+x} ]%3B then + cp $(MSBuildProjectName).dll IL/ + cp $CORE_ROOT/lib*.so $CORE_ROOT/lib*.dylib $(scriptPath) + else + cp *.dll IL/ + fi fi + + local __ResponseFile=$CORE_ROOT/composite-r2r.dll.rsp + rm $__ResponseFile + __Command=$_DebuggerFullPath + __Command+=" $CORE_ROOT/corerun" + __Command+=" $CORE_ROOT/crossgen2/crossgen2.dll" + __Command+=" @$__ResponseFile" + __Command+=" $ExtraCrossGen2Args" + if [ ! -z ${CompositeBuildMode+x} ]%3B then + echo --composite>>$__ResponseFile + echo -o:$(scriptPath)composite-r2r.dll>>$__ResponseFile + echo $PWD/*.dll>>$__ResponseFile + else + echo --inputbubble>>$__ResponseFile + echo -o:$(scriptPath)$(MSBuildProjectName).dll>>$__ResponseFile + echo -r:$PWD/*.dll>>$__ResponseFile + fi + + echo -r:$CORE_ROOT/System.*.dll>>$__ResponseFile + echo -r:$CORE_ROOT/Microsoft.*.dll>>$__ResponseFile + echo -r:$CORE_ROOT/mscorlib.dll>>$__ResponseFile + echo -r:$CORE_ROOT/netstandard.dll>>$__ResponseFile + echo --targetarch=x64>>$__ResponseFile + echo -O>>$__ResponseFile + + echo "Response file: $__ResponseFile" + cat $__ResponseFile + + echo "Running CrossGen2: $__Command" + $__Command + + __cg2ExitCode=$? + if [ $__cg2ExitCode -ne 0 ] + then + echo Crossgen2 failed with exitcode: $__cg2ExitCode + ReleaseLock + exit 1 + fi + + ReleaseLock fi ]]> @@ -124,24 +155,49 @@ if defined RunCrossGen ( REM CrossGen2 Script if defined RunCrossGen2 ( if defined LargeVersionBubble ( set OptionalArguments=!OptionalArguments! /largeversionbubble) - if not exist "$(MSBuildProjectName).org" ( - call :TakeLock - set CrossGen2Status=0 - if not exist "$(MSBuildProjectName).org" ( - mkdir IL - copy $(MSBuildProjectName).dll IL\$(MSBuildProjectName).dll - ren $(MSBuildProjectName).dll $(MSBuildProjectName).org - set __Command=!_DebuggerFullPath! "!CORE_ROOT!\CoreRun.exe" "!CORE_ROOT!\crossgen2\crossgen2.dll" %21scriptPath%21$(MSBuildProjectName).org -o:%21scriptPath%21$(MSBuildProjectName).dll --targetarch:x64 -O --inputbubble %21ExtraCrossGen2Args%21 -r:"!CORE_ROOT!\System.*.dll" -r:"!CORE_ROOT!\Microsoft.*.dll" -r:"!CORE_ROOT!\netstandard.dll" -r:"!CORE_ROOT!\mscorlib.dll" -r:%25cd%25\*.dll - echo "!__Command!" - call !__Command! - set CrossGen2Status=!ERRORLEVEL! - ) - call :ReleaseLock - IF NOT !CrossGen2Status!==0 ( - ECHO Crossgen2 failed with exitcode - !CrossGen2Status! - Exit /b 1 + call :TakeLock + set CrossGen2Status=0 + if not exist "IL" ( + mkdir IL + if defined CompositeBuildMode ( + copy *.dll IL\ + ) else ( + copy $(MSBuildProjectName).dll IL\ ) ) + set __ResponseFile=!scriptPath!\composite-r2r.dll.rsp + del /Y !__ResponseFile! + set __Command=!_DebuggerFullPath! + set __Command=!__Command! "!CORE_ROOT!\CoreRun.exe" + set __Command=!__Command! "!CORE_ROOT!\crossgen2\crossgen2.dll" + set __Command=!__Command! @"!__ResponseFile!" + set __Command=!__Command! !ExtraCrossGen2Args! + if defined CompositeBuildMode ( + echo --composite>>!__ResponseFile! + echo -o:!scriptPath!composite-r2r.dll>>!__ResponseFile! + rem In composite mode, treat all dll's in the test folder as rooting inputs + echo !scriptPath!IL\*.dll>>!__ResponseFile! + ) else ( + echo --inputbubble>>!__ResponseFile! + echo -o:!scriptPath!$(MSBuildProjectName).dll>>!__ResponseFile! + echo -r:%21scriptPath%21IL\*.dll>>!__ResponseFile! + ) + echo --targetarch:x64>>!__ResponseFile! + echo -O>>!__ResponseFile! + echo -r:!CORE_ROOT!\System.*.dll>>!__ResponseFile! + echo -r:!CORE_ROOT!\Microsoft.*.dll>>!__ResponseFile! + echo -r:!CORE_ROOT!\mscorlib.dll>>!__ResponseFile! + echo -r:!CORE_ROOT!\netstandard.dll>>!__ResponseFile! + echo "Response file: !__ReponseFile!" + type !__ResponseFile! + echo "!__Command!" + call !__Command! + set CrossGen2Status=!ERRORLEVEL! + call :ReleaseLock + IF NOT !CrossGen2Status!==0 ( + ECHO Crossgen2 failed with exitcode - !CrossGen2Status! + Exit /b 1 + ) ) ]]> From 4bc6bd7efff9fadc01b6d854a1a4139fb5efa475 Mon Sep 17 00:00:00 2001 From: Tomas Date: Sun, 15 Mar 2020 18:45:53 +0100 Subject: [PATCH 2/9] Crossgen2 / SuperIlc support for composite mode with shared framework This change adds SuperIlc support for the option --framework in --composite mode by first compiling the entire set of framework assemblies into one composite version bubble and then building the test on top of it as another composite version bubble. One practical upside of this mode is that it's naturally much faster than the "one-component" composite mode as that basically requires compiling the framework with each app. Thanks Tomas --- eng/pipelines/coreclr/templates/run-test-job.yml | 4 ++++ src/coreclr/tests/src/CLRTest.CrossGen.targets | 16 ++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/coreclr/templates/run-test-job.yml b/eng/pipelines/coreclr/templates/run-test-job.yml index b1a8b29c2a83e0..91c6d65c8874b3 100644 --- a/eng/pipelines/coreclr/templates/run-test-job.yml +++ b/eng/pipelines/coreclr/templates/run-test-job.yml @@ -145,6 +145,10 @@ jobs: - name: timeoutPerTestInMinutes value: 90 + - ${{ if eq(parameters.compositeBuildMode, true) }}: + - name: crossgenArg + value: 'composite' + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: DotNet-HelixApi-Access diff --git a/src/coreclr/tests/src/CLRTest.CrossGen.targets b/src/coreclr/tests/src/CLRTest.CrossGen.targets index f2da929774fb81..1921d3d8fce119 100644 --- a/src/coreclr/tests/src/CLRTest.CrossGen.targets +++ b/src/coreclr/tests/src/CLRTest.CrossGen.targets @@ -76,36 +76,34 @@ if [ ! -z ${RunCrossGen2+x} ]%3B then fi fi - local __ResponseFile=$CORE_ROOT/composite-r2r.dll.rsp + __ResponseFile=$PWD/composite-r2r.dll.rsp rm $__ResponseFile __Command=$_DebuggerFullPath __Command+=" $CORE_ROOT/corerun" __Command+=" $CORE_ROOT/crossgen2/crossgen2.dll" __Command+=" @$__ResponseFile" __Command+=" $ExtraCrossGen2Args" + if [ ! -z ${CompositeBuildMode+x} ]%3B then echo --composite>>$__ResponseFile echo -o:$(scriptPath)composite-r2r.dll>>$__ResponseFile - echo $PWD/*.dll>>$__ResponseFile + echo $PWD/IL/*.dll>>$__ResponseFile else echo --inputbubble>>$__ResponseFile echo -o:$(scriptPath)$(MSBuildProjectName).dll>>$__ResponseFile - echo -r:$PWD/*.dll>>$__ResponseFile + echo -r:$PWD/IL/*.dll>>$__ResponseFile fi echo -r:$CORE_ROOT/System.*.dll>>$__ResponseFile echo -r:$CORE_ROOT/Microsoft.*.dll>>$__ResponseFile echo -r:$CORE_ROOT/mscorlib.dll>>$__ResponseFile - echo -r:$CORE_ROOT/netstandard.dll>>$__ResponseFile echo --targetarch=x64>>$__ResponseFile echo -O>>$__ResponseFile echo "Response file: $__ResponseFile" cat $__ResponseFile - echo "Running CrossGen2: $__Command" $__Command - __cg2ExitCode=$? if [ $__cg2ExitCode -ne 0 ] then @@ -165,13 +163,16 @@ if defined RunCrossGen2 ( copy $(MSBuildProjectName).dll IL\ ) ) + set __ResponseFile=!scriptPath!\composite-r2r.dll.rsp del /Y !__ResponseFile! + set __Command=!_DebuggerFullPath! set __Command=!__Command! "!CORE_ROOT!\CoreRun.exe" set __Command=!__Command! "!CORE_ROOT!\crossgen2\crossgen2.dll" set __Command=!__Command! @"!__ResponseFile!" set __Command=!__Command! !ExtraCrossGen2Args! + if defined CompositeBuildMode ( echo --composite>>!__ResponseFile! echo -o:!scriptPath!composite-r2r.dll>>!__ResponseFile! @@ -182,14 +183,17 @@ if defined RunCrossGen2 ( echo -o:!scriptPath!$(MSBuildProjectName).dll>>!__ResponseFile! echo -r:%21scriptPath%21IL\*.dll>>!__ResponseFile! ) + echo --targetarch:x64>>!__ResponseFile! echo -O>>!__ResponseFile! echo -r:!CORE_ROOT!\System.*.dll>>!__ResponseFile! echo -r:!CORE_ROOT!\Microsoft.*.dll>>!__ResponseFile! echo -r:!CORE_ROOT!\mscorlib.dll>>!__ResponseFile! echo -r:!CORE_ROOT!\netstandard.dll>>!__ResponseFile! + echo "Response file: !__ReponseFile!" type !__ResponseFile! + echo "!__Command!" call !__Command! set CrossGen2Status=!ERRORLEVEL! From 970c8ab28467eaf915a86406c85c3e83634de3c5 Mon Sep 17 00:00:00 2001 From: Tomas Date: Mon, 16 Mar 2020 23:12:40 +0100 Subject: [PATCH 3/9] Remove library build mode flag as it's currently not needed --- .../Compiler/CompilationModuleGroup.ReadyToRun.cs | 6 ------ .../ReadyToRunCompilationModuleGroupBase.cs | 11 +++++++++-- .../Compiler/ReadyToRunLibraryRootProvider.cs | 4 +--- ...adyToRunSingleAssemblyCompilationModuleGroup.cs | 2 -- .../Compiler/SingleMethodCompilationModuleGroup.cs | 2 -- .../src/tools/crossgen2/crossgen2/Program.cs | 14 +------------- 6 files changed, 11 insertions(+), 28 deletions(-) diff --git a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/CompilationModuleGroup.ReadyToRun.cs b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/CompilationModuleGroup.ReadyToRun.cs index 38ec589d85b583..acd8aa324e2a44 100644 --- a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/CompilationModuleGroup.ReadyToRun.cs +++ b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/CompilationModuleGroup.ReadyToRun.cs @@ -82,12 +82,6 @@ public bool EnforceOwningType(EcmaModule module) /// public abstract bool IsInputBubble { get; } - /// Returns true when compiling in library output mode (e.g. when compiling the shared framework - /// or a 3rd party package into a composite R2R executable). When set to false, we're compiling a runnable - /// executable. - /// - public abstract bool IsLibraryBuildMode { get; } - /// /// List of input modules to use for the compilation. /// diff --git a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunCompilationModuleGroupBase.cs b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunCompilationModuleGroupBase.cs index ba1293ab328525..3a690e1898cafe 100644 --- a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunCompilationModuleGroupBase.cs +++ b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunCompilationModuleGroupBase.cs @@ -20,8 +20,11 @@ public abstract class ReadyToRunCompilationModuleGroupBase : CompilationModuleGr private Dictionary _typeRefsInCompilationModuleSet; private readonly bool _compileGenericDependenciesFromVersionBubbleModuleSet; private readonly bool _isCompositeBuildMode; +<<<<<<< HEAD private readonly bool _isInputBubble; private readonly bool _isLibraryBuildMode; +======= +>>>>>>> Remove library build mode flag as it's currently not needed private readonly ConcurrentDictionary _containsTypeLayoutCache = new ConcurrentDictionary(); private readonly ConcurrentDictionary _versionsWithTypeCache = new ConcurrentDictionary(); private readonly ConcurrentDictionary _versionsWithMethodCache = new ConcurrentDictionary(); @@ -29,16 +32,22 @@ public abstract class ReadyToRunCompilationModuleGroupBase : CompilationModuleGr public ReadyToRunCompilationModuleGroupBase( TypeSystemContext context, bool isCompositeBuildMode, +<<<<<<< HEAD bool isInputBubble, bool isLibraryBuildMode, +======= +>>>>>>> Remove library build mode flag as it's currently not needed IEnumerable compilationModuleSet, IEnumerable versionBubbleModuleSet, bool compileGenericDependenciesFromVersionBubbleModuleSet) { _compilationModuleSet = new HashSet(compilationModuleSet); _isCompositeBuildMode = isCompositeBuildMode; +<<<<<<< HEAD _isInputBubble = isInputBubble; _isLibraryBuildMode = isLibraryBuildMode; +======= +>>>>>>> Remove library build mode flag as it's currently not needed Debug.Assert(_isCompositeBuildMode || _compilationModuleSet.Count == 1); @@ -216,8 +225,6 @@ public sealed override bool TryGetModuleTokenForExternalType(TypeDesc type, out public sealed override bool IsInputBubble => _isInputBubble; - public sealed override bool IsLibraryBuildMode => _isLibraryBuildMode; - public sealed override IEnumerable CompilationModuleSet => _compilationModuleSet; private bool ComputeTypeVersionsWithCode(TypeDesc type) diff --git a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunLibraryRootProvider.cs b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunLibraryRootProvider.cs index 332c5dc18d5a39..1870e823746876 100644 --- a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunLibraryRootProvider.cs +++ b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunLibraryRootProvider.cs @@ -18,14 +18,12 @@ public class ReadyToRunRootProvider : ICompilationRootProvider private EcmaModule _module; private IEnumerable _profileData; private readonly bool _profileDrivenPartialNGen; - private readonly bool _isLibraryBuildMode; - public ReadyToRunRootProvider(EcmaModule module, ProfileDataManager profileDataManager, bool profileDrivenPartialNGen, bool isLibraryBuildMode) + public ReadyToRunRootProvider(EcmaModule module, ProfileDataManager profileDataManager, bool profileDrivenPartialNGen) { _module = module; _profileData = profileDataManager.GetMethodsForModuleDesc(module); _profileDrivenPartialNGen = profileDrivenPartialNGen; - _isLibraryBuildMode = isLibraryBuildMode; } public void AddCompilationRoots(IRootingServiceProvider rootProvider) diff --git a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunSingleAssemblyCompilationModuleGroup.cs b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunSingleAssemblyCompilationModuleGroup.cs index 1a7c6818ba6b98..202e0ed8a30709 100644 --- a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunSingleAssemblyCompilationModuleGroup.cs +++ b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunSingleAssemblyCompilationModuleGroup.cs @@ -21,7 +21,6 @@ public ReadyToRunSingleAssemblyCompilationModuleGroup( TypeSystemContext context, bool isCompositeBuildMode, bool isInputBubble, - bool isLibraryBuildMode, IEnumerable compilationModuleSet, IEnumerable versionBubbleModuleSet, bool compileGenericDependenciesFromVersionBubbleModuleSet) : @@ -29,7 +28,6 @@ public ReadyToRunSingleAssemblyCompilationModuleGroup( isCompositeBuildMode, isInputBubble, isCompositeBuildMode: isCompositeBuildMode, - isLibraryBuildMode: isLibraryBuildMode, compilationModuleSet, versionBubbleModuleSet, compileGenericDependenciesFromVersionBubbleModuleSet) diff --git a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/SingleMethodCompilationModuleGroup.cs b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/SingleMethodCompilationModuleGroup.cs index aa09bb90615b3a..d7e3584c0c37e7 100644 --- a/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/SingleMethodCompilationModuleGroup.cs +++ b/src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/SingleMethodCompilationModuleGroup.cs @@ -21,7 +21,6 @@ public SingleMethodCompilationModuleGroup( TypeSystemContext context, bool isCompositeBuildMode, bool isInputBubble, - bool isLibraryBuildMode, IEnumerable compilationModuleSet, IEnumerable versionBubbleModuleSet, bool compileGenericDependenciesFromVersionBubbleModuleSet, @@ -30,7 +29,6 @@ public SingleMethodCompilationModuleGroup( isCompositeBuildMode, isInputBubble, isCompositeBuildMode: isCompositeBuildMode, - isLibraryBuildMode: isLibraryBuildMode, compilationModuleSet, versionBubbleModuleSet, compileGenericDependenciesFromVersionBubbleModuleSet) diff --git a/src/coreclr/src/tools/crossgen2/crossgen2/Program.cs b/src/coreclr/src/tools/crossgen2/crossgen2/Program.cs index fd8bb55a87647e..6b68e49986bc68 100644 --- a/src/coreclr/src/tools/crossgen2/crossgen2/Program.cs +++ b/src/coreclr/src/tools/crossgen2/crossgen2/Program.cs @@ -313,7 +313,6 @@ private int Run() List inputModules = new List(); List rootingModules = new List(); HashSet versionBubbleModulesHash = new HashSet(); - bool isLibraryBuildMode = true; foreach (var inputFile in inputFilePaths) { @@ -321,10 +320,6 @@ private int Run() inputModules.Add(module); rootingModules.Add(module); versionBubbleModulesHash.Add(module); - if (module.EntryPoint != null) - { - isLibraryBuildMode = false; - } if (!_commandLineOptions.CompositeOrInputBubble) { @@ -337,10 +332,6 @@ private int Run() EcmaModule module = typeSystemContext.GetModuleFromPath(unrootedInputFile.Value); inputModules.Add(module); versionBubbleModulesHash.Add(module); - if (module.EntryPoint != null) - { - isLibraryBuildMode = false; - } } string systemModuleName = _commandLineOptions.SystemModule ?? DefaultSystemModule; @@ -403,7 +394,6 @@ private int Run() typeSystemContext, _commandLineOptions.Composite, _commandLineOptions.InputBubble, - isLibraryBuildMode, inputModules, versionBubbleModules, _commandLineOptions.CompileBubbleGenerics, @@ -418,7 +408,6 @@ private int Run() _commandLineOptions.Composite, _commandLineOptions.InputBubble, isCompositeBuildMode: _commandLineOptions.Composite, - isLibraryBuildMode: isLibraryBuildMode, inputModules, versionBubbleModules, _commandLineOptions.CompileBubbleGenerics); @@ -448,8 +437,7 @@ private int Run() compilationRoots.Add(new ReadyToRunRootProvider( module, profileDataManager, - profileDrivenPartialNGen: _commandLineOptions.Partial, - isLibraryBuildMode: isLibraryBuildMode)); + profileDrivenPartialNGen: _commandLineOptions.Partial)); if (!_commandLineOptions.CompositeOrInputBubble) { From ac69be3f102cb78da802aae65135369f73756057 Mon Sep 17 00:00:00 2001 From: Tomas Date: Sun, 22 Mar 2020 21:24:04 +0100 Subject: [PATCH 4/9] Relax R2R module check in composite build mode --- src/coreclr/src/vm/jithelpers.cpp | 2 +- src/coreclr/src/vm/readytoruninfo.cpp | 5 +++++ src/coreclr/src/vm/readytoruninfo.h | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/coreclr/src/vm/jithelpers.cpp b/src/coreclr/src/vm/jithelpers.cpp index 658e4e6cdd07ed..7cc3cb24e4fdc9 100644 --- a/src/coreclr/src/vm/jithelpers.cpp +++ b/src/coreclr/src/vm/jithelpers.cpp @@ -3231,7 +3231,7 @@ CORINFO_GENERIC_HANDLE JIT_GenericHandleWorker(MethodDesc * pMD, MethodTable * p #ifdef _DEBUG // Only in R2R mode are the module, dictionary index and dictionary slot provided as an input _ASSERTE(dictionaryIndexAndSlot != (DWORD)-1); - _ASSERT(ExecutionManager::FindReadyToRunModule(dac_cast(signature)) == pModule); + _ASSERT(ReadyToRunInfo::IsNativeImageSharedBy(pModule, ExecutionManager::FindReadyToRunModule(dac_cast(signature)))); #endif dictionaryIndex = (dictionaryIndexAndSlot >> 16); } diff --git a/src/coreclr/src/vm/readytoruninfo.cpp b/src/coreclr/src/vm/readytoruninfo.cpp index 18dafd7fa765d1..5080eb327daefe 100644 --- a/src/coreclr/src/vm/readytoruninfo.cpp +++ b/src/coreclr/src/vm/readytoruninfo.cpp @@ -618,6 +618,11 @@ PTR_ReadyToRunInfo ReadyToRunInfo::Initialize(Module * pModule, AllocMemTracker return new (pMemory) ReadyToRunInfo(pModule, pLayout, pHeader, nativeImage, pamTracker); } +bool ReadyToRunInfo::IsNativeImageSharedBy(PTR_Module pModule1, PTR_Module pModule2) +{ + return pModule1->GetReadyToRunInfo()->m_pComposite == pModule2->GetReadyToRunInfo()->m_pComposite; +} + ReadyToRunInfo::ReadyToRunInfo(Module * pModule, PEImageLayout * pLayout, READYTORUN_HEADER * pHeader, NativeImage *pNativeImage, AllocMemTracker *pamTracker) : m_pModule(pModule), m_pHeader(pHeader), diff --git a/src/coreclr/src/vm/readytoruninfo.h b/src/coreclr/src/vm/readytoruninfo.h index 34acae40d3c96f..f4b62b0d48fcfd 100644 --- a/src/coreclr/src/vm/readytoruninfo.h +++ b/src/coreclr/src/vm/readytoruninfo.h @@ -90,6 +90,8 @@ class ReadyToRunInfo bool IsComponentAssembly() const { return m_isComponentAssembly; } + static bool IsNativeImageSharedBy(PTR_Module pModule1, PTR_Module pModule2); + PTR_READYTORUN_HEADER GetReadyToRunHeader() const { return m_pHeader; } PTR_NativeImage GetNativeImage() const { return m_pNativeImage; } From adbe89195d9c8d160c4ef9e2bf079989559ede67 Mon Sep 17 00:00:00 2001 From: Tomas Date: Thu, 26 Mar 2020 20:19:21 +0100 Subject: [PATCH 5/9] Fix generation of Crossgen2 execution script --- src/coreclr/tests/src/CLRTest.CrossGen.targets | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/tests/src/CLRTest.CrossGen.targets b/src/coreclr/tests/src/CLRTest.CrossGen.targets index 1921d3d8fce119..d2b9c77ed47453 100644 --- a/src/coreclr/tests/src/CLRTest.CrossGen.targets +++ b/src/coreclr/tests/src/CLRTest.CrossGen.targets @@ -165,7 +165,7 @@ if defined RunCrossGen2 ( ) set __ResponseFile=!scriptPath!\composite-r2r.dll.rsp - del /Y !__ResponseFile! + del /Q !__ResponseFile! set __Command=!_DebuggerFullPath! set __Command=!__Command! "!CORE_ROOT!\CoreRun.exe" @@ -191,7 +191,7 @@ if defined RunCrossGen2 ( echo -r:!CORE_ROOT!\mscorlib.dll>>!__ResponseFile! echo -r:!CORE_ROOT!\netstandard.dll>>!__ResponseFile! - echo "Response file: !__ReponseFile!" + echo Response file: !__ResponseFile! type !__ResponseFile! echo "!__Command!" From fbe9749f4a1164d378fb7faadfc68cddd8dbc7f9 Mon Sep 17 00:00:00 2001 From: Tomas Date: Fri, 27 Mar 2020 10:04:27 +0100 Subject: [PATCH 6/9] Add instrumentation to limit the tests to a small subset --- src/coreclr/tests/src/dirs.proj | 3 +-- src/coreclr/tests/src/runtest.proj | 5 ----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/coreclr/tests/src/dirs.proj b/src/coreclr/tests/src/dirs.proj index 107e1f4cf29588..a5ba37fe37306d 100644 --- a/src/coreclr/tests/src/dirs.proj +++ b/src/coreclr/tests/src/dirs.proj @@ -19,8 +19,7 @@ - - +