Build the shim assemblies as part of libs.sfx (#89005)

* Build the shim assemblies as part of libs.sfx

The shims build was intentionally kept out of the libs.sfx subset as
some shims reference out-of-band assemblies that also need to be built.

Since that change was made, the shim assemblies now don't reference all out-of-band
assemblies by default anymore and instead use fine grained dependencies.
Because of that, the number of out-of-band projects referenced and built
as part of the shims build is much smaller (~10 projects).

This fixes issues with ouf-of-band source generators not being able to
target the live targeting pack. These source generators need the shims
(more precisely the netstandard.dll shims) as some of the referenced
Microsoft.CodeAnalysis packages don't provide a .NETCoreApp assembly.

* Add fake assemblies for out-of-band type forward destinations

* Fix issues with nested classes and wrong type destinations

* Disable binplacing

* Make stubs private

* Disable symbols for stubs

* Add build protection to validate API compatibility of shims

Author: ViktorHofer

src/libraries/Directory.Build.targets

@@ -111,10 +111,11 @@
   <Import Project="$(RepositoryEngineeringDir)targetingpacks.targets" />
 
   <PropertyGroup>
-      <!-- Libraries non test projects shouldn't reference compat shims. -->
+      <!-- Libraries ref and source projects which don't bring in dependencies from outside the repository shouldn't reference compat shims. -->
     <SkipTargetingPackShimReferences Condition="'$(UseLocalTargetingRuntimePack)' == 'true' and
                                                 '$(IsTestProject)' != 'true' and
-                                                '$(IsTestSupportProject)' != 'true'">true</SkipTargetingPackShimReferences>
+                                                '$(IsTestSupportProject)' != 'true' and
+                                                '$(IsGeneratorProject)' != 'true'">true</SkipTargetingPackShimReferences>
   </PropertyGroup>
 
   <Import Project="$(RepositoryEngineeringDir)codeOptimization.targets" />

src/libraries/System.Windows.Extensions/ref/System.Windows.Extensions.csproj

@@ -1,6 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
+
   <PropertyGroup>
     <TargetFrameworks>$(NetCoreAppCurrent);$(NetCoreAppPrevious);$(NetCoreAppMinimum)</TargetFrameworks>
+    <SkipValidateReferenceAssemblyProjectReferences>true</SkipValidateReferenceAssemblyProjectReferences>
   </PropertyGroup>
 
   <ItemGroup>
@@ -9,7 +11,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <!-- Avoid a PackageReference to System.Drawing.Common this reference is only needed for TypeForwards -->
-    <PackageReference Include="System.Drawing.Common" Version="$(SystemDrawingCommonVersion)" PrivateAssets="All" />
+    <!-- Avoid a product dependency to System.Drawing.Common, this reference is only needed for TypeForwards -->
+    <ProjectReference Include="$(LibrariesProjectRoot)shims\stubs\System.Drawing.Common.csproj" PrivateAssets="all" />
   </ItemGroup>
+
 </Project>

src/libraries/System.Windows.Extensions/src/System.Windows.Extensions.csproj

@@ -1,4 +1,5 @@
 <Project Sdk="Microsoft.NET.Sdk">
+
   <PropertyGroup>
     <TargetFrameworks>$(NetCoreAppCurrent)-windows;$(NetCoreAppCurrent);$(NetCoreAppPrevious)-windows;$(NetCoreAppPrevious);$(NetCoreAppMinimum)-windows;$(NetCoreAppMinimum)</TargetFrameworks>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
@@ -93,7 +94,8 @@ System.Security.Cryptography.X509Certificates.X509SelectionFlag</PackageDescript
   </ItemGroup>
 
   <ItemGroup>
-    <!-- Avoid a PackageReference to System.Drawing.Common this reference is only needed for TypeForwards -->
-    <PackageReference Include="System.Drawing.Common" Version="$(SystemDrawingCommonVersion)" PrivateAssets="All" />
+    <!-- Avoid a product dependency to System.Drawing.Common, this reference is only needed for TypeForwards -->
+    <ProjectReference Include="$(LibrariesProjectRoot)shims\stubs\System.Drawing.Common.csproj" PrivateAssets="all" />
   </ItemGroup>
+
 </Project>

src/libraries/apicompat/ApiCompat.proj

@@ -33,7 +33,6 @@
   <!-- For API comparison, ApiCompat needs the inbox assemblies (including shims) and to resolve type forwards, the out-of-band assemblies. -->
   <ItemGroup>
     <ProjectReference Include="..\sfx-src.proj" OutputItemType="SharedFrameworkAssembly" />
-    <ProjectReference Include="..\shims.proj" OutputItemType="SharedFrameworkAssembly" />
     <ProjectReference Include="..\oob-src.proj" OutputItemType="OOBAssembly" />
   </ItemGroup>
 
@@ -42,18 +41,52 @@
     <PackageDownload Include="NETStandard.Library.Ref" Version="[$(NETStandardLibraryRefVersion)]" />
     <PackageDownload Include="NETStandard.Library" Version="[$(NetStandardLibraryVersion)]" />
     <PackageReference Include="Microsoft.DotNet.ApiCompat.Task" Version="$(MicrosoftDotNetApiCompatTaskVersion)" IsImplicitlyDefined="true" />
+
+    <!-- Restore and reference assemblies not built in runtime anymore, required for resolving type forward on both comparison sides. -->
+    <ApiCompatTypeForwardDestinationPackage Include="System.Data.SqlClient" PackageVersion="$(SystemDataSqlClientVersion)" TargetFramework="netcoreapp2.1" />
+    <ApiCompatTypeForwardDestinationPackage Include="System.Drawing.Common" PackageVersion="$(SystemDrawingCommonVersion)" TargetFramework="net7.0" />
+
+    <!-- Restore and reference assemblies required for resolving type forwards on the baseline (left) side. -->
+    <ApiCompatLeftTypeForwardDestinationPackage Include="@(ApiCompatTypeForwardDestinationPackage)"  />
+    <ApiCompatLeftTypeForwardDestinationPackage
+      Include="
+        Microsoft.Win32.SystemEvents;
+        System.CodeDom;
+        System.Configuration.ConfigurationManager;
+        System.Data.Odbc;
+        System.Data.OleDb;
+        System.Diagnostics.EventLog;
+        System.Diagnostics.PerformanceCounter;
+        System.IO.Packaging;
+        System.IO.Ports;
+        System.Runtime.Serialization.Schema;
+        System.Security.Cryptography.Pkcs;
+        System.Security.Cryptography.ProtectedData;
+        System.Security.Cryptography.Xml;
+        System.Security.Permissions;
+        System.ServiceModel.Syndication;
+        System.ServiceProcess.ServiceController;
+        System.Threading.AccessControl;
+        System.Windows.Extensions"
+      PackageVersion="$(ApiCompatNetCoreAppBaselineVersion)"
+      TargetFramework="$(ApiCompatNetCoreAppBaselineTFM)" />
+    <PackageDownload Include="@(ApiCompatLeftTypeForwardDestinationPackage)" Version="[%(PackageVersion)]" />
   </ItemGroup>
 
   <Target Name="RunApiCompat"
           DependsOnTargets="FindReferenceAssembliesForReferences;CollectApiCompatInputs"
           AfterTargets="Build"
-          Inputs="@(ReferencePathWithRefAssemblies);$(ApiCompatNetStandard20BaselineFile);$(ApiCompatNetStandard21BaselineFile);$(ApiCompatNetCoreAppLatestStableBaselineFile);@(ApiCompatExcludeAttributesFile)"
+          Inputs="$(MSBuildThisFileFullPath);@(ReferencePathWithRefAssemblies);$(ApiCompatNetStandard20BaselineFile);$(ApiCompatNetStandard21BaselineFile);$(ApiCompatNetCoreAppLatestStableBaselineFile);@(ApiCompatExcludeAttributesFile)"
           Outputs="$(IntermediateOutputPath)$(TargetArchitecture)-marker.txt">
     <Message Text="ApiCompat -> Comparing $(NetCoreAppCurrent) reference assemblies against .NETStandard 2.x and .NETCoreApp $(ApiCompatNetCoreAppBaselineVersion)..." Importance="high"  />
 
     <ItemGroup>
+      <ApiCompatNetCoreAppLatestStableAssemblyReference Include="@(ApiCompatLeftTypeForwardDestinationPackage->'$(NuGetPackageRoot)$([System.String]::new('%(ApiCompatLeftTypeForwardDestinationPackage.Identity)').ToLowerInvariant())\%(ApiCompatLeftTypeForwardDestinationPackage.PackageVersion)\lib\%(ApiCompatLeftTypeForwardDestinationPackage.TargetFramework)\%(ApiCompatLeftTypeForwardDestinationPackage.Identity).dll')" />
+
       <ApiCompatRightAssembly Include="@(ReferencePathWithRefAssemblies->WithMetadataValue('IsPrivateAssembly', 'false')->WithMetadataValue('OutputItemType', 'SharedFrameworkAssembly'))" />
-      <ApiCompatRightAssemblyReference Include="@(ReferencePathWithRefAssemblies->WithMetadataValue('IsPrivateAssembly', 'false')->WithMetadataValue('OutputItemType', 'OOBAssembly'), ',')" />
+      <ApiCompatRightAssemblyReferenceFromPackage Include="@(ApiCompatTypeForwardDestinationPackage->'$(NuGetPackageRoot)$([System.String]::new('%(ApiCompatTypeForwardDestinationPackage.Identity)').ToLowerInvariant())\%(ApiCompatTypeForwardDestinationPackage.PackageVersion)\lib\%(ApiCompatTypeForwardDestinationPackage.TargetFramework)\%(ApiCompatTypeForwardDestinationPackage.Identity).dll')" />
+      <ApiCompatRightAssemblyReference Include="@(ReferencePathWithRefAssemblies->WithMetadataValue('IsPrivateAssembly', 'false')->WithMetadataValue('OutputItemType', 'OOBAssembly'));
+                                                @(ApiCompatRightAssemblyReferenceFromPackage)" />
     </ItemGroup>
 
     <!-- ApiCompat: NetCoreAppCurrent <-> .NETCoreApp ApiCompatNetCoreAppBaselineVersion -->
@@ -68,7 +101,8 @@
       ExcludeAttributesFiles="@(ApiCompatExcludeAttributesFile)"
       LeftAssembliesTransformationPattern="@(ApiCompatNetCoreAppLatestStableLeftAssembliesTransformationPattern)"
       RightAssembliesTransformationPattern="@(ApiCompatNetCoreAppCurrentRightAssembliesTransformationPattern)"
-      RightAssembliesReferences="@(ApiCompatRightAssemblyReference)" />
+      LeftAssembliesReferences="@(ApiCompatNetCoreAppLatestStableAssemblyReference, ',')"
+      RightAssembliesReferences="@(ApiCompatRightAssemblyReference, ',')" />
 
     <!-- ApiCompat: NetCoreAppCurrent <-> netstandard2.1 -->
     <Microsoft.DotNet.ApiCompat.Task.ValidateAssembliesTask
@@ -81,7 +115,7 @@
       ExcludeAttributesFiles="@(ApiCompatExcludeAttributesFile)"
       LeftAssembliesTransformationPattern="@(ApiCompatNetStandard21LeftAssembliesTransformationPattern)"
       RightAssembliesTransformationPattern="@(ApiCompatNetCoreAppCurrentRightAssembliesTransformationPattern)"
-      RightAssembliesReferences="@(ApiCompatRightAssemblyReference)" />
+      RightAssembliesReferences="@(ApiCompatRightAssemblyReference, ',')" />
 
     <!-- ApiCompat: NetCoreAppCurrent <-> netstandard2.0 -->
     <Microsoft.DotNet.ApiCompat.Task.ValidateAssembliesTask
@@ -94,7 +128,7 @@
       ExcludeAttributesFiles="@(ApiCompatExcludeAttributesFile)"
       LeftAssembliesTransformationPattern="@(ApiCompatNetStandard20LeftAssembliesTransformationPattern)"
       RightAssembliesTransformationPattern="@(ApiCompatNetCoreAppCurrentRightAssembliesTransformationPattern)"
-      RightAssembliesReferences="@(ApiCompatRightAssemblyReference)" />
+      RightAssembliesReferences="@(ApiCompatRightAssemblyReference, ',')" />
 
     <!-- Create a marker file which serves as the target's output to enable incremental builds. -->
     <Touch Files="$(IntermediateOutputPath)$(TargetArchitecture)-marker.txt"

src/libraries/frameworklist.targets

@@ -7,10 +7,8 @@
   </PropertyGroup>
 
   <ItemGroup Condition="'$(BuildTargetFramework)' == '$(NetCoreAppCurrent)' or '$(BuildTargetFramework)' == ''">
-    <ProjectReference Include="sfx-gen.proj" OutputItemType="SharedFrameworkAnalyzerAssembly" />
     <ProjectReference Include="sfx-src.proj" OutputItemType="SharedFrameworkAssembly" />
     <ProjectReference Include="oob-src.proj" OutputItemType="OOBAssembly" />
-    <ProjectReference Include="shims.proj" OutputItemType="SharedFrameworkAssembly" />
 
     <ProjectReference Include="apicompat\ApiCompat.proj"
                       Condition="'$(DotNetBuildFromSource)' != 'true' and '$(ApiCompatValidateAssemblies)' != 'false'" />
@@ -23,15 +21,11 @@
   <!-- Support building reference projects only. -->
   <ItemGroup Condition="'$(RefOnly)' == 'true'">
     <ProjectReference Remove="@(ProjectReference)" />
-    <ProjectReference Include="sfx-gen.proj" OutputItemType="SharedFrameworkAnalyzerAssembly" />
     <ProjectReference Include="sfx-ref.proj" OutputItemType="SharedFrameworkAssembly" />
     <ProjectReference Include="oob-ref.proj" />
-    <ProjectReference Include="shims.proj" OutputItemType="SharedFrameworkAssembly" />
   </ItemGroup>
 
   <ImportGroup Condition="'$(BuildTargetFramework)' == '$(NetCoreAppCurrent)' or '$(BuildTargetFramework)' == ''">
-    <!-- Re-generate the targeting pack's framework list to include shims. -->
-    <Import Project="frameworklist.targets" />
     <!-- Import the illink file which contains some of the logic required to illink the out-of-band assemblies. -->
     <Import Project="$(RepositoryEngineeringDir)illink.targets" />
   </ImportGroup>

src/libraries/oob.proj

@@ -17,4 +17,12 @@
                                System.Private.CoreLib\ref\System.Private.CoreLib.csproj" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(RefOnly)' == 'true'">
+    <ProjectReference Include="shims\*\src\*.csproj" />
+    <ReferenceShimProject Include="shims\*\ref\*.csproj" />
+    <ProjectReference Include="@(ReferenceShimProject)" />
+    <!-- Omit projects which depend on source projects to be built. -->
+    <ProjectReference Remove="@(ReferenceShimProject->'shims\%(Filename)\src\%(Filename)%(Extension)')" />
+  </ItemGroup>
+
 </Project>

src/libraries/sfx-ref.proj

@@ -15,7 +15,8 @@
                           Exclude="@(NetCoreAppLibrary->'%(Identity)\src\%(Identity).csproj');
                                    Microsoft.VisualBasic.Core\src\Microsoft.VisualBasic.Core.vbproj" />
     <ProjectReference Include="@(AnyProject);
-                               $(CoreLibProject)"
+                               $(CoreLibProject);
+                               shims\*\src\*.csproj"
                       Exclude="@(NonNetCoreAppProject)" />
   </ItemGroup>
 

src/libraries/sfx-src.proj

@@ -1,8 +1,12 @@
 <Project Sdk="Microsoft.Build.NoTargets">
 
+  <Sdk Name="Microsoft.DotNet.SharedFramework.Sdk" />
+
   <PropertyGroup>
     <TargetFramework>$(NetCoreAppCurrent)-$(TargetOS)</TargetFramework>
     <BuildInParallel>false</BuildInParallel>
+    <IsPackable>false</IsPackable>
+    <FrameworkListOutputPath>$(MicrosoftNetCoreAppRefPackDataDir)FrameworkList.xml</FrameworkListOutputPath>
   </PropertyGroup>
 
   <ItemGroup>
@@ -11,16 +15,47 @@
     <ProjectReference Include="sfx-ref.proj">
       <OutputItemType Condition="'$(RefOnly)' == 'true'">SharedFrameworkAssembly</OutputItemType>
     </ProjectReference>
+     <!-- The RefOnly condition exists to support building the reference source projects only. -->
+    <ProjectReference Include="sfx-src.proj" OutputItemType="SharedFrameworkAssembly" Condition="'$(RefOnly)' != 'true'" />
     <ProjectReference Include="sfx-gen.proj" OutputItemType="SharedFrameworkAnalyzerAssembly" />
   </ItemGroup>
 
-  <!-- Support building only the reference assemblies. -->
-  <ItemGroup Condition="'$(RefOnly)' != 'true'">
-    <ProjectReference Include="sfx-src.proj" OutputItemType="SharedFrameworkAssembly" />
-  </ItemGroup>
+  <Target Name="GetGenerateFrameworkListFileInputs">
+    <Error Text="Shared framework assemblies must be built before the framework list can be generated."
+           Condition="'@(SharedFrameworkAssembly)' == '' or '@(SharedFrameworkAnalyzerAssembly)' == ''" />
+
+    <ItemGroup>
+      <SharedFrameworkAssemblyWithReferenceAssembly Include="@(SharedFrameworkAssembly->WithMetadataValue('IsPrivateAssembly', 'false')->HasMetadata('ReferenceAssembly'))" />
+      <SharedFrameworkAssemblyWithoutReferenceAssembly Include="@(SharedFrameworkAssembly->WithMetadataValue('IsPrivateAssembly', 'false'))"
+                                                       Exclude="@(SharedFrameworkAssemblyWithReferenceAssembly)" />
+
+      <FrameworkListInputFile Include="@(SharedFrameworkAssemblyWithReferenceAssembly->Metadata('ReferenceAssembly'));
+                                       @(SharedFrameworkAssemblyWithoutReferenceAssembly)"
+                              TargetPath="ref/$(NetCoreAppCurrent)" />
+      <FrameworkListInputFile Include="@(SharedFrameworkAnalyzerAssembly)"
+                              TargetPath="%(SharedFrameworkAnalyzerAssembly.GeneratorProjectBaseTargetPath)" />
+    </ItemGroup>
+  </Target>
 
   <!-- Generate the targeting pack's framework list so that out-of-band projects can leverage it. -->
-  <Import Project="frameworklist.targets" />
+  <UsingTask TaskName="CreateFrameworkListFile" AssemblyFile="$(DotNetSharedFrameworkTaskFile)"/>
+  <Target Name="GenerateFrameworkListFile"
+          DependsOnTargets="GetGenerateFrameworkListFileInputs"
+          AfterTargets="Build"
+          Inputs="@(FrameworkListInputFile)"
+          Outputs="$(FrameworkListOutputPath)">
+    <ItemGroup>
+      <FrameworkListRootAttribute Include="Name" Value="$(NetCoreAppCurrentBrandName)" />
+      <FrameworkListRootAttribute Include="TargetFrameworkIdentifier" Value="$(NetCoreAppCurrentIdentifier)" />
+      <FrameworkListRootAttribute Include="TargetFrameworkVersion" Value="$(NetCoreAppCurrentVersion)" />
+      <FrameworkListRootAttribute Include="FrameworkName" Value="$(MicrosoftNetCoreAppFrameworkName)" />
+    </ItemGroup>
+
+    <CreateFrameworkListFile Files="@(FrameworkListInputFile)"
+                             TargetFile="$(FrameworkListOutputPath)"
+                             TargetFilePrefixes="ref/;runtimes/;analyzers/"
+                             RootAttributes="@(FrameworkListRootAttribute)" />
+  </Target>
 
   <!-- Copy the intellisense file to the targeting pack directory. -->
   <Target Name="CopyDocumentationFiles"