[wasm] Use source built emsdk packages for the runtime build (#100266)

steveisok · akoeplinger · radekdoulik · web-flow · commit 52638488c45e · 2024-05-09T16:31:41.000+02:00
* Use the emsdk transport packages to build wasm instead of a cloned emsdk Works for the most part out of the box, but fails on the part where we npm install after the libraries build. The reason for the failure is our node transport package is incomplete. If you replace it with a legit node, the build works fully. * Pull in node transport package as opposed to the one packaged in emsdk. Streamline package deps * Make paths friendly for windows * Work in windows transport packages, copy python to its own directory, and adjust a bunch of paths * Bump to latest version of node that contains icu * Add windows deps and add DOTNET_EMSCRIPTEN_NODE_PATH because windows can't see npm without it * Bump emscripten packages to the latest * Fix typos in Version.Details.xml * Switch out to the plain mariner container to ensure no EMSDK already exists * Revert back to the browser-wasm docker image and bump to the latest * Container type-o * Fix condition where the node path isn't set properly * Provision even when building offsets * Fix goofy paths * Update new template to use the latest browser-wasm container * Don't put python.exe on the path for windows, but the folder instead * So that was a bad idea. May have to have emsdk_env.cmd have two entries * Fix offsets generation for browser * Fix emsdk's python path It is not versioned anymore as we use the content of our nugets * Fix EMSDK_PATH on non-windows platforms * Do not add link flags to compile flags To avoid: C:\helix\work\correlation\build\wasm-shared\WasmApp.Common.targets(825,5): error : emcc: warning: linker setting ignored during compilation: 'EXPORT_ES6' [-Wunused-command-line-argument] [C:\helix\work\workitem\e\publish\ProxyProjectForAOTOnHelix.proj] C:\helix\work\correlation\build\wasm-shared\WasmApp.Common.targets(825,5): error : emcc: warning: linker setting ignored during compilation: 'EXPORT_EXCEPTION_HANDLING_HELPERS' [-Wunused-command-line-argument] [C:\helix\work\workitem\e\publish\ProxyProjectForAOTOnHelix.proj] * Set the emsdk paths relative to the script location * Escape few characters and fix the added script * Fix .emscripten script * Use EM_CONFIG intead of __file__ * Fix emsdk_env.cmd script * Revert "Do not add link flags to compile flags" This reverts commit eb19ade. * Revert changes in package-lock.json * Feedback + cleaning * More cleaning * Try not to use the replace in the EMSDK_PATH * Better property names * Use package and emsdk version properties Co-authored-by: Larry Ewing <lewing@microsoft.com> * Use the updated properties * Fix emsdk version * Use a single version of emsdk and take node version from flow * Update eng/Version.Details.xml Co-authored-by: Larry Ewing <lewing@microsoft.com> --------- Co-authored-by: Alexander Köplinger <alex.koeplinger@outlook.com> Co-authored-by: Radek Doulik <radek.doulik@gmail.com> Co-authored-by: Radek Doulik <radekdoulik@gmail.com> Co-authored-by: Larry Ewing <lewing@microsoft.com>
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
@@ -414,5 +414,37 @@
       <Uri>https://github.com/dotnet/installer</Uri>
       <Sha>fa261b952d702c6bd604728fcbdb58ac071a22b1</Sha>
     </Dependency>
+    <Dependency Name="runtime.linux-arm64.Microsoft.NETCore.Runtime.Wasm.Node.Transport" Version="9.0.0-alpha.1.24175.1">
+      <Uri>https://github.com/dotnet/node</Uri>
+      <Sha>308c7d0f1fa19bd1e7b768ad13646f5206133cdb</Sha>
+    </Dependency>
+    <Dependency Name="runtime.linux-musl-arm64.Microsoft.NETCore.Runtime.Wasm.Node.Transport" Version="9.0.0-alpha.1.24175.1">
+      <Uri>https://github.com/dotnet/node</Uri>
+      <Sha>308c7d0f1fa19bd1e7b768ad13646f5206133cdb</Sha>
+    </Dependency>
+    <Dependency Name="runtime.linux-x64.Microsoft.NETCore.Runtime.Wasm.Node.Transport" Version="9.0.0-alpha.1.24175.1">
+      <Uri>https://github.com/dotnet/node</Uri>
+      <Sha>308c7d0f1fa19bd1e7b768ad13646f5206133cdb</Sha>
+    </Dependency>
+    <Dependency Name="runtime.linux-musl-x64.Microsoft.NETCore.Runtime.Wasm.Node.Transport" Version="9.0.0-alpha.1.24175.1">
+      <Uri>https://github.com/dotnet/node</Uri>
+      <Sha>308c7d0f1fa19bd1e7b768ad13646f5206133cdb</Sha>
+    </Dependency>
+    <Dependency Name="runtime.osx-arm64.Microsoft.NETCore.Runtime.Wasm.Node.Transport" Version="9.0.0-alpha.1.24175.1">
+      <Uri>https://github.com/dotnet/node</Uri>
+      <Sha>308c7d0f1fa19bd1e7b768ad13646f5206133cdb</Sha>
+    </Dependency>
+    <Dependency Name="runtime.osx-x64.Microsoft.NETCore.Runtime.Wasm.Node.Transport" Version="9.0.0-alpha.1.24175.1">
+      <Uri>https://github.com/dotnet/node</Uri>
+      <Sha>308c7d0f1fa19bd1e7b768ad13646f5206133cdb</Sha>
+    </Dependency>
+    <Dependency Name="runtime.win-arm64.Microsoft.NETCore.Runtime.Wasm.Node.Transport" Version="9.0.0-alpha.1.24175.1">
+      <Uri>https://github.com/dotnet/node</Uri>
+      <Sha>308c7d0f1fa19bd1e7b768ad13646f5206133cdb</Sha>
+    </Dependency>
+    <Dependency Name="runtime.win-x64.Microsoft.NETCore.Runtime.Wasm.Node.Transport" Version="9.0.0-alpha.1.24175.1">
+      <Uri>https://github.com/dotnet/node</Uri>
+      <Sha>308c7d0f1fa19bd1e7b768ad13646f5206133cdb</Sha>
+    </Dependency>
   </ToolsetDependencies>
 </Dependencies>
diff --git a/eng/Versions.props b/eng/Versions.props
@@ -256,5 +256,9 @@
     <!-- installer version, for testing workloads -->
     <MicrosoftDotnetSdkInternalVersion>9.0.100-preview.5.24253.16</MicrosoftDotnetSdkInternalVersion>
     <SdkVersionForWorkloadTesting>$(MicrosoftDotnetSdkInternalVersion)</SdkVersionForWorkloadTesting>
+    <runtimewinx64MicrosoftNETCoreRuntimeWasmNodeTransportPackageVersion>9.0.0-alpha.1.24175.1</runtimewinx64MicrosoftNETCoreRuntimeWasmNodeTransportPackageVersion>
+    <EmsdkPackageVersion>$(MicrosoftNETRuntimeEmscriptenVersion)</EmsdkPackageVersion>
+    <NodePackageVersion>$(runtimewinx64MicrosoftNETCoreRuntimeWasmNodeTransportPackageVersion)</NodePackageVersion>
+    <EmsdkVersion>3.1.34</EmsdkVersion>
   </PropertyGroup>
 </Project>
diff --git a/eng/native/gen-buildsys.cmd b/eng/native/gen-buildsys.cmd
@@ -56,7 +56,7 @@ if /i "%__Arch%" == "wasm" (
         set "EMSDK_PATH=!EMSDK_PATH:\=/!"
         if not "!EMSDK_PATH:~-1!" == "/" set "EMSDK_PATH=!EMSDK_PATH!/"
 
-        set __ExtraCmakeParams=%__ExtraCmakeParams% "-DCMAKE_TOOLCHAIN_FILE=!EMSDK_PATH!/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake"
+        set __ExtraCmakeParams=%__ExtraCmakeParams% "-DCMAKE_TOOLCHAIN_FILE=!EMSDK_PATH!/emscripten/cmake/Modules/Platform/Emscripten.cmake"
         set __UseEmcmake=1
     )
     if /i "%__Os%" == "wasi" (
@@ -107,7 +107,7 @@ if not "%__ConfigureOnly%" == "1" (
 )
 
 if /i "%__UseEmcmake%" == "1" (
-    call "!EMSDK_PATH!/emsdk_env.bat" > nul 2>&1 && emcmake "%CMakePath%" %__ExtraCmakeParams% --no-warn-unused-cli -G "%__CmakeGenerator%" -B %__IntermediatesDir% -S %__SourceDir%
+    call "!EMSDK_PATH!/emsdk_env.cmd" > nul 2>&1 && emcmake "%CMakePath%" %__ExtraCmakeParams% --no-warn-unused-cli -G "%__CmakeGenerator%" -B %__IntermediatesDir% -S %__SourceDir%
 ) else (
     "%CMakePath%" %__ExtraCmakeParams% --no-warn-unused-cli -G "%__CmakeGenerator%" -B %__IntermediatesDir% -S %__SourceDir%
 )
diff --git a/eng/pipelines/common/templates/pipeline-with-resources.yml b/eng/pipelines/common/templates/pipeline-with-resources.yml
@@ -100,7 +100,7 @@ extends:
         image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8
 
       browser_wasm:
-        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly
         env:
           ROOTFS_DIR: /crossrootfs/x64
 
diff --git a/src/libraries/sendtohelix-browser.targets b/src/libraries/sendtohelix-browser.targets
@@ -52,8 +52,7 @@
     <IncludeHelixCorrelationPayload>false</IncludeHelixCorrelationPayload>
     <EnableDefaultBuildHelixWorkItems>false</EnableDefaultBuildHelixWorkItems>
 
-    <!-- on unix CI has emscripten provisioned in $(EMSDK_PATH) as `/usr/local/emscripten`. -->
-    <EMSDK_PATH Condition="$([MSBuild]::IsOSPlatform('WINDOWS')) and '$(EMSDK_PATH)' == ''">$(RepoRoot)src\mono\browser\emsdk\</EMSDK_PATH>
+    <EMSDK_PATH Condition="'$(EMSDK_PATH)' == ''">$([MSBuild]::NormalizeDirectory($(RepoRoot), 'src', 'mono', 'browser', 'emsdk'))</EMSDK_PATH>
     <EmSdkDirForHelixPayload>$(HelixDependenciesStagingPath)\emsdk</EmSdkDirForHelixPayload>
     <DebuggerHost Condition="'$(DebuggerHost)' == ''">chrome</DebuggerHost>
 
diff --git a/src/mono/browser/Makefile b/src/mono/browser/Makefile
@@ -167,4 +167,4 @@ build-app-host:
 	$(DOTNET) build $(TOP)/src/mono/wasm/host $(_MSBUILD_WASM_BUILD_ARGS) $(MSBUILD_ARGS)
 
 patch-deterministic:
-	cd emsdk/upstream/emscripten/ && patch -p1 < ../../../runtime/deterministic.diff
+	cd emsdk/emscripten/ && patch -p1 < ../../../runtime/deterministic.diff
diff --git a/src/mono/browser/browser.proj b/src/mono/browser/browser.proj
@@ -33,7 +33,7 @@
     <_EmccDefaultsRspPath>$(NativeBinDir)src\emcc-default.rsp</_EmccDefaultsRspPath>
     <_EmccCompileRspPath>$(NativeBinDir)src\emcc-compile.rsp</_EmccCompileRspPath>
     <_EmccLinkRspPath>$(NativeBinDir)src\emcc-link.rsp</_EmccLinkRspPath>
-    <EmSdkLLVMAr>$(EMSDK_PATH)\upstream\bin\llvm-ar</EmSdkLLVMAr>
+    <EmSdkLLVMAr>$(EMSDK_PATH)\bin\llvm-ar</EmSdkLLVMAr>
     <EmSdkLLVMAr Condition="$([MSBuild]::IsOSPlatform('Windows'))">$(EmSdkLLVMAr).exe</EmSdkLLVMAr>
   </PropertyGroup>
 
@@ -397,12 +397,12 @@
       <CMakeBuildRuntimeConfigureCmd Condition="'$(WasmEnableJsInteropByValue)' == 'true'">$(CMakeBuildRuntimeConfigureCmd) -DENABLE_JS_INTEROP_BY_VALUE=1</CMakeBuildRuntimeConfigureCmd>
       <CMakeBuildRuntimeConfigureCmd>$(CMakeBuildRuntimeConfigureCmd) $(CMakeConfigurationEmsdkPath)</CMakeBuildRuntimeConfigureCmd>
 
-      <CMakeBuildRuntimeConfigureCmd Condition="'$(OS)' == 'Windows_NT'">call &quot;$(RepositoryEngineeringDir)native\init-vs-env.cmd&quot; &amp;&amp; call &quot;$([MSBuild]::NormalizePath('$(EMSDK_PATH)', 'emsdk_env.bat'))&quot; &amp;&amp; $(CMakeBuildRuntimeConfigureCmd)</CMakeBuildRuntimeConfigureCmd>
+      <CMakeBuildRuntimeConfigureCmd Condition="'$(OS)' == 'Windows_NT'">call &quot;$(RepositoryEngineeringDir)native\init-vs-env.cmd&quot; &amp;&amp; call &quot;$([MSBuild]::NormalizePath('$(EMSDK_PATH)', 'emsdk_env.cmd'))&quot; &amp;&amp; $(CMakeBuildRuntimeConfigureCmd)</CMakeBuildRuntimeConfigureCmd>
       <CMakeBuildRuntimeConfigureCmd Condition="'$(OS)' != 'Windows_NT'">bash -c 'source $(EMSDK_PATH)/emsdk_env.sh 2>&amp;1 &amp;&amp; $(CMakeBuildRuntimeConfigureCmd)'</CMakeBuildRuntimeConfigureCmd>
 
       <CMakeOptions Condition="'$(MonoVerboseBuild)' != ''">-v</CMakeOptions>
       <CMakeBuildRuntimeCmd>cmake --build . --config $(Configuration) $(CmakeOptions)</CMakeBuildRuntimeCmd>
-      <CMakeBuildRuntimeCmd Condition="'$(OS)' == 'Windows_NT'">call &quot;$(RepositoryEngineeringDir)native\init-vs-env.cmd&quot; &amp;&amp; call &quot;$([MSBuild]::NormalizePath('$(EMSDK_PATH)', 'emsdk_env.bat'))&quot; &amp;&amp; $(CMakeBuildRuntimeCmd)</CMakeBuildRuntimeCmd>
+      <CMakeBuildRuntimeCmd Condition="'$(OS)' == 'Windows_NT'">call &quot;$(RepositoryEngineeringDir)native\init-vs-env.cmd&quot; &amp;&amp; call &quot;$([MSBuild]::NormalizePath('$(EMSDK_PATH)', 'emsdk_env.cmd'))&quot; &amp;&amp; $(CMakeBuildRuntimeCmd)</CMakeBuildRuntimeCmd>
       <CMakeBuildRuntimeCmd Condition="'$(OS)' != 'Windows_NT'">bash -c 'source $(EMSDK_PATH)/emsdk_env.sh 2>&amp;1 &amp;&amp;  $(CMakeBuildRuntimeCmd)'</CMakeBuildRuntimeCmd>
     </PropertyGroup>
     <ItemGroup>
diff --git a/src/mono/browser/build/EmSdkRepo.Defaults.props b/src/mono/browser/build/EmSdkRepo.Defaults.props
@@ -4,25 +4,18 @@
 
   <PropertyGroup>
     <EmscriptenSdkToolsPath>$([MSBuild]::EnsureTrailingSlash($(EMSDK_PATH)))</EmscriptenSdkToolsPath>
-    <EmscriptenUpstreamBinPath>$([MSBuild]::NormalizeDirectory($(EmscriptenSdkToolsPath), 'upstream', 'bin'))</EmscriptenUpstreamBinPath>
-    <EmscriptenUpstreamEmscriptenPath>$([MSBuild]::NormalizeDirectory($(EmscriptenSdkToolsPath), 'upstream', 'emscripten'))</EmscriptenUpstreamEmscriptenPath>
-
-    <_NodeToolsBasePath>$(EmscriptenSdkToolsPath)node</_NodeToolsBasePath>
+    <EmscriptenUpstreamBinPath>$([MSBuild]::NormalizeDirectory($(EmscriptenSdkToolsPath), 'bin'))</EmscriptenUpstreamBinPath>
+    <EmscriptenUpstreamEmscriptenPath>$([MSBuild]::NormalizeDirectory($(EmscriptenSdkToolsPath), 'emscripten'))</EmscriptenUpstreamEmscriptenPath>
 
     <!-- gets the path like emsdk/python/3.7.4-2_64bit -->
-    <_NodeToolsVersionedPath Condition="Exists($(_NodeToolsBasePath))">$([System.IO.Directory]::GetDirectories($(_NodeToolsBasePath)))</_NodeToolsVersionedPath>
-    <EmscriptenNodeToolsPath Condition="'$(_NodeToolsVersionedPath)' != ''">$(_NodeToolsVersionedPath)</EmscriptenNodeToolsPath>
-    <EmscriptenNodeToolsPath Condition="'$(EmscriptenNodeToolsPath)' != ''">$([MSBuild]::NormalizeDirectory($(EmscriptenNodeToolsPath)))</EmscriptenNodeToolsPath>
+    <EmscriptenNodeToolsPath Condition="'$(EmscriptenNodeToolsPath)' == ''">$([MSBuild]::NormalizeDirectory($(EmscriptenSdkToolsPath), 'node', 'bin'))</EmscriptenNodeToolsPath>
 
     <_UsingEMSDK_PATH>true</_UsingEMSDK_PATH>
   </PropertyGroup>
 
   <!-- Emscripten uses system python on Linux, so we don't need $(EmscriptenPythonToolsPath) -->
   <PropertyGroup Condition="!$([MSBuild]::IsOSPlatform('linux'))">
-    <_PythonToolsBasePath>$(EmscriptenSdkToolsPath)python</_PythonToolsBasePath>
-    <_PythonToolsVersionedPath Condition="Exists($(_PythonToolsBasePath))">$([System.IO.Directory]::GetDirectories($(_PythonToolsBasePath)))</_PythonToolsVersionedPath>
-    <EmscriptenPythonToolsPath Condition="'$(_PythonToolsVersionedPath)' != ''">$(_PythonToolsVersionedPath)</EmscriptenPythonToolsPath>
-    <EmscriptenPythonToolsPath Condition="'$(EmscriptenPythonToolsPath)' != ''">$([MSBuild]::NormalizeDirectory($(EmscriptenPythonToolsPath)))</EmscriptenPythonToolsPath>
+    <EmscriptenPythonToolsPath>$([MSBuild]::NormalizeDirectory($(EmscriptenSdkToolsPath)python))</EmscriptenPythonToolsPath>
 
     <_EMSDKMissingPaths Condition="'$(_EMSDKMissingPaths)' == '' and ('$(EmscriptenPythonToolsPath)' == '' or !Exists('$(EmscriptenPythonToolsPath)'))">%24(EmscriptenPythonToolsPath)=$(EmscriptenPythonToolsPath) </_EMSDKMissingPaths>
   </PropertyGroup>
@@ -31,9 +24,9 @@
   <ItemGroup>
     <EmscriptenPrependPATH Include="$(EmscriptenSdkToolsPath)" />
     <EmscriptenPrependPATH Include="$(EmscriptenUpstreamBinPath)" />
-    <EmscriptenPrependPATH Include="$([MSBuild]::NormalizeDirectory($(EmscriptenSdkToolsPath), 'upstream', 'emscripten'))" />
+    <EmscriptenPrependPATH Include="$([MSBuild]::NormalizeDirectory($(EmscriptenSdkToolsPath), 'emscripten'))" />
 
-    <EmscriptenPrependPATH Include="$(EmscriptenNodeToolsPath)bin" />
+    <EmscriptenPrependPATH Include="$(EmscriptenNodeToolsPath)" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(EmscriptenPythonToolsPath)' != ''">
diff --git a/src/mono/browser/runtime/CMakeLists.txt b/src/mono/browser/runtime/CMakeLists.txt
@@ -44,7 +44,7 @@ set(ignoreMeEmsdkPath "${EMSDK_PATH}")
 
 if(CMAKE_BUILD_TYPE STREQUAL "Release")
     add_custom_command(TARGET dotnet.native
-                        POST_BUILD COMMAND ${EMSDK_PATH}/upstream/bin/wasm-opt --enable-exception-handling ${CONFIGURATION_WASM_OPT_FLAGS} --strip-dwarf ${NATIVE_BIN_DIR}/dotnet.native.wasm -o ${NATIVE_BIN_DIR}/dotnet.native.wasm
+                        POST_BUILD COMMAND ${EMSDK_PATH}/bin/wasm-opt --enable-exception-handling ${CONFIGURATION_WASM_OPT_FLAGS} --strip-dwarf ${NATIVE_BIN_DIR}/dotnet.native.wasm -o ${NATIVE_BIN_DIR}/dotnet.native.wasm
                         COMMENT "Stripping debug symbols from dotnet.native.wasm using wasm-opt")
 endif()
 
diff --git a/src/mono/browser/sanitize.py b/src/mono/browser/sanitize.py
@@ -32,7 +32,7 @@ def rewrite_package_json(path):
 
 
 emsdk_path = sys.argv[1]
-emscripten_path = os.path.join(emsdk_path, "upstream", "emscripten")
+emscripten_path = os.path.join(emsdk_path, "emscripten")
 node_root = os.path.join(emsdk_path, "node")
 node_paths = glob(node_root)
 upgrade = True
diff --git a/src/mono/mono.proj b/src/mono/mono.proj
diff --git a/src/tasks/WasmAppBuilder/RunWithEmSdkEnv.cs b/src/tasks/WasmAppBuilder/RunWithEmSdkEnv.cs

Original file line number	Diff line number	Diff line change
`@@ -56,7 +56,7 @@ if /i "%__Arch%" == "wasm" (`
`56`	`56`	`set "EMSDK_PATH=!EMSDK_PATH:\=/!"`
`57`	`57`	`if not "!EMSDK_PATH:~-1!" == "/" set "EMSDK_PATH=!EMSDK_PATH!/"`
`58`	`58`
`59`		`- set __ExtraCmakeParams=%__ExtraCmakeParams% "-DCMAKE_TOOLCHAIN_FILE=!EMSDK_PATH!/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake"`
	`59`	`+ set __ExtraCmakeParams=%__ExtraCmakeParams% "-DCMAKE_TOOLCHAIN_FILE=!EMSDK_PATH!/emscripten/cmake/Modules/Platform/Emscripten.cmake"`
`60`	`60`	`set __UseEmcmake=1`
`61`	`61`	`)`
`62`	`62`	`if /i "%__Os%" == "wasi" (`
`@@ -107,7 +107,7 @@ if not "%__ConfigureOnly%" == "1" (`
`107`	`107`	`)`
`108`	`108`
`109`	`109`	`if /i "%__UseEmcmake%" == "1" (`
`110`		`- call "!EMSDK_PATH!/emsdk_env.bat" > nul 2>&1 && emcmake "%CMakePath%" %__ExtraCmakeParams% --no-warn-unused-cli -G "%__CmakeGenerator%" -B %__IntermediatesDir% -S %__SourceDir%`
	`110`	`+ call "!EMSDK_PATH!/emsdk_env.cmd" > nul 2>&1 && emcmake "%CMakePath%" %__ExtraCmakeParams% --no-warn-unused-cli -G "%__CmakeGenerator%" -B %__IntermediatesDir% -S %__SourceDir%`
`111`	`111`	`) else (`
`112`	`112`	`"%CMakePath%" %__ExtraCmakeParams% --no-warn-unused-cli -G "%__CmakeGenerator%" -B %__IntermediatesDir% -S %__SourceDir%`
`113`	`113`	`)`