diff --git a/src/Microsoft.DotNet.XHarness.CLI/Commands/WASM/WasmTestMessagesProcessor.cs b/src/Microsoft.DotNet.XHarness.CLI/Commands/WASM/WasmTestMessagesProcessor.cs index 937264920..0c568c8ca 100644 --- a/src/Microsoft.DotNet.XHarness.CLI/Commands/WASM/WasmTestMessagesProcessor.cs +++ b/src/Microsoft.DotNet.XHarness.CLI/Commands/WASM/WasmTestMessagesProcessor.cs @@ -5,8 +5,8 @@ using System; using System.IO; using System.Text.Json; +using System.Text.RegularExpressions; using System.Threading.Tasks; -using System.Xml.Linq; using Microsoft.Extensions.Logging; #nullable enable @@ -14,8 +14,7 @@ namespace Microsoft.DotNet.XHarness.CLI.Commands.Wasm { public class WasmTestMessagesProcessor { - private MemoryStream? _xmlResultsMemoryStream; - private StreamWriter? _xmlResultsFileWriter; + private static Regex xmlRx = new Regex(@"^STARTRESULTXML ([0-9]*) ([^ ]*) ENDRESULTXML", RegexOptions.Compiled | RegexOptions.CultureInvariant); private readonly StreamWriter _stdoutFileWriter; private readonly string _xmlResultsFilePath; @@ -77,16 +76,27 @@ private void InvokeInternal(string message) line = message.TrimEnd(); } - if (_xmlResultsFileWriter == null) + var match = xmlRx.Match(line); + if (match.Success) { - if (line.Contains("STARTRESULTXML")) + var expectedLength = Int32.Parse(match.Groups[1].Value); + using (var stream = new FileStream(_xmlResultsFilePath, FileMode.Create)) { - _logger.LogDebug("Reached start of testResults.xml"); - _xmlResultsMemoryStream = new MemoryStream(); - _xmlResultsFileWriter = new StreamWriter(_xmlResultsMemoryStream); - return; + var bytes = System.Convert.FromBase64String(match.Groups[2].Value); + stream.Write(bytes); + if (bytes.Length == expectedLength) + { + _logger.LogInformation($"Received expected {bytes.Length} of {_xmlResultsFilePath}"); + } + else + { + _logger.LogInformation($"Received {bytes.Length} of {_xmlResultsFilePath} but expected {expectedLength}"); + } } - else if (line.StartsWith("[PASS]") || line.StartsWith("[SKIP]")) + } + else + { + if (line.StartsWith("[PASS]") || line.StartsWith("[SKIP]")) { _logger.LogDebug(line); } @@ -113,38 +123,6 @@ private void InvokeInternal(string message) if (_stdoutFileWriter.BaseStream.CanWrite) _stdoutFileWriter.WriteLine(line); } - else - { - if (line.Contains("ENDRESULTXML")) - { - _logger.LogDebug($"Reached end of {_xmlResultsFilePath}"); - _xmlResultsFileWriter.Flush(); - _xmlResultsMemoryStream!.Seek(0, SeekOrigin.Begin); - try - { - // we validate it, to make sure it's not corrupted. - var testResults = XElement.Load(_xmlResultsMemoryStream); - using (var testResultsFile = File.CreateText(_xmlResultsFilePath)) - { - testResults.Save(testResultsFile); - _logger.LogInformation($"Written {_xmlResultsMemoryStream.Length} original bytes of {_xmlResultsFilePath} as {testResultsFile.BaseStream.Length} formatted bytes"); - }; - } - catch (Exception ex) - { - _logger.LogError($"Error while saving testResults.xml {ex}"); - throw; - } - _xmlResultsFileWriter.Dispose(); - _xmlResultsFileWriter = null; - _xmlResultsMemoryStream.Dispose(); - _xmlResultsMemoryStream = null; - return; - } - - if (_xmlResultsFileWriter?.BaseStream.CanWrite == true) - _xmlResultsFileWriter.WriteLine(line); - } // the test runner writes this as the last line, // after the tests have run, and the xml results file diff --git a/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/ThreadlessXunitTestRunner.cs b/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/ThreadlessXunitTestRunner.cs index bb74f19bd..63148dd61 100644 --- a/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/ThreadlessXunitTestRunner.cs +++ b/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/ThreadlessXunitTestRunner.cs @@ -6,10 +6,10 @@ using System; using System.Collections.Generic; -using System.IO; +using System.IO; using System.Linq; using System.Reflection; -using System.Text; +using System.Text; using System.Threading; using System.Threading.Tasks; using System.Xml.Linq; @@ -21,87 +21,84 @@ namespace Microsoft.DotNet.XHarness.TestRunners.Xunit { internal class ThreadlessXunitTestRunner { - public static async Task Run(string assemblyFileName, bool printXml, XunitFilters filters) + public static async Task Run(string assemblyFileName, bool printXml, XunitFilters filters, bool oneLineResults = false) { - try - { - var configuration = new TestAssemblyConfiguration() { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false }; - var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); - var discoverySink = new TestDiscoverySink(); - var diagnosticSink = new ConsoleDiagnosticMessageSink(); - var testOptions = TestFrameworkOptions.ForExecution(configuration); - var testSink = new TestMessageSink(); - var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false); - - discoveryOptions.SetSynchronousMessageReporting(true); - testOptions.SetSynchronousMessageReporting(true); - - Console.WriteLine($"Discovering: {assemblyFileName} (method display = {discoveryOptions.GetMethodDisplayOrDefault()}, method display options = {discoveryOptions.GetMethodDisplayOptionsOrDefault()})"); - var assembly = Assembly.LoadFrom(assemblyFileName); - var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly); - var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink); - - discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions); - discoverySink.Finished.WaitOne(); - var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList(); - Console.WriteLine($"Discovered: {assemblyFileName} (found {testCasesToRun.Count} of {discoverySink.TestCases.Count} test cases)"); - - var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"{Environment.NewLine}=== TEST EXECUTION SUMMARY ==={Environment.NewLine}Total: {summary.Total}, Errors: 0, Failed: {summary.Failed}, Skipped: {summary.Skipped}, Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s{Environment.NewLine}"); }); - var resultsXmlAssembly = new XElement("assembly"); - var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); - - if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null) - { - testSink.Execution.TestStartingEvent += args => { Console.WriteLine($"[STRT] {args.Message.Test.DisplayName}"); }; - } - testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); }; - testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); }; - testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); }; - - testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Starting: {assemblyFileName}"); }; - testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished: {assemblyFileName}"); }; - - controller.RunTests(testCasesToRun, resultsSink, testOptions); - - while (!resultsSink.Finished.WaitOne(0)) - { - await Task.Delay(1); - } - - if (printXml) - { - var resultsXml = new XElement("assemblies"); - resultsXml.Add(resultsXmlAssembly); - using (var ms = new MemoryStream()) - { - using (var tw = new StreamWriter(ms, Encoding.UTF8, 100*1024, true)) - { - // no new line. make it single line, single WS message - resultsXml.Save(tw, SaveOptions.DisableFormatting); - } - ms.Seek(0, SeekOrigin.Begin); - using (var stdout = Console.OpenStandardOutput()) - { - using (var twc = new StreamWriter(stdout)) - { - twc.WriteLine($"STARTRESULTXML {ms.Length}"); - twc.Flush(); - ms.CopyTo(stdout, 100 * 1024); - twc.WriteLine(); - twc.WriteLine("ENDRESULTXML"); - } - } - Console.WriteLine($"Finished writing {ms.Length} bytes of RESULTXML"); - } - } - - var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; - return failed ? 1 : 0; - } - catch (Exception ex) - { - Console.Error.WriteLine($"ThreadlessXunitTestRunner failed: {ex}"); - return 2; + try + { + var configuration = new TestAssemblyConfiguration() { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false }; + var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); + var discoverySink = new TestDiscoverySink(); + var diagnosticSink = new ConsoleDiagnosticMessageSink(); + var testOptions = TestFrameworkOptions.ForExecution(configuration); + var testSink = new TestMessageSink(); + var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false); + + discoveryOptions.SetSynchronousMessageReporting(true); + testOptions.SetSynchronousMessageReporting(true); + + Console.WriteLine($"Discovering: {assemblyFileName} (method display = {discoveryOptions.GetMethodDisplayOrDefault()}, method display options = {discoveryOptions.GetMethodDisplayOptionsOrDefault()})"); + var assembly = Assembly.LoadFrom(assemblyFileName); + var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly); + var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink); + + discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions); + discoverySink.Finished.WaitOne(); + var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList(); + Console.WriteLine($"Discovered: {assemblyFileName} (found {testCasesToRun.Count} of {discoverySink.TestCases.Count} test cases)"); + + var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"{Environment.NewLine}=== TEST EXECUTION SUMMARY ==={Environment.NewLine}Total: {summary.Total}, Errors: 0, Failed: {summary.Failed}, Skipped: {summary.Skipped}, Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s{Environment.NewLine}"); }); + var resultsXmlAssembly = new XElement("assembly"); + var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); + + if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null) + { + testSink.Execution.TestStartingEvent += args => { Console.WriteLine($"[STRT] {args.Message.Test.DisplayName}"); }; + } + testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); }; + testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); }; + testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); }; + + testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Starting: {assemblyFileName}"); }; + testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished: {assemblyFileName}"); }; + + controller.RunTests(testCasesToRun, resultsSink, testOptions); + + while (!resultsSink.Finished.WaitOne(0)) + { + await Task.Delay(1); + } + + if (printXml) + { + if (oneLineResults) + { + var resultsXml = new XElement("assemblies"); + resultsXml.Add(resultsXmlAssembly); + using var sw = new StringWriter(); + resultsXml.Save(sw); + var bytes = System.Text.Encoding.UTF8.GetBytes(sw.ToString()); + var base64 = Convert.ToBase64String(bytes, Base64FormattingOptions.None); + Console.WriteLine($"STARTRESULTXML {bytes.Length} {base64} ENDRESULTXML"); + Console.WriteLine($"Finished writing {bytes.Length} bytes of RESULTXML"); + } + else + { + Console.WriteLine($"STARTRESULTXML"); + var resultsXml = new XElement("assemblies"); + resultsXml.Add(resultsXmlAssembly); + resultsXml.Save(Console.Out); + Console.WriteLine(); + Console.WriteLine($"ENDRESULTXML"); + } + } + + var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; + return failed ? 1 : 0; + } + catch (Exception ex) + { + Console.Error.WriteLine($"ThreadlessXunitTestRunner failed: {ex}"); + return 2; } } } diff --git a/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/WasmApplicationEntryPoint.cs b/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/WasmApplicationEntryPoint.cs index d7aff3882..a355a74f7 100644 --- a/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/WasmApplicationEntryPoint.cs +++ b/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/WasmApplicationEntryPoint.cs @@ -30,7 +30,7 @@ public async Task Run() foreach (var cl in IncludedClasses) filters.IncludedClasses.Add(cl); foreach (var me in IncludedMethods) filters.IncludedMethods.Add(me); - var result = await ThreadlessXunitTestRunner.Run(TestAssembly, printXml: true, filters); + var result = await ThreadlessXunitTestRunner.Run(TestAssembly, printXml: true, filters, true); return result; }