Skip to content

[wasm] write test as single line via base64 #771

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Nov 22, 2021
Merged

[wasm] write test as single line via base64 #771

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
86f5abf
save as single base64 line
pavelsavara Nov 20, 2021
7d1281f
check lenght
pavelsavara Nov 20, 2021
6a3b20d
feedback
pavelsavara Nov 21, 2021
71e8926
back to LF
pavelsavara Nov 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 20 additions & 42 deletions src/Microsoft.DotNet.XHarness.CLI/Commands/WASM/WasmTestMessagesProcessor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,16 @@
using System;
using System.IO;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml.Linq;
using Microsoft.Extensions.Logging;

#nullable enable
namespace Microsoft.DotNet.XHarness.CLI.Commands.Wasm
{
public class WasmTestMessagesProcessor
{
private MemoryStream? _xmlResultsMemoryStream;
private StreamWriter? _xmlResultsFileWriter;
private static Regex xmlRx = new Regex(@"^STARTRESULTXML ([0-9]*) ([^ ]*) ENDRESULTXML", RegexOptions.Compiled | RegexOptions.CultureInvariant);
private readonly StreamWriter _stdoutFileWriter;
private readonly string _xmlResultsFilePath;

Expand Down Expand Up @@ -77,16 +76,27 @@ private void InvokeInternal(string message)
line = message.TrimEnd();
}

if (_xmlResultsFileWriter == null)
var match = xmlRx.Match(line);
if (match.Success)
{
if (line.Contains("STARTRESULTXML"))
var expectedLength = Int32.Parse(match.Groups[1].Value);
using (var stream = new FileStream(_xmlResultsFilePath, FileMode.Create))
{
_logger.LogDebug("Reached start of testResults.xml");
_xmlResultsMemoryStream = new MemoryStream();
_xmlResultsFileWriter = new StreamWriter(_xmlResultsMemoryStream);
return;
var bytes = System.Convert.FromBase64String(match.Groups[2].Value);
stream.Write(bytes);
if (bytes.Length == expectedLength)
{
_logger.LogInformation($"Received expected {bytes.Length} of {_xmlResultsFilePath}");
}
else
{
_logger.LogInformation($"Received {bytes.Length} of {_xmlResultsFilePath} but expected {expectedLength}");
}
}
else if (line.StartsWith("[PASS]") || line.StartsWith("[SKIP]"))
}
else
{
if (line.StartsWith("[PASS]") || line.StartsWith("[SKIP]"))
{
_logger.LogDebug(line);
}
Expand All @@ -113,38 +123,6 @@ private void InvokeInternal(string message)
if (_stdoutFileWriter.BaseStream.CanWrite)
_stdoutFileWriter.WriteLine(line);
}
else
{
if (line.Contains("ENDRESULTXML"))
{
_logger.LogDebug($"Reached end of {_xmlResultsFilePath}");
_xmlResultsFileWriter.Flush();
_xmlResultsMemoryStream!.Seek(0, SeekOrigin.Begin);
try
{
// we validate it, to make sure it's not corrupted.
var testResults = XElement.Load(_xmlResultsMemoryStream);
using (var testResultsFile = File.CreateText(_xmlResultsFilePath))
{
testResults.Save(testResultsFile);
_logger.LogInformation($"Written {_xmlResultsMemoryStream.Length} original bytes of {_xmlResultsFilePath} as {testResultsFile.BaseStream.Length} formatted bytes");
};
}
catch (Exception ex)
{
_logger.LogError($"Error while saving testResults.xml {ex}");
throw;
}
_xmlResultsFileWriter.Dispose();
_xmlResultsFileWriter = null;
_xmlResultsMemoryStream.Dispose();
_xmlResultsMemoryStream = null;
return;
}

if (_xmlResultsFileWriter?.BaseStream.CanWrite == true)
_xmlResultsFileWriter.WriteLine(line);
}

// the test runner writes this as the last line,
// after the tests have run, and the xml results file
Expand Down
161 changes: 79 additions & 82 deletions src/Microsoft.DotNet.XHarness.TestRunners.Xunit/ThreadlessXunitTestRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@

using System;
using System.Collections.Generic;
using System.IO;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
Expand All @@ -21,87 +21,84 @@ namespace Microsoft.DotNet.XHarness.TestRunners.Xunit
{
internal class ThreadlessXunitTestRunner
{
public static async Task<int> Run(string assemblyFileName, bool printXml, XunitFilters filters)
public static async Task<int> Run(string assemblyFileName, bool printXml, XunitFilters filters, bool oneLineResults = false)
{
try
{
var configuration = new TestAssemblyConfiguration() { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false };
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false);

discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);

Console.WriteLine($"Discovering: {assemblyFileName} (method display = {discoveryOptions.GetMethodDisplayOrDefault()}, method display options = {discoveryOptions.GetMethodDisplayOptionsOrDefault()})");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);

discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Console.WriteLine($"Discovered: {assemblyFileName} (found {testCasesToRun.Count} of {discoverySink.TestCases.Count} test cases)");

var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"{Environment.NewLine}=== TEST EXECUTION SUMMARY ==={Environment.NewLine}Total: {summary.Total}, Errors: 0, Failed: {summary.Failed}, Skipped: {summary.Skipped}, Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s{Environment.NewLine}"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);

if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null)
{
testSink.Execution.TestStartingEvent += args => { Console.WriteLine($"[STRT] {args.Message.Test.DisplayName}"); };
}
testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); };

testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Starting: {assemblyFileName}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished: {assemblyFileName}"); };

controller.RunTests(testCasesToRun, resultsSink, testOptions);

while (!resultsSink.Finished.WaitOne(0))
{
await Task.Delay(1);
}

if (printXml)
{
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
using (var ms = new MemoryStream())
{
using (var tw = new StreamWriter(ms, Encoding.UTF8, 100*1024, true))
{
// no new line. make it single line, single WS message
resultsXml.Save(tw, SaveOptions.DisableFormatting);
}
ms.Seek(0, SeekOrigin.Begin);
using (var stdout = Console.OpenStandardOutput())
{
using (var twc = new StreamWriter(stdout))
{
twc.WriteLine($"STARTRESULTXML {ms.Length}");
twc.Flush();
ms.CopyTo(stdout, 100 * 1024);
twc.WriteLine();
twc.WriteLine("ENDRESULTXML");
}
}
Console.WriteLine($"Finished writing {ms.Length} bytes of RESULTXML");
}
}

var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return failed ? 1 : 0;
}
catch (Exception ex)
{
Console.Error.WriteLine($"ThreadlessXunitTestRunner failed: {ex}");
return 2;
try
{
var configuration = new TestAssemblyConfiguration() { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false };
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false);

discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);

Console.WriteLine($"Discovering: {assemblyFileName} (method display = {discoveryOptions.GetMethodDisplayOrDefault()}, method display options = {discoveryOptions.GetMethodDisplayOptionsOrDefault()})");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);

discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Console.WriteLine($"Discovered: {assemblyFileName} (found {testCasesToRun.Count} of {discoverySink.TestCases.Count} test cases)");

var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"{Environment.NewLine}=== TEST EXECUTION SUMMARY ==={Environment.NewLine}Total: {summary.Total}, Errors: 0, Failed: {summary.Failed}, Skipped: {summary.Skipped}, Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s{Environment.NewLine}"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);

if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null)
{
testSink.Execution.TestStartingEvent += args => { Console.WriteLine($"[STRT] {args.Message.Test.DisplayName}"); };
}
testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); };

testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Starting: {assemblyFileName}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished: {assemblyFileName}"); };

controller.RunTests(testCasesToRun, resultsSink, testOptions);

while (!resultsSink.Finished.WaitOne(0))
{
await Task.Delay(1);
}

if (printXml)
{
if (oneLineResults)
{
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
using var sw = new StringWriter();
resultsXml.Save(sw);
var bytes = System.Text.Encoding.UTF8.GetBytes(sw.ToString());
var base64 = Convert.ToBase64String(bytes, Base64FormattingOptions.None);
Console.WriteLine($"STARTRESULTXML {bytes.Length} {base64} ENDRESULTXML");
Console.WriteLine($"Finished writing {bytes.Length} bytes of RESULTXML");
}
else
{
Console.WriteLine($"STARTRESULTXML");
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
resultsXml.Save(Console.Out);
Console.WriteLine();
Console.WriteLine($"ENDRESULTXML");
}
}

var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return failed ? 1 : 0;
}
catch (Exception ex)
{
Console.Error.WriteLine($"ThreadlessXunitTestRunner failed: {ex}");
return 2;
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.DotNet.XHarness.TestRunners.Xunit/WasmApplicationEntryPoint.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public async Task<int> Run()
foreach (var cl in IncludedClasses) filters.IncludedClasses.Add(cl);
foreach (var me in IncludedMethods) filters.IncludedMethods.Add(me);

var result = await ThreadlessXunitTestRunner.Run(TestAssembly, printXml: true, filters);
var result = await ThreadlessXunitTestRunner.Run(TestAssembly, printXml: true, filters, true);

return result;
}
Expand Down