Added retry facts for ci only

test/Lsp.Tests/Integration/DynamicRegistrationTests.cs

@@ -98,7 +98,7 @@ await TestHelper.DelayUntil(
                 client.RegistrationManager.CurrentRegistrations.Should().Contain(x => x.Method == "@/" + TextDocumentNames.SemanticTokensFull);
             }
 
-            [FactWithSkipOn(SkipOnPlatform.All)]
+            [RetryFact]
             public async Task Should_Unregister_Dynamically_While_Server_Is_Running()
             {
                 var (client, server) = await Initialize(new ConfigureClient().Configure, new ConfigureServer().Configure);

test/Lsp.Tests/Integration/PartialItemTests.cs

@@ -24,7 +24,7 @@ public Delegates(ITestOutputHelper testOutputHelper, LanguageProtocolFixture<Def
             {
             }
 
-            [FactWithSkipOn(SkipOnPlatform.All)]
+            [RetryFact]
             public async Task Should_Behave_Like_A_Task()
             {
                 var result = await Client.TextDocument.RequestSemanticTokens(
@@ -34,7 +34,7 @@ public async Task Should_Behave_Like_A_Task()
                 result!.Data.Should().HaveCount(3);
             }
 
-            [FactWithSkipOn(SkipOnPlatform.All)]
+            [RetryFact]
             public async Task Should_Behave_Like_An_Observable()
             {
                 var items = await Client.TextDocument

test/Lsp.Tests/Integration/PartialItemsTests.cs

@@ -28,7 +28,7 @@ public Delegates(ITestOutputHelper testOutputHelper, LanguageProtocolFixture<Def
             {
             }
 
-            [FactWithSkipOn(SkipOnPlatform.All)]
+            [RetryFact]
             public async Task Should_Behave_Like_A_Task()
             {
                 var result = await Client.TextDocument.RequestCodeLens(
@@ -41,7 +41,7 @@ public async Task Should_Behave_Like_A_Task()
                 result.Select(z => z.Command!.Name).Should().ContainInOrder("CodeLens 1", "CodeLens 2", "CodeLens 3");
             }
 
-            [FactWithSkipOn(SkipOnPlatform.All)]
+            [RetryFact]
             public async Task Should_Behave_Like_An_Observable()
             {
                 var items = await Client.TextDocument

test/Lsp.Tests/Integration/TypedCodeActionTests.cs

@@ -204,7 +204,7 @@ public async Task Should_Resolve_With_Data_Capability()
             item.CodeAction!.Command!.Name.Should().Be("resolved");
         }
 
-        [FactWithSkipOn(SkipOnPlatform.Mac)]
+        [RetryFact]
         public async Task Should_Resolve_With_Partial_Data_Capability()
         {
             var (client, _) = await Initialize(
@@ -297,7 +297,7 @@ public async Task Should_Resolve_With_Data_CancellationToken()
             item.CodeAction!.Command!.Name.Should().Be("resolved");
         }
 
-        [FactWithSkipOn(SkipOnPlatform.Mac)]
+        [RetryFact]
         public async Task Should_Resolve_With_Partial_Data_CancellationToken()
         {
             var (client, _) = await Initialize(

test/Lsp.Tests/Integration/TypedCodeLensTests.cs

@@ -195,7 +195,7 @@ public async Task Should_Resolve_With_Data_Capability()
             item.Command!.Name.Should().Be("resolved");
         }
 
-        [FactWithSkipOn(SkipOnPlatform.Mac)]
+        [RetryFact]
         public async Task Should_Resolve_With_Partial_Data_Capability()
         {
             var (client, _) = await Initialize(
@@ -284,7 +284,7 @@ public async Task Should_Resolve_With_Data_CancellationToken()
             item.Command!.Name.Should().Be("resolved");
         }
 
-        [FactWithSkipOn(SkipOnPlatform.Mac)]
+        [RetryFact]
         public async Task Should_Resolve_With_Partial_Data_CancellationToken()
         {
             var (client, _) = await Initialize(

test/Lsp.Tests/Integration/TypedCompletionTests.cs

@@ -195,7 +195,7 @@ public async Task Should_Resolve_With_Data_Capability()
             item.Detail.Should().Be("resolved");
         }
 
-        [FactWithSkipOn(SkipOnPlatform.Mac)]
+        [RetryFact]
         public async Task Should_Resolve_With_Partial_Data_Capability()
         {
             var (client, _) = await Initialize(
@@ -284,7 +284,7 @@ public async Task Should_Resolve_With_Data_CancellationToken()
             item.Detail.Should().Be("resolved");
         }
 
-        [FactWithSkipOn(SkipOnPlatform.Mac)]
+        [RetryFact]
         public async Task Should_Resolve_With_Partial_Data_CancellationToken()
         {
             var (client, _) = await Initialize(

test/Lsp.Tests/Integration/TypedDocumentLinkTests.cs

@@ -178,7 +178,7 @@ public async Task Should_Resolve_With_Data_Capability()
             item.Tooltip.Should().Be("resolved");
         }
 
-        [FactWithSkipOn(SkipOnPlatform.Mac)]
+        [RetryFact]
         public async Task Should_Resolve_With_Partial_Data_Capability()
         {
             var (client, _) = await Initialize(
@@ -261,7 +261,7 @@ public async Task Should_Resolve_With_Data_CancellationToken()
             item.Tooltip.Should().Be("resolved");
         }
 
-        [FactWithSkipOn(SkipOnPlatform.Mac)]
+        [RetryFact]
         public async Task Should_Resolve_With_Partial_Data_CancellationToken()
         {
             var (client, _) = await Initialize(

test/TestingUtils/AutoNSubstitute/TestLoggerFactory.cs

@@ -46,7 +46,7 @@ public void Swap(ITestOutputHelper testOutputHelper)
             _testOutputHelper.Swap(testOutputHelper);
         }
 
-        class InnerTestOutputHelper : ITestOutputHelper
+        private class InnerTestOutputHelper : ITestOutputHelper
         {
             private ITestOutputHelper? _testOutputHelper;
 

test/TestingUtils/BlockingMessageBus.cs

@@ -0,0 +1,52 @@
+// See https://github.com/JoshKeegan/xRetry
+
+using System.Collections.Concurrent;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace TestingUtils
+{
+    /// <summary>
+    /// An XUnit message bus that can block messages from being passed until we want them to be.
+    /// </summary>
+    public class BlockingMessageBus : IMessageBus
+    {
+        private readonly IMessageBus _underlyingMessageBus;
+        private ConcurrentQueue<IMessageSinkMessage> _messageQueue = new ConcurrentQueue<IMessageSinkMessage>();
+
+        public BlockingMessageBus(IMessageBus underlyingMessageBus)
+        {
+            _underlyingMessageBus = underlyingMessageBus;
+        }
+
+        public bool QueueMessage(IMessageSinkMessage message)
+        {
+            _messageQueue.Enqueue(message);
+
+            // Returns if execution should continue. Since we are intercepting the message, we
+            //  have no way of checking this so always continue...
+            return true;
+        }
+
+        public void Clear()
+        {
+            _messageQueue = new ConcurrentQueue<IMessageSinkMessage>();
+        }
+
+        /// <summary>
+        /// Write the cached messages to the underlying message bus
+        /// </summary>
+        public void Flush()
+        {
+            while (_messageQueue.TryDequeue(out IMessageSinkMessage message))
+            {
+                _underlyingMessageBus.QueueMessage(message);
+            }
+        }
+
+        public void Dispose()
+        {
+            // Do not dispose of the underlying message bus - it is an externally owned resource
+        }
+    }
+}

test/TestingUtils/IRetryableTestCase.cs

@@ -0,0 +1,12 @@
+// See https://github.com/JoshKeegan/xRetry
+
+using Xunit.Sdk;
+
+namespace TestingUtils
+{
+    public interface IRetryableTestCase : IXunitTestCase
+    {
+        int MaxRetries { get; }
+        int DelayBetweenRetriesMs { get; }
+    }
+}

test/TestingUtils/RetryFactAttribute.cs

@@ -0,0 +1,40 @@
+// See https://github.com/JoshKeegan/xRetry
+
+using System;
+using Xunit;
+using Xunit.Sdk;
+
+namespace TestingUtils
+{
+    /// <summary>
+    /// Attribute that is applied to a method to indicate that it is a fact that should be run
+    /// by the test runner up to MaxRetries times, until it succeeds.
+    /// </summary>
+    [XunitTestCaseDiscoverer("xRetry.RetryFactDiscoverer", "xRetry")]
+    [AttributeUsage(AttributeTargets.Method)]
+    public class RetryFactAttribute : FactAttribute
+    {
+        public readonly int MaxRetries;
+        public readonly int DelayBetweenRetriesMs;
+
+        /// <summary>
+        /// Ctor
+        /// </summary>
+        /// <param name="maxRetries">The number of times to run a test for until it succeeds</param>
+        /// <param name="delayBetweenRetriesMs">The amount of time (in ms) to wait between each test run attempt</param>
+        public RetryFactAttribute(int maxRetries = 3, int delayBetweenRetriesMs = 0)
+        {
+            if (maxRetries < 1)
+            {
+                throw new ArgumentOutOfRangeException(nameof(maxRetries) + " must be >= 1");
+            }
+            if (delayBetweenRetriesMs < 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(delayBetweenRetriesMs) + " must be >= 0");
+            }
+
+            MaxRetries = UnitTestDetector.IsCI() ? maxRetries : 1;
+            DelayBetweenRetriesMs = delayBetweenRetriesMs;
+        }
+    }
+}

test/TestingUtils/RetryFactDiscoverer.cs

@@ -0,0 +1,46 @@
+using System.Collections.Generic;
+using System.Linq;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace TestingUtils
+{
+    public class RetryFactDiscoverer : IXunitTestCaseDiscoverer
+    {
+        private readonly IMessageSink _messageSink;
+
+        public RetryFactDiscoverer(IMessageSink messageSink)
+        {
+            _messageSink = messageSink;
+        }
+
+        public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod,
+            IAttributeInfo factAttribute)
+        {
+            IXunitTestCase testCase;
+
+            if (testMethod.Method.GetParameters().Any())
+            {
+                testCase = new ExecutionErrorTestCase(_messageSink, discoveryOptions.MethodDisplayOrDefault(),
+                    discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod,
+                    "[RetryFact] methods are not allowed to have parameters. Did you mean to use [RetryTheory]?");
+            }
+            else if (testMethod.Method.IsGenericMethodDefinition)
+            {
+                testCase = new ExecutionErrorTestCase(_messageSink, discoveryOptions.MethodDisplayOrDefault(),
+                    discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod,
+                    "[RetryFact] methods are not allowed to be generic.");
+            }
+            else
+            {
+                var maxRetries = factAttribute.GetNamedArgument<int>(nameof(RetryFactAttribute.MaxRetries));
+                var delayBetweenRetriesMs =
+                    factAttribute.GetNamedArgument<int>(nameof(RetryFactAttribute.DelayBetweenRetriesMs));
+                testCase = new RetryTestCase(_messageSink, discoveryOptions.MethodDisplayOrDefault(),
+                    discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, maxRetries, delayBetweenRetriesMs);
+            }
+
+            return new[] { testCase };
+        }
+    }
+}

test/TestingUtils/RetryTestCase.cs

@@ -0,0 +1,60 @@
+using System;
+using System.ComponentModel;
+using System.Threading;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace TestingUtils
+{
+    [Serializable]
+    public class RetryTestCase : XunitTestCase, IRetryableTestCase
+    {
+        public int MaxRetries { get; private set; }
+        public int DelayBetweenRetriesMs { get; private set; }
+
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        [Obsolete(
+            "Called by the de-serializer; should only be called by deriving classes for de-serialization purposes", true)]
+        public RetryTestCase() { }
+
+        public RetryTestCase(
+            IMessageSink diagnosticMessageSink,
+            TestMethodDisplay defaultMethodDisplay,
+            TestMethodDisplayOptions defaultMethodDisplayOptions,
+            ITestMethod testMethod,
+            int maxRetries,
+            int delayBetweenRetriesMs,
+            object[]? testMethodArguments = null)
+            : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod,
+                testMethodArguments)
+        {
+            MaxRetries = maxRetries;
+            DelayBetweenRetriesMs = delayBetweenRetriesMs;
+        }
+
+        public override Task<RunSummary> RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus,
+            object[] constructorArguments, ExceptionAggregator aggregator,
+            CancellationTokenSource cancellationTokenSource) =>
+            RetryTestCaseRunner.RunAsync(this, diagnosticMessageSink, messageBus, cancellationTokenSource,
+                blockingMessageBus => new XunitTestCaseRunner(this, DisplayName, SkipReason, constructorArguments,
+                        TestMethodArguments, blockingMessageBus, aggregator, cancellationTokenSource)
+                    .RunAsync());
+
+        public override void Serialize(IXunitSerializationInfo data)
+        {
+            base.Serialize(data);
+
+            data.AddValue("MaxRetries", MaxRetries);
+            data.AddValue("DelayBetweenRetriesMs", DelayBetweenRetriesMs);
+        }
+
+        public override void Deserialize(IXunitSerializationInfo data)
+        {
+            base.Deserialize(data);
+
+            MaxRetries = data.GetValue<int>("MaxRetries");
+            DelayBetweenRetriesMs = data.GetValue<int>("DelayBetweenRetriesMs");
+        }
+    }
+}