Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Commit defbb8a

Browse files
davidshdavidsh
committed
Disable most networking tests for default CI runs
Lately, the CI system and test server infrastructure has been overloaded with many PRs. This is causing many networking tests to randomly fail due to resource exhaustion caused by so many simultaneous PR runs on the same CI machines. This occurrs not only when using an external test server endpoint (corefx-net.cloudapp.net) but also loopback servers. To improve engineering efficiency for the team, we are moving most of the tests in the networking libraries from the default CI runs (innerloop). For developers working on networking libraries, the networking tests can be run by invoking the "outerloop" tests. We will not be allowing any merges of PRs in the networking libraries unless the outerloop tests are also run by the developer. This affects tests in the following libraries right now: * System.Net.Http * System.Net.Http.WinHttpHandler * System.Net.Sockets * System.Net.Security * System.Net.WebSockets.Client I will be working on fine tuning these exclusions (issue #11345) to try to bring back tests into the innerloop runs when we have begun to address the infrastructure issues.
1 parent 11139d6 commit defbb8a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+321
-4
lines changed

‎src/System.Net.Http.WinHttpHandler/tests/FunctionalTests/ServerCertificateTest.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ public ServerCertificateTest(ITestOutputHelper output)
2828
_validationCallbackHistory = new ValidationCallbackHistory();
2929
}
3030

31+
[OuterLoop] // TODO: Issue #11345
3132
[Fact]
3233
public async Task NoCallback_ValidCertificate_CallbackNotCalled()
3334
{
@@ -40,6 +41,7 @@ public async Task NoCallback_ValidCertificate_CallbackNotCalled()
4041
}
4142
}
4243

44+
[OuterLoop] // TODO: Issue #11345
4345
[Fact]
4446
public async Task UseCallback_NotSecureConnection_CallbackNotCalled()
4547
{
@@ -53,6 +55,7 @@ public async Task UseCallback_NotSecureConnection_CallbackNotCalled()
5355
}
5456
}
5557

58+
[OuterLoop] // TODO: Issue #11345
5659
[Fact]
5760
public async Task UseCallback_ValidCertificate_ExpectedValuesDuringCallback()
5861
{
@@ -68,6 +71,7 @@ public async Task UseCallback_ValidCertificate_ExpectedValuesDuringCallback()
6871
}
6972
}
7073

74+
[OuterLoop] // TODO: Issue #11345
7175
[Fact]
7276
public async Task UseCallback_RedirectandValidCertificate_ExpectedValuesDuringCallback()
7377
{
@@ -85,6 +89,7 @@ public async Task UseCallback_RedirectandValidCertificate_ExpectedValuesDuringCa
8589
}
8690
}
8791

92+
[OuterLoop] // TODO: Issue #11345
8893
[Fact]
8994
public async Task UseCallback_CallbackReturnsFailure_ThrowsInnerSecurityFailureException()
9095
{
@@ -103,6 +108,7 @@ public async Task UseCallback_CallbackReturnsFailure_ThrowsInnerSecurityFailureE
103108
}
104109
}
105110

111+
[OuterLoop] // TODO: Issue #11345
106112
[Fact]
107113
public async Task UseCallback_CallbackThrowsSpecificException_ThrowsInnerSpecificException()
108114
{

‎src/System.Net.Http.WinHttpHandler/tests/FunctionalTests/WinHttpHandlerTest.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ public WinHttpHandlerTest(ITestOutputHelper output)
3030
_output = output;
3131
}
3232

33+
[OuterLoop] // TODO: Issue #11345
3334
[Fact]
3435
public void SendAsync_SimpleGet_Success()
3536
{
@@ -44,6 +45,7 @@ public void SendAsync_SimpleGet_Success()
4445
}
4546
}
4647

48+
[OuterLoop] // TODO: Issue #11345
4749
[Theory]
4850
[InlineData(CookieUsePolicy.UseInternalCookieStoreOnly, "cookieName1", "cookieValue1")]
4951
[InlineData(CookieUsePolicy.UseSpecifiedCookieContainer, "cookieName2", "cookieValue2")]
@@ -75,6 +77,7 @@ public async Task GetAsync_RedirectResponseHasCookie_CookieSentToFinalUri(
7577
}
7678
}
7779

80+
[OuterLoop] // TODO: Issue #11345
7881
[Fact]
7982
[OuterLoop]
8083
public async Task SendAsync_SlowServerAndCancel_ThrowsTaskCanceledException()
@@ -93,6 +96,7 @@ public async Task SendAsync_SlowServerAndCancel_ThrowsTaskCanceledException()
9396
}
9497
}
9598

99+
[OuterLoop] // TODO: Issue #11345
96100
[Fact]
97101
[OuterLoop]
98102
public void SendAsync_SlowServerRespondsAfterDefaultReceiveTimeout_ThrowsHttpRequestException()

‎src/System.Net.Http/tests/FunctionalTests/DefaultCredentialsTest.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ public DefaultCredentialsTest(ITestOutputHelper output)
4040
_output.WriteLine(s_authenticatedServer.ToString());
4141
}
4242

43+
[OuterLoop] // TODO: Issue #11345
4344
[ActiveIssue(10041)]
4445
[ConditionalTheory(nameof(DomainJoinedTestsEnabled))]
4546
[InlineData(false)]
@@ -56,6 +57,7 @@ public async Task UseDefaultCredentials_DefaultValue_Unauthorized(bool useProxy)
5657
}
5758
}
5859

60+
[OuterLoop] // TODO: Issue #11345
5961
[ActiveIssue(10041)]
6062
[ConditionalTheory(nameof(DomainJoinedTestsEnabled))]
6163
[InlineData(false)]
@@ -71,6 +73,7 @@ public async Task UseDefaultCredentials_SetFalse_Unauthorized(bool useProxy)
7173
}
7274
}
7375

76+
[OuterLoop] // TODO: Issue #11345
7477
[ActiveIssue(10041)]
7578
[ConditionalTheory(nameof(DomainJoinedTestsEnabled))]
7679
[InlineData(false)]
@@ -91,6 +94,7 @@ public async Task UseDefaultCredentials_SetTrue_ConnectAsCurrentIdentity(bool us
9194
}
9295
}
9396

97+
[OuterLoop] // TODO: Issue #11345
9498
[ActiveIssue(10041)]
9599
[ConditionalTheory(nameof(DomainJoinedTestsEnabled))]
96100
[InlineData(false)]
@@ -111,6 +115,7 @@ public async Task UseDefaultCredentials_SetTrueAndServerOffersMultipleSchemes_Ok
111115
}
112116
}
113117

118+
[OuterLoop] // TODO: Issue #11345
114119
[ActiveIssue(10041)]
115120
[ConditionalTheory(nameof(DomainJoinedTestsEnabled))]
116121
[InlineData(false)]
@@ -132,6 +137,7 @@ public async Task Credentials_SetToSpecificCredential_ConnectAsSpecificIdentity(
132137
}
133138
}
134139

140+
[OuterLoop] // TODO: Issue #11345
135141
[ActiveIssue(10041)]
136142
[ConditionalTheory(nameof(DomainJoinedTestsEnabled))]
137143
[InlineData(false)]
@@ -157,6 +163,7 @@ public async Task Credentials_SetToWrappedDefaultCredential_ConnectAsCurrentIden
157163
}
158164
}
159165

166+
[OuterLoop] // TODO: Issue #11345
160167
[ActiveIssue(10041)]
161168
[ConditionalFact(nameof(DomainProxyTestsEnabled))]
162169
public async Task Proxy_UseAuthenticatedProxyWithNoCredentials_ProxyAuthenticationRequired()
@@ -171,6 +178,7 @@ public async Task Proxy_UseAuthenticatedProxyWithNoCredentials_ProxyAuthenticati
171178
}
172179
}
173180

181+
[OuterLoop] // TODO: Issue #11345
174182
[ActiveIssue(10041)]
175183
[ConditionalFact(nameof(DomainProxyTestsEnabled))]
176184
public async Task Proxy_UseAuthenticatedProxyWithDefaultCredentials_OK()
@@ -185,6 +193,7 @@ public async Task Proxy_UseAuthenticatedProxyWithDefaultCredentials_OK()
185193
}
186194
}
187195

196+
[OuterLoop] // TODO: Issue #11345
188197
[ConditionalFact(nameof(DomainProxyTestsEnabled))]
189198
public async Task Proxy_UseAuthenticatedProxyWithWrappedDefaultCredentials_OK()
190199
{

‎src/System.Net.Http/tests/FunctionalTests/DiagnosticsTests.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ public class DiagnosticsTest : RemoteExecutorTestBase
2525
/// This test must be in the same test collection as any others testing HttpClient/WinHttpHandler
2626
/// DiagnosticSources, since the global logging mechanism makes them conflict inherently.
2727
/// </remarks>
28+
[OuterLoop] // TODO: Issue #11345
2829
[Fact]
2930
public void SendAsync_ExpectedDiagnosticSourceLogging()
3031
{
@@ -79,6 +80,7 @@ public void SendAsync_ExpectedDiagnosticSourceLogging()
7980
/// This test must be in the same test collection as any others testing HttpClient/WinHttpHandler
8081
/// DiagnosticSources, since the global logging mechanism makes them conflict inherently.
8182
/// </remarks>
83+
[OuterLoop] // TODO: Issue #11345
8284
[Fact]
8385
public void SendAsync_ExpectedDiagnosticSourceNoLogging()
8486
{
@@ -113,6 +115,7 @@ public void SendAsync_ExpectedDiagnosticSourceNoLogging()
113115
}).Dispose();
114116
}
115117

118+
[OuterLoop] // TODO: Issue #11345
116119
[Fact]
117120
public void SendAsync_HttpTracingEnabled_Succeeds()
118121
{

‎src/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.ClientCertificates.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ public void ClientCertificateOptions_ValueArg_Roundtrips(ClientCertificateOption
4646
}
4747
}
4848

49+
[OuterLoop] // TODO: Issue #11345
4950
[ConditionalFact(nameof(BackendDoesNotSupportCustomCertificateHandling))]
5051
public async Task Automatic_SSLBackendNotSupported_ThrowsPlatformNotSupportedException()
5152
{
@@ -55,6 +56,7 @@ public async Task Automatic_SSLBackendNotSupported_ThrowsPlatformNotSupportedExc
5556
}
5657
}
5758

59+
[OuterLoop] // TODO: Issue #11345
5860
[ConditionalFact(nameof(BackendDoesNotSupportCustomCertificateHandling))]
5961
public async Task Manual_SSLBackendNotSupported_ThrowsPlatformNotSupportedException()
6062
{
@@ -66,6 +68,7 @@ public async Task Manual_SSLBackendNotSupported_ThrowsPlatformNotSupportedExcept
6668
}
6769
}
6870

71+
[OuterLoop] // TODO: Issue #11345
6972
[ActiveIssue(9543, PlatformID.Windows)] // reuseClient==false fails in debug/release, reuseClient==true fails sporadically in release
7073
[ConditionalTheory(nameof(BackendSupportsCustomCertificateHandling))]
7174
[InlineData(6, false)]

‎src/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.DefaultProxyCredentials.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ public void SetGet_Roundtrips()
3535
}
3636
}
3737

38+
[OuterLoop] // TODO: Issue #11345
3839
[Fact]
3940
public void ProxyExplicitlyProvided_DefaultCredentials_Ignored()
4041
{
@@ -63,6 +64,7 @@ public void ProxyExplicitlyProvided_DefaultCredentials_Ignored()
6364
}
6465
}
6566

67+
[OuterLoop] // TODO: Issue #11345
6668
[Fact]
6769
[PlatformSpecific(PlatformID.AnyUnix)] // proxies set via the http_proxy environment variable are specific to Unix
6870
public void ProxySetViaEnvironmentVariable_DefaultProxyCredentialsUsed()

‎src/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.MaxResponseHeadersLength.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ public async Task SetAfterUse_Throws()
5959
}
6060
}
6161

62+
[OuterLoop] // TODO: Issue #11345
6263
[Theory]
6364
[InlineData("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", 37, false)]
6465
[InlineData("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", 38, true)]

‎src/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.ServerCertificates.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ namespace System.Net.Http.Functional.Tests
1515
{
1616
public class HttpClientHandler_ServerCertificates_Test
1717
{
18+
[OuterLoop] // TODO: Issue #11345
1819
[Fact]
1920
public async Task NoCallback_ValidCertificate_CallbackNotCalled()
2021
{
@@ -34,6 +35,7 @@ public async Task NoCallback_ValidCertificate_CallbackNotCalled()
3435
}
3536
}
3637

38+
[OuterLoop] // TODO: Issue #11345
3739
[ConditionalFact(nameof(BackendSupportsCustomCertificateHandling))]
3840
public async Task UseCallback_NotSecureConnection_CallbackNotCalled()
3941
{
@@ -67,6 +69,7 @@ public static IEnumerable<object[]> UseCallback_ValidCertificate_ExpectedValuesD
6769
}
6870
}
6971

72+
[OuterLoop] // TODO: Issue #11345
7073
[ConditionalTheory(nameof(BackendSupportsCustomCertificateHandling))]
7174
[MemberData(nameof(UseCallback_ValidCertificate_ExpectedValuesDuringCallback_Urls))]
7275
public async Task UseCallback_ValidCertificate_ExpectedValuesDuringCallback(Uri url, bool checkRevocation)
@@ -95,6 +98,7 @@ public async Task UseCallback_ValidCertificate_ExpectedValuesDuringCallback(Uri
9598
}
9699
}
97100

101+
[OuterLoop] // TODO: Issue #11345
98102
[ConditionalFact(nameof(BackendSupportsCustomCertificateHandling))]
99103
public async Task UseCallback_CallbackReturnsFailure_ThrowsException()
100104
{
@@ -106,6 +110,7 @@ public async Task UseCallback_CallbackReturnsFailure_ThrowsException()
106110
}
107111
}
108112

113+
[OuterLoop] // TODO: Issue #11345
109114
[ConditionalFact(nameof(BackendSupportsCustomCertificateHandling))]
110115
public async Task UseCallback_CallbackThrowsException_ExceptionPropagates()
111116
{
@@ -125,6 +130,7 @@ public async Task UseCallback_CallbackThrowsException_ExceptionPropagates()
125130
new object[] { Configuration.Http.WrongHostNameCertRemoteServer },
126131
};
127132

133+
[OuterLoop] // TODO: Issue #11345
128134
[Theory]
129135
[MemberData(nameof(CertificateValidationServers))]
130136
public async Task NoCallback_BadCertificate_ThrowsException(string url)
@@ -135,6 +141,7 @@ public async Task NoCallback_BadCertificate_ThrowsException(string url)
135141
}
136142
}
137143

144+
[OuterLoop] // TODO: Issue #11345
138145
[Fact]
139146
public async Task NoCallback_RevokedCertificate_NoRevocationChecking_Succeeds()
140147
{
@@ -145,6 +152,7 @@ public async Task NoCallback_RevokedCertificate_NoRevocationChecking_Succeeds()
145152
}
146153
}
147154

155+
[OuterLoop] // TODO: Issue #11345
148156
[ConditionalFact(nameof(BackendSupportsCustomCertificateHandling))]
149157
public async Task NoCallback_RevokedCertificate_RevocationChecking_Fails()
150158
{
@@ -162,6 +170,7 @@ public async Task NoCallback_RevokedCertificate_RevocationChecking_Fails()
162170
new object[] { Configuration.Http.WrongHostNameCertRemoteServer , SslPolicyErrors.RemoteCertificateNameMismatch},
163171
};
164172

173+
[OuterLoop] // TODO: Issue #11345
165174
[ActiveIssue(7812, PlatformID.Windows)]
166175
[ConditionalTheory(nameof(BackendSupportsCustomCertificateHandling))]
167176
[MemberData(nameof(CertificateValidationServersAndExpectedPolicies))]
@@ -191,6 +200,7 @@ public async Task UseCallback_BadCertificate_ExpectedPolicyErrors(string url, Ss
191200
}
192201
}
193202

203+
[OuterLoop] // TODO: Issue #11345
194204
[ConditionalFact(nameof(BackendDoesNotSupportCustomCertificateHandling))]
195205
public async Task SSLBackendNotSupported_Callback_ThrowsPlatformNotSupportedException()
196206
{
@@ -200,6 +210,7 @@ public async Task SSLBackendNotSupported_Callback_ThrowsPlatformNotSupportedExce
200210
}
201211
}
202212

213+
[OuterLoop] // TODO: Issue #11345
203214
[ConditionalFact(nameof(BackendDoesNotSupportCustomCertificateHandling))]
204215
public async Task SSLBackendNotSupported_Revocation_ThrowsPlatformNotSupportedException()
205216
{
@@ -209,6 +220,7 @@ public async Task SSLBackendNotSupported_Revocation_ThrowsPlatformNotSupportedEx
209220
}
210221
}
211222

223+
[OuterLoop] // TODO: Issue #11345
212224
[PlatformSpecific(PlatformID.Windows)] // CopyToAsync(Stream, TransportContext) isn't used on unix
213225
[Fact]
214226
public async Task PostAsync_Post_ChannelBinding_ConfiguredCorrectly()

‎src/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.SslProtocols.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ public void SetGetProtocols_Roundtrips(SslProtocols protocols)
4343
}
4444
}
4545

46+
[OuterLoop] // TODO: Issue #11345
4647
[ConditionalFact(nameof(BackendSupportsSslConfiguration))]
4748
public async Task SetProtocols_AfterRequest_ThrowsException()
4849
{
@@ -59,6 +60,7 @@ await TestHelper.WhenAllCompletedOrAnyFailed(
5960
}
6061
}
6162

63+
[OuterLoop] // TODO: Issue #11345
6264
[Theory]
6365
[InlineData(~SslProtocols.None)]
6466
#pragma warning disable 0618 // obsolete warning
@@ -75,6 +77,7 @@ public void DisabledProtocols_SetSslProtocols_ThrowsException(SslProtocols disab
7577
}
7678
}
7779

80+
[OuterLoop] // TODO: Issue #11345
7881
[ConditionalTheory(nameof(BackendSupportsSslConfiguration))]
7982
[InlineData(SslProtocols.Tls, false)]
8083
[InlineData(SslProtocols.Tls, true)]
@@ -144,6 +147,7 @@ public async Task GetAsync_UnsupportedSSLVersion_Throws(string name, string url)
144147
}
145148
}
146149

150+
[OuterLoop] // TODO: Issue #11345
147151
[ConditionalFact(nameof(BackendSupportsSslConfiguration), nameof(SslDefaultsToTls12))]
148152
public async Task GetAsync_NoSpecifiedProtocol_DefaultsToTls12()
149153
{
@@ -165,6 +169,7 @@ await TestHelper.WhenAllCompletedOrAnyFailed(
165169
}
166170
}
167171

172+
[OuterLoop] // TODO: Issue #11345
168173
[ConditionalTheory(nameof(BackendSupportsSslConfiguration))]
169174
[InlineData(SslProtocols.Tls11, SslProtocols.Tls, typeof(IOException))]
170175
[InlineData(SslProtocols.Tls12, SslProtocols.Tls11, typeof(IOException))]
@@ -185,6 +190,7 @@ await TestHelper.WhenAllCompletedOrAnyFailed(
185190
}
186191
}
187192

193+
[OuterLoop] // TODO: Issue #11345
188194
[ActiveIssue(8538, PlatformID.Windows)]
189195
[Fact]
190196
public async Task GetAsync_DisallowTls10_AllowTls11_AllowTls12()

0 commit comments

Comments
 (0)