Skip to content

Commit 8a5350d

Browse files
anujmodi2021anujmodi2021
authored
HADOOP-18960: [ABFS] Making Contract tests run in sequential and Other Test Fixes (#7104) (#7153)
Contributed by: Anuj Modi
1 parent 5a9404e commit 8a5350d

File tree

9 files changed

+123
-50
lines changed

9 files changed

+123
-50
lines changed

‎hadoop-tools/hadoop-azure/pom.xml‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -661,7 +661,6 @@
661661
<fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
662662
</systemPropertyVariables>
663663
<includes>
664-
<include>**/azurebfs/contract/ITest*.java</include>
665664
<include>**/azurebfs/ITestAzureBlobFileSystemE2EScale.java</include>
666665
<include>**/azurebfs/ITestAbfsReadWriteAndSeek.java</include>
667666
<include>**/azurebfs/ITestAzureBlobFileSystemListStatus.java</include>
@@ -673,6 +672,33 @@
673672
</includes>
674673
</configuration>
675674
</execution>
675+
<execution>
676+
<id>integration-test-abfs-sequential-contract-classes</id>
677+
<goals>
678+
<goal>integration-test</goal>
679+
<goal>verify</goal>
680+
</goals>
681+
<configuration>
682+
<!--NOTICE: hadoop contract tests methods can not be run in parallel-->
683+
<forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
684+
<trimStackTrace>false</trimStackTrace>
685+
<systemPropertyVariables>
686+
<!-- Tell tests that they are being executed sequentially -->
687+
<test.parallel.execution>false</test.parallel.execution>
688+
<!-- Propagate scale parameters -->
689+
<fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
690+
<fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
691+
<fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
692+
<fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
693+
<fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
694+
<!-- http connection pool size -->
695+
<http.maxConnections>${http.maxConnections}</http.maxConnections>
696+
</systemPropertyVariables>
697+
<includes>
698+
<include>**/azurebfs/contract/ITest*.java</include>
699+
</includes>
700+
</configuration>
701+
</execution>
676702
</executions>
677703
</plugin>
678704
</plugins>

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsCustomEncryption.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -401,9 +401,11 @@ private AzureBlobFileSystem getAzureBlobFileSystem(final Configuration conf) {
401401
AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(
402402
conf);
403403
fileSystemsOpenedInTest.add(fs);
404+
// Default for this config should be true here as FNS Accounts would have failed initialization.
405+
// This is needed to make sure test runs even if test config is missing.
404406
Assertions.assertThat(
405407
getConfiguration().getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT,
406-
false))
408+
true))
407409
.describedAs("Encryption tests should run only on namespace enabled account")
408410
.isTrue();
409411
return fs;

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java‎

Lines changed: 27 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,14 @@
3030
import org.apache.hadoop.fs.FileSystem;
3131
import org.apache.hadoop.fs.Path;
3232

33-
import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DOT;
3433
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION;
34+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT;
3535
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME;
36+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED;
3637
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME;
38+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty;
3739
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ABFS_ACCOUNT_NAME;
40+
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT;
3841
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
3942

4043
/**
@@ -140,19 +143,8 @@ public void testCustomTokenFetchRetryCount() throws Exception {
140143

141144
public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exception {
142145
AzureBlobFileSystem fs = this.getFileSystem();
143-
144-
Configuration config = new Configuration(this.getRawConfiguration());
145-
String accountName = config.get("fs.azure.abfs.account.name");
146-
// Setup to configure custom token provider.
147-
config.set("fs.azure.account.auth.type." + accountName, "Custom");
148-
config.set("fs.azure.account.oauth.provider.type." + accountName, "org.apache.hadoop.fs"
149-
+ ".azurebfs.oauth2.RetryTestTokenProvider");
150-
config.set("fs.azure.custom.token.fetch.retry.count", Integer.toString(numOfRetries));
151-
// Stop filesystem creation as it will lead to calls to store.
152-
config.set("fs.azure.createRemoteFileSystemDuringInitialization", "false");
153-
154-
try (final AzureBlobFileSystem fs1 =
155-
(AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(),
146+
Configuration config = getCustomAuthConfiguration(numOfRetries);
147+
try (AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(),
156148
config)) {
157149
RetryTestTokenProvider retryTestTokenProvider
158150
= RetryTestTokenProvider.getCurrentRetryTestProviderInstance(
@@ -174,18 +166,7 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce
174166

175167
@Test
176168
public void testAuthFailException() throws Exception {
177-
Configuration config = new Configuration(getRawConfiguration());
178-
String accountName = config
179-
.get(FS_AZURE_ABFS_ACCOUNT_NAME);
180-
// Setup to configure custom token provider
181-
config.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + DOT
182-
+ accountName, "Custom");
183-
config.set(
184-
FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + DOT + accountName,
185-
RETRY_TEST_TOKEN_PROVIDER);
186-
// Stop filesystem creation as it will lead to calls to store.
187-
config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false");
188-
169+
Configuration config = getCustomAuthConfiguration(0);
189170
final AzureBlobFileSystem fs = getFileSystem(config);
190171
AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> {
191172
fs.getFileStatus(new Path("/"));
@@ -201,4 +182,24 @@ public void testAuthFailException() throws Exception {
201182
.describedAs("Incorrect error message: " + errorDesc)
202183
.contains("Auth failure: ");
203184
}
185+
186+
/**
187+
* Returns a configuration with a custom token provider configured. {@link RetryTestTokenProvider}
188+
* @param numOfRetries Number of retries to be configured for token fetch.
189+
* @return Configuration
190+
*/
191+
private Configuration getCustomAuthConfiguration(final int numOfRetries) {
192+
Configuration config = new Configuration(this.getRawConfiguration());
193+
String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME);
194+
// Setup to configure custom token provider.
195+
config.set(accountProperty(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, accountName), "Custom");
196+
config.set(accountProperty(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, accountName),
197+
RETRY_TEST_TOKEN_PROVIDER);
198+
config.setInt(AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, numOfRetries);
199+
// Stop filesystem creation as it will lead to calls to store.
200+
config.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, false);
201+
config.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean(
202+
FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true));
203+
return config;
204+
}
204205
}

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java‎

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,16 @@
2828
import org.apache.hadoop.conf.Configuration;
2929

3030
import org.apache.hadoop.fs.Path;
31-
import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
3231
import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys;
3332
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException;
3433
import org.apache.hadoop.fs.azurebfs.extensions.MockSASTokenProvider;
3534
import org.apache.hadoop.fs.azurebfs.services.AuthType;
3635
import org.apache.hadoop.fs.permission.FsAction;
3736
import org.apache.hadoop.fs.permission.FsPermission;
3837

38+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME;
39+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED;
40+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE;
3941
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.MOCK_SASTOKENPROVIDER_FAIL_INIT;
4042
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN;
4143
import static org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers.aclEntry;
@@ -61,12 +63,12 @@ public ITestAzureBlobFileSystemAuthorization() throws Exception {
6163

6264
@Override
6365
public void setup() throws Exception {
64-
boolean isHNSEnabled = this.getConfiguration().getBoolean(
66+
boolean isHNSEnabled = getConfiguration().getBoolean(
6567
TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false);
6668
Assume.assumeTrue(isHNSEnabled);
6769
loadConfiguredFileSystem();
68-
this.getConfiguration().set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_AUTHZ_CLASS);
69-
this.getConfiguration().set(ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, "SAS");
70+
getConfiguration().set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_AUTHZ_CLASS);
71+
getConfiguration().set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SAS.toString());
7072
super.setup();
7173
}
7274

@@ -75,13 +77,15 @@ public void testSASTokenProviderInitializeException() throws Exception {
7577
final AzureBlobFileSystem fs = this.getFileSystem();
7678

7779
final AzureBlobFileSystem testFs = new AzureBlobFileSystem();
78-
Configuration testConfig = this.getConfiguration().getRawConfiguration();
79-
testConfig.set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
80-
testConfig.set(MOCK_SASTOKENPROVIDER_FAIL_INIT, "true");
80+
Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration());
81+
testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
82+
testConfig.setBoolean(MOCK_SASTOKENPROVIDER_FAIL_INIT, true);
83+
// Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check.
84+
testConfig.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs));
8185

8286
intercept(SASTokenProviderException.class,
8387
()-> {
84-
testFs.initialize(fs.getUri(), this.getConfiguration().getRawConfiguration());
88+
testFs.initialize(fs.getUri(), testConfig);
8589
});
8690
}
8791

@@ -90,12 +94,13 @@ public void testSASTokenProviderEmptySASToken() throws Exception {
9094
final AzureBlobFileSystem fs = this.getFileSystem();
9195

9296
final AzureBlobFileSystem testFs = new AzureBlobFileSystem();
93-
Configuration testConfig = this.getConfiguration().getRawConfiguration();
94-
testConfig.set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
95-
testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true");
97+
Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration());
98+
testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
99+
testConfig.setBoolean(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, true);
100+
// Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check.
101+
testConfig.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs));
96102

97-
testFs.initialize(fs.getUri(),
98-
this.getConfiguration().getRawConfiguration());
103+
testFs.initialize(fs.getUri(), testConfig);
99104
intercept(SASTokenProviderException.class,
100105
() -> {
101106
testFs.create(new org.apache.hadoop.fs.Path("/testFile")).close();
@@ -107,10 +112,13 @@ public void testSASTokenProviderNullSASToken() throws Exception {
107112
final AzureBlobFileSystem fs = this.getFileSystem();
108113

109114
final AzureBlobFileSystem testFs = new AzureBlobFileSystem();
110-
Configuration testConfig = this.getConfiguration().getRawConfiguration();
111-
testConfig.set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
115+
Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration());
116+
testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
117+
testConfig.setBoolean(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, true);
118+
// Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check.
119+
testConfig.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs));
112120

113-
testFs.initialize(fs.getUri(), this.getConfiguration().getRawConfiguration());
121+
testFs.initialize(fs.getUri(), testConfig);
114122
intercept(SASTokenProviderException.class,
115123
()-> {
116124
testFs.create(new org.apache.hadoop.fs.Path("/testFile")).close();

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@
3737
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN;
3838
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE;
3939
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty;
40+
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_ID;
41+
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_SECRET;
42+
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID;
43+
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_TENANT_ID;
4044
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
4145

4246
/**
@@ -62,8 +66,8 @@ public ITestAzureBlobFileSystemChooseSAS() throws Exception {
6266

6367
@Override
6468
public void setup() throws Exception {
65-
createFilesystemWithTestFileForSASTests(new Path(TEST_PATH));
6669
super.setup();
70+
createFilesystemWithTestFileForSASTests(new Path(TEST_PATH));
6771
generateAccountSAS();
6872
}
6973

@@ -95,6 +99,11 @@ public void testBothProviderFixedTokenConfigured() throws Exception {
9599
// Configuring a SASTokenProvider class which provides a user delegation SAS.
96100
testAbfsConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE,
97101
MockDelegationSASTokenProvider.class.getName());
102+
// Make sure test configs required by MockDelegationSASTokenProvider are set.
103+
assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_TEST_APP_ID);
104+
assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_TEST_APP_SECRET);
105+
assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_TENANT_ID);
106+
assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID);
98107

99108
// configuring the Fixed SAS token which is an Account SAS.
100109
testAbfsConfig.set(FS_AZURE_SAS_FIXED_TOKEN, accountSAS);

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
4343
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES;
4444
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty;
45+
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ACCOUNT_KEY;
4546
import static org.mockito.ArgumentMatchers.any;
4647
import static org.mockito.ArgumentMatchers.anyString;
4748
import static org.mockito.Mockito.doReturn;
@@ -279,12 +280,15 @@ public void testAccountSpecificConfig() throws Exception {
279280
this.getAccountName()));
280281
String testAccountName = "testAccount.dfs.core.windows.net";
281282
String otherAccountName = "otherAccount.dfs.core.windows.net";
283+
String dummyAcountKey = "dummyKey";
282284
String defaultUri = this.getTestUrl().replace(this.getAccountName(), testAccountName);
283285
String otherUri = this.getTestUrl().replace(this.getAccountName(), otherAccountName);
284286

285287
// Set both account specific and account agnostic config for test account
286288
rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR);
287289
rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, TRUE_STR);
290+
rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_KEY, testAccountName), dummyAcountKey);
291+
rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_KEY, otherAccountName), dummyAcountKey);
288292
// Assert that account specific config takes precedence
289293
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
290294
assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_AHEAD_RANGE;
3636
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_BUFFER_SIZE;
3737
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE;
38+
import static org.apache.hadoop.fs.azurebfs.utils.AbfsTestUtils.disableFilesystemCaching;
3839
import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
3940
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
4041
import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
@@ -68,6 +69,7 @@ protected Configuration createConfiguration() {
6869
protected AbstractFSContract createContract(final Configuration conf) {
6970
conf.setInt(AZURE_READ_AHEAD_RANGE, MIN_BUFFER_SIZE);
7071
conf.setInt(AZURE_READ_BUFFER_SIZE, MIN_BUFFER_SIZE);
72+
disableFilesystemCaching(conf);
7173
return new AbfsFileSystemContract(conf, isSecure);
7274
}
7375

0 commit comments

Comments
 (0)