From afa9b7957a239c0195ed92d8f20b02d5ae80e175 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Thu, 3 Apr 2025 09:58:23 +0200 Subject: [PATCH 1/7] HADOOP-19528. Update Surefire plugin to 3.5.3 --- hadoop-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index b957a6a148c5e..545e40213e0c1 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -185,7 +185,7 @@ -Xmx2048m -Xss2m -XX:+HeapDumpOnOutOfMemoryError ${extraJavaTestArgs} - 3.0.0-M4 + 3.5.3 ${maven-surefire-plugin.version} ${maven-surefire-plugin.version} From 029cd588a350001661bb32c3dc4ecd8e8213279a Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Thu, 3 Apr 2025 19:57:35 +0200 Subject: [PATCH 2/7] add -Dsurefire.failIfNoSpecifiedTests=false in hadoop.sh --- dev-support/bin/hadoop.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/bin/hadoop.sh b/dev-support/bin/hadoop.sh index dbda5cad8cfb7..a00ee4d810809 100755 --- a/dev-support/bin/hadoop.sh +++ b/dev-support/bin/hadoop.sh @@ -619,7 +619,7 @@ function shadedclient_rebuild echo_and_redirect "${logfile}" \ "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \ "${modules[@]}" \ - -DskipShade -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true \ + -DskipShade -Dtest=NoUnitTests -Dsurefire.failIfNoSpecifiedTests=false -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true \ -Dspotbugs.skip=true ${extra[*]} count=$("${GREP}" -c '\[ERROR\]' "${logfile}") From bc74a833f81020829a3dad6fa1fd0e80db93e6da Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Fri, 4 Apr 2025 06:58:17 +0200 Subject: [PATCH 3/7] add another missing -Dsurefire.failIfNoSpecifiedTests=false --- dev-support/bin/hadoop.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-support/bin/hadoop.sh b/dev-support/bin/hadoop.sh index a00ee4d810809..aa3f68ea710bb 100755 --- a/dev-support/bin/hadoop.sh +++ b/dev-support/bin/hadoop.sh @@ -582,6 +582,7 @@ function shadedclient_rebuild extra=( "-Dtest=NoUnitTests" + "-Dsurefire.failIfNoSpecifiedTests=false" "-Dmaven.javadoc.skip=true" "-Dcheckstyle.skip=true" "-Dspotbugs.skip=true" From 02a7ddc872a31fb957af99014c40e2acbff67384 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Fri, 11 Apr 2025 19:30:56 +0200 Subject: [PATCH 4/7] Add timeout to TestPipeApplication --- .../org/apache/hadoop/mapred/pipes/TestPipeApplication.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java index 46ac5cacae62a..a7c8de7b9691e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; @@ -69,12 +70,14 @@ import org.apache.hadoop.util.Progressable; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +@Timeout(value=10, unit = TimeUnit.MINUTES) public class TestPipeApplication { private static File workSpace = new File("target", TestPipeApplication.class.getName() + "-workSpace"); From cb7bf9f277f28add71a925d6aa735ea79926d65e Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Fri, 11 Apr 2025 19:47:26 +0200 Subject: [PATCH 5/7] add timeout to the rest of the tests that silently timed out with Surefire 3.0.0-M4 --- .../hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java | 2 ++ .../server/nodemanager/amrmproxy/TestFederationInterceptor.java | 2 ++ .../nodemanager/amrmproxy/TestFederationInterceptorSecure.java | 2 ++ 3 files changed, 6 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java index bb6944e1034e0..4ff8087c35220 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java @@ -25,6 +25,7 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.apache.hadoop.conf.Configuration; @@ -67,6 +68,7 @@ /** * Unit test for UnmanagedApplicationManager. */ +@Timeout(value=10, unit = TimeUnit.MINUTES) public class TestUnmanagedApplicationManager { private static final Logger LOG = LoggerFactory.getLogger(TestUnmanagedApplicationManager.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java index 0182ac9c38c98..f97f9330d4fd9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java @@ -33,6 +33,7 @@ import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.hadoop.registry.client.api.RegistryOperations; @@ -102,6 +103,7 @@ * class. The tests for AMRMProxyService has been written cleverly so that it * can be reused to validate different request interceptor chains. */ +@Timeout(value=10, unit = TimeUnit.MINUTES) public class TestFederationInterceptor extends BaseAMRMProxyTest { private static final Logger LOG = LoggerFactory.getLogger(TestFederationInterceptor.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java index 57ca49a0e121a..f480cae7d5dac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.fs.Path; import org.apache.hadoop.ipc.Server; @@ -84,6 +85,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +@Timeout(value=10, unit = TimeUnit.MINUTES) public class TestFederationInterceptorSecure extends BaseAMRMProxyTest { private static final Logger LOG = LoggerFactory.getLogger(TestFederationInterceptor.class); From 70950d63de6e422abc4b1faec7af23fd97bbb881 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Tue, 15 Apr 2025 14:40:59 +0200 Subject: [PATCH 6/7] upgrade Junit5 to 5.12.1 and add SEPARATE_THREAD timeout to TestPipeApplication --- .../org/apache/hadoop/mapred/pipes/TestPipeApplication.java | 3 ++- hadoop-project/pom.xml | 6 +++--- .../yarn/server/uam/TestUnmanagedApplicationManager.java | 2 -- .../nodemanager/amrmproxy/TestFederationInterceptor.java | 2 -- .../amrmproxy/TestFederationInterceptorSecure.java | 2 -- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java index a7c8de7b9691e..45e90b3799181 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java @@ -76,8 +76,9 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Timeout.ThreadMode.SEPARATE_THREAD; -@Timeout(value=10, unit = TimeUnit.MINUTES) +@Timeout(value=1, unit = TimeUnit.MINUTES, threadMode = SEPARATE_THREAD) public class TestPipeApplication { private static File workSpace = new File("target", TestPipeApplication.class.getName() + "-workSpace"); diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 545e40213e0c1..7db5c389c1e24 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -223,9 +223,9 @@ 2.11.0 2.6.1-hadoop3 4.13.2 - 5.8.2 - 5.8.2 - 1.8.2 + 5.12.2 + 5.12.2 + 1.12.2 3.12.2 3.9.0 2.0.9 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java index 4ff8087c35220..bb6944e1034e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java @@ -25,7 +25,6 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.apache.hadoop.conf.Configuration; @@ -68,7 +67,6 @@ /** * Unit test for UnmanagedApplicationManager. */ -@Timeout(value=10, unit = TimeUnit.MINUTES) public class TestUnmanagedApplicationManager { private static final Logger LOG = LoggerFactory.getLogger(TestUnmanagedApplicationManager.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java index f97f9330d4fd9..0182ac9c38c98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java @@ -33,7 +33,6 @@ import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.hadoop.registry.client.api.RegistryOperations; @@ -103,7 +102,6 @@ * class. The tests for AMRMProxyService has been written cleverly so that it * can be reused to validate different request interceptor chains. */ -@Timeout(value=10, unit = TimeUnit.MINUTES) public class TestFederationInterceptor extends BaseAMRMProxyTest { private static final Logger LOG = LoggerFactory.getLogger(TestFederationInterceptor.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java index f480cae7d5dac..57ca49a0e121a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java @@ -28,7 +28,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; import org.apache.hadoop.fs.Path; import org.apache.hadoop.ipc.Server; @@ -85,7 +84,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; -@Timeout(value=10, unit = TimeUnit.MINUTES) public class TestFederationInterceptorSecure extends BaseAMRMProxyTest { private static final Logger LOG = LoggerFactory.getLogger(TestFederationInterceptor.class); From d8dc4321b267a544aa221fa69849b88c09f0dbd4 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Tue, 15 Apr 2025 18:45:13 +0200 Subject: [PATCH 7/7] add timeout to TestConfiguredFailoverProxyProvider --- .../namenode/ha/TestConfiguredFailoverProxyProvider.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java index cceb01b31cd16..6204f3ddc260c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java @@ -28,6 +28,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.slf4j.event.Level; @@ -38,6 +39,7 @@ import java.net.URISyntaxException; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -46,6 +48,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Timeout.ThreadMode.SEPARATE_THREAD; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -344,6 +347,8 @@ private void testResolveDomainNameUsingDNS(boolean useFQDN) throws Exception { assertTrue(nn2Count.get() > 0, "nn2 should have been selected: " + nn2Count.get()); } + // FIXME Sometimes java.net.Inet4AddressImpl.lookupAllHostAddr() hangs. Why ? + @Timeout(value=1, unit = TimeUnit.MINUTES, threadMode = SEPARATE_THREAD) @Test public void testResolveDomainNameUsingDNS() throws Exception { // test resolving to IP