Skip to content

MAPREDUCE-7419. Upgrade Junit 4 to 5 in hadoop-mapreduce-client-common #5028

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
May 23, 2023
Merged

MAPREDUCE-7419. Upgrade Junit 4 to 5 in hadoop-mapreduce-client-common #5028

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
93e161c
MAPREDUCE-7419. Upgrade Junit 4 to 5 in hadoop-mapreduce-client-common
Oct 14, 2022
fa10a07
stylecheck fix
Oct 15, 2022
e9dea30
addressing comments
May 16, 2023
7e9273d
temp commit to run all tests
May 16, 2023
8cbbccc
temp commit to trigger all tests
May 17, 2023
3f6f8e8
Addressing missed comments
May 20, 2023
08d4657
Revert "temp commit to run all tests"
May 21, 2023
da420c1
Revert "temp commit to trigger all tests"
May 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,21 @@
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-launcher</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down
42 changes: 24 additions & 18 deletions .../hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,22 +22,27 @@
import java.util.Collection;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.ClusterStatus.BlackListInfo;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

public class TestJobClient {

final static String TEST_DIR = new File("target",
TestJobClient.class.getSimpleName()).getAbsolutePath();

@After
@AfterEach
public void tearDown() {
FileUtil.fullyDelete(new File(TEST_DIR));
}
Expand All @@ -51,47 +56,48 @@ public void testGetClusterStatusWithLocalJobRunner() throws Exception {
ClusterStatus clusterStatus = client.getClusterStatus(true);
Collection<String> activeTrackerNames = clusterStatus
.getActiveTrackerNames();
Assert.assertEquals(0, activeTrackerNames.size());
assertEquals(0, activeTrackerNames.size());
int blacklistedTrackers = clusterStatus.getBlacklistedTrackers();
Assert.assertEquals(0, blacklistedTrackers);
assertEquals(0, blacklistedTrackers);
Collection<BlackListInfo> blackListedTrackersInfo = clusterStatus
.getBlackListedTrackersInfo();
Assert.assertEquals(0, blackListedTrackersInfo.size());
assertEquals(0, blackListedTrackersInfo.size());
}

@Test(timeout = 10000)
@Test
@Timeout(10000)
public void testIsJobDirValid() throws IOException {
Copy link
Member

@ayushtkn ayushtkn May 12, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is removing public mandatory? if things can work with this and others staying public, Would prefer it stay as public only

Configuration conf = new Configuration();
FileSystem fs = FileSystem.getLocal(conf);
Path testDir = new Path(TEST_DIR);
fs.mkdirs(testDir);
Assert.assertFalse(JobClient.isJobDirValid(testDir, fs));
assertFalse(JobClient.isJobDirValid(testDir, fs));

Path jobconf = new Path(testDir, "job.xml");
Path jobsplit = new Path(testDir, "job.split");
fs.create(jobconf);
fs.create(jobsplit);
Assert.assertTrue(JobClient.isJobDirValid(testDir, fs));
assertTrue(JobClient.isJobDirValid(testDir, fs));

fs.delete(jobconf, true);
fs.delete(jobsplit, true);
}

@Test(timeout = 10000)

@Test
@Timeout(10000)
public void testGetStagingAreaDir() throws IOException, InterruptedException {
Configuration conf = new Configuration();
JobClient client = new JobClient(conf);

Assert.assertTrue(
"Mismatch in paths",
client.getClusterHandle().getStagingAreaDir().toString()
.equals(client.getStagingAreaDir().toString()));
assertEquals(client.getClusterHandle().getStagingAreaDir().toString(),
client.getStagingAreaDir().toString());
}

/**
* Asks the compiler to check if JobClient is AutoClosable.
*/
@Test(timeout = 10000)
@Test
@Timeout(10000)
public void testAutoClosable() throws IOException {
Configuration conf = new Configuration();
try (JobClient jobClient = new JobClient(conf)) {
Expand Down
13 changes: 7 additions & 6 deletions ...p-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,16 @@

package org.apache.hadoop.mapred;

import static org.junit.Assert.assertNotNull;

import java.io.IOException;

import org.junit.jupiter.api.Test;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.Test;

import static org.junit.jupiter.api.Assertions.assertNotNull;

public class TestJobClientGetJob {

Expand All @@ -42,7 +43,7 @@ private Path createTempFile(String filename, String contents)
os.close();
return path;
}

@SuppressWarnings("deprecation")
@Test
public void testGetRunningJobFromJobClient() throws Exception {
Expand All @@ -54,10 +55,10 @@ public void testGetRunningJobFromJobClient() throws Exception {
FileOutputFormat.setOutputPath(conf, outputDir);
JobClient jc = new JobClient(conf);
RunningJob runningJob = jc.submitJob(conf);
assertNotNull("Running job", runningJob);
assertNotNull(runningJob, "Running job");
// Check that the running job can be retrieved by ID
RunningJob newRunningJob = jc.getJob(runningJob.getID());
assertNotNull("New running job", newRunningJob);
assertNotNull(newRunningJob, "New running job");
}

}
32 changes: 16 additions & 16 deletions ...lient-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@

package org.apache.hadoop.mapred;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
Expand Down Expand Up @@ -55,9 +55,9 @@
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.util.functional.CallableRaisingIOE;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;

Expand Down Expand Up @@ -109,7 +109,7 @@ private static void delete(File file) throws IOException {
}
}

@Before
@BeforeEach
public void setup() throws Exception {
mockfs = mock(FileSystem.class);
localDir = new File(System.getProperty("test.build.dir", "target/test-dir"),
Expand All @@ -118,7 +118,7 @@ public void setup() throws Exception {
localDir.mkdirs();
}

@After
@AfterEach
public void cleanup() throws Exception {
delete(localDir);
}
Expand Down Expand Up @@ -163,8 +163,8 @@ public void testDownload() throws Exception {
when(mockfs.getFileStatus(any(Path.class))).thenAnswer(new Answer<FileStatus>() {
@Override
public FileStatus answer(InvocationOnMock args) throws Throwable {
Path p = (Path)args.getArguments()[0];
if("file.txt".equals(p.getName())) {
Path p = (Path) args.getArguments()[0];
if ("file.txt".equals(p.getName())) {
return createMockTestFileStatus(filePath);
} else {
throw notMocked(p);
Expand All @@ -180,7 +180,7 @@ public FileStatus answer(InvocationOnMock args) throws Throwable {
// anything else: FNFE
when(mockfs.openFile(any(Path.class))).thenAnswer(
(Answer<FutureDataInputStreamBuilder>) args -> {
Path src = (Path)args.getArguments()[0];
Path src = (Path) args.getArguments()[0];
if ("file.txt".equals(src.getName())) {
return new MockOpenFileBuilder(mockfs, src,
() -> CompletableFuture.completedFuture(in));
Expand Down Expand Up @@ -228,15 +228,15 @@ public void testEmptyDownload() throws Exception {

when(mockfs.getFileStatus(any(Path.class))).thenAnswer(
(Answer<FileStatus>) args -> {
Path p = (Path)args.getArguments()[0];
Path p = (Path) args.getArguments()[0];
throw notMocked(p);
});

when(mockfs.getConf()).thenReturn(conf);
when(mockfs.openFile(any(Path.class))).thenAnswer(
(Answer<FutureDataInputStreamBuilder>) args -> {
Path src = (Path)args.getArguments()[0];
throw notMocked(src);
Path src = (Path) args.getArguments()[0];
throw notMocked(src);
});
conf.set(MRJobConfig.CACHE_FILES, "");
conf.set(MRConfig.LOCAL_DIR, localDir.getAbsolutePath());
Expand Down Expand Up @@ -272,8 +272,8 @@ public void testDuplicateDownload() throws Exception {
when(mockfs.getFileStatus(any(Path.class))).thenAnswer(new Answer<FileStatus>() {
@Override
public FileStatus answer(InvocationOnMock args) throws Throwable {
Path p = (Path)args.getArguments()[0];
if("file.txt".equals(p.getName())) {
Path p = (Path) args.getArguments()[0];
if ("file.txt".equals(p.getName())) {
return createMockTestFileStatus(filePath);
} else {
throw notMocked(p);
Expand All @@ -286,7 +286,7 @@ public FileStatus answer(InvocationOnMock args) throws Throwable {
new FSDataInputStream(new MockInputStream(TEST_DATA));
when(mockfs.openFile(any(Path.class))).thenAnswer(
(Answer<FutureDataInputStreamBuilder>) args -> {
Path src = (Path)args.getArguments()[0];
Path src = (Path) args.getArguments()[0];
if ("file.txt".equals(src.getName())) {
return new MockOpenFileBuilder(mockfs, src,
() -> CompletableFuture.completedFuture(in));
Expand Down
17 changes: 9 additions & 8 deletions ...reduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalModeWithNewApis.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
*/
package org.apache.hadoop.mapred;

import static org.junit.Assert.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.io.BufferedReader;
import java.io.DataOutputStream;
Expand All @@ -38,9 +39,9 @@
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -51,13 +52,13 @@ public class TestLocalModeWithNewApis {

Configuration conf;

@Before
@BeforeEach
public void setUp() throws Exception {
conf = new Configuration();
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
}

@After
@AfterEach
public void tearDown() throws Exception {
}

Expand Down Expand Up @@ -93,8 +94,8 @@ public void testNewApis() throws Exception {

String output = readOutput(outDir, conf);
assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" +
"quick\t1\nred\t1\nsilly\t1\nsox\t1\n", output);
"quick\t1\nred\t1\nsilly\t1\nsox\t1\n", output);

outFs.delete(tmpBaseDir, true);
}

Expand Down
Loading