Skip to content

Commit 4d1fac5

Browse files
jlowejlowe
committed
YARN-6411. Clean up the overwrite of createDispatcher() in subclass of MockRM. Contributed by Yufei Gu
1 parent 28cdc5a commit 4d1fac5

File tree

15 files changed

+277
-475
lines changed

15 files changed

+277
-475
lines changed

‎hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java‎

Lines changed: 202 additions & 251 deletions
Large diffs are not rendered by default.

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java‎

Lines changed: 21 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,6 @@
4949
import org.apache.hadoop.yarn.client.api.AMRMClient;
5050
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
5151
import org.apache.hadoop.yarn.conf.YarnConfiguration;
52-
import org.apache.hadoop.yarn.event.Dispatcher;
53-
import org.apache.hadoop.yarn.event.DrainDispatcher;
5452
import org.apache.hadoop.yarn.event.EventHandler;
5553
import org.apache.hadoop.yarn.ipc.YarnRPC;
5654
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -126,22 +124,20 @@ public void testAMRMClientResendsRequestsOnRMRestart() throws Exception {
126124
// Phase-1 Start 1st RM
127125
MyResourceManager rm1 = new MyResourceManager(conf, memStore);
128126
rm1.start();
129-
DrainDispatcher dispatcher =
130-
(DrainDispatcher) rm1.getRMContext().getDispatcher();
131127

132128
// Submit the application
133129
RMApp app = rm1.submitApp(1024);
134-
dispatcher.await();
130+
rm1.drainEvents();
135131

136132
MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
137133
nm1.registerNode();
138134
nm1.nodeHeartbeat(true); // Node heartbeat
139-
dispatcher.await();
135+
rm1.drainEvents();
140136

141137
ApplicationAttemptId appAttemptId =
142138
app.getCurrentAppAttempt().getAppAttemptId();
143139
rm1.sendAMLaunched(appAttemptId);
144-
dispatcher.await();
140+
rm1.drainEvents();
145141

146142
org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token =
147143
rm1.getRMContext().getRMApps().get(appAttemptId.getApplicationId())
@@ -176,7 +172,7 @@ public void testAMRMClientResendsRequestsOnRMRestart() throws Exception {
176172
blacklistAdditions.remove("h2");// remove from local list
177173

178174
AllocateResponse allocateResponse = amClient.allocate(0.1f);
179-
dispatcher.await();
175+
rm1.drainEvents();
180176
Assert.assertEquals("No of assignments must be 0", 0, allocateResponse
181177
.getAllocatedContainers().size());
182178

@@ -189,10 +185,10 @@ public void testAMRMClientResendsRequestsOnRMRestart() throws Exception {
189185
// Step-2 : NM heart beat is sent.
190186
// On 2nd AM allocate request, RM allocates 3 containers to AM
191187
nm1.nodeHeartbeat(true); // Node heartbeat
192-
dispatcher.await();
188+
rm1.drainEvents();
193189

194190
allocateResponse = amClient.allocate(0.2f);
195-
dispatcher.await();
191+
rm1.drainEvents();
196192
// 3 containers are allocated i.e for cRequest1, cRequest2 and cRequest3.
197193
Assert.assertEquals("No of assignments must be 0", 3, allocateResponse
198194
.getAllocatedContainers().size());
@@ -207,7 +203,7 @@ public void testAMRMClientResendsRequestsOnRMRestart() throws Exception {
207203
amClient.removeContainerRequest(cRequest3);
208204

209205
allocateResponse = amClient.allocate(0.2f);
210-
dispatcher.await();
206+
rm1.drainEvents();
211207
Assert.assertEquals("No of assignments must be 0", 0, allocateResponse
212208
.getAllocatedContainers().size());
213209
assertAsksAndReleases(4, 0, rm1);
@@ -233,7 +229,7 @@ public void testAMRMClientResendsRequestsOnRMRestart() throws Exception {
233229
// request
234230
nm1.nodeHeartbeat(containerId.getApplicationAttemptId(),
235231
containerId.getContainerId(), ContainerState.RUNNING);
236-
dispatcher.await();
232+
rm1.drainEvents();
237233
amClient.requestContainerUpdate(
238234
container, UpdateContainerRequest.newInstance(
239235
container.getVersion(), container.getId(),
@@ -242,7 +238,7 @@ public void testAMRMClientResendsRequestsOnRMRestart() throws Exception {
242238
it.remove();
243239

244240
allocateResponse = amClient.allocate(0.3f);
245-
dispatcher.await();
241+
rm1.drainEvents();
246242
Assert.assertEquals("No of assignments must be 0", 0, allocateResponse
247243
.getAllocatedContainers().size());
248244
assertAsksAndReleases(3, pendingRelease, rm1);
@@ -258,7 +254,6 @@ public void testAMRMClientResendsRequestsOnRMRestart() throws Exception {
258254
rm2.start();
259255
nm1.setResourceTrackerService(rm2.getResourceTrackerService());
260256
((MyAMRMClientImpl) amClient).updateRMProxy(rm2);
261-
dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher();
262257

263258
// NM should be rebooted on heartbeat, even first heartbeat for nm2
264259
NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true);
@@ -274,7 +269,7 @@ public void testAMRMClientResendsRequestsOnRMRestart() throws Exception {
274269
Collections.singletonList(
275270
containerId.getApplicationAttemptId().getApplicationId()));
276271
nm1.nodeHeartbeat(true);
277-
dispatcher.await();
272+
rm2.drainEvents();
278273

279274
blacklistAdditions.add("h3");
280275
amClient.updateBlacklist(blacklistAdditions, null);
@@ -296,7 +291,7 @@ public void testAMRMClientResendsRequestsOnRMRestart() throws Exception {
296291
// containerRequest and blacklisted nodes.
297292
// Intern RM send resync command,AMRMClient resend allocate request
298293
allocateResponse = amClient.allocate(0.3f);
299-
dispatcher.await();
294+
rm2.drainEvents();
300295

301296
completedContainer =
302297
allocateResponse.getCompletedContainersStatuses().size();
@@ -313,7 +308,7 @@ public void testAMRMClientResendsRequestsOnRMRestart() throws Exception {
313308

314309
// Step-5 : Allocater after resync command
315310
allocateResponse = amClient.allocate(0.5f);
316-
dispatcher.await();
311+
rm2.drainEvents();
317312
Assert.assertEquals("No of assignments must be 0", 0, allocateResponse
318313
.getAllocatedContainers().size());
319314

@@ -326,10 +321,10 @@ public void testAMRMClientResendsRequestsOnRMRestart() throws Exception {
326321
int count = 5;
327322
while (count-- > 0) {
328323
nm1.nodeHeartbeat(true);
329-
dispatcher.await();
324+
rm2.drainEvents();
330325

331326
allocateResponse = amClient.allocate(0.5f);
332-
dispatcher.await();
327+
rm2.drainEvents();
333328
noAssignedContainer += allocateResponse.getAllocatedContainers().size();
334329
if (noAssignedContainer == 3) {
335330
break;
@@ -358,22 +353,20 @@ public void testAMRMClientForUnregisterAMOnRMRestart() throws Exception {
358353
// Phase-1 Start 1st RM
359354
MyResourceManager rm1 = new MyResourceManager(conf, memStore);
360355
rm1.start();
361-
DrainDispatcher dispatcher =
362-
(DrainDispatcher) rm1.getRMContext().getDispatcher();
363356

364357
// Submit the application
365358
RMApp app = rm1.submitApp(1024);
366-
dispatcher.await();
359+
rm1.drainEvents();
367360

368361
MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
369362
nm1.registerNode();
370363
nm1.nodeHeartbeat(true); // Node heartbeat
371-
dispatcher.await();
364+
rm1.drainEvents();
372365

373366
ApplicationAttemptId appAttemptId =
374367
app.getCurrentAppAttempt().getAppAttemptId();
375368
rm1.sendAMLaunched(appAttemptId);
376-
dispatcher.await();
369+
rm1.drainEvents();
377370

378371
org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token =
379372
rm1.getRMContext().getRMApps().get(appAttemptId.getApplicationId())
@@ -393,7 +386,6 @@ public void testAMRMClientForUnregisterAMOnRMRestart() throws Exception {
393386
rm2.start();
394387
nm1.setResourceTrackerService(rm2.getResourceTrackerService());
395388
((MyAMRMClientImpl) amClient).updateRMProxy(rm2);
396-
dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher();
397389

398390
// NM should be rebooted on heartbeat, even first heartbeat for nm2
399391
NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true);
@@ -409,7 +401,7 @@ public void testAMRMClientForUnregisterAMOnRMRestart() throws Exception {
409401
Priority.newInstance(0), 0);
410402
nm1.registerNode(Arrays.asList(containerReport), null);
411403
nm1.nodeHeartbeat(true);
412-
dispatcher.await();
404+
rm2.drainEvents();
413405

414406
amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
415407
null, null);
@@ -421,7 +413,6 @@ public void testAMRMClientForUnregisterAMOnRMRestart() throws Exception {
421413
amClient.stop();
422414
rm1.stop();
423415
rm2.stop();
424-
425416
}
426417

427418

@@ -439,22 +430,20 @@ public void testAMRMClientOnAMRMTokenRollOverOnRMRestart() throws Exception {
439430
// start first RM
440431
MyResourceManager2 rm1 = new MyResourceManager2(conf, memStore);
441432
rm1.start();
442-
DrainDispatcher dispatcher =
443-
(DrainDispatcher) rm1.getRMContext().getDispatcher();
444433
Long startTime = System.currentTimeMillis();
445434
// Submit the application
446435
RMApp app = rm1.submitApp(1024);
447-
dispatcher.await();
436+
rm1.drainEvents();
448437

449438
MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
450439
nm1.registerNode();
451440
nm1.nodeHeartbeat(true); // Node heartbeat
452-
dispatcher.await();
441+
rm1.drainEvents();
453442

454443
ApplicationAttemptId appAttemptId =
455444
app.getCurrentAppAttempt().getAppAttemptId();
456445
rm1.sendAMLaunched(appAttemptId);
457-
dispatcher.await();
446+
rm1.drainEvents();
458447

459448
AMRMTokenSecretManager amrmTokenSecretManagerForRM1 =
460449
rm1.getRMContext().getAMRMTokenSecretManager();
@@ -513,7 +502,6 @@ public void testAMRMClientOnAMRMTokenRollOverOnRMRestart() throws Exception {
513502
rm2.start();
514503
nm1.setResourceTrackerService(rm2.getResourceTrackerService());
515504
((MyAMRMClientImpl) amClient).updateRMProxy(rm2);
516-
dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher();
517505

518506
AMRMTokenSecretManager amrmTokenSecretManagerForRM2 =
519507
rm2.getRMContext().getAMRMTokenSecretManager();
@@ -615,11 +603,6 @@ public void serviceStart() throws Exception {
615603
MyResourceManager.setClusterTimeStamp(fakeClusterTimeStamp);
616604
}
617605

618-
@Override
619-
protected Dispatcher createDispatcher() {
620-
return new DrainDispatcher();
621-
}
622-
623606
@Override
624607
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
625608
// Dispatch inline for test sanity

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,9 @@
3030
import org.apache.hadoop.service.Service.STATE;
3131
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
3232
import org.apache.hadoop.yarn.conf.YarnConfiguration;
33-
import org.apache.hadoop.yarn.event.Dispatcher;
34-
import org.apache.hadoop.yarn.event.DrainDispatcher;
3533
import org.apache.hadoop.yarn.ipc.YarnRPC;
3634
import org.junit.Before;
3735

38-
import static org.mockito.Mockito.mock;
39-
import static org.mockito.Mockito.when;
40-
4136
public abstract class ACLsTestBase {
4237

4338
protected static final String COMMON_USER = "common_user";
@@ -80,11 +75,6 @@ this.queueACLsManager, getRMContext()
8075
.getRMDelegationTokenSecretManager());
8176
}
8277

83-
@Override
84-
protected Dispatcher createDispatcher() {
85-
return new DrainDispatcher();
86-
}
87-
8878
@Override
8979
protected void doSecureLogin() throws IOException {
9080
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java‎

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -26,22 +26,17 @@
2626
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
2727
import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
2828
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
29-
import org.apache.hadoop.yarn.api.records.ResourceRequest;
3029
import org.apache.hadoop.yarn.conf.HAUtil;
3130
import org.apache.hadoop.yarn.conf.YarnConfiguration;
32-
import org.apache.hadoop.yarn.event.Dispatcher;
33-
import org.apache.hadoop.yarn.event.DrainDispatcher;
3431
import org.apache.hadoop.yarn.exceptions.YarnException;
3532
import org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore;
3633
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
3734
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
3835
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
3936
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
40-
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
4137
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
4238
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
4339
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
44-
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
4540
import org.junit.After;
4641
import org.junit.Assert;
4742
import org.junit.Before;
@@ -108,20 +103,9 @@ protected MockAM launchAM(RMApp app, MockRM rm, MockNM nm)
108103
}
109104

110105
protected void startRMs() throws IOException {
111-
rm1 = new MockRM(confForRM1, null, false, false){
112-
@Override
113-
protected Dispatcher createDispatcher() {
114-
return new DrainDispatcher();
115-
}
116-
};
117-
rm2 = new MockRM(confForRM2, null, false, false){
118-
@Override
119-
protected Dispatcher createDispatcher() {
120-
return new DrainDispatcher();
121-
}
122-
};
106+
rm1 = new MockRM(confForRM1, null, false, false);
107+
rm2 = new MockRM(confForRM2, null, false, false);
123108
startRMs(rm1, confForRM1, rm2, confForRM2);
124-
125109
}
126110

127111
protected void startRMsWithCustomizedRMAppManager() throws IOException {

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@
4747
import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
4848
import org.apache.hadoop.yarn.api.records.ReservationRequests;
4949
import org.apache.hadoop.yarn.conf.YarnConfiguration;
50-
import org.apache.hadoop.yarn.event.DrainDispatcher;
5150
import org.apache.hadoop.yarn.exceptions.YarnException;
5251
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
5352
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@@ -463,9 +462,7 @@ private void registerNode(String host, int memory, int vCores) throws
463462
int attempts = 10;
464463
Collection<Plan> plans;
465464
do {
466-
DrainDispatcher dispatcher =
467-
(DrainDispatcher) resourceManager.getRMContext().getDispatcher();
468-
dispatcher.await();
465+
resourceManager.drainEvents();
469466
LOG.info("Waiting for node capacity to be added to plan");
470467
plans = resourceManager.getRMContext().getReservationSystem()
471468
.getAllPlans().values();

0 commit comments

Comments
 (0)