Skip to content

Commit 04abd0e

Browse files
szilard-nemethszilard-nemeth
committed
YARN-10330. Add missing test scenarios to TestUserGroupMappingPlacementRule and TestAppNameMappingPlacementRule. Contributed by Peter Bacsko
1 parent 9b5557a commit 04abd0e

File tree

3 files changed

+263
-23
lines changed

3 files changed

+263
-23
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java‎

Lines changed: 86 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,15 @@
3737
import static org.mockito.Mockito.when;
3838

3939
public class TestAppNameMappingPlacementRule {
40+
private static final String ROOT_QUEUE = "root";
41+
private static final String Q2_QUEUE = "q2";
42+
private static final String Q1_QUEUE = "q1";
43+
private static final String USER_NAME = "user";
44+
private static final String DEFAULT_QUEUE = "default";
45+
private static final String APPLICATION_PLACEHOLDER = "%application";
46+
private static final String AMBIGUOUS_QUEUE = "ambiguousQueue";
4047
private static final String APP_NAME = "DistributedShell";
48+
private static final String MAPREDUCE_APP_NAME = "MAPREDUCE";
4149

4250
private YarnConfiguration conf = new YarnConfiguration();
4351

@@ -62,16 +70,20 @@ private void verifyQueueMapping(QueueMapping queueMapping,
6270
CapacitySchedulerQueueManager qm =
6371
mock(CapacitySchedulerQueueManager.class);
6472
when(qm.isAmbiguous(Mockito.isA(String.class))).thenReturn(false);
73+
when(qm.isAmbiguous(AMBIGUOUS_QUEUE)).thenReturn(true);
74+
6575
rule.queueManager = qm;
6676

6777
ApplicationSubmissionContext asc = Records.newRecord(
6878
ApplicationSubmissionContext.class);
69-
if (inputQueue.equals("%application")) {
79+
if (inputQueue.equals(APPLICATION_PLACEHOLDER)) {
7080
inputQueue = APP_NAME;
7181
}
7282
asc.setQueue(inputQueue);
7383
String appName = queueMapping.getSource();
74-
if (appName.equals("%application")) {
84+
// to create a scenario when source != appName
85+
if (appName.equals(APPLICATION_PLACEHOLDER)
86+
|| appName.equals(MAPREDUCE_APP_NAME)) {
7587
appName = APP_NAME;
7688
}
7789
asc.setApplicationName(appName);
@@ -81,31 +93,85 @@ private void verifyQueueMapping(QueueMapping queueMapping,
8193
ctx != null ? ctx.getQueue() : inputQueue);
8294
}
8395

84-
public QueueMapping queueMappingBuilder(String source, String queue) {
96+
public QueueMapping getQueueMapping(String source, String queue) {
97+
return getQueueMapping(source, null, queue);
98+
}
99+
100+
public QueueMapping getQueueMapping(String source, String parent,
101+
String queue) {
85102
return QueueMapping.QueueMappingBuilder.create()
86103
.type(QueueMapping.MappingType.APPLICATION)
87104
.source(source)
88105
.queue(queue)
106+
.parentQueue(parent)
89107
.build();
90108
}
91109

92110
@Test
93-
public void testMapping() throws YarnException {
94-
// simple base case for mapping user to queue
95-
verifyQueueMapping(queueMappingBuilder(APP_NAME,
96-
"q1"), "user_1", "q1");
97-
verifyQueueMapping(queueMappingBuilder("%application", "q2"), "user_1",
98-
"q2");
99-
verifyQueueMapping(queueMappingBuilder("%application", "%application"),
100-
"user_1", APP_NAME);
101-
102-
// specify overwritten, and see if user specified a queue, and it will be
103-
// overridden
104-
verifyQueueMapping(queueMappingBuilder(APP_NAME,
105-
"q1"), "1", "q2", "q1", true);
106-
107-
// if overwritten not specified, it should be which user specified
108-
verifyQueueMapping(queueMappingBuilder(APP_NAME,
109-
"q1"), "1", "q2", "q2", false);
111+
public void testSpecificAppNameMappedToDefinedQueue() throws YarnException {
112+
verifyQueueMapping(getQueueMapping(APP_NAME, Q1_QUEUE),
113+
USER_NAME, Q1_QUEUE);
114+
}
115+
116+
@Test
117+
public void testPlaceholderAppSourceMappedToQueue() throws YarnException {
118+
verifyQueueMapping(getQueueMapping(APPLICATION_PLACEHOLDER, Q2_QUEUE),
119+
USER_NAME, Q2_QUEUE);
120+
}
121+
122+
@Test
123+
public void testPlaceHolderAppSourceAndQueueMappedToAppNameQueue()
124+
throws YarnException {
125+
verifyQueueMapping(getQueueMapping(APPLICATION_PLACEHOLDER,
126+
APPLICATION_PLACEHOLDER), USER_NAME, APP_NAME);
127+
}
128+
129+
@Test
130+
public void testQueueInMappingOverridesSpecifiedQueue()
131+
throws YarnException {
132+
verifyQueueMapping(getQueueMapping(APP_NAME,
133+
Q1_QUEUE), USER_NAME, Q2_QUEUE, Q1_QUEUE, true);
134+
}
135+
136+
@Test
137+
public void testQueueInMappingDoesNotOverrideSpecifiedQueue()
138+
throws YarnException {
139+
verifyQueueMapping(getQueueMapping(APP_NAME,
140+
Q1_QUEUE), USER_NAME, Q2_QUEUE, Q2_QUEUE, false);
110141
}
142+
143+
@Test
144+
public void testDefaultQueueInMappingIsNotUsedWithoutOverride()
145+
throws YarnException {
146+
verifyQueueMapping(getQueueMapping(APP_NAME,
147+
DEFAULT_QUEUE), USER_NAME, Q2_QUEUE, Q2_QUEUE, false);
148+
}
149+
150+
@Test
151+
public void testDefaultQueueInMappingEqualsToInputQueue()
152+
throws YarnException {
153+
verifyQueueMapping(getQueueMapping(APP_NAME,
154+
DEFAULT_QUEUE), USER_NAME, DEFAULT_QUEUE, DEFAULT_QUEUE, false);
155+
}
156+
157+
@Test
158+
public void testMappingSourceDiffersFromInputQueue() throws YarnException {
159+
verifyQueueMapping(getQueueMapping(MAPREDUCE_APP_NAME,
160+
Q1_QUEUE), USER_NAME, DEFAULT_QUEUE, DEFAULT_QUEUE, false);
161+
}
162+
163+
@Test(expected = YarnException.class)
164+
public void testMappingContainsAmbiguousLeafQueueWithoutParent()
165+
throws YarnException {
166+
verifyQueueMapping(getQueueMapping(APP_NAME, AMBIGUOUS_QUEUE),
167+
USER_NAME, DEFAULT_QUEUE, DEFAULT_QUEUE, false);
168+
}
169+
170+
@Test
171+
public void testMappingContainsAmbiguousLeafQueueWithParent()
172+
throws YarnException {
173+
verifyQueueMapping(getQueueMapping(APP_NAME, ROOT_QUEUE, AMBIGUOUS_QUEUE),
174+
USER_NAME, DEFAULT_QUEUE, AMBIGUOUS_QUEUE, false);
175+
}
176+
111177
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java‎

Lines changed: 168 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ private AbstractCSQueue createRootQueue(String rootQueueName) {
149149
ParentQueue root = mock(ParentQueue.class);
150150
when(root.getQueuePath()).thenReturn(rootQueueName);
151151
when(queueManager.getQueue(rootQueueName)).thenReturn(root);
152+
when(queueManager.getQueueByFullName(rootQueueName)).thenReturn(root);
152153
return root;
153154
}
154155

@@ -220,11 +221,13 @@ private void verifyQueueMapping(QueueMappingTestData queueMappingTestData)
220221
.withQueue("root.agroup.a")
221222
.withQueue("root.asubgroup2")
222223
.withQueue("root.bsubgroup2.b")
224+
.withQueue("root.users.primarygrouponly")
225+
.withQueue("root.admins.primarygrouponly")
223226
.withManagedParentQueue("root.managedParent")
224227
.build();
225228

226229
when(queueManager.getQueue(isNull())).thenReturn(null);
227-
230+
when(queueManager.isAmbiguous("primarygrouponly")).thenReturn(true);
228231
rule.setQueueManager(queueManager);
229232
ApplicationSubmissionContext asc = Records.newRecord(
230233
ApplicationSubmissionContext.class);
@@ -375,6 +378,170 @@ public void testUserMappingToQueueNamedAsUsernameWithPrimaryGroupAsParentQueue()
375378
.build());
376379
}
377380

381+
@Test
382+
public void testUserMappingToPrimaryGroupInvalidNestedPlaceholder()
383+
throws YarnException {
384+
// u:%user:%primary_group.%random, no matching queue
385+
verifyQueueMapping(
386+
QueueMappingTestDataBuilder.create()
387+
.queueMapping(QueueMappingBuilder.create()
388+
.type(MappingType.USER)
389+
.source("%user")
390+
.queue("%random")
391+
.parentQueue("%primary_group")
392+
.build())
393+
.inputUser("a")
394+
.expectedQueue("default")
395+
.build());
396+
}
397+
398+
@Test
399+
public void testUserMappingToSecondaryGroupInvalidNestedPlaceholder()
400+
throws YarnException {
401+
// u:%user:%secondary_group.%random, no matching queue
402+
verifyQueueMapping(
403+
QueueMappingTestDataBuilder.create()
404+
.queueMapping(QueueMappingBuilder.create()
405+
.type(MappingType.USER)
406+
.source("%user")
407+
.queue("%random")
408+
.parentQueue("%secondary_group")
409+
.build())
410+
.inputUser("a")
411+
.expectedQueue("default")
412+
.build());
413+
}
414+
415+
@Test
416+
public void testUserMappingDiffersFromSubmitterQueueDoesNotExist()
417+
throws YarnException {
418+
// u:a:%random, submitter: xyz, no matching queue
419+
verifyQueueMapping(
420+
QueueMappingTestDataBuilder.create()
421+
.queueMapping(QueueMappingBuilder.create()
422+
.type(MappingType.USER)
423+
.source("a")
424+
.queue("%random")
425+
.build())
426+
.inputUser("xyz")
427+
.expectedQueue("default")
428+
.build());
429+
}
430+
431+
@Test
432+
public void testSpecificUserMappingToPrimaryGroup() throws YarnException {
433+
// u:a:%primary_group
434+
verifyQueueMapping(
435+
QueueMappingTestDataBuilder.create()
436+
.queueMapping(QueueMappingBuilder.create()
437+
.type(MappingType.USER)
438+
.source("a")
439+
.queue("%primary_group")
440+
.build())
441+
.inputUser("a")
442+
.expectedQueue("agroup")
443+
.build());
444+
}
445+
446+
@Test
447+
public void testSpecificUserMappingToSecondaryGroup()
448+
throws YarnException {
449+
// u:a:%secondary_group
450+
verifyQueueMapping(
451+
QueueMappingTestDataBuilder.create()
452+
.queueMapping(QueueMappingBuilder.create()
453+
.type(MappingType.USER)
454+
.source("a")
455+
.queue("%secondary_group")
456+
.build())
457+
.inputUser("a")
458+
.expectedQueue("asubgroup2")
459+
.build());
460+
}
461+
462+
@Test
463+
public void testSpecificUserMappingWithNoSecondaryGroup()
464+
throws YarnException {
465+
// u:nosecondarygroupuser:%secondary_group, no matching queue
466+
verifyQueueMapping(
467+
QueueMappingTestDataBuilder.create()
468+
.queueMapping(QueueMappingBuilder.create()
469+
.type(MappingType.USER)
470+
.source("nosecondarygroupuser")
471+
.queue("%secondary_group")
472+
.build())
473+
.inputUser("nosecondarygroupuser")
474+
.expectedQueue("default")
475+
.build());
476+
}
477+
478+
@Test
479+
public void testGenericUserMappingWithNoSecondaryGroup()
480+
throws YarnException {
481+
// u:%user:%user, no matching queue
482+
verifyQueueMapping(
483+
QueueMappingTestDataBuilder.create()
484+
.queueMapping(QueueMappingBuilder.create()
485+
.type(MappingType.USER)
486+
.source("%user")
487+
.queue("%user")
488+
.parentQueue("%secondary_group")
489+
.build())
490+
.inputUser("nosecondarygroupuser")
491+
.expectedQueue("default")
492+
.build());
493+
}
494+
495+
@Test(expected = YarnException.class)
496+
public void testUserMappingToNestedUserPrimaryGroupWithAmbiguousQueues()
497+
throws YarnException {
498+
// u:%user:%user, submitter nosecondarygroupuser, queue is ambiguous
499+
verifyQueueMapping(
500+
QueueMappingTestDataBuilder.create()
501+
.queueMapping(QueueMappingBuilder.create()
502+
.type(MappingType.USER)
503+
.source("%user")
504+
.queue("%user")
505+
.parentQueue("%primary_group")
506+
.build())
507+
.inputUser("nosecondarygroupuser")
508+
.build());
509+
}
510+
511+
@Test(expected = YarnException.class)
512+
public void testResolvedQueueIsNotManaged()
513+
throws YarnException {
514+
// u:%user:%primary_group.%user, "admins" group will be "root",
515+
// resulting parent queue will be "root" which is not managed
516+
verifyQueueMapping(
517+
QueueMappingTestDataBuilder.create()
518+
.queueMapping(QueueMappingBuilder.create()
519+
.type(MappingType.USER)
520+
.source("%user")
521+
.queue("%user")
522+
.parentQueue("%primary_group")
523+
.build())
524+
.inputUser("admins")
525+
.build());
526+
}
527+
528+
@Test(expected = YarnException.class)
529+
public void testUserMappingToPrimaryGroupWithAmbiguousQueues()
530+
throws YarnException {
531+
// u:%user:%primary_group, submitter nosecondarygroupuser,
532+
// queue is ambiguous
533+
verifyQueueMapping(
534+
QueueMappingTestDataBuilder.create()
535+
.queueMapping(QueueMappingBuilder.create()
536+
.type(MappingType.USER)
537+
.source("%user")
538+
.queue("%primary_group")
539+
.build())
540+
.inputUser("nosecondarygroupuser")
541+
.expectedQueue("default")
542+
.build());
543+
}
544+
378545
@Test
379546
public void testUserMappingToQueueNamedAsUsernameWithSecondaryGroupAsParentQueue()
380547
throws YarnException {

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,17 @@
2525
import org.apache.hadoop.security.GroupMappingServiceProvider;
2626

2727
public class SimpleGroupsMapping implements GroupMappingServiceProvider {
28-
28+
2929
@Override
3030
public List<String> getGroups(String user) {
31-
return Arrays.asList(user + "group", user + "subgroup1", user + "subgroup2");
31+
if ("admins".equals(user)) {
32+
return Arrays.asList("root");
33+
} else if ("nosecondarygroupuser".equals(user)) {
34+
return Arrays.asList("primarygrouponly");
35+
} else {
36+
return Arrays.asList(
37+
user + "group", user + "subgroup1", user + "subgroup2");
38+
}
3239
}
3340

3441
@Override

0 commit comments

Comments
 (0)