Skip to content

Commit c394051

Browse files
author
Inigo Goiri
committed
HDFS-13326. RBF: Improve the interfaces to modify and view mount tables. Contributed by Gang Li.
1 parent ed4e067 commit c394051

File tree

3 files changed

+276
-10
lines changed

3 files changed

+276
-10
lines changed

‎hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java‎

Lines changed: 122 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,10 @@ public RouterAdmin(Configuration conf) {
9494
*/
9595
public void printUsage() {
9696
String usage = "Federation Admin Tools:\n"
97-
+ "\t[-add <source> <nameservice> <destination> "
97+
+ "\t[-add <source> <nameservice1, nameservice2, ...> <destination> "
98+
+ "[-readonly] [-order HASH|LOCAL|RANDOM|HASH_ALL] "
99+
+ "-owner <owner> -group <group> -mode <mode>]\n"
100+
+ "\t[-update <source> <nameservice1, nameservice2, ...> <destination> "
98101
+ "[-readonly] [-order HASH|LOCAL|RANDOM|HASH_ALL] "
99102
+ "-owner <owner> -group <group> -mode <mode>]\n"
100103
+ "\t[-rm <source>]\n"
@@ -112,7 +115,7 @@ public void printUsage() {
112115
@Override
113116
public int run(String[] argv) throws Exception {
114117
if (argv.length < 1) {
115-
System.err.println("Not enough parameters specificed");
118+
System.err.println("Not enough parameters specified");
116119
printUsage();
117120
return -1;
118121
}
@@ -124,31 +127,37 @@ public int run(String[] argv) throws Exception {
124127
// Verify that we have enough command line parameters
125128
if ("-add".equals(cmd)) {
126129
if (argv.length < 4) {
127-
System.err.println("Not enough parameters specificed for cmd " + cmd);
130+
System.err.println("Not enough parameters specified for cmd " + cmd);
131+
printUsage();
132+
return exitCode;
133+
}
134+
} else if ("-update".equals(cmd)) {
135+
if (argv.length < 4) {
136+
System.err.println("Not enough parameters specified for cmd " + cmd);
128137
printUsage();
129138
return exitCode;
130139
}
131140
} else if ("-rm".equalsIgnoreCase(cmd)) {
132141
if (argv.length < 2) {
133-
System.err.println("Not enough parameters specificed for cmd " + cmd);
142+
System.err.println("Not enough parameters specified for cmd " + cmd);
134143
printUsage();
135144
return exitCode;
136145
}
137146
} else if ("-setQuota".equalsIgnoreCase(cmd)) {
138147
if (argv.length < 4) {
139-
System.err.println("Not enough parameters specificed for cmd " + cmd);
148+
System.err.println("Not enough parameters specified for cmd " + cmd);
140149
printUsage();
141150
return exitCode;
142151
}
143152
} else if ("-clrQuota".equalsIgnoreCase(cmd)) {
144153
if (argv.length < 2) {
145-
System.err.println("Not enough parameters specificed for cmd " + cmd);
154+
System.err.println("Not enough parameters specified for cmd " + cmd);
146155
printUsage();
147156
return exitCode;
148157
}
149158
} else if ("-safemode".equalsIgnoreCase(cmd)) {
150159
if (argv.length < 2) {
151-
System.err.println("Not enough parameters specificed for cmd " + cmd);
160+
System.err.println("Not enough parameters specified for cmd " + cmd);
152161
printUsage();
153162
return exitCode;
154163
}
@@ -181,7 +190,11 @@ public int run(String[] argv) throws Exception {
181190
try {
182191
if ("-add".equals(cmd)) {
183192
if (addMount(argv, i)) {
184-
System.out.println("Successfuly added mount point " + argv[i]);
193+
System.out.println("Successfully added mount point " + argv[i]);
194+
}
195+
} else if ("-update".equals(cmd)) {
196+
if (updateMount(argv, i)) {
197+
System.out.println("Successfully updated mount point " + argv[i]);
185198
}
186199
} else if ("-rm".equals(cmd)) {
187200
if (removeMount(argv[i])) {
@@ -398,6 +411,107 @@ public boolean addMount(String mount, String[] nss, String dest,
398411
}
399412
}
400413

414+
/**
415+
* Update a mount table entry.
416+
*
417+
* @param parameters Parameters for the mount point.
418+
* @param i Index in the parameters.
419+
*/
420+
public boolean updateMount(String[] parameters, int i) throws IOException {
421+
// Mandatory parameters
422+
String mount = parameters[i++];
423+
String[] nss = parameters[i++].split(",");
424+
String dest = parameters[i++];
425+
426+
// Optional parameters
427+
boolean readOnly = false;
428+
String owner = null;
429+
String group = null;
430+
FsPermission mode = null;
431+
DestinationOrder order = null;
432+
while (i < parameters.length) {
433+
if (parameters[i].equals("-readonly")) {
434+
readOnly = true;
435+
} else if (parameters[i].equals("-order")) {
436+
i++;
437+
try {
438+
order = DestinationOrder.valueOf(parameters[i]);
439+
} catch(Exception e) {
440+
System.err.println("Cannot parse order: " + parameters[i]);
441+
}
442+
} else if (parameters[i].equals("-owner")) {
443+
i++;
444+
owner = parameters[i];
445+
} else if (parameters[i].equals("-group")) {
446+
i++;
447+
group = parameters[i];
448+
} else if (parameters[i].equals("-mode")) {
449+
i++;
450+
short modeValue = Short.parseShort(parameters[i], 8);
451+
mode = new FsPermission(modeValue);
452+
}
453+
454+
i++;
455+
}
456+
457+
return updateMount(mount, nss, dest, readOnly, order,
458+
new ACLEntity(owner, group, mode));
459+
}
460+
461+
/**
462+
* Update a mount table entry.
463+
*
464+
* @param mount Mount point.
465+
* @param nss Nameservices where this is mounted to.
466+
* @param dest Destination path.
467+
* @param readonly If the mount point is read only.
468+
* @param order Order of the destination locations.
469+
* @param aclInfo the ACL info for mount point.
470+
* @return If the mount point was updated.
471+
* @throws IOException Error updating the mount point.
472+
*/
473+
public boolean updateMount(String mount, String[] nss, String dest,
474+
boolean readonly, DestinationOrder order, ACLEntity aclInfo)
475+
throws IOException {
476+
MountTableManager mountTable = client.getMountTableManager();
477+
478+
// Create a new entry
479+
Map<String, String> destMap = new LinkedHashMap<>();
480+
for (String ns : nss) {
481+
destMap.put(ns, dest);
482+
}
483+
MountTable newEntry = MountTable.newInstance(mount, destMap);
484+
485+
newEntry.setReadOnly(readonly);
486+
487+
if (order != null) {
488+
newEntry.setDestOrder(order);
489+
}
490+
491+
// Update ACL info of mount table entry
492+
if (aclInfo.getOwner() != null) {
493+
newEntry.setOwnerName(aclInfo.getOwner());
494+
}
495+
496+
if (aclInfo.getGroup() != null) {
497+
newEntry.setGroupName(aclInfo.getGroup());
498+
}
499+
500+
if (aclInfo.getMode() != null) {
501+
newEntry.setMode(aclInfo.getMode());
502+
}
503+
504+
UpdateMountTableEntryRequest updateRequest =
505+
UpdateMountTableEntryRequest.newInstance(newEntry);
506+
UpdateMountTableEntryResponse updateResponse =
507+
mountTable.updateMountTableEntry(updateRequest);
508+
boolean updated = updateResponse.getStatus();
509+
if (!updated) {
510+
System.err.println("Cannot update mount point " + mount);
511+
}
512+
return updated;
513+
}
514+
401515
/**
402516
* Remove mount point.
403517
*

‎hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -529,4 +529,154 @@ public Boolean get() {
529529
}
530530
}, 1000, 30000);
531531
}
532+
533+
@Test
534+
public void testUpdateNonExistingMountTable() throws Exception {
535+
System.setOut(new PrintStream(out));
536+
String nsId = "ns0";
537+
String src = "/test-updateNonExistingMounttable";
538+
String dest = "/updateNonExistingMounttable";
539+
String[] argv = new String[] {"-update", src, nsId, dest};
540+
assertEquals(0, ToolRunner.run(admin, argv));
541+
542+
stateStore.loadCache(MountTableStoreImpl.class, true);
543+
GetMountTableEntriesRequest getRequest =
544+
GetMountTableEntriesRequest.newInstance(src);
545+
GetMountTableEntriesResponse getResponse =
546+
client.getMountTableManager().getMountTableEntries(getRequest);
547+
// Ensure the destination updated successfully
548+
MountTable mountTable = getResponse.getEntries().get(0);
549+
assertEquals(src, mountTable.getSourcePath());
550+
assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId());
551+
assertEquals(dest, mountTable.getDestinations().get(0).getDest());
552+
}
553+
554+
@Test
555+
public void testUpdateNameserviceDestinationForExistingMountTable() throws
556+
Exception {
557+
// Add a mount table firstly
558+
String nsId = "ns0";
559+
String src = "/test-updateNameserviceDestinationForExistingMountTable";
560+
String dest = "/UpdateNameserviceDestinationForExistingMountTable";
561+
String[] argv = new String[] {"-add", src, nsId, dest};
562+
assertEquals(0, ToolRunner.run(admin, argv));
563+
564+
stateStore.loadCache(MountTableStoreImpl.class, true);
565+
GetMountTableEntriesRequest getRequest =
566+
GetMountTableEntriesRequest.newInstance(src);
567+
GetMountTableEntriesResponse getResponse =
568+
client.getMountTableManager().getMountTableEntries(getRequest);
569+
// Ensure mount table added successfully
570+
MountTable mountTable = getResponse.getEntries().get(0);
571+
assertEquals(src, mountTable.getSourcePath());
572+
assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId());
573+
assertEquals(dest, mountTable.getDestinations().get(0).getDest());
574+
575+
// Update the destination
576+
String newNsId = "ns1";
577+
String newDest = "/newDestination";
578+
argv = new String[] {"-update", src, newNsId, newDest};
579+
assertEquals(0, ToolRunner.run(admin, argv));
580+
581+
stateStore.loadCache(MountTableStoreImpl.class, true);
582+
getResponse = client.getMountTableManager()
583+
.getMountTableEntries(getRequest);
584+
// Ensure the destination updated successfully
585+
mountTable = getResponse.getEntries().get(0);
586+
assertEquals(src, mountTable.getSourcePath());
587+
assertEquals(newNsId,
588+
mountTable.getDestinations().get(0).getNameserviceId());
589+
assertEquals(newDest, mountTable.getDestinations().get(0).getDest());
590+
}
591+
592+
@Test
593+
public void testUpdateReadonlyUserGroupPermissionMountable()
594+
throws Exception {
595+
// Add a mount table
596+
String nsId = "ns0";
597+
String src = "/test-updateReadonlyUserGroupPermissionMountTable";
598+
String dest = "/UpdateReadonlyUserGroupPermissionMountTable";
599+
String[] argv = new String[] {"-add", src, nsId, dest};
600+
assertEquals(0, ToolRunner.run(admin, argv));
601+
602+
stateStore.loadCache(MountTableStoreImpl.class, true);
603+
GetMountTableEntriesRequest getRequest =
604+
GetMountTableEntriesRequest.newInstance(src);
605+
GetMountTableEntriesResponse getResponse =
606+
client.getMountTableManager().getMountTableEntries(getRequest);
607+
// Ensure mount table added successfully
608+
MountTable mountTable = getResponse.getEntries().get(0);
609+
assertEquals(src, mountTable.getSourcePath());
610+
assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId());
611+
assertEquals(dest, mountTable.getDestinations().get(0).getDest());
612+
assertFalse(mountTable.isReadOnly());
613+
614+
// Update the readonly, owner, group and permission
615+
String testOwner = "test_owner";
616+
String testGroup = "test_group";
617+
argv = new String[] {"-update", src, nsId, dest, "-readonly",
618+
"-owner", testOwner, "-group", testGroup, "-mode", "0455"};
619+
assertEquals(0, ToolRunner.run(admin, argv));
620+
621+
stateStore.loadCache(MountTableStoreImpl.class, true);
622+
getResponse = client.getMountTableManager()
623+
.getMountTableEntries(getRequest);
624+
625+
// Ensure the destination updated successfully
626+
mountTable = getResponse.getEntries().get(0);
627+
assertEquals(src, mountTable.getSourcePath());
628+
assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId());
629+
assertEquals(dest, mountTable.getDestinations().get(0).getDest());
630+
assertTrue(mountTable.isReadOnly());
631+
assertEquals(testOwner, mountTable.getOwnerName());
632+
assertEquals(testGroup, mountTable.getGroupName());
633+
assertEquals((short)0455, mountTable.getMode().toShort());
634+
}
635+
636+
@Test
637+
public void testUpdateOrderMountTable() throws Exception {
638+
testUpdateOrderMountTable(DestinationOrder.HASH);
639+
testUpdateOrderMountTable(DestinationOrder.LOCAL);
640+
testUpdateOrderMountTable(DestinationOrder.RANDOM);
641+
testUpdateOrderMountTable(DestinationOrder.HASH_ALL);
642+
}
643+
644+
private void testUpdateOrderMountTable(DestinationOrder order)
645+
throws Exception {
646+
// Add a mount table
647+
String nsId = "ns0";
648+
String src = "/test-updateOrderMountTable-"+order.toString();
649+
String dest = "/UpdateOrderMountTable";
650+
String[] argv = new String[] {"-add", src, nsId, dest};
651+
assertEquals(0, ToolRunner.run(admin, argv));
652+
653+
stateStore.loadCache(MountTableStoreImpl.class, true);
654+
GetMountTableEntriesRequest getRequest =
655+
GetMountTableEntriesRequest.newInstance(src);
656+
GetMountTableEntriesResponse getResponse =
657+
client.getMountTableManager().getMountTableEntries(getRequest);
658+
659+
// Ensure mount table added successfully
660+
MountTable mountTable = getResponse.getEntries().get(0);
661+
assertEquals(src, mountTable.getSourcePath());
662+
assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId());
663+
assertEquals(dest, mountTable.getDestinations().get(0).getDest());
664+
assertEquals(DestinationOrder.HASH, mountTable.getDestOrder());
665+
666+
// Update the order
667+
argv = new String[] {"-update", src, nsId, dest, "-order",
668+
order.toString()};
669+
assertEquals(0, ToolRunner.run(admin, argv));
670+
671+
stateStore.loadCache(MountTableStoreImpl.class, true);
672+
getResponse = client.getMountTableManager()
673+
.getMountTableEntries(getRequest);
674+
675+
// Ensure the destination updated successfully
676+
mountTable = getResponse.getEntries().get(0);
677+
assertEquals(src, mountTable.getSourcePath());
678+
assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId());
679+
assertEquals(dest, mountTable.getDestinations().get(0).getDest());
680+
assertEquals(order, mountTable.getDestOrder());
681+
}
532682
}

‎hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -425,7 +425,8 @@ Runs the DFS router. See [Router](./HDFSRouterFederation.html#Router) for more i
425425
Usage:
426426

427427
hdfs dfsrouteradmin
428-
[-add <source> <nameservice> <destination> [-readonly] -owner <owner> -group <group> -mode <mode>]
428+
[-add <source> <nameservice1, nameservice2, ...> <destination> [-readonly] [-order HASH|LOCAL|RANDOM|HASH_ALL] -owner <owner> -group <group> -mode <mode>]
429+
[-update <source> <nameservice1, nameservice2, ...> <destination> [-readonly] [-order HASH|LOCAL|RANDOM|HASH_ALL] -owner <owner> -group <group> -mode <mode>]
429430
[-rm <source>]
430431
[-ls <path>]
431432
[-setQuota <path> -nsQuota <nsQuota> -ssQuota <quota in bytes or quota size string>]
@@ -436,7 +437,8 @@ Usage:
436437

437438
| COMMAND\_OPTION | Description |
438439
|:---- |:---- |
439-
| `-add` *source* *nameservice* *destination* | Add a mount table entry or update if it exists. |
440+
| `-add` *source* *nameservices* *destination* | Add a mount table entry or update if it exists. |
441+
| `-update` *source* *nameservices* *destination* | Update a mount table entry or create one if it does not exist. |
440442
| `-rm` *source* | Remove mount point of specified path. |
441443
| `-ls` *path* | List mount points under specified path. |
442444
| `-setQuota` *path* `-nsQuota` *nsQuota* `-ssQuota` *ssQuota* | Set quota for specified path. See [HDFS Quotas Guide](./HdfsQuotaAdminGuide.html) for the quota detail. |

0 commit comments

Comments
 (0)