Skip to content

Commit b09d389

Browse files
bharatviswa504bharatviswa504
authored
HDDS-2019. Handle Set DtService of token in S3Gateway for OM HA. (#1489)
1 parent 53ed78b commit b09d389

File tree

4 files changed

+230
-5
lines changed

4 files changed

+230
-5
lines changed

‎hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@ public class OzoneClientProducer {
6262
@Inject
6363
private Text omService;
6464

65+
@Inject
66+
private String omServiceID;
67+
6568

6669
@Produces
6770
public OzoneClient createClient() throws IOException {
@@ -105,7 +108,13 @@ private OzoneClient getClient(OzoneConfiguration config) throws IOException {
105108
} catch (Exception e) {
106109
LOG.error("Error: ", e);
107110
}
108-
return OzoneClientFactory.getClient(ozoneConfiguration);
111+
112+
if (omServiceID == null) {
113+
return OzoneClientFactory.getClient(ozoneConfiguration);
114+
} else {
115+
// As in HA case, we need to pass om service ID.
116+
return OzoneClientFactory.getRpcClient(omServiceID, ozoneConfiguration);
117+
}
109118
}
110119

111120
@VisibleForTesting

‎hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneServiceProvider.java‎

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,33 +20,75 @@
2020
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
2121
import org.apache.hadoop.io.Text;
2222
import org.apache.hadoop.ozone.OmUtils;
23+
import org.apache.hadoop.ozone.s3.util.OzoneS3Util;
2324
import org.apache.hadoop.security.SecurityUtil;
2425

2526
import javax.annotation.PostConstruct;
2627
import javax.enterprise.context.ApplicationScoped;
2728
import javax.enterprise.inject.Produces;
2829
import javax.inject.Inject;
30+
31+
import java.util.Arrays;
32+
import java.util.Collection;
33+
34+
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_NODES_KEY;
35+
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY;
36+
2937
/**
3038
* This class creates the OM service .
3139
*/
3240
@ApplicationScoped
3341
public class OzoneServiceProvider {
3442

35-
private Text omServiceAdd;
43+
private Text omServiceAddr;
44+
45+
private String omserviceID;
3646

3747
@Inject
3848
private OzoneConfiguration conf;
3949

4050
@PostConstruct
4151
public void init() {
42-
omServiceAdd = SecurityUtil.buildTokenService(OmUtils.
43-
getOmAddressForClients(conf));
52+
Collection<String> serviceIdList =
53+
conf.getTrimmedStringCollection(OZONE_OM_SERVICE_IDS_KEY);
54+
if (serviceIdList.size() == 0) {
55+
// Non-HA cluster
56+
omServiceAddr = SecurityUtil.buildTokenService(OmUtils.
57+
getOmAddressForClients(conf));
58+
} else {
59+
// HA cluster.
60+
//For now if multiple service id's are configured we throw exception.
61+
// As if multiple service id's are configured, S3Gateway will not be
62+
// knowing which one to talk to. In future, if OM federation is supported
63+
// we can resolve this by having another property like
64+
// ozone.om.internal.service.id.
65+
// TODO: Revisit this later.
66+
if (serviceIdList.size() > 1) {
67+
throw new IllegalArgumentException("Multiple serviceIds are " +
68+
"configured. " + Arrays.toString(serviceIdList.toArray()));
69+
} else {
70+
String serviceId = serviceIdList.iterator().next();
71+
Collection<String> omNodeIds = OmUtils.getOMNodeIds(conf, serviceId);
72+
if (omNodeIds.size() == 0) {
73+
throw new IllegalArgumentException(OZONE_OM_NODES_KEY
74+
+ "." + serviceId + " is not defined");
75+
}
76+
omServiceAddr = new Text(OzoneS3Util.buildServiceNameForToken(conf,
77+
serviceId, omNodeIds));
78+
omserviceID = serviceId;
79+
}
80+
}
4481
}
4582

4683

4784
@Produces
4885
public Text getService() {
49-
return omServiceAdd;
86+
return omServiceAddr;
87+
}
88+
89+
@Produces
90+
public String getOmServiceID() {
91+
return omserviceID;
5092
}
5193

5294
}

‎hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,17 @@
1919
package org.apache.hadoop.ozone.s3.util;
2020

2121
import org.apache.commons.codec.digest.DigestUtils;
22+
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
23+
import org.apache.hadoop.net.NetUtils;
24+
import org.apache.hadoop.ozone.OmUtils;
25+
import org.apache.hadoop.security.SecurityUtil;
26+
27+
import javax.annotation.Nonnull;
28+
import java.util.Collection;
2229
import java.util.Objects;
2330

31+
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
32+
2433
/**
2534
* Ozone util for S3 related operations.
2635
*/
@@ -33,4 +42,39 @@ public static String getVolumeName(String userName) {
3342
Objects.requireNonNull(userName);
3443
return DigestUtils.md5Hex(userName);
3544
}
45+
46+
/**
47+
* Generate service Name for token.
48+
* @param configuration
49+
* @param serviceId - ozone manager service ID
50+
* @param omNodeIds - list of node ids for the given OM service.
51+
* @return service Name.
52+
*/
53+
public static String buildServiceNameForToken(
54+
@Nonnull OzoneConfiguration configuration, @Nonnull String serviceId,
55+
@Nonnull Collection<String> omNodeIds) {
56+
StringBuilder rpcAddress = new StringBuilder();
57+
58+
int nodesLength = omNodeIds.size();
59+
int counter = 0;
60+
for (String nodeId : omNodeIds) {
61+
counter++;
62+
String rpcAddrKey = OmUtils.addKeySuffixes(OZONE_OM_ADDRESS_KEY,
63+
serviceId, nodeId);
64+
String rpcAddrStr = OmUtils.getOmRpcAddress(configuration, rpcAddrKey);
65+
if (rpcAddrStr == null || rpcAddrStr.isEmpty()) {
66+
throw new IllegalArgumentException("Could not find rpcAddress for " +
67+
OZONE_OM_ADDRESS_KEY + "." + serviceId + "." + nodeId);
68+
}
69+
70+
if (counter != nodesLength) {
71+
rpcAddress.append(SecurityUtil.buildTokenService(
72+
NetUtils.createSocketAddr(rpcAddrStr)) + ",");
73+
} else {
74+
rpcAddress.append(SecurityUtil.buildTokenService(
75+
NetUtils.createSocketAddr(rpcAddrStr)));
76+
}
77+
}
78+
return rpcAddress.toString();
79+
}
3680
}

‎hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestOzoneS3Util.java‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
* <p>
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
* <p>
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.ozone.s3.util;
20+
21+
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
22+
import org.apache.hadoop.ozone.OmUtils;
23+
import org.apache.hadoop.ozone.om.OMConfigKeys;
24+
import org.apache.hadoop.security.SecurityUtil;
25+
import org.apache.hadoop.test.GenericTestUtils;
26+
import org.junit.Assert;
27+
import org.junit.Before;
28+
import org.junit.Test;
29+
30+
import java.util.Collection;
31+
32+
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_TOKEN_SERVICE_USE_IP;
33+
import static org.junit.Assert.fail;
34+
35+
/**
36+
* Class used to test OzoneS3Util.
37+
*/
38+
public class TestOzoneS3Util {
39+
40+
41+
private OzoneConfiguration configuration;
42+
private String serviceID = "omService";
43+
44+
@Before
45+
public void setConf() {
46+
configuration = new OzoneConfiguration();
47+
48+
String nodeIDs = "om1,om2,om3";
49+
configuration.set(OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY, serviceID);
50+
configuration.set(OMConfigKeys.OZONE_OM_NODES_KEY + "." + serviceID,
51+
nodeIDs);
52+
configuration.setBoolean(HADOOP_SECURITY_TOKEN_SERVICE_USE_IP, false);
53+
}
54+
55+
@Test
56+
public void testBuildServiceNameForToken() {
57+
58+
Collection<String> nodeIDList = OmUtils.getOMNodeIds(configuration,
59+
serviceID);
60+
61+
configuration.set(OmUtils.addKeySuffixes(OMConfigKeys.OZONE_OM_ADDRESS_KEY,
62+
serviceID, "om1"), "om1:9862");
63+
configuration.set(OmUtils.addKeySuffixes(OMConfigKeys.OZONE_OM_ADDRESS_KEY,
64+
serviceID, "om2"), "om2:9862");
65+
configuration.set(OmUtils.addKeySuffixes(OMConfigKeys.OZONE_OM_ADDRESS_KEY,
66+
serviceID, "om3"), "om3:9862");
67+
68+
String expectedOmServiceAddress = buildServiceAddress(nodeIDList);
69+
70+
SecurityUtil.setConfiguration(configuration);
71+
String omserviceAddr = OzoneS3Util.buildServiceNameForToken(configuration,
72+
serviceID, nodeIDList);
73+
74+
Assert.assertEquals(expectedOmServiceAddress, omserviceAddr);
75+
}
76+
77+
78+
@Test
79+
public void testBuildServiceNameForTokenIncorrectConfig() {
80+
81+
Collection<String> nodeIDList = OmUtils.getOMNodeIds(configuration,
82+
serviceID);
83+
84+
// Don't set om3 node rpc address. Here we are skipping setting of one of
85+
// the OM address. So buildServiceNameForToken will fail.
86+
configuration.set(OmUtils.addKeySuffixes(OMConfigKeys.OZONE_OM_ADDRESS_KEY,
87+
serviceID, "om1"), "om1:9862");
88+
configuration.set(OmUtils.addKeySuffixes(OMConfigKeys.OZONE_OM_ADDRESS_KEY,
89+
serviceID, "om2"), "om2:9862");
90+
91+
92+
SecurityUtil.setConfiguration(configuration);
93+
94+
try {
95+
OzoneS3Util.buildServiceNameForToken(configuration,
96+
serviceID, nodeIDList);
97+
fail("testBuildServiceNameForTokenIncorrectConfig failed");
98+
} catch (IllegalArgumentException ex) {
99+
GenericTestUtils.assertExceptionContains("Could not find rpcAddress " +
100+
"for", ex);
101+
}
102+
103+
104+
}
105+
106+
/**
107+
* Build serviceName from list of node ids.
108+
* @param nodeIDList
109+
* @return service name for token.
110+
*/
111+
private String buildServiceAddress(Collection<String> nodeIDList) {
112+
StringBuilder omServiceAddrBuilder = new StringBuilder();
113+
int nodesLength = nodeIDList.size();
114+
int counter = 0;
115+
for (String nodeID : nodeIDList) {
116+
counter++;
117+
String addr = configuration.get(OmUtils.addKeySuffixes(
118+
OMConfigKeys.OZONE_OM_ADDRESS_KEY, serviceID, nodeID));
119+
120+
if (counter != nodesLength) {
121+
omServiceAddrBuilder.append(addr + ",");
122+
} else {
123+
omServiceAddrBuilder.append(addr);
124+
}
125+
}
126+
127+
return omServiceAddrBuilder.toString();
128+
}
129+
130+
}

0 commit comments

Comments
 (0)