Merge branch 'trunk' of github.com:apache/hadoop into HADOOP-19072-trunk

Author: virajjasani

LICENSE-binary

@@ -379,7 +379,7 @@ hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h
 
 com.github.luben:zstd-jni:1.5.2-1
-dnsjava:dnsjava:2.1.7
+dnsjava:dnsjava:3.6.1
 org.codehaus.woodstox:stax2-api:4.2.1
 
 

dev-support/docker/pkg-resolver/install-yasm.sh

@@ -40,7 +40,7 @@ fi
 
 if [ "$version_to_install" == "1.2.0-4" ]; then
   mkdir -p /tmp/yasm &&
-    curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \
+    curl -L -s -S https://archives.fedoraproject.org/pub/archive/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \
       -o /tmp/yasm-1.2.0-4.el7.x86_64.rpm &&
     rpm -Uvh /tmp/yasm-1.2.0-4.el7.x86_64.rpm
 else

hadoop-client-modules/hadoop-client-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh

@@ -51,6 +51,8 @@ allowed_expr+="|^[^-]*-default.xml$"
 allowed_expr+="|^[^-]*-version-info.properties$"
 #   * Hadoop's application classloader properties file.
 allowed_expr+="|^org.apache.hadoop.application-classloader.properties$"
+# Comes from dnsjava, not sure if relocatable.
+allowed_expr+="|^messages.properties$"
 # public suffix list used by httpcomponents
 allowed_expr+="|^mozilla/$"
 allowed_expr+="|^mozilla/public-suffix-list.txt$"

hadoop-client-modules/hadoop-client-runtime/pom.xml

@@ -229,6 +229,8 @@
                         <exclude>jnamed*</exclude>
                         <exclude>lookup*</exclude>
                         <exclude>update*</exclude>
+                        <exclude>META-INF/versions/21/*</exclude>
+                        <exclude>META-INF/versions/21/**/*</exclude>
                       </excludes>
                     </filter>
                     <filter>
@@ -243,6 +245,7 @@
                       <excludes>
                         <exclude>META-INF/versions/9/module-info.class</exclude>
                         <exclude>META-INF/versions/11/module-info.class</exclude>
+                        <exclude>META-INF/versions/21/module-info.class</exclude>
                       </excludes>
                     </filter>
 

hadoop-common-project/hadoop-common/pom.xml

@@ -708,6 +708,17 @@
           </filesets>
         </configuration>
       </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <configuration>
+          <sourceFileExcludes>
+            <sourceFileExclude>**/FSProtos.java</sourceFileExclude>
+          </sourceFileExcludes>
+          <excludePackageNames>*.proto:*.tracing:*.protobuf</excludePackageNames>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 
@@ -1279,16 +1290,6 @@
               </execution>
              </executions>
           </plugin>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-javadoc-plugin</artifactId>
-            <configuration>
-              <sourceFileExcludes>
-                <sourceFileExclude>**/FSProtos.java</sourceFileExclude>
-              </sourceFileExcludes>
-              <excludePackageNames>*.proto:*.tracing:*.protobuf</excludePackageNames>
-            </configuration>
-          </plugin>
         </plugins>
       </build>
     </profile>

hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java

@@ -1682,7 +1682,7 @@ public void exec(Zone zone, Record record) throws IOException {
                   DNSSEC.sign(rRset, dnskeyRecord, privateKey,
                       inception, expiration);
               LOG.info("Adding {}", rrsigRecord);
-              rRset.addRR(rrsigRecord);
+              zone.addRecord(rrsigRecord);
 
               //addDSRecord(zone, record.getName(), record.getDClass(),
               //  record.getTTL(), inception, expiration);

hadoop-common-project/hadoop-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java

@@ -350,7 +350,7 @@ public void testMissingReverseLookup() throws Exception {
     Name name = Name.fromString("19.1.17.172.in-addr.arpa.");
     Record question = Record.newRecord(name, Type.PTR, DClass.IN);
     Message query = Message.newQuery(question);
-    OPTRecord optRecord = new OPTRecord(4096, 0, 0, Flags.DO, null);
+    OPTRecord optRecord = new OPTRecord(4096, 0, 0, Flags.DO);
     query.addRecord(optRecord, Section.ADDITIONAL);
     byte[] responseBytes = getRegistryDNS().generateReply(query, null);
     Message response = new Message(responseBytes);
@@ -392,7 +392,7 @@ private List<Record> assertDNSQuery(String lookup, int type, int numRecs)
     Name name = Name.fromString(lookup);
     Record question = Record.newRecord(name, type, DClass.IN);
     Message query = Message.newQuery(question);
-    OPTRecord optRecord = new OPTRecord(4096, 0, 0, Flags.DO, null);
+    OPTRecord optRecord = new OPTRecord(4096, 0, 0, Flags.DO);
     query.addRecord(optRecord, Section.ADDITIONAL);
     byte[] responseBytes = getRegistryDNS().generateReply(query, null);
     Message response = new Message(responseBytes);
@@ -421,7 +421,7 @@ private List<Record> assertDNSQueryNotNull(
     Name name = Name.fromString(lookup);
     Record question = Record.newRecord(name, type, DClass.IN);
     Message query = Message.newQuery(question);
-    OPTRecord optRecord = new OPTRecord(4096, 0, 0, Flags.DO, null);
+    OPTRecord optRecord = new OPTRecord(4096, 0, 0, Flags.DO);
     query.addRecord(optRecord, Section.ADDITIONAL);
     byte[] responseBytes = getRegistryDNS().generateReply(query, null);
     Message response = new Message(responseBytes);
@@ -592,7 +592,7 @@ public void testReadMasterFile() throws Exception {
     Name name = Name.fromString("5.0.17.172.in-addr.arpa.");
     Record question = Record.newRecord(name, Type.PTR, DClass.IN);
     Message query = Message.newQuery(question);
-    OPTRecord optRecord = new OPTRecord(4096, 0, 0, Flags.DO, null);
+    OPTRecord optRecord = new OPTRecord(4096, 0, 0, Flags.DO);
     query.addRecord(optRecord, Section.ADDITIONAL);
     byte[] responseBytes = getRegistryDNS().generateReply(query, null);
     Message response = new Message(responseBytes);

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java

@@ -20,6 +20,7 @@
 import java.net.InetSocketAddress;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.server.common.JspHelper;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer;
@@ -86,6 +87,16 @@ protected void serviceStart() throws Exception {
         RBFConfigKeys.DFS_ROUTER_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY,
         RBFConfigKeys.DFS_ROUTER_KEYTAB_FILE_KEY);
 
+    final boolean xFrameEnabled = conf.getBoolean(
+        DFSConfigKeys.DFS_XFRAME_OPTION_ENABLED,
+        DFSConfigKeys.DFS_XFRAME_OPTION_ENABLED_DEFAULT);
+
+    final String xFrameOptionValue = conf.getTrimmed(
+        DFSConfigKeys.DFS_XFRAME_OPTION_VALUE,
+        DFSConfigKeys.DFS_XFRAME_OPTION_VALUE_DEFAULT);
+
+    builder.configureXFrame(xFrameEnabled).setXFrameOption(xFrameOptionValue);
+
     this.httpServer = builder.build();
 
     NameNodeHttpServer.initWebHdfs(conf, httpServer,

hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterHttpServerXFrame.java

@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URL;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+
+import static org.apache.hadoop.http.HttpServer2.XFrameOption.SAMEORIGIN;
+
+/**
+ * A class to test the XFrame options of Router HTTP Server.
+ */
+public class TestRouterHttpServerXFrame {
+
+  @Test
+  public void testRouterXFrame() throws IOException {
+    Configuration conf = new HdfsConfiguration();
+    conf.setBoolean(DFSConfigKeys.DFS_XFRAME_OPTION_ENABLED, true);
+    conf.set(DFSConfigKeys.DFS_XFRAME_OPTION_VALUE, SAMEORIGIN.toString());
+
+    Router router = new Router();
+    try {
+      router.init(conf);
+      router.start();
+
+      InetSocketAddress httpAddress = router.getHttpServerAddress();
+      URL url =
+          URI.create("http://" + httpAddress.getHostName() + ":" + httpAddress.getPort()).toURL();
+      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+      conn.connect();
+
+      String xfoHeader = conn.getHeaderField("X-FRAME-OPTIONS");
+      Assert.assertNotNull("X-FRAME-OPTIONS is absent in the header", xfoHeader);
+      Assert.assertTrue(xfoHeader.endsWith(SAMEORIGIN.toString()));
+    } finally {
+      router.stop();
+      router.close();
+    }
+  }
+}

hadoop-project/pom.xml

@@ -106,7 +106,7 @@
     <zookeeper.version>3.8.4</zookeeper.version>
     <curator.version>5.2.0</curator.version>
     <findbugs.version>3.0.5</findbugs.version>
-    <dnsjava.version>3.4.0</dnsjava.version>
+    <dnsjava.version>3.6.1</dnsjava.version>
 
     <guava.version>27.0-jre</guava.version>
     <guice.version>5.1.0</guice.version>