[DCOS-45820] Upgrade of Hadoop, ZooKeeper, and Jackson libraries to fix CVEs

core/pom.xml

@@ -274,6 +274,10 @@
       <groupId>com.fasterxml.jackson.module</groupId>
       <artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.scala-lang.modules</groupId>
+      <artifactId>scala-xml_${scala.binary.version}</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.derby</groupId>
       <artifactId>derby</artifactId>

core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala

@@ -41,7 +41,7 @@ import org.apache.spark.internal.Logging
  * There is no particular relationship between an operation scope and a stage or a job.
  * A scope may live inside one stage (e.g. map) or span across multiple jobs (e.g. take).
  */
-@JsonInclude(Include.NON_NULL)
+@JsonInclude(Include.NON_ABSENT)
 @JsonPropertyOrder(Array("id", "name", "parent"))
 private[spark] class RDDOperationScope(
     val name: String,

core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala

@@ -49,7 +49,7 @@ private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{
   }
   mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
   mapper.enable(SerializationFeature.INDENT_OUTPUT)
-  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
+  mapper.setSerializationInclusion(JsonInclude.Include.NON_ABSENT)
   mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)
 
   override def isWriteable(

core/src/test/scala/org/apache/spark/JsonTestUtils.scala

@@ -21,7 +21,7 @@ import org.json4s.jackson.JsonMethods
 
 trait JsonTestUtils {
   def assertValidDataInJson(validateJson: JValue, expectedJson: JValue) {
-    val Diff(c, a, d) = validateJson.diff(expectedJson)
+    val Diff(c, a, d) = expectedJson.diff(validateJson)
     val validatePretty = JsonMethods.pretty(validateJson)
     val expectedPretty = JsonMethods.pretty(expectedJson)
     val errorMessage = s"Expected:\n$expectedPretty\nFound:\n$validatePretty"

core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala

@@ -491,7 +491,7 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
       json match {
         case JNothing => Seq()
         case apps: JArray =>
-          apps.filter(app => {
+          apps.children.filter(app => {
             (app \ "attempts") match {
               case attempts: JArray =>
                 val state = (attempts.children.head \ "completed").asInstanceOf[JBool]

core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala

@@ -131,7 +131,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
 
       val storageJson = getJson(ui, "storage/rdd")
       storageJson.children.length should be (1)
-      (storageJson \ "storageLevel").extract[String] should be (StorageLevels.DISK_ONLY.description)
+      (storageJson.children.head \ "storageLevel").extract[String] should be (StorageLevels.DISK_ONLY.description)
       val rddJson = getJson(ui, "storage/rdd/0")
       (rddJson  \ "storageLevel").extract[String] should be (StorageLevels.DISK_ONLY.description)
 
@@ -150,7 +150,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
 
       val updatedStorageJson = getJson(ui, "storage/rdd")
       updatedStorageJson.children.length should be (1)
-      (updatedStorageJson \ "storageLevel").extract[String] should be (
+      (updatedStorageJson.children.head \ "storageLevel").extract[String] should be (
         StorageLevels.MEMORY_ONLY.description)
       val updatedRddJson = getJson(ui, "storage/rdd/0")
       (updatedRddJson  \ "storageLevel").extract[String] should be (
@@ -204,7 +204,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
       }
       val stageJson = getJson(sc.ui.get, "stages")
       stageJson.children.length should be (1)
-      (stageJson \ "status").extract[String] should be (StageStatus.FAILED.name())
+      (stageJson.children.head \ "status").extract[String] should be (StageStatus.FAILED.name())
 
       // Regression test for SPARK-2105
       class NotSerializable
@@ -325,11 +325,11 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
         find(cssSelector(".progress-cell .progress")).get.text should be ("2/2 (1 failed)")
       }
       val jobJson = getJson(sc.ui.get, "jobs")
-      (jobJson \ "numTasks").extract[Int]should be (2)
-      (jobJson \ "numCompletedTasks").extract[Int] should be (3)
-      (jobJson \ "numFailedTasks").extract[Int] should be (1)
-      (jobJson \ "numCompletedStages").extract[Int] should be (2)
-      (jobJson \ "numFailedStages").extract[Int] should be (1)
+      (jobJson.children.head \ "numTasks").extract[Int]should be (2)
+      (jobJson.children.head \ "numCompletedTasks").extract[Int] should be (3)
+      (jobJson.children.head \ "numFailedTasks").extract[Int] should be (1)
+      (jobJson.children.head \ "numCompletedStages").extract[Int] should be (2)
+      (jobJson.children.head \ "numFailedStages").extract[Int] should be (1)
       val stageJson = getJson(sc.ui.get, "stages")
 
       for {
@@ -656,11 +656,11 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
         sc.ui.get.webUrl + "/api/v1/applications"))
       val appListJsonAst = JsonMethods.parse(appListRawJson)
       appListJsonAst.children.length should be (1)
-      val attempts = (appListJsonAst \ "attempts").children
+      val attempts = (appListJsonAst.children.head \ "attempts").children
       attempts.size should be (1)
-      (attempts(0) \ "completed").extract[Boolean] should be (false)
-      parseDate(attempts(0) \ "startTime") should be (sc.startTime)
-      parseDate(attempts(0) \ "endTime") should be (-1)
+      (attempts.head \ "completed").extract[Boolean] should be (false)
+      parseDate(attempts.head \ "startTime") should be (sc.startTime)
+      parseDate(attempts.head \ "endTime") should be (-1)
       val oneAppJsonAst = getJson(sc.ui.get, "")
       oneAppJsonAst should be (appListJsonAst.children(0))
     }

dev/deps/spark-deps-hadoop-2.6

@@ -17,6 +17,7 @@ arpack_combined_all-0.1.jar
 arrow-format-0.8.0.jar
 arrow-memory-0.8.0.jar
 arrow-vector-0.8.0.jar
+audience-annotations-0.5.0.jar
 automaton-1.11-8.jar
 avro-1.7.7.jar
 avro-ipc-1.7.7.jar
@@ -89,16 +90,16 @@ htrace-core-3.0.4.jar
 httpclient-4.5.4.jar
 httpcore-4.4.8.jar
 ivy-2.4.0.jar
-jackson-annotations-2.6.7.jar
-jackson-core-2.6.7.jar
+jackson-annotations-2.9.6.jar
+jackson-core-2.9.6.jar
 jackson-core-asl-1.9.13.jar
-jackson-databind-2.6.7.1.jar
-jackson-dataformat-yaml-2.6.7.jar
+jackson-databind-2.9.6.jar
+jackson-dataformat-yaml-2.9.6.jar
 jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl-1.9.13.jar
-jackson-module-jaxb-annotations-2.6.7.jar
-jackson-module-paranamer-2.7.9.jar
-jackson-module-scala_2.11-2.6.7.1.jar
+jackson-module-jaxb-annotations-2.9.6.jar
+jackson-module-paranamer-2.9.6.jar
+jackson-module-scala_2.11-2.9.6.jar
 jackson-xc-1.9.13.jar
 janino-3.0.8.jar
 java-xmlbuilder-1.1.jar
@@ -126,9 +127,10 @@ jline-2.12.1.jar
 joda-time-2.9.3.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar
-json4s-ast_2.11-3.2.11.jar
-json4s-core_2.11-3.2.11.jar
-json4s-jackson_2.11-3.2.11.jar
+json4s-ast_2.11-3.6.2.jar
+json4s-core_2.11-3.6.2.jar
+json4s-jackson_2.11-3.6.2.jar
+json4s-scalap_2.11-3.6.2.jar
 jsr305-1.3.9.jar
 jta-1.1.jar
 jtransforms-2.4.0.jar
@@ -173,14 +175,13 @@ py4j-0.10.7.jar
 pyrolite-4.13.jar
 scala-compiler-2.11.8.jar
 scala-library-2.11.8.jar
-scala-parser-combinators_2.11-1.0.4.jar
+scala-parser-combinators_2.11-1.1.1.jar
 scala-reflect-2.11.8.jar
-scala-xml_2.11-1.0.5.jar
-scalap-2.11.8.jar
+scala-xml_2.11-1.1.1.jar
 shapeless_2.11-2.3.2.jar
 slf4j-api-1.7.16.jar
 slf4j-log4j12-1.7.16.jar
-snakeyaml-1.15.jar
+snakeyaml-1.18.jar
 snappy-0.2.jar
 snappy-java-1.1.2.6.jar
 spire-macros_2.11-0.13.0.jar
@@ -197,5 +198,5 @@ xercesImpl-2.9.1.jar
 xmlenc-0.52.jar
 xz-1.0.jar
 zjsonpatch-0.3.0.jar
-zookeeper-3.4.6.jar
+zookeeper-3.4.13.jar
 zstd-jni-1.3.2-2.jar

dev/deps/spark-deps-hadoop-2.7

@@ -17,6 +17,7 @@ arpack_combined_all-0.1.jar
 arrow-format-0.8.0.jar
 arrow-memory-0.8.0.jar
 arrow-vector-0.8.0.jar
+audience-annotations-0.5.0.jar
 automaton-1.11-8.jar
 avro-1.7.7.jar
 avro-ipc-1.7.7.jar
@@ -66,21 +67,21 @@ gson-2.2.4.jar
 guava-14.0.1.jar
 guice-3.0.jar
 guice-servlet-3.0.jar
-hadoop-annotations-2.7.3.jar
-hadoop-auth-2.7.3.jar
-hadoop-client-2.7.3.jar
-hadoop-common-2.7.3.jar
-hadoop-hdfs-2.7.3.jar
-hadoop-mapreduce-client-app-2.7.3.jar
-hadoop-mapreduce-client-common-2.7.3.jar
-hadoop-mapreduce-client-core-2.7.3.jar
-hadoop-mapreduce-client-jobclient-2.7.3.jar
-hadoop-mapreduce-client-shuffle-2.7.3.jar
-hadoop-yarn-api-2.7.3.jar
-hadoop-yarn-client-2.7.3.jar
-hadoop-yarn-common-2.7.3.jar
-hadoop-yarn-server-common-2.7.3.jar
-hadoop-yarn-server-web-proxy-2.7.3.jar
+hadoop-annotations-2.7.7.jar
+hadoop-auth-2.7.7.jar
+hadoop-client-2.7.7.jar
+hadoop-common-2.7.7.jar
+hadoop-hdfs-2.7.7.jar
+hadoop-mapreduce-client-app-2.7.7.jar
+hadoop-mapreduce-client-common-2.7.7.jar
+hadoop-mapreduce-client-core-2.7.7.jar
+hadoop-mapreduce-client-jobclient-2.7.7.jar
+hadoop-mapreduce-client-shuffle-2.7.7.jar
+hadoop-yarn-api-2.7.7.jar
+hadoop-yarn-client-2.7.7.jar
+hadoop-yarn-common-2.7.7.jar
+hadoop-yarn-server-common-2.7.7.jar
+hadoop-yarn-server-web-proxy-2.7.7.jar
 hk2-api-2.4.0-b34.jar
 hk2-locator-2.4.0-b34.jar
 hk2-utils-2.4.0-b34.jar
@@ -89,16 +90,16 @@ htrace-core-3.1.0-incubating.jar
 httpclient-4.5.4.jar
 httpcore-4.4.8.jar
 ivy-2.4.0.jar
-jackson-annotations-2.6.7.jar
-jackson-core-2.6.7.jar
+jackson-annotations-2.9.6.jar
+jackson-core-2.9.6.jar
 jackson-core-asl-1.9.13.jar
-jackson-databind-2.6.7.1.jar
-jackson-dataformat-yaml-2.6.7.jar
+jackson-databind-2.9.6.jar
+jackson-dataformat-yaml-2.9.6.jar
 jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl-1.9.13.jar
-jackson-module-jaxb-annotations-2.6.7.jar
-jackson-module-paranamer-2.7.9.jar
-jackson-module-scala_2.11-2.6.7.1.jar
+jackson-module-jaxb-annotations-2.9.6.jar
+jackson-module-paranamer-2.9.6.jar
+jackson-module-scala_2.11-2.9.6.jar
 jackson-xc-1.9.13.jar
 janino-3.0.8.jar
 java-xmlbuilder-1.1.jar
@@ -121,14 +122,16 @@ jersey-media-jaxb-2.22.2.jar
 jersey-server-2.22.2.jar
 jets3t-0.9.4.jar
 jetty-6.1.26.jar
+jetty-sslengine-6.1.26.jar
 jetty-util-6.1.26.jar
 jline-2.12.1.jar
 joda-time-2.9.3.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar
-json4s-ast_2.11-3.2.11.jar
-json4s-core_2.11-3.2.11.jar
-json4s-jackson_2.11-3.2.11.jar
+json4s-ast_2.11-3.6.2.jar
+json4s-core_2.11-3.6.2.jar
+json4s-jackson_2.11-3.6.2.jar
+json4s-scalap_2.11-3.6.2.jar
 jsp-api-2.1.jar
 jsr305-1.3.9.jar
 jta-1.1.jar
@@ -174,14 +177,13 @@ py4j-0.10.7.jar
 pyrolite-4.13.jar
 scala-compiler-2.11.8.jar
 scala-library-2.11.8.jar
-scala-parser-combinators_2.11-1.0.4.jar
+scala-parser-combinators_2.11-1.1.1.jar
 scala-reflect-2.11.8.jar
-scala-xml_2.11-1.0.5.jar
-scalap-2.11.8.jar
+scala-xml_2.11-1.1.1.jar
 shapeless_2.11-2.3.2.jar
 slf4j-api-1.7.16.jar
 slf4j-log4j12-1.7.16.jar
-snakeyaml-1.15.jar
+snakeyaml-1.18.jar
 snappy-0.2.jar
 snappy-java-1.1.2.6.jar
 spire-macros_2.11-0.13.0.jar
@@ -198,5 +200,5 @@ xercesImpl-2.9.1.jar
 xmlenc-0.52.jar
 xz-1.0.jar
 zjsonpatch-0.3.0.jar
-zookeeper-3.4.6.jar
+zookeeper-3.4.13.jar
 zstd-jni-1.3.2-2.jar

pom.xml

@@ -121,7 +121,7 @@
     <protobuf.version>2.5.0</protobuf.version>
     <yarn.version>${hadoop.version}</yarn.version>
     <flume.version>1.6.0</flume.version>
-    <zookeeper.version>3.4.6</zookeeper.version>
+    <zookeeper.version>3.4.13</zookeeper.version>
     <curator.version>2.6.0</curator.version>
     <hive.group>org.spark-project.hive</hive.group>
     <!-- Version used in Maven Hive dependency -->
@@ -157,9 +157,11 @@
     <commons.collections.version>3.2.2</commons.collections.version>
     <scala.version>2.11.8</scala.version>
     <scala.binary.version>2.11</scala.binary.version>
+    <scala.xml.version>1.1.1</scala.xml.version>
+    <scala.parser.combinators.version>1.1.1</scala.parser.combinators.version>
     <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
-    <fasterxml.jackson.version>2.6.7</fasterxml.jackson.version>
-    <fasterxml.jackson.databind.version>2.6.7.1</fasterxml.jackson.databind.version>
+    <fasterxml.jackson.version>2.9.6</fasterxml.jackson.version>
+    <fasterxml.jackson.databind.version>2.9.6</fasterxml.jackson.databind.version>
     <snappy.version>1.1.2.6</snappy.version>
     <netlib.java.version>1.1.2</netlib.java.version>
     <calcite.version>1.2.0-incubating</calcite.version>
@@ -708,7 +710,7 @@
       <dependency>
         <groupId>org.json4s</groupId>
         <artifactId>json4s-jackson_${scala.binary.version}</artifactId>
-        <version>3.2.11</version>
+        <version>3.6.2</version>
       </dependency>
       <dependency>
         <groupId>org.scala-lang</groupId>
@@ -733,7 +735,7 @@
       <dependency>
         <groupId>org.scala-lang.modules</groupId>
         <artifactId>scala-parser-combinators_${scala.binary.version}</artifactId>
-        <version>1.0.4</version>
+        <version>${scala.parser.combinators.version}</version>
       </dependency>
       <dependency>
         <groupId>org.scala-lang</groupId>
@@ -746,6 +748,11 @@
         <artifactId>jline</artifactId>
         <version>2.12.1</version>
       </dependency>
+      <dependency>
+        <groupId>org.scala-lang.modules</groupId>
+        <artifactId>scala-xml_${scala.binary.version}</artifactId>
+        <version>${scala.xml.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.scalatest</groupId>
         <artifactId>scalatest_${scala.binary.version}</artifactId>
@@ -2664,7 +2671,7 @@
     <profile>
       <id>hadoop-2.7</id>
       <properties>
-        <hadoop.version>2.7.3</hadoop.version>
+        <hadoop.version>2.7.7</hadoop.version>
         <curator.version>2.7.1</curator.version>
       </properties>
     </profile>

sql/catalyst/pom.xml

@@ -43,8 +43,11 @@
       <groupId>org.scala-lang.modules</groupId>
       <artifactId>scala-parser-combinators_${scala.binary.version}</artifactId>
     </dependency>
-
     <dependency>
+      <groupId>org.scala-lang</groupId>
+      <artifactId>scala-compiler</artifactId>
+    </dependency>
+      <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-core_${scala.binary.version}</artifactId>
       <version>${project.version}</version>