From 0f29656cd2a933fad37af33e59115d376026d09d Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Fri, 14 Jul 2017 15:01:55 -0700 Subject: [PATCH] [SPARK-21422][BUILD] Depend on Apache ORC 1.4.0 --- assembly/pom.xml | 6 +++++ dev/deps/spark-deps-hadoop-2.6 | 3 +++ dev/deps/spark-deps-hadoop-2.7 | 3 +++ pom.xml | 44 ++++++++++++++++++++++++++++++++++ sql/core/pom.xml | 10 ++++++++ 5 files changed, 66 insertions(+) diff --git a/assembly/pom.xml b/assembly/pom.xml index 464af16e46f6..cd8366a17552 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -220,6 +220,12 @@ provided + + orc-provided + + provided + + parquet-provided diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index 1a6515be51cf..68039788f8ea 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -2,6 +2,7 @@ JavaEWAH-0.3.2.jar RoaringBitmap-0.5.11.jar ST4-4.0.4.jar activation-1.1.1.jar +aircompressor-0.3.jar antlr-2.7.7.jar antlr-runtime-3.4.jar antlr4-runtime-4.5.3.jar @@ -148,6 +149,8 @@ netty-3.9.9.Final.jar netty-all-4.0.43.Final.jar objenesis-2.1.jar opencsv-2.3.jar +orc-core-1.4.0-nohive.jar +orc-mapreduce-1.4.0-nohive.jar oro-2.0.8.jar osgi-resource-locator-1.0.1.jar paranamer-2.6.jar diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index 09e5a4288ca5..12580037ec92 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -2,6 +2,7 @@ JavaEWAH-0.3.2.jar RoaringBitmap-0.5.11.jar ST4-4.0.4.jar activation-1.1.1.jar +aircompressor-0.3.jar antlr-2.7.7.jar antlr-runtime-3.4.jar antlr4-runtime-4.5.3.jar @@ -149,6 +150,8 @@ netty-3.9.9.Final.jar netty-all-4.0.43.Final.jar objenesis-2.1.jar opencsv-2.3.jar +orc-core-1.4.0-nohive.jar +orc-mapreduce-1.4.0-nohive.jar oro-2.0.8.jar osgi-resource-locator-1.0.1.jar paranamer-2.6.jar diff --git a/pom.xml b/pom.xml index 11cd028c9914..b155ebe62e94 100644 --- a/pom.xml +++ b/pom.xml @@ -132,6 +132,8 @@ 1.2.1 10.12.1.1 1.8.2 + 1.4.0 + nohive 1.6.0 9.3.20.v20170531 3.1.0 @@ -207,6 +209,7 @@ compile compile compile + compile compile test @@ -1677,6 +1680,44 @@ + + org.apache.orc + orc-core + ${orc.version} + ${orc.classifier} + ${orc.deps.scope} + + + org.apache.hadoop + hadoop-common + + + org.apache.hive + hive-storage-api + + + + + org.apache.orc + orc-mapreduce + ${orc.version} + ${orc.classifier} + ${orc.deps.scope} + + + org.apache.hadoop + hadoop-common + + + org.apache.orc + orc-core + + + org.apache.hive + hive-storage-api + + + org.apache.parquet parquet-column @@ -2710,6 +2751,9 @@ hive-provided + + orc-provided + parquet-provided diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 661c31ded714..eb45422f77e3 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -86,6 +86,16 @@ test + + org.apache.orc + orc-core + ${orc.classifier} + + + org.apache.orc + orc-mapreduce + ${orc.classifier} + org.apache.parquet parquet-column