From d73b6ec37bd7a8dfbe4010830ee3061130bc8664 Mon Sep 17 00:00:00 2001 From: witgo Date: Mon, 19 May 2014 12:19:31 +0800 Subject: [PATCH] HOTFIX SPARK-1875:NoClassDefFoundError: StringUtils when building against Hadoop 1 --- docs/building-with-maven.md | 3 ++- make-distribution.sh | 1 + pom.xml | 12 ++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/building-with-maven.md b/docs/building-with-maven.md index 8b44535d82404..07415ea576ac3 100644 --- a/docs/building-with-maven.md +++ b/docs/building-with-maven.md @@ -37,7 +37,8 @@ Because HDFS is not protocol-compatible across versions, if you want to read fro 0.23.xhadoop-0.23 - 1.x to 2.1.x(none) + 1.xhadoop-1 + 2.0.x to 2.1.x(none) 2.2.xhadoop-2.2 2.3.xhadoop-2.3 2.4.xhadoop-2.4 diff --git a/make-distribution.sh b/make-distribution.sh index 1c89027d68bed..ff8f402156645 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -140,6 +140,7 @@ if [[ "$SPARK_HADOOP_VERSION" =~ ^0\.23\. ]]; then BUILD_COMMAND="$BUILD_COMMAND if [[ "$SPARK_HADOOP_VERSION" =~ ^2\.2\. ]]; then BUILD_COMMAND="$BUILD_COMMAND -Phadoop-2.2"; fi if [[ "$SPARK_HADOOP_VERSION" =~ ^2\.3\. ]]; then BUILD_COMMAND="$BUILD_COMMAND -Phadoop-2.3"; fi if [[ "$SPARK_HADOOP_VERSION" =~ ^2\.4\. ]]; then BUILD_COMMAND="$BUILD_COMMAND -Phadoop-2.4"; fi +if [[ "$SPARK_HADOOP_VERSION" =~ ^1\. ]]; then BUILD_COMMAND="$BUILD_COMMAND -Phadoop-1"; fi if [[ "$SPARK_HIVE" == "true" ]]; then BUILD_COMMAND="$BUILD_COMMAND -Phive"; fi if [[ "$SPARK_YARN" == "true" ]]; then # For hadoop versions 0.23.x to 2.1.x, use the yarn-alpha profile diff --git a/pom.xml b/pom.xml index 86264d1132ec4..dfc0630b80ab9 100644 --- a/pom.xml +++ b/pom.xml @@ -951,6 +951,18 @@ + + hadoop-1 + + + + commons-lang + commons-lang + 2.4 + + + + hadoop-0.23