- Removed setting SPARK_HOME

- Changed the location of pyspark's directory into interpreter/spark

Author: jongyoul

bin/interpreter.sh

@@ -73,16 +73,20 @@ if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then
   $(mkdir -p "${ZEPPELIN_LOG_DIR}")
 fi
 
-if [[ x"" == x${PYTHONPATH} ]]; then
-  export PYTHONPATH="${ZEPPELIN_HOME}/python/lib/pyspark.zip:${ZEPPELIN_HOME}/python/lib/py4j-0.8.2.1-src.zip"
+if [[ ! -z "${SPARK_HOME}" ]]; then
+  PYSPARKPATH="${SPARK_HOME}/python/lib/pyspark.zip:${SPARK_HOME}/python/lib/py4j-0.8.2.1-src.zip"
 else
-  export PYTHONPATH="$PYTHONPATH:${ZEPPELIN_HOME}/lib/pyspark.zip:${ZEPPELIN_HOME}/python/lib/py4j-0.8.2.1-src.zip"
+  PYSPARKPATH="${ZEPPELIN_HOME}/interpreter/spark/pyspark/pyspark.zip:${ZEPPELIN_HOME}/interpreter/spark/pyspark/py4j-0.8.2.1-src.zip"
 fi
 
-if [[ x"" == x${SPARK_HOME} ]]; then
-  export SPARK_HOME=${ZEPPELIN_HOME}
+
+if [[ x"" == x"${PYTHONPATH}" ]]; then
+  export PYTHONPATH="${PYSPARKPATH}"
+else
+  export PYTHONPATH="${PYTHONPATH}:${PYSPARKPATH}"
 fi
 
+unset PYSPARKPATH
 
 ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} -cp ${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} &
 pid=$!

spark/pom.xml

@@ -779,13 +779,14 @@
                 </goals>
                 <configuration>
                   <target>
-                    <delete dir="../python"/>
-                    <copy todir="../python">
-                      <fileset dir="${project.build.directory}/spark-dist/spark-${spark.version}/python"/>
-                    </copy>
-                    <unzip src="../python/lib/py4j-0.8.2.1-src.zip"
-                           dest="../python/build"/>
-                    <zip destfile="${project.build.directory}/../../python/lib/pyspark.zip"
+                    <delete dir="../interpreter/spark/pyspark"/>
+                    <copy todir="../interpreter/spark/pyspark"
+                        file="${project.build.directory}/spark-dist/spark-${spark.version}/python/lib/py4j-0.8.2.1-src.zip"/>
+                      <!--<fileset dir="${project.build.directory}/spark-dist/spark-${spark.version}/python"/>
+                    </copy>-->
+                    <!--<unzip src="../interpreter/spark/python/py4j-0.8.2.1-src.zip"
+                           dest="../interpreter/spark/python/build"/>-->
+                    <zip destfile="${project.build.directory}/../../interpreter/spark/pyspark/pyspark.zip"
                          basedir="${project.build.directory}/spark-dist/spark-${spark.version}/python"
                          includes="pyspark/*.py,pyspark/**/*.py"/>
                   </target>

spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java

@@ -271,14 +271,27 @@ public SparkContext createSparkContext() {
     }
 
     //TODO(jongyoul): Move these codes into PySparkInterpreter.java
-    String zeppelinHome = getSystemDefault("ZEPPELIN_HOME", "zeppelin.home", "../");
-    File zeppelinPythonLibPath = new File(zeppelinHome, "python/lib");
+//    String zeppelinHome = getSystemDefault("ZEPPELIN_HOME", "zeppelin.home", "../");
+//    File zeppelinPythonLibPath = new File(zeppelinHome, "python/lib");
+    
+    String pysparkBasePath = getSystemDefault("SPARK_HOME", "spark.home", null);
+    File pysparkPath;
+    if (null == pysparkBasePath) {
+      pysparkBasePath = getSystemDefault("ZEPPELIN_HOME", "zeppelin.home", "../");
+      pysparkPath = new File(pysparkBasePath,
+          "interpreter" + File.separator + "spark" + File.separator + "pyspark");
+    } else {
+      pysparkPath = new File(pysparkBasePath,
+          "python" + File.separator + "lib");
+    }
+
     String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.8.2.1-src.zip"};
     ArrayList<String> pythonLibUris = new ArrayList<>();
     for (String lib : pythonLibs) {
-      File libFile = new File(zeppelinPythonLibPath, lib);
+//      File libFile = new File(zeppelinPythonLibPath, lib);
+      File libFile = new File(pysparkPath, lib);
       if (libFile.exists()) {
-        pythonLibUris.add(libFile.toURI().toString()); 
+        pythonLibUris.add(libFile.toURI().toString());
       }
     }
     pythonLibUris.trimToSize();