From 49ce5a8329db43293b6ef4b22b2601594e7f4092 Mon Sep 17 00:00:00 2001 From: Mark Pavey Date: Wed, 21 Nov 2018 10:07:16 +0000 Subject: [PATCH 1/5] [SPARK-26137][CORE] Use Java system property "file.separator" instead of hard coded "/" in DependencyUtils --- .../main/scala/org/apache/spark/deploy/DependencyUtils.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala b/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala index 178bdcfccb603..a6d9d6572b327 100644 --- a/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala +++ b/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala @@ -19,6 +19,7 @@ package org.apache.spark.deploy import java.io.File import java.net.URI +import java.util.regex.Pattern import org.apache.commons.lang3.StringUtils import org.apache.hadoop.conf.Configuration @@ -61,11 +62,12 @@ private[deploy] object DependencyUtils extends Logging { hadoopConf: Configuration, secMgr: SecurityManager): String = { val targetDir = Utils.createTempDir() + val fileSeparator = Pattern.quote(System.getProperty("file.separator")) Option(jars) .map { resolveGlobPaths(_, hadoopConf) .split(",") - .filterNot(_.contains(userJar.split("/").last)) + .filterNot(_.contains(userJar.split(fileSeparator).last)) .mkString(",") } .filterNot(_ == "") From 24c05e3524c0b41a11aca6d02b6d75fde2e16381 Mon Sep 17 00:00:00 2001 From: Mark Pavey Date: Mon, 26 Nov 2018 16:57:30 +0000 Subject: [PATCH 2/5] [SPARK-26137][CORE] Use File.separator instead of System.getProperty("file.separator") --- .../scala/org/apache/spark/deploy/DependencyUtils.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala b/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala index a6d9d6572b327..e5d9bf211ebca 100644 --- a/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala +++ b/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala @@ -24,7 +24,6 @@ import java.util.regex.Pattern import org.apache.commons.lang3.StringUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} - import org.apache.spark.{SecurityManager, SparkConf, SparkException} import org.apache.spark.internal.Logging import org.apache.spark.util.{MutableURLClassLoader, Utils} @@ -62,12 +61,13 @@ private[deploy] object DependencyUtils extends Logging { hadoopConf: Configuration, secMgr: SecurityManager): String = { val targetDir = Utils.createTempDir() - val fileSeparator = Pattern.quote(System.getProperty("file.separator")) + val fileSeparator = Pattern.quote(File.separator) + val userJarName = userJar.split(fileSeparator).last Option(jars) .map { resolveGlobPaths(_, hadoopConf) .split(",") - .filterNot(_.contains(userJar.split(fileSeparator).last)) + .filterNot(_.contains(userJarName)) .mkString(",") } .filterNot(_ == "") From bfabd4fb37c9140fa7b0603205b274222333f084 Mon Sep 17 00:00:00 2001 From: Mark Pavey Date: Mon, 26 Nov 2018 16:57:51 +0000 Subject: [PATCH 3/5] [SPARK-26137][CORE] Add unit test --- .../spark/deploy/SparkSubmitSuite.scala | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 652c36ffa6e71..c093789244bfe 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -962,6 +962,25 @@ class SparkSubmitSuite } } + test("remove copies of application jar from classpath") { + val fs = File.separator + val sparkConf = new SparkConf(false) + val hadoopConf = new Configuration() + val secMgr = new SecurityManager(sparkConf) + + val appJarName = "myApp.jar" + val jar1Name = "myJar1.jar" + val jar2Name = "myJar2.jar" + val userJar = s"file:/path${fs}to${fs}app${fs}jar$fs$appJarName" + val jars = s"file:/$jar1Name,file:/$appJarName,file:/$jar2Name" + + val resolvedJars = DependencyUtils + .resolveAndDownloadJars(jars, userJar, sparkConf, hadoopConf, secMgr) + + assert(!resolvedJars.contains(appJarName)) + assert(resolvedJars.contains(jar1Name) && resolvedJars.contains(jar2Name)) + } + test("Avoid re-upload remote resources in yarn client mode") { val hadoopConf = new Configuration() updateConfWithFakeS3Fs(hadoopConf) From 4ab68f08dbf1a0489adcd3a6657251a836fea8bc Mon Sep 17 00:00:00 2001 From: Mark Pavey Date: Mon, 26 Nov 2018 17:04:26 +0000 Subject: [PATCH 4/5] [SPARK-26137][CORE] Add missing blank line to comply with style --- .../src/main/scala/org/apache/spark/deploy/DependencyUtils.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala b/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala index e5d9bf211ebca..cc436e2049503 100644 --- a/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala +++ b/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala @@ -24,6 +24,7 @@ import java.util.regex.Pattern import org.apache.commons.lang3.StringUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} + import org.apache.spark.{SecurityManager, SparkConf, SparkException} import org.apache.spark.internal.Logging import org.apache.spark.util.{MutableURLClassLoader, Utils} From e1e97aabb45cf611221ee7baa8d0351b32fbb770 Mon Sep 17 00:00:00 2001 From: Mark Pavey Date: Tue, 27 Nov 2018 09:39:09 +0000 Subject: [PATCH 5/5] [SPARK-26137][CORE] Use File.separatorChar instead of File.separator --- .../main/scala/org/apache/spark/deploy/DependencyUtils.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala b/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala index cc436e2049503..5a17a6b6e169c 100644 --- a/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala +++ b/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala @@ -19,7 +19,6 @@ package org.apache.spark.deploy import java.io.File import java.net.URI -import java.util.regex.Pattern import org.apache.commons.lang3.StringUtils import org.apache.hadoop.conf.Configuration @@ -62,8 +61,7 @@ private[deploy] object DependencyUtils extends Logging { hadoopConf: Configuration, secMgr: SecurityManager): String = { val targetDir = Utils.createTempDir() - val fileSeparator = Pattern.quote(File.separator) - val userJarName = userJar.split(fileSeparator).last + val userJarName = userJar.split(File.separatorChar).last Option(jars) .map { resolveGlobPaths(_, hadoopConf)