Skip to content

[spark-11026][yarn]spark.yarn.user.classpath.first does work for 'spark-submit --jars hdfs://user/foo.jar' #9045

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 6 commits into from
Closed

[spark-11026][yarn]spark.yarn.user.classpath.first does work for 'spark-submit --jars hdfs://user/foo.jar' #9045

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9b2b612
init commit
lianhuiwang Oct 9, 2015
3023e1e
update comments
lianhuiwang Oct 9, 2015
4c5290a
remove debug info
lianhuiwang Oct 9, 2015
812e169
fix minor bug
lianhuiwang Oct 10, 2015
6f395d6
fix minor bug
lianhuiwang Oct 10, 2015
d4049a7
fix minor nit
lianhuiwang Oct 13, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 15 additions & 8 deletions yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1164,7 +1164,7 @@ object Client extends Logging {
} else {
getMainJarUri(sparkConf.getOption(CONF_SPARK_USER_JAR))
}
mainJar.foreach(addFileToClasspath(sparkConf, _, APP_JAR, env))
mainJar.foreach(addFileToClasspath(sparkConf, conf, _, APP_JAR, env))

val secondaryJars =
if (args != null) {
Expand All @@ -1173,10 +1173,10 @@ object Client extends Logging {
getSecondaryJarUris(sparkConf.getOption(CONF_SPARK_YARN_SECONDARY_JARS))
}
secondaryJars.foreach { x =>
addFileToClasspath(sparkConf, x, null, env)
addFileToClasspath(sparkConf, conf, x, null, env)
}
}
addFileToClasspath(sparkConf, new URI(sparkJar(sparkConf)), SPARK_JAR, env)
addFileToClasspath(sparkConf, conf, new URI(sparkJar(sparkConf)), SPARK_JAR, env)
populateHadoopClasspath(conf, env)
sys.env.get(ENV_DIST_CLASSPATH).foreach { cp =>
addClasspathEntry(getClusterPath(sparkConf, cp), env)
Expand Down Expand Up @@ -1211,15 +1211,17 @@ object Client extends Logging {
* If an alternate name for the file is given, and it's not a "local:" file, the alternate
* name will be added to the classpath (relative to the job's work directory).
*
* If not a "local:" file and no alternate name, the environment is not modified.
* If not a "local:" file and no alternate name, the linkName will be added to the classpath.
*
* @param conf Spark configuration.
* @param uri URI to add to classpath (optional).
* @param fileName Alternate name for the file (optional).
* @param env Map holding the environment variables.
* @param conf Spark configuration.
* @param hadoopConf Hadoop configuration.
* @param uri URI to add to classpath (optional).
* @param fileName Alternate name for the file (optional).
* @param env Map holding the environment variables.
*/
private def addFileToClasspath(
conf: SparkConf,
hadoopConf: Configuration,
uri: URI,
fileName: String,
env: HashMap[String, String]): Unit = {
Expand All @@ -1228,6 +1230,11 @@ object Client extends Logging {
} else if (fileName != null) {
addClasspathEntry(buildPath(
YarnSparkHadoopUtil.expandEnvironment(Environment.PWD), fileName), env)
} else if (uri != null) {
val localPath = getQualifiedLocalPath(uri, hadoopConf)
val linkName = Option(uri.getFragment()).getOrElse(localPath.getName())
addClasspathEntry(buildPath(
YarnSparkHadoopUtil.expandEnvironment(Environment.PWD), linkName), env)
}
}

Expand Down