From c02140f67c8e7d36675b3ec20f27d25e456900be Mon Sep 17 00:00:00 2001 From: lianhuiwang Date: Thu, 12 Feb 2015 15:11:19 +0800 Subject: [PATCH 1/2] ExecutorRunnable should catch YarnException while NMClient start container --- .../apache/spark/deploy/yarn/ExecutorRunnable.scala | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala index 6d5b8fda76ab8..792122112766f 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api._ import org.apache.hadoop.yarn.api.records._ import org.apache.hadoop.yarn.client.api.NMClient import org.apache.hadoop.yarn.conf.YarnConfiguration +import org.apache.hadoop.yarn.exceptions.YarnException import org.apache.hadoop.yarn.ipc.YarnRPC import org.apache.hadoop.yarn.util.{ConverterUtils, Records} @@ -109,7 +110,14 @@ class ExecutorRunnable( } // Send the start request to the ContainerManager - nmClient.startContainer(container, ctx) + try { + nmClient.startContainer(container, ctx) + } catch { + case ex: YarnException => + logError("Exception while start container %s on host %s:" + .format(container.getId, hostname), ex) + throw ex + } } private def prepareCommand( From caf5a9969bf24e2f886c801274908dfa7fd135fc Mon Sep 17 00:00:00 2001 From: lianhuiwang Date: Thu, 12 Feb 2015 16:29:39 +0800 Subject: [PATCH 2/2] use SparkException to warp exception --- .../apache/spark/deploy/yarn/ExecutorRunnable.scala | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala index 792122112766f..5488d96ccaf56 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala @@ -35,11 +35,10 @@ import org.apache.hadoop.yarn.api._ import org.apache.hadoop.yarn.api.records._ import org.apache.hadoop.yarn.client.api.NMClient import org.apache.hadoop.yarn.conf.YarnConfiguration -import org.apache.hadoop.yarn.exceptions.YarnException import org.apache.hadoop.yarn.ipc.YarnRPC import org.apache.hadoop.yarn.util.{ConverterUtils, Records} -import org.apache.spark.{SecurityManager, SparkConf, Logging} +import org.apache.spark.{SparkException, SecurityManager, SparkConf, Logging} import org.apache.spark.network.util.JavaUtils class ExecutorRunnable( @@ -113,10 +112,9 @@ class ExecutorRunnable( try { nmClient.startContainer(container, ctx) } catch { - case ex: YarnException => - logError("Exception while start container %s on host %s:" - .format(container.getId, hostname), ex) - throw ex + case ex: Exception => + throw new SparkException("Exception while starting container ${container.getId}" + + " on host $hostname", ex) } }