Skip to content
This repository was archived by the owner on Jan 9, 2020. It is now read-only.

Log the driver exit status explicitly #282

Closed
wants to merge 2 commits into from
Closed

Log the driver exit status explicitly #282

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
df2fc00
Log the driver exit status explicitly.
lins05 May 18, 2017
21be5d3
Merge branch 'branch-2.1-kubernetes' into k8s-improve-driver-exit-output
lins05 May 19, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion .../kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/Client.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,15 @@ private[spark] class Client(
if (waitForAppCompletion) {
logInfo(s"Waiting for application $kubernetesAppId to finish...")
driverPodCompletedLatch.await()
logInfo(s"Application $kubernetesAppId finished.")

val exitMessage = loggingWatch.getDriverPodExitStatus match {
case (Some(error), _) => s"failed with error: $error"
case (None, 0) => "finished successfully"
case (None, code) =>
s"failed with exit code $code. You may want to check the driver pod logs."
}

logInfo(s"Application $kubernetesAppId $exitMessage")
} else {
logInfo(s"Application $kubernetesAppId successfully launched.")
}
Expand Down
12 changes: 12 additions & 0 deletions ...src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/LoggingPodStatusWatcher.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ private[kubernetes] class LoggingPodStatusWatcher(podCompletedFuture: CountDownL
private def status: String = pod.map(_.getStatus().getContainerStatuses().toString())
.getOrElse("unknown")

private var driverPodExitCode: Int = 0
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

make this an Option[Int] and only set when we get an exit code. Otherwise if something goes wrong clients of LoggingPodStatusWatcher might think exit code is 0 when it's not

private var driverPodErrorMessage: Option[String] = None

def start(): Unit = {
if (interval > 0) {
scheduler.scheduleAtFixedRate(logRunnable, 0, interval, TimeUnit.MILLISECONDS)
Expand All @@ -62,14 +65,17 @@ private[kubernetes] class LoggingPodStatusWatcher(podCompletedFuture: CountDownL
this.pod = Option(pod)
action match {
case Action.DELETED =>
driverPodErrorMessage = Some("the driver pod was deleted")
closeWatch()

case Action.ERROR =>
driverPodErrorMessage = Some("error happened with the driver pod")
closeWatch()

case _ =>
logLongStatus()
if (hasCompleted()) {
driverPodExitCode = getDriverPodExitCode
closeWatch()
}
}
Expand Down Expand Up @@ -129,4 +135,10 @@ private[kubernetes] class LoggingPodStatusWatcher(podCompletedFuture: CountDownL
s"\n\t $k: $newValue"
}.mkString("")
}

def getDriverPodExitCode: Int = {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

private

pod.get.getStatus().getContainerStatuses().asScala.last.getState.getTerminated.getExitCode
}

def getDriverPodExitStatus: (Option[String], Int) = (driverPodErrorMessage, driverPodExitCode)
}