History Server: updated order for multiple attempts

rekhajoshm · rekhajoshm · commit a41ac4bda10c · 2015-07-08T12:05:42.000-07:00
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -414,7 +414,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
   /**
    * Comparison function that defines the sort order for application attempts within the same
    * application. Order is: running attempts before complete attempts, running attempts sorted
-   * by start time, completed attempts sorted by end time.
+   * by start time showing whichever started first,
+   * completed attempts sorted by end time showing whichever ended first.
    *
    * Normally applications should have a single running attempt; but failure to call sc.stop()
    * may cause multiple running attempts to show up.
@@ -425,9 +426,9 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
       a1: FsApplicationAttemptInfo,
       a2: FsApplicationAttemptInfo): Boolean = {
     if (a1.completed == a2.completed) {
-      if (a1.completed) a1.endTime >= a2.endTime else a1.startTime >= a2.startTime
+      if (a1.completed) a1.endTime <= a2.endTime else a1.startTime <= a2.startTime
     } else {
-      !a1.completed
+      a1.completed
     }
   }
 
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
@@ -35,6 +35,7 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("")
       Option(request.getParameter("showIncomplete")).getOrElse("false").toBoolean
 
     val allApps = parent.getApplicationList()
+      .filter(_.attempts.head.completed != requestedIncomplete)
     val allAppsSize = allApps.size
 
     val actualFirst = if (requestedFirst < allAppsSize) requestedFirst else 0
@@ -50,15 +51,9 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("")
     val hasMultipleAttempts = appsToShow.exists(_.attempts.size > 1)
     val appTable =
       if (hasMultipleAttempts) {
-        UIUtils.listingTable(
-          appWithAttemptHeader,
-          appWithAttemptRow(_, requestedIncomplete),
-          appsToShow)
+        UIUtils.listingTable(appWithAttemptHeader, appWithAttemptRow, appsToShow)
       } else {
-        UIUtils.listingTable(
-          appHeader,
-          appRow(_, requestedIncomplete),
-          appsToShow)
+        UIUtils.listingTable(appHeader, appRow, appsToShow)
       }
 
     val providerConfig = parent.getProviderConfig()
@@ -69,61 +64,61 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("")
             {providerConfig.map { case (k, v) => <li><strong>{k}:</strong> {v}</li> }}
           </ul>
           {
-            // This displays the indices of pages that are within `plusOrMinus` pages of
-            // the current page. Regardless of where the current page is, this also links
-            // to the first and last page. If the current page +/- `plusOrMinus` is greater
-            // than the 2nd page from the first page or less than the 2nd page from the last
-            // page, `...` will be displayed.
-            if (allAppsSize > 0) {
-              val leftSideIndices =
-                rangeIndices(actualPage - plusOrMinus until actualPage, 1 < _, requestedIncomplete)
-              val rightSideIndices =
-                rangeIndices(actualPage + 1 to actualPage + plusOrMinus, _ < pageCount,
-                  requestedIncomplete)
-
-              <h4>
-                Showing {actualFirst + 1}-{last + 1} of {allAppsSize}
-                {if (requestedIncomplete) "(Incomplete applications)"}
-                <span style="float: right">
-                  {
-                    if (actualPage > 1) {
-                      <a href={makePageLink(actualPage - 1, requestedIncomplete)}>&lt; </a>
-                      <a href={makePageLink(1, requestedIncomplete)}>1</a>
-                    }
-                  }
-                  {if (actualPage - plusOrMinus > secondPageFromLeft) " ... "}
-                  {leftSideIndices}
-                  {actualPage}
-                  {rightSideIndices}
-                  {if (actualPage + plusOrMinus < secondPageFromRight) " ... "}
-                  {
-                    if (actualPage < pageCount) {
-                      <a href={makePageLink(pageCount, requestedIncomplete)}>{pageCount}</a>
-                      <a href={makePageLink(actualPage + 1, requestedIncomplete)}> &gt;</a>
-                    }
-                  }
-                </span>
-              </h4> ++
+          // This displays the indices of pages that are within `plusOrMinus` pages of
+          // the current page. Regardless of where the current page is, this also links
+          // to the first and last page. If the current page +/- `plusOrMinus` is greater
+          // than the 2nd page from the first page or less than the 2nd page from the last
+          // page, `...` will be displayed.
+          if (allAppsSize > 0) {
+            val leftSideIndices =
+              rangeIndices(actualPage - plusOrMinus until actualPage, 1 < _, requestedIncomplete)
+            val rightSideIndices =
+              rangeIndices(actualPage + 1 to actualPage + plusOrMinus, _ < pageCount,
+                requestedIncomplete)
+
+            <h4>
+              Showing {actualFirst + 1}-{last + 1} of {allAppsSize}
+              {if (requestedIncomplete) "(Incomplete applications)"}
+              <span style="float: right">
+                {
+                if (actualPage > 1) {
+                  <a href={makePageLink(actualPage - 1, requestedIncomplete)}>&lt; </a>
+                    <a href={makePageLink(1, requestedIncomplete)}>1</a>
+                }
+                }
+                {if (actualPage - plusOrMinus > secondPageFromLeft) " ... "}
+                {leftSideIndices}
+                {actualPage}
+                {rightSideIndices}
+                {if (actualPage + plusOrMinus < secondPageFromRight) " ... "}
+                {
+                if (actualPage < pageCount) {
+                  <a href={makePageLink(pageCount, requestedIncomplete)}>{pageCount}</a>
+                    <a href={makePageLink(actualPage + 1, requestedIncomplete)}> &gt;</a>
+                }
+                }
+              </span>
+            </h4> ++
               appTable
-            } else if (requestedIncomplete) {
-              <h4>No incomplete applications found!</h4>
-            } else {
-              <h4>No completed applications found!</h4> ++
+          } else if (requestedIncomplete) {
+            <h4>No incomplete applications found!</h4>
+          } else {
+            <h4>No completed applications found!</h4> ++
               <p>Did you specify the correct logging directory?
                 Please verify your setting of <span style="font-style:italic">
                 spark.history.fs.logDirectory</span> and whether you have the permissions to
                 access it.<br /> It is also possible that your application did not run to
                 completion or did not stop the SparkContext.
               </p>
-            }
+          }
           }
           <a href={makePageLink(actualPage, !requestedIncomplete)}>
             {
-              if (requestedIncomplete) {
-                "Back to completed applications"
-              } else {
-                "Show incomplete applications"
-              }
+            if (requestedIncomplete) {
+              "Back to completed applications"
+            } else {
+              "Show incomplete applications"
+            }
             }
           </a>
         </div>
@@ -151,19 +146,18 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("")
     "Last Updated")
 
   private def rangeIndices(
-      range: Seq[Int],
-      condition: Int => Boolean,
-      showIncomplete: Boolean): Seq[Node] = {
+                            range: Seq[Int],
+                            condition: Int => Boolean,
+                            showIncomplete: Boolean): Seq[Node] = {
     range.filter(condition).map(nextPage =>
       <a href={makePageLink(nextPage, showIncomplete)}> {nextPage} </a>)
   }
 
   private def attemptRow(
-      renderAttemptIdColumn: Boolean,
-      info: ApplicationHistoryInfo,
-      attempt: ApplicationAttemptInfo,
-      isFirst: Boolean,
-      requestedIncomplete: Boolean): Seq[Node] = {
+                          renderAttemptIdColumn: Boolean,
+                          info: ApplicationHistoryInfo,
+                          attempt: ApplicationAttemptInfo,
+                          isFirst: Boolean): Seq[Node] = {
     val uiAddress = HistoryServer.getAttemptURI(info.id, attempt.attemptId)
     val startTime = UIUtils.formatDate(attempt.startTime)
     val endTime = if (attempt.endTime > 0) UIUtils.formatDate(attempt.endTime) else "-"
@@ -174,67 +168,50 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("")
         "-"
       }
     val lastUpdated = UIUtils.formatDate(attempt.lastUpdated)
-    var someAttemptCompleted = false
-    info.attempts.foreach{ attempt =>
-      if (attempt.completed) someAttemptCompleted = true
-    }
     <tr>
       {
-        if (isFirst) {
-          if (info.attempts.size > 1 || renderAttemptIdColumn) {
-            <td rowspan={info.attempts.size.toString} style="background-color: #ffffff">
-              <a href={uiAddress}>{info.id}</a></td>
+      if (isFirst) {
+        if (info.attempts.size > 1 || renderAttemptIdColumn) {
+          <td rowspan={info.attempts.size.toString} style="background-color: #ffffff">
+            <a href={uiAddress}>{info.id}</a></td>
             <td rowspan={info.attempts.size.toString} style="background-color: #ffffff">
               {info.name}</td>
-          } else {
-            <td><a href={uiAddress}>{info.id}</a></td>
-            <td>{info.name}</td>
-          }
         } else {
-          Nil
+          <td><a href={uiAddress}>{info.id}</a></td>
+            <td>{info.name}</td>
         }
+      } else {
+        Nil
       }
-      {
-        if (renderAttemptIdColumn &&
-            (requestedIncomplete || (!requestedIncomplete && someAttemptCompleted))) {
-          if (info.attempts.size > 1 && attempt.attemptId.isDefined) {
-            <td><a href={HistoryServer.getAttemptURI(info.id, attempt.attemptId)}>
-              {attempt.attemptId.get}</a></td>
-          } else {
-            <td>&nbsp;</td>
-          }
-        } else {
-          Nil
-        }
       }
       {
-        if (requestedIncomplete || (!requestedIncomplete && someAttemptCompleted)) {
-          <td sorttable_customkey={attempt.startTime.toString}>{startTime}</td>
-          <td sorttable_customkey={attempt.endTime.toString}>{endTime}</td>
-          <td sorttable_customkey={(attempt.endTime - attempt.startTime).toString}>
-              {duration}</td>
-          <td>{attempt.sparkUser}</td>
-          <td sorttable_customkey={attempt.lastUpdated.toString}>{lastUpdated}</td>
+      if (renderAttemptIdColumn) {
+        if (info.attempts.size > 1 && attempt.attemptId.isDefined) {
+          <td><a href={HistoryServer.getAttemptURI(info.id, attempt.attemptId)}>
+            {attempt.attemptId.get}</a></td>
         } else {
-          Nil
+          <td>&nbsp;</td>
         }
+      } else {
+        Nil
+      }
       }
+      <td sorttable_customkey={attempt.startTime.toString}>{startTime}</td>
+      <td sorttable_customkey={attempt.endTime.toString}>{endTime}</td>
+      <td sorttable_customkey={(attempt.endTime - attempt.startTime).toString}>
+        {duration}</td>
+      <td>{attempt.sparkUser}</td>
+      <td sorttable_customkey={attempt.lastUpdated.toString}>{lastUpdated}</td>
     </tr>
   }
 
-
-
-  private def appRow(
-      info: ApplicationHistoryInfo,
-      requestedIncomplete: Boolean): Seq[Node] = {
-    attemptRow(false, info, info.attempts.head, true, requestedIncomplete)
+  private def appRow(info: ApplicationHistoryInfo): Seq[Node] = {
+    attemptRow(false, info, info.attempts.head, true)
   }
 
-  private def appWithAttemptRow(
-      info: ApplicationHistoryInfo,
-      requestedIncomplete: Boolean): Seq[Node] = {
-    attemptRow(true, info, info.attempts.head, true, requestedIncomplete) ++
-      info.attempts.drop(1).flatMap(attemptRow(true, info, _, false, requestedIncomplete))
+  private def appWithAttemptRow(info: ApplicationHistoryInfo): Seq[Node] = {
+    attemptRow(true, info, info.attempts.head, true) ++
+      info.attempts.drop(1).flatMap(attemptRow(true, info, _, false))
   }
 
   private def makePageLink(linkPage: Int, showIncomplete: Boolean): String = {
