Skip to content

[SPARK-5329][WebUI] UIWorkloadGenerator should stop SparkContext. #4112

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

[SPARK-5329][WebUI] UIWorkloadGenerator should stop SparkContext. #4112

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
86a3b95
Fixed UIWorkloadGenerator to stop SparkContext in it
sarutak Jan 20, 2015
bcc0fa9
Disabled scalastyle for a bock comment
sarutak Jan 20, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions core/src/main/scala/org/apache/spark/ui/UIWorkloadGenerator.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,20 +22,23 @@ import scala.util.Random
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.scheduler.SchedulingMode

// scalastyle:off
/**
* Continuously generates jobs that expose various features of the WebUI (internal testing tool).
*
* Usage: ./bin/spark-class org.apache.spark.ui.UIWorkloadGenerator [master] [FIFO|FAIR]
* Usage: ./bin/spark-class org.apache.spark.ui.UIWorkloadGenerator [master] [FIFO|FAIR] [#job set (4 jobs per set)]
*/
// scalastyle:on
private[spark] object UIWorkloadGenerator {

val NUM_PARTITIONS = 100
val INTER_JOB_WAIT_MS = 5000

def main(args: Array[String]) {
if (args.length < 2) {
if (args.length < 3) {
println(
"usage: ./bin/spark-class org.apache.spark.ui.UIWorkloadGenerator [master] [FIFO|FAIR]")
"usage: ./bin/spark-class org.apache.spark.ui.UIWorkloadGenerator " +
"[master] [FIFO|FAIR] [#job set (4 jobs per set)]")
System.exit(1)
}

Expand All @@ -45,6 +48,7 @@ private[spark] object UIWorkloadGenerator {
if (schedulingMode == SchedulingMode.FAIR) {
conf.set("spark.scheduler.mode", "FAIR")
}
val nJobSet = args(2).toInt
val sc = new SparkContext(conf)

def setProperties(s: String) = {
Expand Down Expand Up @@ -84,7 +88,7 @@ private[spark] object UIWorkloadGenerator {
("Job with delays", baseData.map(x => Thread.sleep(100)).count)
)

while (true) {
(1 to nJobSet).foreach { _ =>
for ((desc, job) <- jobs) {
new Thread {
override def run() {
Expand All @@ -101,5 +105,6 @@ private[spark] object UIWorkloadGenerator {
Thread.sleep(INTER_JOB_WAIT_MS)
}
}
sc.stop()
}
}