Skip to content

Commit 63ab5bd

Browse files
author
Branden Smith
committed
internalize testdata for JsonSuite SPARK-26745 test
1 parent b7a4783 commit 63ab5bd

File tree

2 files changed

+9
-15
lines changed

2 files changed

+9
-15
lines changed

sql/core/src/test/resources/test-data/with-empty-line.json

Lines changed: 0 additions & 7 deletions
This file was deleted.

sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2427,14 +2427,15 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData {
24272427
}
24282428

24292429
test("SPARK-26745: count() for non-multiline input with empty lines") {
2430-
val df = spark.read.json(testFile("test-data/with-empty-line.json"))
2431-
val withEmptyLineData = Row(1, 2, 3) :: Row(4, 5, 6) :: Row(7, 8, 9) :: Nil
2432-
// important to do this .count() first, prior to caching/persisting/computing/collecting, to
2433-
// test the non-parsed-count pathway
2434-
assert(df.count() === withEmptyLineData.length,
2435-
"JSON DataFrame unparsed-count should exclude whitespace-only lines")
2436-
df.createOrReplaceTempView("jsonWithEmptyLineTable")
2437-
checkAnswer(sql("select * from jsonWithEmptyLineTable"), withEmptyLineData)
2430+
withTempPath { tempPath =>
2431+
val path = tempPath.getCanonicalPath
2432+
Seq("""{ "a" : 1 }""", "", """ { "a" : 2 }""", " \t ")
2433+
.toDS()
2434+
.repartition(1)
2435+
.write
2436+
.text(path)
2437+
assert(spark.read.json(path).count() === 2)
2438+
}
24382439
}
24392440

24402441
test("SPARK-25040: empty strings should be disallowed") {

0 commit comments

Comments
 (0)