From 01cb3b2fe1e9d7f2c616f89669b81ce586f9c828 Mon Sep 17 00:00:00 2001 From: uncleGen Date: Fri, 3 Mar 2017 11:52:06 +0800 Subject: [PATCH 1/3] Log the row type when query type dose not match --- .../test/scala/org/apache/spark/sql/QueryTest.scala | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala index 34fa626e00e31..f0757cd057413 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala @@ -312,13 +312,23 @@ object QueryTest { sparkAnswer: Seq[Row], isSorted: Boolean = false): Option[String] = { if (prepareAnswer(expectedAnswer, isSorted) != prepareAnswer(sparkAnswer, isSorted)) { + val expectedAnswerType = "RowType" + expectedAnswer.headOption + .map(row => s"[${row.schema.fields.map(_.dataType.typeName).mkString(",")}]") + .getOrElse("[]") + + val sparkAnswerType = "RowType" + sparkAnswer.headOption + .map(row => s"[${row.schema.fields.map(_.dataType.typeName).mkString(",")}]") + .getOrElse("[]") + val errorMessage = s""" |== Results == |${sideBySide( s"== Correct Answer - ${expectedAnswer.size} ==" +: + expectedAnswerType +: prepareAnswer(expectedAnswer, isSorted).map(_.toString()), s"== Spark Answer - ${sparkAnswer.size} ==" +: + sparkAnswerType +: prepareAnswer(sparkAnswer, isSorted).map(_.toString())).mkString("\n")} """.stripMargin return Some(errorMessage) From 160e4068fe6f2becebb8e6a642df6383d72c6d2b Mon Sep 17 00:00:00 2001 From: uncleGen Date: Fri, 3 Mar 2017 13:59:40 +0800 Subject: [PATCH 2/3] bug fix --- .../scala/org/apache/spark/sql/QueryTest.scala | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala index f0757cd057413..f019854375f44 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala @@ -312,23 +312,23 @@ object QueryTest { sparkAnswer: Seq[Row], isSorted: Boolean = false): Option[String] = { if (prepareAnswer(expectedAnswer, isSorted) != prepareAnswer(sparkAnswer, isSorted)) { - val expectedAnswerType = "RowType" + expectedAnswer.headOption - .map(row => s"[${row.schema.fields.map(_.dataType.typeName).mkString(",")}]") - .getOrElse("[]") - - val sparkAnswerType = "RowType" + sparkAnswer.headOption - .map(row => s"[${row.schema.fields.map(_.dataType.typeName).mkString(",")}]") - .getOrElse("[]") + val getRowType: Option[Row] => String = row => + "RowType" + row.map(row => + if (row.schema == null) { + "[]" + } else { + s"[${row.schema.fields.map(_.dataType.typeName).mkString(",")}]" + }).getOrElse("[]") val errorMessage = s""" |== Results == |${sideBySide( s"== Correct Answer - ${expectedAnswer.size} ==" +: - expectedAnswerType +: + getRowType(expectedAnswer.headOption) +: prepareAnswer(expectedAnswer, isSorted).map(_.toString()), s"== Spark Answer - ${sparkAnswer.size} ==" +: - sparkAnswerType +: + getRowType(sparkAnswer.headOption) +: prepareAnswer(sparkAnswer, isSorted).map(_.toString())).mkString("\n")} """.stripMargin return Some(errorMessage) From 2cff2b2e3261bb988391200c366a10ca0f274fc8 Mon Sep 17 00:00:00 2001 From: uncleGen Date: Sun, 5 Mar 2017 13:07:15 +0800 Subject: [PATCH 3/3] address comment --- .../src/test/scala/org/apache/spark/sql/QueryTest.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala index f019854375f44..f9808834df4a5 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala @@ -313,12 +313,12 @@ object QueryTest { isSorted: Boolean = false): Option[String] = { if (prepareAnswer(expectedAnswer, isSorted) != prepareAnswer(sparkAnswer, isSorted)) { val getRowType: Option[Row] => String = row => - "RowType" + row.map(row => + row.map(row => if (row.schema == null) { - "[]" + "struct<>" } else { - s"[${row.schema.fields.map(_.dataType.typeName).mkString(",")}]" - }).getOrElse("[]") + s"${row.schema.catalogString}" + }).getOrElse("struct<>") val errorMessage = s"""