From 358ac0d2e9b27bcf7c3d0448555497b60fc20dd5 Mon Sep 17 00:00:00 2001 From: bomeng Date: Thu, 16 Jun 2016 16:21:39 -0700 Subject: [PATCH 1/6] improve the disply of CatalogTable information --- .../org/apache/spark/sql/catalyst/catalog/interface.scala | 5 +++-- .../org/apache/spark/sql/execution/command/tables.scala | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index 6197acab33786..3262e84becad9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -127,7 +127,7 @@ case class CatalogTable( sortColumnNames: Seq[String] = Seq.empty, bucketColumnNames: Seq[String] = Seq.empty, numBuckets: Int = -1, - owner: String = "", + owner: String = System.getProperty("user.name"), createTime: Long = System.currentTimeMillis, lastAccessTime: Long = -1, properties: Map[String, String] = Map.empty, @@ -180,7 +180,8 @@ case class CatalogTable( Seq(s"Table: ${identifier.quotedString}", if (owner.nonEmpty) s"Owner: $owner" else "", s"Created: ${new Date(createTime).toString}", - s"Last Access: ${new Date(lastAccessTime).toString}", + "Last Access: " + + (if (lastAccessTime == -1) "UNKNOWN" else new Date(lastAccessTime).toString), s"Type: ${tableType.name}", if (schema.nonEmpty) s"Schema: ${schema.mkString("[", ", ", "]")}" else "", if (partitionColumnNames.nonEmpty) s"Partition Columns: $partitionColumns" else "", diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 3eb93a2922708..9ce6f63bb1134 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -468,7 +468,8 @@ case class DescribeTableCommand(table: TableIdentifier, isExtended: Boolean, isF append(buffer, "Database:", table.database, "") append(buffer, "Owner:", table.owner, "") append(buffer, "Create Time:", new Date(table.createTime).toString, "") - append(buffer, "Last Access Time:", new Date(table.lastAccessTime).toString, "") + append(buffer, "Last Access Time:", + if (table.lastAccessTime == -1) "UNKNOWN" else new Date(table.lastAccessTime).toString, "") append(buffer, "Location:", table.storage.locationUri.getOrElse(""), "") append(buffer, "Table Type:", table.tableType.name, "") @@ -522,7 +523,7 @@ case class DescribeTableCommand(table: TableIdentifier, isExtended: Boolean, isF private def describeSchema(schema: Seq[CatalogColumn], buffer: ArrayBuffer[Row]): Unit = { schema.foreach { column => - append(buffer, column.name, column.dataType.toLowerCase, column.comment.orNull) + append(buffer, column.name, column.dataType.toLowerCase, column.comment.getOrElse("")) } } From 0fe835d876cb7af8f23f9c1a37158caa15f71cdd Mon Sep 17 00:00:00 2001 From: bomeng Date: Thu, 16 Jun 2016 18:59:26 -0700 Subject: [PATCH 2/6] fix the tests failure --- .../execution/HiveOperatorQueryableSuite.scala | 8 ++++---- .../sql/hive/execution/HiveQuerySuite.scala | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala index 0e89e990e564e..5d98aa7287ec3 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala @@ -41,13 +41,13 @@ class HiveOperatorQueryableSuite extends QueryTest with TestHiveSingleton { checkAnswer( sql("select * from mydesc"), Seq( - Row("key", "int", null), - Row("value", "string", null))) + Row("key", "int", ""), + Row("value", "string", ""))) checkAnswer( sql("select col_name, data_type, comment from mydesc"), Seq( - Row("key", "int", null), - Row("value", "string", null))) + Row("key", "int", ""), + Row("value", "string", ""))) } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index e0f6ccf04dd33..79ef0717cd435 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -798,12 +798,12 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { // Describe a table assertResult( Array( - Row("key", "int", null), - Row("value", "string", null), - Row("dt", "string", null), + Row("key", "int", ""), + Row("value", "string", ""), + Row("dt", "string", ""), Row("# Partition Information", "", ""), Row("# col_name", "data_type", "comment"), - Row("dt", "string", null)) + Row("dt", "string", "")) ) { sql("DESCRIBE test_describe_commands1") .select('col_name, 'data_type, 'comment) @@ -813,12 +813,12 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { // Describe a table with a fully qualified table name assertResult( Array( - Row("key", "int", null), - Row("value", "string", null), - Row("dt", "string", null), + Row("key", "int", ""), + Row("value", "string", ""), + Row("dt", "string", ""), Row("# Partition Information", "", ""), Row("# col_name", "data_type", "comment"), - Row("dt", "string", null)) + Row("dt", "string", "")) ) { sql("DESCRIBE default.test_describe_commands1") .select('col_name, 'data_type, 'comment) From 90c3ff0ac39729b2b50552a5033d9a82a866f89e Mon Sep 17 00:00:00 2001 From: bomeng Date: Tue, 21 Jun 2016 14:52:08 -0700 Subject: [PATCH 3/6] revert user name --- .../scala/org/apache/spark/sql/catalyst/catalog/interface.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index 3262e84becad9..5e7e4634577ef 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -127,7 +127,7 @@ case class CatalogTable( sortColumnNames: Seq[String] = Seq.empty, bucketColumnNames: Seq[String] = Seq.empty, numBuckets: Int = -1, - owner: String = System.getProperty("user.name"), + owner: String = "", createTime: Long = System.currentTimeMillis, lastAccessTime: Long = -1, properties: Map[String, String] = Map.empty, From 4c9e295feb58958248e37ab24b78cbefbc485463 Mon Sep 17 00:00:00 2001 From: bomeng Date: Fri, 24 Jun 2016 13:49:21 -0700 Subject: [PATCH 4/6] revert comment to use null --- .../spark/sql/execution/command/tables.scala | 2 +- .../execution/HiveOperatorQueryableSuite.scala | 8 ++++---- .../sql/hive/execution/HiveQuerySuite.scala | 16 ++++++++-------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 35840e112b976..e87fd35730c90 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -523,7 +523,7 @@ case class DescribeTableCommand(table: TableIdentifier, isExtended: Boolean, isF private def describeSchema(schema: Seq[CatalogColumn], buffer: ArrayBuffer[Row]): Unit = { schema.foreach { column => - append(buffer, column.name, column.dataType.toLowerCase, column.comment.getOrElse("")) + append(buffer, column.name, column.dataType.toLowerCase, column.comment.orNull) } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala index 5d98aa7287ec3..76d98506d949e 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala @@ -41,13 +41,13 @@ class HiveOperatorQueryableSuite extends QueryTest with TestHiveSingleton { checkAnswer( sql("select * from mydesc"), Seq( - Row("key", "int", ""), - Row("value", "string", ""))) + Row("key", "int", null ), + Row("value", "string", null))) checkAnswer( sql("select col_name, data_type, comment from mydesc"), Seq( - Row("key", "int", ""), - Row("value", "string", ""))) + Row("key", "int", null), + Row("value", "string", null))) } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 2cdeb30359abc..f8c55ec45650e 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -798,12 +798,12 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { // Describe a table assertResult( Array( - Row("key", "int", ""), - Row("value", "string", ""), - Row("dt", "string", ""), + Row("key", "int", null), + Row("value", "string", null), + Row("dt", "string", null), Row("# Partition Information", "", ""), Row("# col_name", "data_type", "comment"), - Row("dt", "string", "")) + Row("dt", "string", null)) ) { sql("DESCRIBE test_describe_commands1") .select('col_name, 'data_type, 'comment) @@ -813,12 +813,12 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { // Describe a table with a fully qualified table name assertResult( Array( - Row("key", "int", ""), - Row("value", "string", ""), - Row("dt", "string", ""), + Row("key", "int", null), + Row("value", "string", null), + Row("dt", "string", null), Row("# Partition Information", "", ""), Row("# col_name", "data_type", "comment"), - Row("dt", "string", "")) + Row("dt", "string", null)) ) { sql("DESCRIBE default.test_describe_commands1") .select('col_name, 'data_type, 'comment) From e93b72a12312358e1c379be45c4743568268a17c Mon Sep 17 00:00:00 2001 From: bomeng Date: Fri, 24 Jun 2016 13:50:30 -0700 Subject: [PATCH 5/6] remove space --- .../spark/sql/hive/execution/HiveOperatorQueryableSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala index 76d98506d949e..0e89e990e564e 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala @@ -41,7 +41,7 @@ class HiveOperatorQueryableSuite extends QueryTest with TestHiveSingleton { checkAnswer( sql("select * from mydesc"), Seq( - Row("key", "int", null ), + Row("key", "int", null), Row("value", "string", null))) checkAnswer( From a72800c63970f7a3300bb53309cb655665f6e3ec Mon Sep 17 00:00:00 2001 From: bomeng Date: Mon, 27 Jun 2016 11:12:39 -0700 Subject: [PATCH 6/6] add test cases --- .../spark/sql/execution/command/DDLSuite.scala | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 47d8a28f49927..413af675b5372 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -265,6 +265,23 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { } } + test("Describe Table") { + val tabName = "tab1" + withTable(tabName) { + sql(s"CREATE TABLE $tabName(a int comment 'test')") + + assert(sql(s"DESC $tabName").collect().length == 1) + + assert( + sql(s"DESC FORMATTED $tabName").collect() + .exists(_.getString(0) == "# Storage Information")) + + assert( + sql(s"DESC EXTENDED $tabName").collect() + .exists(_.getString(0) == "# Detailed Table Information")) + } + } + test("Alter/Describe Database") { withTempDir { tmpDir => val path = tmpDir.toString