Skip to content

Commit 3772d93

Browse files
gatorsmilecloud-fan
authored andcommitted
[SPARK-26307][SQL] Fix CTAS when INSERT a partitioned table using Hive serde
This is a Spark 2.3 regression introduced in #20521. We should add the partition info for InsertIntoHiveTable in CreateHiveTableAsSelectCommand. Otherwise, we will hit the following error by running the newly added test case: ``` [info] - CTAS: INSERT a partitioned table using Hive serde *** FAILED *** (829 milliseconds) [info] org.apache.spark.SparkException: Requested partitioning does not match the tab1 table: [info] Requested partitions: [info] Table partitions: part [info] at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.processInsert(InsertIntoHiveTable.scala:179) [info] at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.run(InsertIntoHiveTable.scala:107) ``` Added a test case. Closes #23255 from gatorsmile/fixCTAS. Authored-by: gatorsmile <[email protected]> Signed-off-by: Wenchen Fan <[email protected]> (cherry picked from commit 3bc83de) Signed-off-by: Wenchen Fan <[email protected]>
1 parent 1899dd2 commit 3772d93

File tree

2 files changed

+13
-1
lines changed

2 files changed

+13
-1
lines changed

sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/CreateHiveTableAsSelectCommand.scala

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,11 @@ case class CreateHiveTableAsSelectCommand(
5757
return Seq.empty
5858
}
5959

60+
// For CTAS, there is no static partition values to insert.
61+
val partition = tableDesc.partitionColumnNames.map(_ -> None).toMap
6062
InsertIntoHiveTable(
6163
tableDesc,
62-
Map.empty,
64+
partition,
6365
query,
6466
overwrite = false,
6567
ifPartitionNotExists = false,

sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -750,4 +750,14 @@ class InsertSuite extends QueryTest with TestHiveSingleton with BeforeAndAfter
750750
}
751751
}
752752
}
753+
754+
test("SPARK-26307: CTAS - INSERT a partitioned table using Hive serde") {
755+
withTable("tab1") {
756+
withSQLConf("hive.exec.dynamic.partition.mode" -> "nonstrict") {
757+
val df = Seq(("a", 100)).toDF("part", "id")
758+
df.write.format("hive").partitionBy("part").mode("overwrite").saveAsTable("tab1")
759+
df.write.format("hive").partitionBy("part").mode("append").saveAsTable("tab1")
760+
}
761+
}
762+
}
753763
}

0 commit comments

Comments
 (0)