File tree Expand file tree Collapse file tree 2 files changed +28
-2
lines changed
main/scala/org/apache/spark/sql/hive
test/scala/org/apache/spark/sql/parquet Expand file tree Collapse file tree 2 files changed +28
-2
lines changed Original file line number Diff line number Diff line change @@ -45,6 +45,9 @@ private[hive] trait HiveStrategies {
4545 *
4646 * TODO: Much of this logic is duplicated in HiveTableScan. Ideally we would do some refactoring
4747 * but since this is after the code freeze for 1.1 all logic is here to minimize disruption.
48+ *
49+ * Other issues:
50+ * - Much of this logic assumes case insensitive resolution.
4851 */
4952 @ Experimental
5053 object ParquetConversion extends Strategy {
@@ -60,8 +63,14 @@ private[hive] trait HiveStrategies {
6063 })
6164 }
6265
63- implicit class PhysicalPlanHacks (s : SparkPlan ) {
64- def fakeOutput (newOutput : Seq [Attribute ]) = OutputFaker (newOutput, s)
66+ implicit class PhysicalPlanHacks (originalPlan : SparkPlan ) {
67+ def fakeOutput (newOutput : Seq [Attribute ]) =
68+ OutputFaker (
69+ originalPlan.output.map(a =>
70+ newOutput.find(a.name.toLowerCase == _.name.toLowerCase)
71+ .getOrElse(
72+ sys.error(s " Can't find attribute $a to fake in set ${newOutput.mkString(" ," )}" ))),
73+ originalPlan)
6574 }
6675
6776 def apply (plan : LogicalPlan ): Seq [SparkPlan ] = plan match {
Original file line number Diff line number Diff line change @@ -103,6 +103,23 @@ class ParquetMetastoreSuite extends QueryTest with BeforeAndAfterAll {
103103 )
104104 }
105105
106+ test(" project partitioning and non-partitioning columns" ) {
107+ checkAnswer(
108+ sql(" SELECT stringField, p, count(intField) " +
109+ " FROM partitioned_parquet GROUP BY p, stringField" ),
110+ (" part-1" , 1 , 10 ) ::
111+ (" part-2" , 2 , 10 ) ::
112+ (" part-3" , 3 , 10 ) ::
113+ (" part-4" , 4 , 10 ) ::
114+ (" part-5" , 5 , 10 ) ::
115+ (" part-6" , 6 , 10 ) ::
116+ (" part-7" , 7 , 10 ) ::
117+ (" part-8" , 8 , 10 ) ::
118+ (" part-9" , 9 , 10 ) ::
119+ (" part-10" , 10 , 10 ) :: Nil
120+ )
121+ }
122+
106123 test(" simple count" ) {
107124 checkAnswer(
108125 sql(" SELECT COUNT(*) FROM partitioned_parquet" ),
You can’t perform that action at this time.
0 commit comments