Skip to content

Commit bd35ee7

Browse files
committed
address comments.
1 parent 6a59b42 commit bd35ee7

File tree

1 file changed

+8
-6
lines changed
  • sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer

1 file changed

+8
-6
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -320,15 +320,12 @@ object ColumnPruning extends Rule[LogicalPlan] {
320320
output1.zip(output2).forall(pair => pair._1.semanticEquals(pair._2))
321321

322322
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
323-
// Prunes the unused columns from project list of Project/Aggregate/Window/Expand
323+
// Prunes the unused columns from project list of Project/Aggregate/Expand
324324
case p @ Project(_, p2: Project) if (p2.outputSet -- p.references).nonEmpty =>
325325
p.copy(child = p2.copy(projectList = p2.projectList.filter(p.references.contains)))
326326
case p @ Project(_, a: Aggregate) if (a.outputSet -- p.references).nonEmpty =>
327327
p.copy(
328328
child = a.copy(aggregateExpressions = a.aggregateExpressions.filter(p.references.contains)))
329-
case p @ Project(_, w: Window) if (w.windowOutputSet -- p.references).nonEmpty =>
330-
p.copy(child = w.copy(
331-
windowExpressions = w.windowExpressions.filter(p.references.contains)))
332329
case a @ Project(_, e @ Expand(_, _, grandChild)) if (e.outputSet -- a.references).nonEmpty =>
333330
val newOutput = e.output.filter(a.references.contains(_))
334331
val newProjects = e.projections.map { proj =>
@@ -378,12 +375,17 @@ object ColumnPruning extends Rule[LogicalPlan] {
378375
p
379376
}
380377

381-
// Eliminate no-op Projects
382-
case p @ Project(projectList, child) if sameOutput(child.output, p.output) => child
378+
// Prune unnecessary window expressions
379+
case p @ Project(_, w: Window) if (w.windowOutputSet -- p.references).nonEmpty =>
380+
p.copy(child = w.copy(
381+
windowExpressions = w.windowExpressions.filter(p.references.contains)))
383382

384383
// Eliminate no-op Window
385384
case w: Window if w.windowExpressions.isEmpty => w.child
386385

386+
// Eliminate no-op Projects
387+
case p @ Project(projectList, child) if sameOutput(child.output, p.output) => child
388+
387389
// Can't prune the columns on LeafNode
388390
case p @ Project(_, l: LeafNode) => p
389391

0 commit comments

Comments
 (0)