Skip to content

Commit 2dd37d8

Browse files
cloud-fanhvanhovell
authored andcommitted
[SPARK-21826][SQL] outer broadcast hash join should not throw NPE
## What changes were proposed in this pull request? This is a bug introduced by https://github.com/apache/spark/pull/11274/files#diff-7adb688cbfa583b5711801f196a074bbL274 . Non-equal join condition should only be applied when the equal-join condition matches. ## How was this patch tested? regression test Author: Wenchen Fan <[email protected]> Closes #19036 from cloud-fan/bug.
1 parent 183d4cb commit 2dd37d8

File tree

2 files changed

+21
-1
lines changed

2 files changed

+21
-1
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -283,8 +283,8 @@ case class BroadcastHashJoinExec(
283283
s"""
284284
|boolean $conditionPassed = true;
285285
|${eval.trim}
286-
|${ev.code}
287286
|if ($matched != null) {
287+
| ${ev.code}
288288
| $conditionPassed = !${ev.isNull} && ${ev.value};
289289
|}
290290
""".stripMargin

sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.spark.sql
1919

20+
import scala.collection.JavaConverters._
2021
import scala.collection.mutable.ListBuffer
2122
import scala.language.existentials
2223

@@ -26,6 +27,7 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
2627
import org.apache.spark.sql.execution.joins._
2728
import org.apache.spark.sql.internal.SQLConf
2829
import org.apache.spark.sql.test.SharedSQLContext
30+
import org.apache.spark.sql.types.StructType
2931

3032
class JoinSuite extends QueryTest with SharedSQLContext {
3133
import testImplicits._
@@ -767,4 +769,22 @@ class JoinSuite extends QueryTest with SharedSQLContext {
767769
}
768770
}
769771
}
772+
773+
test("outer broadcast hash join should not throw NPE") {
774+
withTempView("v1", "v2") {
775+
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true") {
776+
Seq(2 -> 2).toDF("x", "y").createTempView("v1")
777+
778+
spark.createDataFrame(
779+
Seq(Row(1, "a")).asJava,
780+
new StructType().add("i", "int", nullable = false).add("j", "string", nullable = false)
781+
).createTempView("v2")
782+
783+
checkAnswer(
784+
sql("select x, y, i, j from v1 left join v2 on x = i and y < length(j)"),
785+
Row(2, 2, null, null)
786+
)
787+
}
788+
}
789+
}
770790
}

0 commit comments

Comments
 (0)