Skip to content

[SPARK-7952][SQL] use internal Decimal instead of java.math.BigDecimal #6574

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

[SPARK-7952][SQL] use internal Decimal instead of java.math.BigDecimal #6574

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f93c420
compare literal
cloud-fan Jun 2, 2015
1987b37
use Decimal instead of java.math.BigDecimal
cloud-fan Jun 2, 2015
b0e3549
rename to BooleanEquality
cloud-fan Jun 2, 2015
d7b60c8
another bug
cloud-fan Jun 8, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 22 additions & 22 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ trait HiveTypeCoercion {
WidenTypes ::
PromoteStrings ::
DecimalPrecision ::
BooleanEqualization ::
BooleanEquality ::
StringToIntegralCasts ::
FunctionArgumentConversion ::
CaseWhenCoercion ::
Expand Down Expand Up @@ -445,10 +445,10 @@ trait HiveTypeCoercion {
e2 @ DecimalType.Expression(p2, s2)) if p1 != p2 || s1 != s2 =>
val resultType = DecimalType(max(p1, p2), max(s1, s2))
b.makeCopy(Array(Cast(e1, resultType), Cast(e2, resultType)))
case b @ BinaryComparison(e1 @ DecimalType.Fixed(_, _), e2)
case b @ BinaryComparison(e1 @ DecimalType.Expression(_, _), e2)
if e2.dataType == DecimalType.Unlimited =>
b.makeCopy(Array(Cast(e1, DecimalType.Unlimited), e2))
case b @ BinaryComparison(e1, e2 @ DecimalType.Fixed(_, _))
case b @ BinaryComparison(e1, e2 @ DecimalType.Expression(_, _))
if e1.dataType == DecimalType.Unlimited =>
b.makeCopy(Array(e1, Cast(e2, DecimalType.Unlimited)))

Expand Down Expand Up @@ -479,9 +479,9 @@ trait HiveTypeCoercion {
/**
* Changes numeric values to booleans so that expressions like true = 1 can be evaluated.
*/
object BooleanEqualization extends Rule[LogicalPlan] {
private val trueValues = Seq(1.toByte, 1.toShort, 1, 1L, new java.math.BigDecimal(1))
private val falseValues = Seq(0.toByte, 0.toShort, 0, 0L, new java.math.BigDecimal(0))
object BooleanEquality extends Rule[LogicalPlan] {
private val trueValues = Seq(1.toByte, 1.toShort, 1, 1L, Decimal(1))
private val falseValues = Seq(0.toByte, 0.toShort, 0, 0L, Decimal(0))

private def buildCaseKeyWhen(booleanExpr: Expression, numericExpr: Expression) = {
CaseKeyWhen(numericExpr, Seq(
Expand Down Expand Up @@ -512,22 +512,22 @@ trait HiveTypeCoercion {
// all other cases are considered as false.

// We may simplify the expression if one side is literal numeric values
case EqualTo(left @ BooleanType(), Literal(value, _: NumericType))
if trueValues.contains(value) => left
case EqualTo(left @ BooleanType(), Literal(value, _: NumericType))
if falseValues.contains(value) => Not(left)
case EqualTo(Literal(value, _: NumericType), right @ BooleanType())
if trueValues.contains(value) => right
case EqualTo(Literal(value, _: NumericType), right @ BooleanType())
if falseValues.contains(value) => Not(right)
case EqualNullSafe(left @ BooleanType(), Literal(value, _: NumericType))
if trueValues.contains(value) => And(IsNotNull(left), left)
case EqualNullSafe(left @ BooleanType(), Literal(value, _: NumericType))
if falseValues.contains(value) => And(IsNotNull(left), Not(left))
case EqualNullSafe(Literal(value, _: NumericType), right @ BooleanType())
if trueValues.contains(value) => And(IsNotNull(right), right)
case EqualNullSafe(Literal(value, _: NumericType), right @ BooleanType())
if falseValues.contains(value) => And(IsNotNull(right), Not(right))
case EqualTo(bool @ BooleanType(), Literal(value, _: NumericType))
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We only care about boolean type and literal type here, not left and right, so I use bool instead.

if trueValues.contains(value) => bool
case EqualTo(bool @ BooleanType(), Literal(value, _: NumericType))
if falseValues.contains(value) => Not(bool)
case EqualTo(Literal(value, _: NumericType), bool @ BooleanType())
if trueValues.contains(value) => bool
case EqualTo(Literal(value, _: NumericType), bool @ BooleanType())
if falseValues.contains(value) => Not(bool)
case EqualNullSafe(bool @ BooleanType(), Literal(value, _: NumericType))
if trueValues.contains(value) => And(IsNotNull(bool), bool)
case EqualNullSafe(bool @ BooleanType(), Literal(value, _: NumericType))
if falseValues.contains(value) => And(IsNotNull(bool), Not(bool))
case EqualNullSafe(Literal(value, _: NumericType), bool @ BooleanType())
if trueValues.contains(value) => And(IsNotNull(bool), bool)
case EqualNullSafe(Literal(value, _: NumericType), bool @ BooleanType())
if falseValues.contains(value) => And(IsNotNull(bool), Not(bool))

case EqualTo(left @ BooleanType(), right @ NumericType()) =>
transform(left , right)
Expand Down
24 changes: 23 additions & 1 deletion ...atalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,8 @@ class HiveTypeCoercionSuite extends PlanTest {
}

test("type coercion simplification for equal to") {
val be = new HiveTypeCoercion {}.BooleanEqualization
val be = new HiveTypeCoercion {}.BooleanEquality

ruleTest(be,
EqualTo(Literal(true), Literal(1)),
Literal(true)
Expand All @@ -164,5 +165,26 @@ class HiveTypeCoercionSuite extends PlanTest {
EqualNullSafe(Literal(true), Literal(0)),
And(IsNotNull(Literal(true)), Not(Literal(true)))
)

ruleTest(be,
EqualTo(Literal(true), Literal(1L)),
Literal(true)
)
ruleTest(be,
EqualTo(Literal(new java.math.BigDecimal(1)), Literal(true)),
Literal(true)
)
ruleTest(be,
EqualTo(Literal(BigDecimal(0)), Literal(true)),
Not(Literal(true))
)
ruleTest(be,
EqualTo(Literal(Decimal(1)), Literal(true)),
Literal(true)
)
ruleTest(be,
EqualTo(Literal.create(Decimal(1), DecimalType(8, 0)), Literal(true)),
Literal(true)
)
}
}