Skip to content

Commit 37d7a18

Browse files
rxinnemccarthy
authored andcommitted
[SQL] Fixed expression data type matching.
Also took the chance to improve documentation for various types. Author: Reynold Xin <[email protected]> Closes apache#5675 from rxin/data-type-matching-expr and squashes the following commits: 0f31856 [Reynold Xin] One more function documentation. 27c1973 [Reynold Xin] Added more documentation. 336a36d [Reynold Xin] [SQL] Fixed expression data type matching.
1 parent aacac2d commit 37d7a18

File tree

2 files changed

+42
-10
lines changed

2 files changed

+42
-10
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,7 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
279279
org.apache.spark.sql.types.UTF8String(${eval.primitiveTerm}.toString)
280280
""".children
281281

282-
case EqualTo(e1: BinaryType, e2: BinaryType) =>
282+
case EqualTo(e1 @ BinaryType(), e2 @ BinaryType()) =>
283283
(e1, e2).evaluateAs (BooleanType) {
284284
case (eval1, eval2) =>
285285
q"""

sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala

Lines changed: 41 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -40,32 +40,46 @@ import org.apache.spark.util.Utils
4040
*/
4141
@DeveloperApi
4242
abstract class DataType {
43-
/** Matches any expression that evaluates to this DataType */
44-
def unapply(a: Expression): Boolean = a match {
43+
/**
44+
* Enables matching against NumericType for expressions:
45+
* {{{
46+
* case Cast(child @ BinaryType(), StringType) =>
47+
* ...
48+
* }}}
49+
*/
50+
private[sql] def unapply(a: Expression): Boolean = a match {
4551
case e: Expression if e.dataType == this => true
4652
case _ => false
4753
}
4854

49-
/** The default size of a value of this data type. */
55+
/**
56+
* The default size of a value of this data type, used internally for size estimation.
57+
*/
5058
def defaultSize: Int
5159

60+
/** Name of the type used in JSON serialization. */
5261
def typeName: String = this.getClass.getSimpleName.stripSuffix("$").dropRight(4).toLowerCase
5362

5463
private[sql] def jsonValue: JValue = typeName
5564

65+
/** The compact JSON representation of this data type. */
5666
def json: String = compact(render(jsonValue))
5767

68+
/** The pretty (i.e. indented) JSON representation of this data type. */
5869
def prettyJson: String = pretty(render(jsonValue))
5970

71+
/** Readable string representation for the type. */
6072
def simpleString: String = typeName
6173

62-
/** Check if `this` and `other` are the same data type when ignoring nullability
63-
* (`StructField.nullable`, `ArrayType.containsNull`, and `MapType.valueContainsNull`).
74+
/**
75+
* Check if `this` and `other` are the same data type when ignoring nullability
76+
* (`StructField.nullable`, `ArrayType.containsNull`, and `MapType.valueContainsNull`).
6477
*/
6578
private[spark] def sameType(other: DataType): Boolean =
6679
DataType.equalsIgnoreNullability(this, other)
6780

68-
/** Returns the same data type but set all nullability fields are true
81+
/**
82+
* Returns the same data type but set all nullability fields are true
6983
* (`StructField.nullable`, `ArrayType.containsNull`, and `MapType.valueContainsNull`).
7084
*/
7185
private[spark] def asNullable: DataType
@@ -104,12 +118,25 @@ abstract class NumericType extends AtomicType {
104118

105119

106120
private[sql] object NumericType {
121+
/**
122+
* Enables matching against NumericType for expressions:
123+
* {{{
124+
* case Cast(child @ NumericType(), StringType) =>
125+
* ...
126+
* }}}
127+
*/
107128
def unapply(e: Expression): Boolean = e.dataType.isInstanceOf[NumericType]
108129
}
109130

110131

111-
/** Matcher for any expressions that evaluate to [[IntegralType]]s */
112132
private[sql] object IntegralType {
133+
/**
134+
* Enables matching against IntegralType for expressions:
135+
* {{{
136+
* case Cast(child @ IntegralType(), StringType) =>
137+
* ...
138+
* }}}
139+
*/
113140
def unapply(a: Expression): Boolean = a match {
114141
case e: Expression if e.dataType.isInstanceOf[IntegralType] => true
115142
case _ => false
@@ -122,9 +149,14 @@ private[sql] abstract class IntegralType extends NumericType {
122149
}
123150

124151

125-
126-
/** Matcher for any expressions that evaluate to [[FractionalType]]s */
127152
private[sql] object FractionalType {
153+
/**
154+
* Enables matching against FractionalType for expressions:
155+
* {{{
156+
* case Cast(child @ FractionalType(), StringType) =>
157+
* ...
158+
* }}}
159+
*/
128160
def unapply(a: Expression): Boolean = a match {
129161
case e: Expression if e.dataType.isInstanceOf[FractionalType] => true
130162
case _ => false

0 commit comments

Comments
 (0)