Skip to content

Commit 956b0a4

Browse files
author
Davies Liu
committed
fix hive tests
1 parent 73e4363 commit 956b0a4

File tree

5 files changed

+34
-27
lines changed

5 files changed

+34
-27
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ object Literal {
2929
case f: Float => Literal(f, FloatType)
3030
case b: Byte => Literal(b, ByteType)
3131
case s: Short => Literal(s, ShortType)
32-
case s: String => Literal(s, StringType)
32+
case s: String => Literal(UTF8String(s), StringType)
3333
case b: Boolean => Literal(b, BooleanType)
3434
case d: BigDecimal => Literal(Decimal(d), DecimalType.Unlimited)
3535
case d: java.math.BigDecimal => Literal(Decimal(d), DecimalType.Unlimited)

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ object LikeSimplification extends Rule[LogicalPlan] {
209209
case equalTo(pattern) =>
210210
EqualTo(l, Literal(pattern))
211211
case _ =>
212-
Like(l, Literal(utf, StringType))
212+
Like(l, Literal.create(utf, StringType))
213213
}
214214
}
215215
}

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1054,7 +1054,7 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
10541054
checkEvaluation(Sqrt(d), expected, row)
10551055
}
10561056

1057-
checkEvaluation(Sqrt(Literal.create(null, DoubleType)), null, new create_row(Array[Any](null)))
1057+
checkEvaluation(Sqrt(Literal.create(null, DoubleType)), null, create_row(Array[Any](null)))
10581058
checkEvaluation(Sqrt(-1), null, EmptyRow)
10591059
checkEvaluation(Sqrt(-1.5), null, EmptyRow)
10601060
}

‎sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetPartitionDiscoverySuite.scala‎

Lines changed: 26 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import scala.collection.mutable.ArrayBuffer
2020

2121
import org.apache.hadoop.fs.Path
2222

23-
import org.apache.spark.sql.catalyst.expressions.Literal
23+
import org.apache.spark.sql.catalyst.expressions._
2424
import org.apache.spark.sql.parquet.ParquetRelation2._
2525
import org.apache.spark.sql.test.TestSQLContext
2626
import org.apache.spark.sql.types._
@@ -40,6 +40,10 @@ class ParquetPartitionDiscoverySuite extends QueryTest with ParquetTest {
4040

4141
val defaultPartitionName = "__NULL__"
4242

43+
def create_row(values: Any*): Row = {
44+
new GenericRow(values.map(Literal.convertToUTF8String).toArray)
45+
}
46+
4347
test("column type inference") {
4448
def check(raw: String, literal: Literal): Unit = {
4549
assert(inferPartitionColumnValue(raw, defaultPartitionName) === literal)
@@ -107,7 +111,7 @@ class ParquetPartitionDiscoverySuite extends QueryTest with ParquetTest {
107111
StructType(Seq(
108112
StructField("a", IntegerType),
109113
StructField("b", StringType))),
110-
Seq(Partition(Row(10, "hello"), "hdfs://host:9000/path/a=10/b=hello"))))
114+
Seq(Partition(create_row(10, "hello"), "hdfs://host:9000/path/a=10/b=hello"))))
111115

112116
check(Seq(
113117
"hdfs://host:9000/path/a=10/b=20",
@@ -117,8 +121,8 @@ class ParquetPartitionDiscoverySuite extends QueryTest with ParquetTest {
117121
StructField("a", FloatType),
118122
StructField("b", StringType))),
119123
Seq(
120-
Partition(Row(10, "20"), "hdfs://host:9000/path/a=10/b=20"),
121-
Partition(Row(10.5, "hello"), "hdfs://host:9000/path/a=10.5/b=hello"))))
124+
Partition(create_row(10, "20"), "hdfs://host:9000/path/a=10/b=20"),
125+
Partition(create_row(10.5, "hello"), "hdfs://host:9000/path/a=10.5/b=hello"))))
122126

123127
check(Seq(
124128
s"hdfs://host:9000/path/a=10/b=20",
@@ -128,8 +132,8 @@ class ParquetPartitionDiscoverySuite extends QueryTest with ParquetTest {
128132
StructField("a", IntegerType),
129133
StructField("b", StringType))),
130134
Seq(
131-
Partition(Row(10, "20"), s"hdfs://host:9000/path/a=10/b=20"),
132-
Partition(Row(null, "hello"), s"hdfs://host:9000/path/a=$defaultPartitionName/b=hello"))))
135+
Partition(create_row(10, "20"), s"hdfs://host:9000/path/a=10/b=20"),
136+
Partition(create_row(null, "hello"), s"hdfs://host:9000/path/a=$defaultPartitionName/b=hello"))))
133137

134138
check(Seq(
135139
s"hdfs://host:9000/path/a=10/b=$defaultPartitionName",
@@ -139,8 +143,8 @@ class ParquetPartitionDiscoverySuite extends QueryTest with ParquetTest {
139143
StructField("a", FloatType),
140144
StructField("b", StringType))),
141145
Seq(
142-
Partition(Row(10, null), s"hdfs://host:9000/path/a=10/b=$defaultPartitionName"),
143-
Partition(Row(10.5, null), s"hdfs://host:9000/path/a=10.5/b=$defaultPartitionName"))))
146+
Partition(create_row(10, null), s"hdfs://host:9000/path/a=10/b=$defaultPartitionName"),
147+
Partition(create_row(10.5, null), s"hdfs://host:9000/path/a=10.5/b=$defaultPartitionName"))))
144148
}
145149

146150
test("read partitioned table - normal case") {
@@ -163,29 +167,29 @@ class ParquetPartitionDiscoverySuite extends QueryTest with ParquetTest {
163167
i <- 1 to 10
164168
pi <- Seq(1, 2)
165169
ps <- Seq("foo", "bar")
166-
} yield Row(i, i.toString, pi, ps))
170+
} yield create_row(i, i.toString, pi, ps))
167171

168172
checkAnswer(
169173
sql("SELECT intField, pi FROM t"),
170174
for {
171175
i <- 1 to 10
172176
pi <- Seq(1, 2)
173177
_ <- Seq("foo", "bar")
174-
} yield Row(i, pi))
178+
} yield create_row(i, pi))
175179

176180
checkAnswer(
177181
sql("SELECT * FROM t WHERE pi = 1"),
178182
for {
179183
i <- 1 to 10
180184
ps <- Seq("foo", "bar")
181-
} yield Row(i, i.toString, 1, ps))
185+
} yield create_row(i, i.toString, 1, ps))
182186

183187
checkAnswer(
184188
sql("SELECT * FROM t WHERE ps = 'foo'"),
185189
for {
186190
i <- 1 to 10
187191
pi <- Seq(1, 2)
188-
} yield Row(i, i.toString, pi, "foo"))
192+
} yield create_row(i, i.toString, pi, "foo"))
189193
}
190194
}
191195
}
@@ -210,29 +214,29 @@ class ParquetPartitionDiscoverySuite extends QueryTest with ParquetTest {
210214
i <- 1 to 10
211215
pi <- Seq(1, 2)
212216
ps <- Seq("foo", "bar")
213-
} yield Row(i, pi, i.toString, ps))
217+
} yield create_row(i, pi, i.toString, ps))
214218

215219
checkAnswer(
216220
sql("SELECT intField, pi FROM t"),
217221
for {
218222
i <- 1 to 10
219223
pi <- Seq(1, 2)
220224
_ <- Seq("foo", "bar")
221-
} yield Row(i, pi))
225+
} yield create_row(i, pi))
222226

223227
checkAnswer(
224228
sql("SELECT * FROM t WHERE pi = 1"),
225229
for {
226230
i <- 1 to 10
227231
ps <- Seq("foo", "bar")
228-
} yield Row(i, 1, i.toString, ps))
232+
} yield create_row(i, 1, i.toString, ps))
229233

230234
checkAnswer(
231235
sql("SELECT * FROM t WHERE ps = 'foo'"),
232236
for {
233237
i <- 1 to 10
234238
pi <- Seq(1, 2)
235-
} yield Row(i, pi, i.toString, "foo"))
239+
} yield create_row(i, pi, i.toString, "foo"))
236240
}
237241
}
238242
}
@@ -264,21 +268,21 @@ class ParquetPartitionDiscoverySuite extends QueryTest with ParquetTest {
264268
i <- 1 to 10
265269
pi <- Seq(1, null.asInstanceOf[Integer])
266270
ps <- Seq("foo", null.asInstanceOf[String])
267-
} yield Row(i, i.toString, pi, ps))
271+
} yield create_row(i, i.toString, pi, ps))
268272

269273
checkAnswer(
270274
sql("SELECT * FROM t WHERE pi IS NULL"),
271275
for {
272276
i <- 1 to 10
273277
ps <- Seq("foo", null.asInstanceOf[String])
274-
} yield Row(i, i.toString, null, ps))
278+
} yield create_row(i, i.toString, null, ps))
275279

276280
checkAnswer(
277281
sql("SELECT * FROM t WHERE ps IS NULL"),
278282
for {
279283
i <- 1 to 10
280284
pi <- Seq(1, null.asInstanceOf[Integer])
281-
} yield Row(i, i.toString, pi, null))
285+
} yield create_row(i, i.toString, pi, null))
282286
}
283287
}
284288
}
@@ -309,14 +313,14 @@ class ParquetPartitionDiscoverySuite extends QueryTest with ParquetTest {
309313
i <- 1 to 10
310314
pi <- Seq(1, 2)
311315
ps <- Seq("foo", null.asInstanceOf[String])
312-
} yield Row(i, pi, i.toString, ps))
316+
} yield create_row(i, pi, i.toString, ps))
313317

314318
checkAnswer(
315319
sql("SELECT * FROM t WHERE ps IS NULL"),
316320
for {
317321
i <- 1 to 10
318322
pi <- Seq(1, 2)
319-
} yield Row(i, pi, i.toString, null))
323+
} yield create_row(i, pi, i.toString, null))
320324
}
321325
}
322326
}
@@ -336,7 +340,7 @@ class ParquetPartitionDiscoverySuite extends QueryTest with ParquetTest {
336340
withTempTable("t") {
337341
checkAnswer(
338342
sql("SELECT * FROM t"),
339-
(1 to 10).map(i => Row(i, null, 1)) ++ (1 to 10).map(i => Row(i, i.toString, 2)))
343+
(1 to 10).map(i => create_row(i, null, 1)) ++ (1 to 10).map(i => create_row(i, i.toString, 2)))
340344
}
341345
}
342346
}

‎sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ import org.apache.spark.sql.execution._
3535
import org.apache.spark.sql.hive.execution._
3636
import org.apache.spark.sql.parquet.ParquetRelation
3737
import org.apache.spark.sql.sources.{CreateTableUsingAsSelect, CreateTableUsing}
38-
import org.apache.spark.sql.types.StringType
38+
import org.apache.spark.sql.types.{UTF8String, StringType}
3939

4040

4141
private[hive] trait HiveStrategies {
@@ -131,7 +131,10 @@ private[hive] trait HiveStrategies {
131131
val partitionValues = part.getValues
132132
var i = 0
133133
while (i < partitionValues.size()) {
134-
inputData(i) = partitionValues(i)
134+
inputData(i) = partitionValues(i) match {
135+
case s: String => UTF8String(s)
136+
case other => other
137+
}
135138
i += 1
136139
}
137140
pruningCondition(inputData)

0 commit comments

Comments
 (0)