Skip to content

Commit 0736e72

Browse files
mgaido91cloud-fan
authored andcommitted
[SPARK-25371][SQL] struct() should allow being called with 0 args
## What changes were proposed in this pull request? SPARK-21281 introduced a check for the inputs of `CreateStructLike` to be non-empty. This means that `struct()`, which was previously considered valid, now throws an Exception. This behavior change was introduced in 2.3.0. The change may break users' application on upgrade and it causes `VectorAssembler` to fail when an empty `inputCols` is defined. The PR removes the added check making `struct()` valid again. ## How was this patch tested? added UT Closes #22373 from mgaido91/SPARK-25371. Authored-by: Marco Gaido <[email protected]> Signed-off-by: Wenchen Fan <[email protected]>
1 parent da5685b commit 0736e72

File tree

3 files changed

+6
-6
lines changed

3 files changed

+6
-6
lines changed

mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,4 +256,9 @@ class VectorAssemblerSuite
256256
assert(runWithMetadata("keep", additional_filter = "id1 > 2").count() == 4)
257257
}
258258

259+
test("SPARK-25371: VectorAssembler with empty inputCols") {
260+
val vectorAssembler = new VectorAssembler().setInputCols(Array()).setOutputCol("a")
261+
val output = vectorAssembler.transform(dfWithNullsAndNaNs)
262+
assert(output.select("a").limit(1).collect().head == Row(Vectors.sparse(0, Seq.empty)))
263+
}
259264
}

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -379,10 +379,7 @@ trait CreateNamedStructLike extends Expression {
379379
}
380380

381381
override def checkInputDataTypes(): TypeCheckResult = {
382-
if (children.length < 1) {
383-
TypeCheckResult.TypeCheckFailure(
384-
s"input to function $prettyName requires at least one argument")
385-
} else if (children.size % 2 != 0) {
382+
if (children.size % 2 != 0) {
386383
TypeCheckResult.TypeCheckFailure(s"$prettyName expects an even number of arguments.")
387384
} else {
388385
val invalidNames = nameExprs.filterNot(e => e.foldable && e.dataType == StringType)

sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2677,8 +2677,6 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext {
26772677
val funcsMustHaveAtLeastOneArg =
26782678
("coalesce", (df: DataFrame) => df.select(coalesce())) ::
26792679
("coalesce", (df: DataFrame) => df.selectExpr("coalesce()")) ::
2680-
("named_struct", (df: DataFrame) => df.select(struct())) ::
2681-
("named_struct", (df: DataFrame) => df.selectExpr("named_struct()")) ::
26822680
("hash", (df: DataFrame) => df.select(hash())) ::
26832681
("hash", (df: DataFrame) => df.selectExpr("hash()")) :: Nil
26842682
funcsMustHaveAtLeastOneArg.foreach { case (name, func) =>

0 commit comments

Comments
 (0)