From 2354f100901ab49ded8eb8052344c014af7e990f Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 2 Oct 2022 20:07:18 +0800 Subject: [PATCH 1/5] upgrade --- dev/deps/spark-deps-hadoop-2-hive-2.3 | 4 ++-- dev/deps/spark-deps-hadoop-3-hive-2.3 | 4 ++-- pom.xml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2-hive-2.3 b/dev/deps/spark-deps-hadoop-2-hive-2.3 index 8a9621e5364be..4739163b6af8f 100644 --- a/dev/deps/spark-deps-hadoop-2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2-hive-2.3 @@ -40,7 +40,7 @@ commons-cli/1.5.0//commons-cli-1.5.0.jar commons-codec/1.15//commons-codec-1.15.jar commons-collections/3.2.2//commons-collections-3.2.2.jar commons-collections4/4.4//commons-collections4-4.4.jar -commons-compiler/3.1.7//commons-compiler-3.1.7.jar +commons-compiler/3.1.8//commons-compiler-3.1.8.jar commons-compress/1.21//commons-compress-1.21.jar commons-configuration/1.6//commons-configuration-1.6.jar commons-crypto/1.1.0//commons-crypto-1.1.0.jar @@ -140,7 +140,7 @@ jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar -janino/3.1.7//janino-3.1.7.jar +janino/3.1.8//janino-3.1.8.jar javassist/3.25.0-GA//javassist-3.25.0-GA.jar javax.inject/1//javax.inject-1.jar javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index c26dfa3f9ce7f..d145fb153b8e9 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -41,7 +41,7 @@ commons-cli/1.5.0//commons-cli-1.5.0.jar commons-codec/1.15//commons-codec-1.15.jar commons-collections/3.2.2//commons-collections-3.2.2.jar commons-collections4/4.4//commons-collections4-4.4.jar -commons-compiler/3.1.7//commons-compiler-3.1.7.jar +commons-compiler/3.1.8//commons-compiler-3.1.8.jar commons-compress/1.21//commons-compress-1.21.jar commons-crypto/1.1.0//commons-crypto-1.1.0.jar commons-dbcp/1.4//commons-dbcp-1.4.jar @@ -126,7 +126,7 @@ jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar -janino/3.1.7//janino-3.1.7.jar +janino/3.1.8//janino-3.1.8.jar javassist/3.25.0-GA//javassist-3.25.0-GA.jar javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar javolution/5.5.1//javolution-5.5.1.jar diff --git a/pom.xml b/pom.xml index 89f9807ca200d..957cb87ec89b0 100644 --- a/pom.xml +++ b/pom.xml @@ -187,7 +187,7 @@ 2.11.1 4.1.17 14.0.1 - 3.1.7 + 3.1.8 2.36 2.10.14 3.5.2 From 864aad068c5e24278cafb1b26819591d0fae6c33 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 18 Nov 2022 17:45:27 +0800 Subject: [PATCH 2/5] init to null --- .../apache/spark/sql/catalyst/expressions/objects/objects.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala index a644b90a96ff6..c8a8e78ef1d73 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala @@ -1310,7 +1310,7 @@ case class CatalystToExternalMap private( val tupleClass = classOf[(_, _)].getName val appendToBuilder = s""" - $tupleClass $tupleLoopValue; + $tupleClass $tupleLoopValue = null; if (${genValueFunction.isNull}) { $tupleLoopValue = new $tupleClass($genKeyFunctionValue, null); From 8a0b6562561f8cdafe4f254fb2deabf7be7720be Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 18 Nov 2022 20:55:18 +0800 Subject: [PATCH 3/5] fix --- .../spark/sql/catalyst/expressions/objects/objects.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala index c8a8e78ef1d73..e5d63ef5efaf9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala @@ -1275,7 +1275,6 @@ case class CatalystToExternalMap private( val genInputData = inputData.genCode(ctx) val dataLength = ctx.freshName("dataLength") val loopIndex = ctx.freshName("loopIndex") - val tupleLoopValue = ctx.freshName("tupleLoopValue") val builderValue = ctx.freshName("builderValue") val keyArray = ctx.freshName("keyArray") @@ -1310,15 +1309,13 @@ case class CatalystToExternalMap private( val tupleClass = classOf[(_, _)].getName val appendToBuilder = s""" - $tupleClass $tupleLoopValue = null; if (${genValueFunction.isNull}) { - $tupleLoopValue = new $tupleClass($genKeyFunctionValue, null); + $builderValue.$$plus$$eq(new $tupleClass($genKeyFunctionValue, null)); } else { - $tupleLoopValue = new $tupleClass($genKeyFunctionValue, $genValueFunctionValue); + $builderValue.$$plus$$eq(new $tupleClass($genKeyFunctionValue, $genValueFunctionValue)); } - $builderValue.$$plus$$eq($tupleLoopValue); """ val getBuilderResult = s"${ev.value} = (${collClass.getName}) $builderValue.result();" From c00c4e69357abeeb9a3ba3cc5c4de6b5afca7cbe Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 21 Nov 2022 16:01:11 +0800 Subject: [PATCH 4/5] upgrade to 3.1.9 --- dev/deps/spark-deps-hadoop-2-hive-2.3 | 4 ++-- dev/deps/spark-deps-hadoop-3-hive-2.3 | 4 ++-- pom.xml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2-hive-2.3 b/dev/deps/spark-deps-hadoop-2-hive-2.3 index aeaa37f51a914..b583b99bd3d82 100644 --- a/dev/deps/spark-deps-hadoop-2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2-hive-2.3 @@ -38,7 +38,7 @@ commons-cli/1.5.0//commons-cli-1.5.0.jar commons-codec/1.15//commons-codec-1.15.jar commons-collections/3.2.2//commons-collections-3.2.2.jar commons-collections4/4.4//commons-collections4-4.4.jar -commons-compiler/3.1.8//commons-compiler-3.1.8.jar +commons-compiler/3.1.9//commons-compiler-3.1.9.jar commons-compress/1.21//commons-compress-1.21.jar commons-configuration/1.6//commons-configuration-1.6.jar commons-crypto/1.1.0//commons-crypto-1.1.0.jar @@ -127,7 +127,7 @@ jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar -janino/3.1.8//janino-3.1.8.jar +janino/3.1.9//janino-3.1.9.jar javassist/3.25.0-GA//javassist-3.25.0-GA.jar javax.inject/1//javax.inject-1.jar javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 7b01e60ae1b28..0cc57f083801c 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -39,7 +39,7 @@ commons-cli/1.5.0//commons-cli-1.5.0.jar commons-codec/1.15//commons-codec-1.15.jar commons-collections/3.2.2//commons-collections-3.2.2.jar commons-collections4/4.4//commons-collections4-4.4.jar -commons-compiler/3.1.8//commons-compiler-3.1.8.jar +commons-compiler/3.1.9//commons-compiler-3.1.9.jar commons-compress/1.21//commons-compress-1.21.jar commons-crypto/1.1.0//commons-crypto-1.1.0.jar commons-dbcp/1.4//commons-dbcp-1.4.jar @@ -113,7 +113,7 @@ jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar -janino/3.1.8//janino-3.1.8.jar +janino/3.1.9//janino-3.1.9.jar javassist/3.25.0-GA//javassist-3.25.0-GA.jar javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar javolution/5.5.1//javolution-5.5.1.jar diff --git a/pom.xml b/pom.xml index d384d20429276..a2d9240b2c2dd 100644 --- a/pom.xml +++ b/pom.xml @@ -190,7 +190,7 @@ 2.11.1 4.1.17 14.0.1 - 3.1.8 + 3.1.9 2.36 2.12.1 3.5.2 From 2f35797b8532eba990234e275a7374703a9ff938 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 21 Nov 2022 16:02:28 +0800 Subject: [PATCH 5/5] revert objects.scala for test --- .../spark/sql/catalyst/expressions/objects/objects.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala index e5d63ef5efaf9..a644b90a96ff6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala @@ -1275,6 +1275,7 @@ case class CatalystToExternalMap private( val genInputData = inputData.genCode(ctx) val dataLength = ctx.freshName("dataLength") val loopIndex = ctx.freshName("loopIndex") + val tupleLoopValue = ctx.freshName("tupleLoopValue") val builderValue = ctx.freshName("builderValue") val keyArray = ctx.freshName("keyArray") @@ -1309,13 +1310,15 @@ case class CatalystToExternalMap private( val tupleClass = classOf[(_, _)].getName val appendToBuilder = s""" + $tupleClass $tupleLoopValue; if (${genValueFunction.isNull}) { - $builderValue.$$plus$$eq(new $tupleClass($genKeyFunctionValue, null)); + $tupleLoopValue = new $tupleClass($genKeyFunctionValue, null); } else { - $builderValue.$$plus$$eq(new $tupleClass($genKeyFunctionValue, $genValueFunctionValue)); + $tupleLoopValue = new $tupleClass($genKeyFunctionValue, $genValueFunctionValue); } + $builderValue.$$plus$$eq($tupleLoopValue); """ val getBuilderResult = s"${ev.value} = (${collClass.getName}) $builderValue.result();"