Skip to content

Commit ed20b70

Browse files
Prashant Singhsenthh
authored andcommitted
[SPARK-35579][SQL] Bump janino to 3.1.7
### What changes were proposed in this pull request? upgrade janino to 3.1.7 from 3.0.16 ### Why are the changes needed? - The proposed version contains bug fix in janino by maropu. - janino-compiler/janino#148 - contains `getBytecodes` method which can be used to simplify the way to get bytecodes from ClassBodyEvaluator in CodeGenerator#updateAndGetCompilationStats method. (by LuciferYang) - apache#32536 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Existing UTs Closes apache#37202 from singhpk234/upgrade/bump-janino. Authored-by: Prashant Singh <[email protected]> Signed-off-by: Sean Owen <[email protected]> (cherry picked from commit 29ed337)
1 parent 29bb1ea commit ed20b70

File tree

6 files changed

+7
-17
lines changed

6 files changed

+7
-17
lines changed

dev/deps/spark-deps-hadoop-2-hive-2.3

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar
126126
jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
127127
jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
128128
jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
129-
janino/3.1.9//janino-3.1.9.jar
129+
janino/3.1.7//janino-3.1.7.jar
130130
javassist/3.25.0-GA//javassist-3.25.0-GA.jar
131131
javax.inject/1//javax.inject-1.jar
132132
javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar

dev/deps/spark-deps-hadoop-3-hive-2.3

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar
112112
jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
113113
jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
114114
jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
115-
janino/3.1.9//janino-3.1.9.jar
115+
janino/3.1.7//janino-3.1.7.jar
116116
javassist/3.25.0-GA//javassist-3.25.0-GA.jar
117117
javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
118118
javolution/5.5.1//javolution-5.5.1.jar

pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@
204204
<commons-pool2.version>2.11.1</commons-pool2.version>
205205
<datanucleus-core.version>4.1.17</datanucleus-core.version>
206206
<guava.version>14.0.1</guava.version>
207-
<janino.version>3.0.16</janino.version>
207+
<janino.version>3.1.7</janino.version>
208208
<jersey.version>2.36</jersey.version>
209209
<joda.version>2.10.13</joda.version>
210210
<jodd.version>3.5.2</jodd.version>

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
package org.apache.spark.sql.catalyst.expressions.codegen
1919

2020
import java.io.ByteArrayInputStream
21-
import java.util.{Map => JavaMap}
2221

2322
import scala.annotation.tailrec
2423
import scala.collection.JavaConverters._
@@ -28,8 +27,8 @@ import scala.util.control.NonFatal
2827

2928
import com.google.common.cache.{CacheBuilder, CacheLoader}
3029
import com.google.common.util.concurrent.{ExecutionError, UncheckedExecutionException}
31-
import org.codehaus.commons.compiler.CompileException
32-
import org.codehaus.janino.{ByteArrayClassLoader, ClassBodyEvaluator, InternalCompilerException, SimpleCompiler}
30+
import org.codehaus.commons.compiler.{CompileException, InternalCompilerException}
31+
import org.codehaus.janino.ClassBodyEvaluator
3332
import org.codehaus.janino.util.ClassFile
3433

3534
import org.apache.spark.{TaskContext, TaskKilledException}
@@ -1521,14 +1520,7 @@ object CodeGenerator extends Logging {
15211520
*/
15221521
private def updateAndGetCompilationStats(evaluator: ClassBodyEvaluator): ByteCodeStats = {
15231522
// First retrieve the generated classes.
1524-
val classes = {
1525-
val resultField = classOf[SimpleCompiler].getDeclaredField("result")
1526-
resultField.setAccessible(true)
1527-
val loader = resultField.get(evaluator).asInstanceOf[ByteArrayClassLoader]
1528-
val classesField = loader.getClass.getDeclaredField("classes")
1529-
classesField.setAccessible(true)
1530-
classesField.get(loader).asInstanceOf[JavaMap[String, Array[Byte]]].asScala
1531-
}
1523+
val classes = evaluator.getBytecodes.asScala
15321524

15331525
// Then walk the classes to get at the method bytecode.
15341526
val codeAttr = Utils.classForName("org.codehaus.janino.util.ClassFile$CodeAttribute")

sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,7 @@ import java.util.concurrent.TimeoutException
3131
import com.fasterxml.jackson.core.{JsonParser, JsonToken}
3232
import org.apache.hadoop.fs.{FileAlreadyExistsException, FileStatus, Path}
3333
import org.apache.hadoop.fs.permission.FsPermission
34-
import org.codehaus.commons.compiler.CompileException
35-
import org.codehaus.janino.InternalCompilerException
34+
import org.codehaus.commons.compiler.{CompileException, InternalCompilerException}
3635

3736
import org.apache.spark.{Partition, SparkArithmeticException, SparkArrayIndexOutOfBoundsException, SparkClassNotFoundException, SparkConcurrentModificationException, SparkDateTimeException, SparkException, SparkFileAlreadyExistsException, SparkFileNotFoundException, SparkIllegalArgumentException, SparkIndexOutOfBoundsException, SparkNoSuchElementException, SparkNoSuchMethodException, SparkNumberFormatException, SparkRuntimeException, SparkSecurityException, SparkSQLException, SparkSQLFeatureNotSupportedException, SparkUnsupportedOperationException, SparkUpgradeException}
3837
import org.apache.spark.executor.CommitDeniedException

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -568,7 +568,6 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper {
568568
assert(refTerm.contains("scala.math.LowPriorityOrderingImplicits$$anon$"))
569569
}
570570

571-
// TODO (SPARK-35579): Fix this bug in janino and upgrade janino in Spark.
572571
test("SPARK-35578: final local variable bug in janino") {
573572
val code =
574573
"""

0 commit comments

Comments
 (0)