From 972dc9b88088f71bd88876e16ac18d76ffe54fbc Mon Sep 17 00:00:00 2001 From: Denis Bolshakov Date: Thu, 30 Mar 2017 19:18:08 +0300 Subject: [PATCH 1/3] [SPARK-9002][CORE] KryoSerializer initialization does not include 'Array[Int]' --- .../org/apache/spark/serializer/KryoSerializer.scala | 1 + .../apache/spark/serializer/KryoSerializerSuite.scala | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala index 03815631a604c..3557ce28d7c0d 100644 --- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala +++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala @@ -386,6 +386,7 @@ private[serializer] object KryoSerializer { classOf[BlockManagerId], classOf[Array[Byte]], classOf[Array[Short]], + classOf[Array[Int]], classOf[Array[Long]], classOf[BoundedPriorityQueue[_]], classOf[SparkConf] diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala index a30653bb36fa1..87a93cef17f50 100644 --- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala +++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala @@ -339,6 +339,16 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext { } } + test("registration of Array[Int]") { + val conf = new SparkConf(false) + conf.set("spark.kryo.registrationRequired", "true") + + val ser = new KryoSerializer(conf).newInstance() + + val t = Array(0, 1, 2) + assert(ser.deserialize[Array[Int]](ser.serialize(t)) === t) + } + test("serialization buffer overflow reporting") { import org.apache.spark.SparkException val kryoBufferMaxProperty = "spark.kryoserializer.buffer.max" From 20517a5dda6566d1f94f929f432d43af10cd1b2a Mon Sep 17 00:00:00 2001 From: Denis Bolshakov Date: Thu, 30 Mar 2017 19:39:59 +0300 Subject: [PATCH 2/3] [SPARK-9002][CORE] KryoSerializer initialization does not include 'Array[Int]' making 'basic types' passed with spark.kryo.registrationRequired --- .../apache/spark/serializer/KryoSerializer.scala | 6 ++++++ .../spark/serializer/KryoSerializerSuite.scala | 13 +++---------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala index 3557ce28d7c0d..6fc66e2374bd9 100644 --- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala +++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala @@ -384,10 +384,16 @@ private[serializer] object KryoSerializer { classOf[HighlyCompressedMapStatus], classOf[CompactBuffer[_]], classOf[BlockManagerId], + classOf[Array[Boolean]], classOf[Array[Byte]], classOf[Array[Short]], classOf[Array[Int]], classOf[Array[Long]], + classOf[Array[Float]], + classOf[Array[Double]], + classOf[Array[Char]], + classOf[Array[String]], + classOf[Array[Array[String]]], classOf[BoundedPriorityQueue[_]], classOf[SparkConf] ) diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala index 87a93cef17f50..457c9c63b5d3d 100644 --- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala +++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala @@ -76,6 +76,9 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext { } test("basic types") { + val conf = new SparkConf(false) + conf.set("spark.kryo.registrationRequired", "true") + val ser = new KryoSerializer(conf).newInstance() def check[T: ClassTag](t: T) { assert(ser.deserialize[T](ser.serialize(t)) === t) @@ -339,16 +342,6 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext { } } - test("registration of Array[Int]") { - val conf = new SparkConf(false) - conf.set("spark.kryo.registrationRequired", "true") - - val ser = new KryoSerializer(conf).newInstance() - - val t = Array(0, 1, 2) - assert(ser.deserialize[Array[Int]](ser.serialize(t)) === t) - } - test("serialization buffer overflow reporting") { import org.apache.spark.SparkException val kryoBufferMaxProperty = "spark.kryoserializer.buffer.max" From 89eb1b6e6aa52157bb2ff38a7663592b09dcb475 Mon Sep 17 00:00:00 2001 From: Denis Bolshakov Date: Thu, 30 Mar 2017 21:26:42 +0300 Subject: [PATCH 3/3] [SPARK-9002][CORE] KryoSerializer initialization does not include 'Array[Int]' extending list of checked classes for Kryo registration --- .../org/apache/spark/serializer/KryoSerializerSuite.scala | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala index 457c9c63b5d3d..7c3922e47fbb9 100644 --- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala +++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala @@ -109,6 +109,9 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext { } test("pairs") { + val conf = new SparkConf(false) + conf.set("spark.kryo.registrationRequired", "true") + val ser = new KryoSerializer(conf).newInstance() def check[T: ClassTag](t: T) { assert(ser.deserialize[T](ser.serialize(t)) === t) @@ -133,12 +136,16 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext { } test("Scala data structures") { + val conf = new SparkConf(false) + conf.set("spark.kryo.registrationRequired", "true") + val ser = new KryoSerializer(conf).newInstance() def check[T: ClassTag](t: T) { assert(ser.deserialize[T](ser.serialize(t)) === t) } check(List[Int]()) check(List[Int](1, 2, 3)) + check(Seq[Int](1, 2, 3)) check(List[String]()) check(List[String]("x", "y", "z")) check(None)