@@ -31,6 +31,7 @@ import org.roaringbitmap.{ArrayContainer, BitmapContainer, RoaringArray, Roaring
3131import org .apache .spark ._
3232import org .apache .spark .api .python .PythonBroadcast
3333import org .apache .spark .broadcast .HttpBroadcast
34+ import org .apache .spark .network .util .ByteUnit
3435import org .apache .spark .network .nio .{GetBlock , GotBlock , PutBlock }
3536import org .apache .spark .scheduler .{CompressedMapStatus , HighlyCompressedMapStatus }
3637import org .apache .spark .storage ._
@@ -51,18 +52,18 @@ class KryoSerializer(conf: SparkConf)
5152
5253 private val bufferSizeKb = conf.getSizeAsKb(" spark.kryoserializer.buffer" , " 64k" )
5354
54- if (bufferSizeKb >= 2048 * 1024 ) {
55+ if (bufferSizeKb >= ByteUnit . GiB .toKiB( 2 ) ) {
5556 throw new IllegalArgumentException (" spark.kryoserializer.buffer must be less than " +
56- s " 2048 mb, got: + ${bufferSizeKb/ 1024 } mb. " )
57+ s " 2048 mb, got: + ${ByteUnit . KiB .toMiB( bufferSizeKb) } mb. " )
5758 }
58- private val bufferSize = (bufferSizeKb * 1024 ).toInt
59+ private val bufferSize = ByteUnit . KiB .toBytes (bufferSizeKb).toInt
5960
6061 val maxBufferSizeMb = conf.getSizeAsMb(" spark.kryoserializer.buffer.max" , " 64m" ).toInt
61- if (maxBufferSizeMb >= 2048 ) {
62+ if (maxBufferSizeMb >= ByteUnit . GiB .toMiB( 2 ) ) {
6263 throw new IllegalArgumentException (" spark.kryoserializer.buffer.max must be less than " +
6364 s " 2048 mb, got: + $maxBufferSizeMb mb. " )
6465 }
65- private val maxBufferSize = maxBufferSizeMb * 1024 * 1024
66+ private val maxBufferSize = ByteUnit . MiB .toBytes( maxBufferSizeMb).toInt
6667
6768 private val referenceTracking = conf.getBoolean(" spark.kryo.referenceTracking" , true )
6869 private val registrationRequired = conf.getBoolean(" spark.kryo.registrationRequired" , false )
0 commit comments