Skip to content

Commit 5d9982b

Browse files
committed
Minor fixes
- style: (v,null) => (v, null) - mention the shuffle in Javadoc
1 parent b86d02f commit 5d9982b

File tree

1 file changed

+9
-3
lines changed
  • core/src/main/scala/org/apache/spark/rdd

1 file changed

+9
-3
lines changed

core/src/main/scala/org/apache/spark/rdd/RDD.scala

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -374,31 +374,37 @@ abstract class RDD[T: ClassTag](
374374
/**
375375
* Return the intersection of this RDD and another one. The output will not contain any duplicate
376376
* elements, even if the input RDDs did.
377+
*
378+
* Note that this method performs a shuffle internally
377379
*/
378380
def intersection(other: RDD[T]): RDD[T] =
379-
this.map(v => (v,null)).cogroup(other.map(v => (v,null)))
381+
this.map(v => (v, null)).cogroup(other.map(v => (v, null)))
380382
.filter { case (_, (leftGroup, rightGroup)) => leftGroup.nonEmpty && rightGroup.nonEmpty }
381383
.keys
382384

383385
/**
384386
* Return the intersection of this RDD and another one. The output will not contain any duplicate
385387
* elements, even if the input RDDs did.
386388
*
389+
* Note that this method performs a shuffle internally
390+
*
387391
* @param partitioner Partitioner to use for the resulting RDD
388392
*/
389393
def intersection(other: RDD[T], partitioner: Partitioner): RDD[T] =
390-
this.map(v => (v,null)).cogroup(other.map(v => (v,null)), partitioner)
394+
this.map(v => (v, null)).cogroup(other.map(v => (v, null)), partitioner)
391395
.filter { case (_, (leftGroup, rightGroup)) => leftGroup.nonEmpty && rightGroup.nonEmpty }
392396
.keys
393397

394398
/**
395399
* Return the intersection of this RDD and another one. The output will not contain any duplicate
396400
* elements, even if the input RDDs did. Performs a hash partition across the cluster
397401
*
402+
* Note that this method performs a shuffle internally
403+
*
398404
* @param numPartitions How many partitions to use in the resulting RDD
399405
*/
400406
def intersection(other: RDD[T], numPartitions: Int): RDD[T] =
401-
this.map(v => (v,null)).cogroup(other.map(v => (v,null)), new HashPartitioner(numPartitions))
407+
this.map(v => (v, null)).cogroup(other.map(v => (v, null)), new HashPartitioner(numPartitions))
402408
.filter { case (_, (leftGroup, rightGroup)) => leftGroup.nonEmpty && rightGroup.nonEmpty }
403409
.keys
404410

0 commit comments

Comments
 (0)