Skip to content

Commit 40cdcb2

Browse files
committed
Bug fix, and addressed PR comments from @mridulm
Fixed bug: getRddId(blockId) returns an Option[Int], which never equals to rddId: Int.
1 parent 62c92ac commit 40cdcb2

File tree

1 file changed

+5
-8
lines changed

1 file changed

+5
-8
lines changed

core/src/main/scala/org/apache/spark/storage/MemoryStore.scala

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -244,14 +244,9 @@ private class MemoryStore(blockManager: BlockManager, maxMemory: Long)
244244
// accessed RDD, unless this is the same RDD as the one with the
245245
// new partition. In that case, we keep the old partition in memory
246246
// to prevent cycling partitions from the same RDD in and out.
247-
//
248-
// TODO implement LRU eviction
249-
rddToAdd match {
250-
case Some(rddId) if rddId == getRddId(blockId) =>
251-
// no-op
252-
case _ =>
253-
selectedBlocks += blockId
254-
selectedMemory += pair.getValue.size
247+
if (rddToAdd.isEmpty || rddToAdd != getRddId(blockId)) {
248+
selectedBlocks += blockId
249+
selectedMemory += pair.getValue.size
255250
}
256251
}
257252
}
@@ -274,6 +269,8 @@ private class MemoryStore(blockManager: BlockManager, maxMemory: Long)
274269
}
275270
return true
276271
} else {
272+
logInfo(s"Will not store $blockIdToAdd as it would require dropping another block " +
273+
"from the same RDD")
277274
return false
278275
}
279276
}

0 commit comments

Comments
 (0)