Skip to content

Commit 1433b42

Browse files
committed
Store record length as int instead of long.
1 parent 026b497 commit 1433b42

File tree

1 file changed

+5
-5
lines changed

1 file changed

+5
-5
lines changed

core/src/main/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleWriter.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -159,16 +159,16 @@ private Iterator<RecordPointerAndKeyPrefix> sortRecords(
159159
// TODO: we should run the partition extraction function _now_, at insert time, rather than
160160
// requiring it to be stored alongisde the data, since this may lead to double storage
161161
// Need 8 bytes to store the prefix (for later retrieval in the prefix computer), plus
162-
// 8 to store the record length (TODO: can store as an int instead).
163-
ensureSpaceInDataPage(serializedRecordSize + 8 + 8);
162+
// 4 to store the record length.
163+
ensureSpaceInDataPage(serializedRecordSize + 8 + 4);
164164

165165
final long recordAddress =
166166
memoryManager.encodePageNumberAndOffset(currentPage, currentPagePosition);
167167
final Object baseObject = currentPage.getBaseObject();
168168
PlatformDependent.UNSAFE.putLong(baseObject, currentPagePosition, partitionId);
169169
currentPagePosition += 8;
170-
PlatformDependent.UNSAFE.putLong(baseObject, currentPagePosition, serializedRecordSize);
171-
currentPagePosition += 8;
170+
PlatformDependent.UNSAFE.putInt(baseObject, currentPagePosition, serializedRecordSize);
171+
currentPagePosition += 4;
172172
PlatformDependent.copyMemory(
173173
serArray,
174174
PlatformDependent.BYTE_ARRAY_OFFSET,
@@ -214,7 +214,7 @@ private long[] writeSortedRecordsToFile(
214214
final int recordLength = (int) PlatformDependent.UNSAFE.getLong(baseObject, baseOffset + 8);
215215
PlatformDependent.copyMemory(
216216
baseObject,
217-
baseOffset + 16,
217+
baseOffset + 8 + 4,
218218
arr,
219219
PlatformDependent.BYTE_ARRAY_OFFSET,
220220
recordLength);

0 commit comments

Comments
 (0)