diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index e62acc1e462f..a2dbdeda8c2e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -144,14 +144,14 @@ public static UnsignedWord getObjectOffset(AlignedHeader that, Pointer objectPoi return objectPointer.subtract(objectsStart); } - static boolean walkObjects(AlignedHeader that, ObjectVisitor visitor) { - return HeapChunk.walkObjectsFrom(that, getObjectsStart(that), visitor); + static void walkObjects(AlignedHeader that, ObjectVisitor visitor) { + HeapChunk.walkObjectsFrom(that, getObjectsStart(that), visitor); } @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - static boolean walkObjectsFromInline(AlignedHeader that, Pointer start, ObjectVisitor visitor) { - return HeapChunk.walkObjectsFromInline(that, start, visitor); + static void walkObjectsFromInline(AlignedHeader that, Pointer start, GreyToBlackObjectVisitor visitor) { + HeapChunk.walkObjectsFromInline(that, start, visitor); } @Fold diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AuxiliaryImageHeap.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AuxiliaryImageHeap.java index 9079a1134ec6..6b3cfcd7c00e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AuxiliaryImageHeap.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AuxiliaryImageHeap.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import jdk.graal.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.word.Pointer; @@ -32,6 +31,8 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.ObjectVisitor; +import jdk.graal.compiler.api.replacements.Fold; + public interface AuxiliaryImageHeap { @Fold static boolean isPresent() { @@ -46,10 +47,10 @@ static AuxiliaryImageHeap singleton() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean containsObject(Pointer address); - boolean walkObjects(ObjectVisitor visitor); + void walkObjects(ObjectVisitor visitor); @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - boolean walkRegions(MemoryWalker.ImageHeapRegionVisitor visitor); + void walkRegions(MemoryWalker.ImageHeapRegionVisitor visitor); ImageHeapInfo getImageHeapInfo(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java index dbf0ffd9016b..20dc53c42432 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java @@ -166,7 +166,7 @@ boolean scanGreyObjects(boolean incrementalGc) { } GreyToBlackObjectVisitor visitor = GCImpl.getGCImpl().getGreyToBlackObjectVisitor(); do { - visitor.visitObjectInline(markStack.pop()); + visitor.visitObject(markStack.pop()); } while (!markStack.isEmpty()); } return true; @@ -362,7 +362,7 @@ private void fixupUnalignedChunkReferences(ChunkReleaser chunkReleaser) { while (uChunk.isNonNull()) { UnalignedHeapChunk.UnalignedHeader next = HeapChunk.getNext(uChunk); Pointer objPointer = UnalignedHeapChunk.getObjectStart(uChunk); - Object obj = objPointer.toObject(); + Object obj = objPointer.toObjectNonNull(); if (ObjectHeaderImpl.isMarked(obj)) { ObjectHeaderImpl.unsetMarkedAndKeepRememberedSetBit(obj); RememberedSet.get().clearRememberedSet(uChunk); @@ -447,8 +447,8 @@ boolean printLocationInfo(Log log, Pointer ptr) { } @Override - public boolean walkObjects(ObjectVisitor visitor) { - return space.walkObjects(visitor); + public void walkObjects(ObjectVisitor visitor) { + space.walkObjects(visitor); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CopyingOldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CopyingOldGeneration.java index 08723ee4da1c..4df260b6cf91 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CopyingOldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CopyingOldGeneration.java @@ -65,8 +65,9 @@ void tearDown() { } @Override - public boolean walkObjects(ObjectVisitor visitor) { - return getFromSpace().walkObjects(visitor) && getToSpace().walkObjects(visitor); + public void walkObjects(ObjectVisitor visitor) { + getFromSpace().walkObjects(visitor); + getToSpace().walkObjects(visitor); } /** Promote an Object to ToSpace if it is not already in ToSpace. */ diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java index 6a811a62e5eb..9d3ec4dd07ab 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java @@ -41,13 +41,7 @@ abstract class Generation { this.name = name; } - /** - * Walk the Objects in this Space, passing each to a Visitor. - * - * @param visitor An ObjectVisitor. - * @return True if all visits returned true, false otherwise. - */ - public abstract boolean walkObjects(ObjectVisitor visitor); + public abstract void walkObjects(ObjectVisitor visitor); public String getName() { return name; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java index 760c0093fec3..4c711bb7649e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; @@ -32,7 +31,8 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.util.VMError; + +import jdk.graal.compiler.word.Word; /** * Apply an ObjectVisitor to all the new Objects in a Space since a snapshot. @@ -103,9 +103,7 @@ private void walkAlignedGreyObjects() { AlignedHeapChunk.AlignedHeader lastChunk; do { lastChunk = aChunk; - if (!AlignedHeapChunk.walkObjectsFromInline(aChunk, aStart, visitor)) { - throw VMError.shouldNotReachHereAtRuntime(); - } + AlignedHeapChunk.walkObjectsFromInline(aChunk, aStart, visitor); aChunk = HeapChunk.getNext(aChunk); aStart = (aChunk.isNonNull() ? AlignedHeapChunk.getObjectsStart(aChunk) : Word.nullPointer()); } while (aChunk.isNonNull()); @@ -131,9 +129,7 @@ private void walkUnalignedGreyObjects() { UnalignedHeapChunk.UnalignedHeader lastChunk; do { lastChunk = uChunk; - if (!UnalignedHeapChunk.walkObjectsInline(uChunk, visitor)) { - throw VMError.shouldNotReachHereAtRuntime(); - } + UnalignedHeapChunk.walkObjectsInline(uChunk, visitor); uChunk = HeapChunk.getNext(uChunk); } while (uChunk.isNonNull()); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index 4ea88ba6194a..63f8656ff4ae 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -33,7 +33,6 @@ import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.ReferenceAccess; -import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; import jdk.graal.compiler.word.Word; @@ -60,31 +59,32 @@ final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { } @Override + @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { - return visitObjectReferenceInline(objRef, 0, compressed, holderObject); + public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count) { + Pointer pos = firstObjRef; + Pointer end = firstObjRef.add(Word.unsigned(count).multiply(referenceSize)); + while (pos.belowThan(end)) { + visitObjectReference(pos, compressed, holderObject); + pos = pos.add(referenceSize); + } } - @Override @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { - assert innerOffset >= 0; + private void visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { assert !objRef.isNull(); counters.noteObjRef(); - Pointer offsetP = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); - assert offsetP.isNonNull() || innerOffset == 0; - - Pointer p = offsetP.subtract(innerOffset); + Pointer p = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); if (p.isNull()) { counters.noteNullReferent(); - return true; + return; } if (HeapImpl.getHeapImpl().isInImageHeap(p)) { counters.noteNonHeapReferent(); - return true; + return; } // This is the most expensive check as it accesses the heap fairly randomly, which results @@ -97,26 +97,23 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole counters.noteForwardedReferent(); // Update the reference to point to the forwarded Object. Object obj = ohi.getForwardedObject(p, header); - Object offsetObj = (innerOffset == 0) ? obj : Word.objectToUntrackedPointer(obj).add(innerOffset).toObject(); - ReferenceAccess.singleton().writeObjectAt(objRef, offsetObj, compressed); + ReferenceAccess.singleton().writeObjectAt(objRef, obj, compressed); RememberedSet.get().dirtyCardIfNecessary(holderObject, obj); - return true; + return; } - Object obj = p.toObject(); + Object obj = p.toObjectNonNull(); if (SerialGCOptions.useCompactingOldGen() && ObjectHeaderImpl.isMarkedHeader(header)) { RememberedSet.get().dirtyCardIfNecessary(holderObject, obj); - return true; + return; } // Promote the Object if necessary, making it at least grey, and ... - assert innerOffset < LayoutEncoding.getSizeFromObjectInGC(obj).rawValue(); Object copy = GCImpl.getGCImpl().promoteObject(obj, header); if (copy != obj) { // ... update the reference to point to the copy, making the reference black. counters.noteCopiedReferent(); - Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject(); - ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed); + ReferenceAccess.singleton().writeObjectAt(objRef, copy, compressed); } else { counters.noteUnmodifiedReference(); } @@ -125,7 +122,6 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole // might have to dirty the card. RememberedSet.get().dirtyCardIfNecessary(holderObject, copy); } - return true; } public Counters openCounters() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index defb148dcd43..a07f09f67ba8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -28,11 +28,9 @@ import org.graalvm.nativeimage.Platforms; import com.oracle.svm.core.AlwaysInline; -import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.UninterruptibleObjectVisitor; import com.oracle.svm.core.hub.InteriorObjRefWalker; -import com.oracle.svm.core.util.VMError; /** * Run an ObjectReferenceVisitor ({@link GreyToBlackObjRefVisitor}) over any interior object @@ -48,19 +46,11 @@ public final class GreyToBlackObjectVisitor implements UninterruptibleObjectVisi this.objRefVisitor = greyToBlackObjRefVisitor; } - @Override - @NeverInline("Non-performance critical version") - @Uninterruptible(reason = "Visitor requires uninterruptible walk.", callerMustBe = true) - public boolean visitObject(Object o) { - throw VMError.shouldNotReachHere("For performance reasons, this should not be called."); - } - @Override @AlwaysInline("GC performance") @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).", callerMustBe = true) - public boolean visitObjectInline(Object o) { + public void visitObject(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); - return true; } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index e8d2bfc102ea..e2f0bd078926 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -301,28 +301,25 @@ private static SignedWord offsetFromPointer(Header that, PointerBase pointer) @NeverInline("Not performance critical") @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).") - public static boolean walkObjectsFrom(Header that, Pointer start, ObjectVisitor visitor) { - return walkObjectsFromInline(that, start, visitor); + public static void walkObjectsFrom(Header that, Pointer start, ObjectVisitor visitor) { + walkObjectsFromInline(that, start, visitor); } @AlwaysInline("GC performance") @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).", callerMustBe = true) - public static boolean walkObjectsFromInline(Header that, Pointer start, ObjectVisitor visitor) { + public static void walkObjectsFromInline(Header that, Pointer start, ObjectVisitor visitor) { Pointer p = start; while (p.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read - Object obj = p.toObject(); - if (!callVisitor(visitor, obj)) { - return false; - } + Object obj = p.toObjectNonNull(); + callVisitor(visitor, obj); p = p.add(LayoutEncoding.getSizeFromObjectInlineInGC(obj)); } - return true; } @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) - private static boolean callVisitor(ObjectVisitor visitor, Object obj) { - return visitor.visitObjectInline(obj); + private static void callVisitor(ObjectVisitor visitor, Object obj) { + visitor.visitObject(obj); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index cda37ba72456..a8d3171daae1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.genscavenge; +import java.io.Serial; import java.lang.ref.Reference; import java.util.ArrayList; import java.util.List; @@ -204,9 +205,10 @@ public void resumeAllocation() { } @Override - public boolean walkObjects(ObjectVisitor visitor) { + public void walkObjects(ObjectVisitor visitor) { VMOperation.guaranteeInProgressAtSafepoint("must only be executed at a safepoint"); - return walkImageHeapObjects(visitor) && walkCollectedHeapObjects(visitor); + walkImageHeapObjects(visitor); + walkCollectedHeapObjects(visitor); } /** Tear down the heap and release its memory. */ @@ -340,14 +342,28 @@ private ArrayList> findAllDynamicHubs() { ArrayList> list = new ArrayList<>(dynamicHubCount); for (ImageHeapInfo info : HeapImpl.getImageHeapInfos()) { - ImageHeapWalker.walkRegions(info, new ClassListBuilderVisitor(list.size() + info.dynamicHubCount, list)); + collectDynamicHubs(info, list); } VMError.guarantee(dynamicHubCount == list.size(), "Found fewer DynamicHubs in the image heap than expected."); return list; } + private static void collectDynamicHubs(ImageHeapInfo info, ArrayList> list) { + try { + ClassListBuilderVisitor visitor = new ClassListBuilderVisitor(list.size() + info.dynamicHubCount, list); + ImageHeapWalker.walkRegions(info, visitor); + } catch (FoundAllHubsException ignored) { + } + } + + /** + * Iterates over the image heap parts that may contain dynamic hubs. After collecting all + * dynamic hubs were collected, this visitor throws a {@link FoundAllHubsException}. + */ private static class ClassListBuilderVisitor implements MemoryWalker.ImageHeapRegionVisitor, ObjectVisitor { + private static final FoundAllHubsException FOUND_ALL_HUBS_EXCEPTION = new FoundAllHubsException(); + private final int dynamicHubCount; private final List> list; @@ -357,21 +373,32 @@ private static class ClassListBuilderVisitor implements MemoryWalker.ImageHeapRe } @Override - public boolean visitNativeImageHeapRegion(T region, MemoryWalker.NativeImageHeapRegionAccess access) { + public void visitNativeImageHeapRegion(T region, MemoryWalker.NativeImageHeapRegionAccess access) { if (!access.isWritable(region) && !access.consistsOfHugeObjects(region)) { - return access.visitObjects(region, this); + access.visitObjects(region, this); } - return true; } @Override @RestrictHeapAccess(access = RestrictHeapAccess.Access.UNRESTRICTED, reason = "Allocation is fine: this method traverses only the image heap.") - public boolean visitObject(Object o) { + public void visitObject(Object o) { if (o instanceof Class) { list.add((Class) o); - return list.size() != dynamicHubCount; + if (list.size() == dynamicHubCount) { + throw FOUND_ALL_HUBS_EXCEPTION; + } } - return true; + } + } + + private static final class FoundAllHubsException extends RuntimeException { + @Serial private static final long serialVersionUID = 1L; + + @Override + @SuppressWarnings("sync-override") + public Throwable fillInStackTrace() { + /* No stacktrace needed. */ + return this; } } @@ -458,24 +485,26 @@ public UnsignedWord getImageHeapCommittedBytes() { } @Override - public boolean walkImageHeapObjects(ObjectVisitor visitor) { + public void walkImageHeapObjects(ObjectVisitor visitor) { VMOperation.guaranteeInProgressAtSafepoint("Must only be called at a safepoint"); if (visitor == null) { - return true; + return; } + for (ImageHeapInfo info : HeapImpl.getImageHeapInfos()) { - if (!ImageHeapWalker.walkImageHeapObjects(info, visitor)) { - return false; - } + ImageHeapWalker.walkImageHeapObjects(info, visitor); + } + if (AuxiliaryImageHeap.isPresent()) { + AuxiliaryImageHeap.singleton().walkObjects(visitor); } - return !AuxiliaryImageHeap.isPresent() || AuxiliaryImageHeap.singleton().walkObjects(visitor); } @Override - public boolean walkCollectedHeapObjects(ObjectVisitor visitor) { + public void walkCollectedHeapObjects(ObjectVisitor visitor) { VMOperation.guaranteeInProgressAtSafepoint("Must only be called at a safepoint"); ThreadLocalAllocation.disableAndFlushForAllThreads(); - return getYoungGeneration().walkObjects(visitor) && getOldGeneration().walkObjects(visitor); + getYoungGeneration().walkObjects(visitor); + getOldGeneration().walkObjects(visitor); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java index ac96a9616a33..3363bad35440 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java @@ -408,9 +408,8 @@ void initialize(AlignedHeader aChunk, UnalignedHeader uChunk) { } @Override - public boolean visitObject(Object object) { + public void visitObject(Object object) { result &= verifyObject(object, aChunk, uChunk); - return true; } } @@ -426,9 +425,17 @@ public void initialize() { } @Override - public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { + public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count) { + Pointer pos = firstObjRef; + Pointer end = firstObjRef.add(Word.unsigned(count).multiply(referenceSize)); + while (pos.belowThan(end)) { + visitObjectReference(pos, compressed, holderObject); + pos = pos.add(referenceSize); + } + } + + private void visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { result &= verifyReference(holderObject, objRef, compressed); - return true; } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java index 68fd58ea2638..f9f6b360bb60 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java @@ -31,6 +31,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.MemoryWalker; +import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.heap.ObjectVisitor; @@ -49,39 +50,34 @@ public final class ImageHeapWalker { private ImageHeapWalker() { } - public static boolean walkRegions(ImageHeapInfo heapInfo, MemoryWalker.ImageHeapRegionVisitor visitor) { - return visitor.visitNativeImageHeapRegion(heapInfo, READ_ONLY_REGULAR_WALKER) && - visitor.visitNativeImageHeapRegion(heapInfo, READ_ONLY_RELOCATABLE_WALKER) && - visitor.visitNativeImageHeapRegion(heapInfo, WRITABLE_REGULAR_WALKER) && - visitor.visitNativeImageHeapRegion(heapInfo, WRITABLE_HUGE_WALKER) && - visitor.visitNativeImageHeapRegion(heapInfo, READ_ONLY_HUGE_WALKER); + public static void walkRegions(ImageHeapInfo heapInfo, MemoryWalker.ImageHeapRegionVisitor visitor) { + visitor.visitNativeImageHeapRegion(heapInfo, READ_ONLY_REGULAR_WALKER); + visitor.visitNativeImageHeapRegion(heapInfo, READ_ONLY_RELOCATABLE_WALKER); + visitor.visitNativeImageHeapRegion(heapInfo, WRITABLE_REGULAR_WALKER); + visitor.visitNativeImageHeapRegion(heapInfo, WRITABLE_HUGE_WALKER); + visitor.visitNativeImageHeapRegion(heapInfo, READ_ONLY_HUGE_WALKER); } - public static boolean walkImageHeapObjects(ImageHeapInfo heapInfo, ObjectVisitor visitor) { - return walkPartition(heapInfo.firstReadOnlyRegularObject, heapInfo.lastReadOnlyRegularObject, visitor, true) && - walkPartition(heapInfo.firstReadOnlyRelocatableObject, heapInfo.lastReadOnlyRelocatableObject, visitor, true) && - walkPartition(heapInfo.firstWritableRegularObject, heapInfo.lastWritableRegularObject, visitor, true) && - walkPartition(heapInfo.firstWritableHugeObject, heapInfo.lastWritableHugeObject, visitor, false) && - walkPartition(heapInfo.firstReadOnlyHugeObject, heapInfo.lastReadOnlyHugeObject, visitor, false); + public static void walkImageHeapObjects(ImageHeapInfo heapInfo, ObjectVisitor visitor) { + walkPartition(heapInfo.firstReadOnlyRegularObject, heapInfo.lastReadOnlyRegularObject, visitor, true); + walkPartition(heapInfo.firstReadOnlyRelocatableObject, heapInfo.lastReadOnlyRelocatableObject, visitor, true); + walkPartition(heapInfo.firstWritableRegularObject, heapInfo.lastWritableRegularObject, visitor, true); + walkPartition(heapInfo.firstWritableHugeObject, heapInfo.lastWritableHugeObject, visitor, false); + walkPartition(heapInfo.firstReadOnlyHugeObject, heapInfo.lastReadOnlyHugeObject, visitor, false); } + @NeverInline("Not performance critical") @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).") - static boolean walkPartition(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks) { - return walkPartitionInline(firstObject, lastObject, visitor, alignedChunks, false); + static void walkPartition(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks) { + walkPartitionInline(firstObject, lastObject, visitor, alignedChunks); } @AlwaysInline("GC performance") @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).", callerMustBe = true) - static boolean walkPartitionInline(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks) { - return walkPartitionInline(firstObject, lastObject, visitor, alignedChunks, true); - } - - @AlwaysInline("GC performance") - @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).", callerMustBe = true) - private static boolean walkPartitionInline(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks, boolean inlineObjectVisit) { + static void walkPartitionInline(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks) { if (firstObject == null || lastObject == null) { assert firstObject == null && lastObject == null; - return true; + return; } Pointer firstPointer = Word.objectToUntrackedPointer(firstObject); Pointer lastPointer = Word.objectToUntrackedPointer(lastObject); @@ -104,15 +100,9 @@ private static boolean walkPartitionInline(Object firstObject, Object lastObject limit = chunkTop.subtract(1); // lastObject in another chunk, visit all objects } while (current.belowOrEqual(limit)) { - Object currentObject = current.toObject(); - if (inlineObjectVisit) { - if (!visitObjectInline(visitor, currentObject)) { - return false; - } - } else if (!visitObject(visitor, currentObject)) { - return false; - } - current = LayoutEncoding.getImageHeapObjectEnd(current.toObject()); + Object currentObject = current.toObjectNonNull(); + visitObjectInline(visitor, currentObject); + current = LayoutEncoding.getImageHeapObjectEnd(currentObject); } if (current.belowThan(lastPointer)) { currentChunk = HeapChunk.getNext(currentChunk); @@ -121,18 +111,12 @@ private static boolean walkPartitionInline(Object firstObject, Object lastObject // Note: current can be equal to lastPointer now, despite not having visited it yet } } while (current.belowOrEqual(lastPointer)); - return true; - } - - @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) - private static boolean visitObject(ObjectVisitor visitor, Object currentObject) { - return visitor.visitObject(currentObject); } @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) - private static boolean visitObjectInline(ObjectVisitor visitor, Object currentObject) { - return visitor.visitObjectInline(currentObject); + private static void visitObjectInline(ObjectVisitor visitor, Object currentObject) { + visitor.visitObject(currentObject); } } @@ -174,9 +158,9 @@ public boolean consistsOfHugeObjects(ImageHeapInfo region) { } @Override - public final boolean visitObjects(ImageHeapInfo region, ObjectVisitor visitor) { + public final void visitObjects(ImageHeapInfo region, ObjectVisitor visitor) { boolean alignedChunks = !consistsOfHugeObjects; - return ImageHeapWalker.walkPartition(getFirstObject(region), getLastObject(region), visitor, alignedChunks); + ImageHeapWalker.walkPartition(getFirstObject(region), getLastObject(region), visitor, alignedChunks); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PathExhibitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PathExhibitor.java deleted file mode 100644 index ad6321a6ad84..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PathExhibitor.java +++ /dev/null @@ -1,574 +0,0 @@ -/* - * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -package com.oracle.svm.core.genscavenge; - -import java.util.ArrayList; - -import org.graalvm.nativeimage.CurrentIsolate; -import org.graalvm.nativeimage.IsolateThread; -import org.graalvm.nativeimage.c.function.CodePointer; -import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; - -import com.oracle.svm.core.NeverInline; -import com.oracle.svm.core.code.CodeInfo; -import com.oracle.svm.core.code.CodeInfoTable; -import com.oracle.svm.core.deopt.DeoptimizedFrame; -import com.oracle.svm.core.heap.Heap; -import com.oracle.svm.core.heap.ObjectReferenceVisitor; -import com.oracle.svm.core.heap.ObjectVisitor; -import com.oracle.svm.core.heap.ReferenceAccess; -import com.oracle.svm.core.heap.RestrictHeapAccess; -import com.oracle.svm.core.heap.VMOperationInfos; -import com.oracle.svm.core.hub.InteriorObjRefWalker; -import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.log.StringBuilderLog; -import com.oracle.svm.core.snippets.KnownIntrinsics; -import com.oracle.svm.core.stack.JavaStackWalker; -import com.oracle.svm.core.stack.StackFrameVisitor; -import com.oracle.svm.core.thread.JavaVMOperation; -import com.oracle.svm.core.thread.VMOperation; -import com.oracle.svm.core.thread.VMThreads; - -import jdk.graal.compiler.word.Word; - -/** Determines paths from roots to objects or heap regions. */ -public final class PathExhibitor { - private static final FrameSlotVisitor frameSlotVisitor = new FrameSlotVisitor(); - - private static final FrameVisitor stackFrameVisitor = new FrameVisitor(); - private static final BootImageHeapObjRefVisitor bootImageHeapObjRefVisitor = new BootImageHeapObjRefVisitor(); - private static final HeapObjRefVisitor heapObjRefVisitor = new HeapObjRefVisitor(); - private static final HeapObjectVisitor heapObjectVisitor = new HeapObjectVisitor(); - private final ArrayList path = new ArrayList<>(); - - public PathExhibitor() { - } - - public boolean hasPath() { - return path.size() > 1; - } - - public PathElement[] getPath() { - return path.toArray(new PathElement[0]); - } - - @NeverInline("Starting a stack walk in the caller frame") - public void findPathToRange(Pointer rangeBegin, Pointer rangeEnd) { - assert VMOperation.isInProgressAtSafepoint(); - if (rangeBegin.isNull() || rangeBegin.aboveThan(rangeEnd)) { - return; - } - PathEdge edge = new PathEdge(); - findPathToTarget(new RangeTargetMatcher(rangeBegin, rangeEnd), edge, KnownIntrinsics.readCallerStackPointer()); - if (edge.isFilled()) { - path.add(edge.getTo()); - path.add(edge.getFrom()); - Object fromObj = edge.getFrom().getObject(); - - // Add the rest of the path - findPathToRoot(fromObj, edge, KnownIntrinsics.readCallerStackPointer()); - } - } - - public void toLog(Log log) { - for (PathElement element : path) { - log.newline(); - element.toLog(log); - } - } - - private void findPathToRoot(Object leaf, PathEdge currentEdge, Pointer currentThreadWalkStackPointer) { - assert VMOperation.isInProgressAtSafepoint(); - if (leaf == null) { - return; - } - Object currentTargetObj = leaf; - for (; /* break */;) { - // Walk backwards one step. - currentEdge.reset(); - findPathToTarget(new ObjectTargetMatcher(currentTargetObj), currentEdge, currentThreadWalkStackPointer); - - PathElement currentElement = null; - if (currentEdge.isFilled()) { - currentElement = currentEdge.getFrom(); - if (path.isEmpty()) { - path.add(currentEdge.getTo()); - } - } - if (currentElement == null) { - // No pointer to current object: The path ends here. - break; - } - currentTargetObj = currentElement.getObject(); - if (currentTargetObj == null) { - // Current element is a root: Add element to path and stop. - path.add(currentElement); - break; - } - if (checkForCycles(currentTargetObj)) { // seen before - CyclicElement cyclic = new CyclicElement(currentTargetObj); - path.add(cyclic); - break; - } - path.add(currentElement); - } - } - - private static void findPathToTarget(TargetMatcher target, PathEdge edge, Pointer currentThreadWalkStackPointer) { - assert target != null && !edge.isFilled(); - findPathInHeap(target, edge); - findPathInImageHeap(target, edge); - findPathInStack(target, edge, currentThreadWalkStackPointer); - } - - private static void findPathInStack(TargetMatcher target, PathEdge edge, Pointer currentThreadWalkStackPointer) { - if (edge.isFilled()) { - return; - } - - stackFrameVisitor.initialize(target, edge); - JavaStackWalker.walkCurrentThread(currentThreadWalkStackPointer, stackFrameVisitor); - stackFrameVisitor.reset(); - - IsolateThread thread = VMThreads.firstThread(); - while (!edge.isFilled() && thread.isNonNull()) { - if (thread.notEqual(CurrentIsolate.getCurrentThread())) { // walked above - stackFrameVisitor.initialize(target, edge); - JavaStackWalker.walkThread(thread, stackFrameVisitor); - stackFrameVisitor.reset(); - } - thread = VMThreads.nextThread(thread); - } - } - - private static void findPathInImageHeap(TargetMatcher target, PathEdge result) { - Heap.getHeap().walkImageHeapObjects(new ObjectVisitor() { - @Override - public boolean visitObject(Object obj) { - if (result.isFilled()) { - return false; - } - bootImageHeapObjRefVisitor.initialize(obj, target, result); - return InteriorObjRefWalker.walkObject(obj, bootImageHeapObjRefVisitor); - } - }); - } - - private static void findPathInHeap(TargetMatcher target, PathEdge result) { - if (result.isFilled()) { - return; - } - heapObjectVisitor.initialize(target, result); - HeapImpl.getHeapImpl().walkObjects(heapObjectVisitor); - } - - private boolean checkForCycles(Object currentObject) { - boolean result = false; - for (PathElement seen : path) { - Object seenObject = seen.getObject(); - if (currentObject == seenObject) { - result = true; - break; - } - } - return result; - } - - public abstract static class PathElement { - - public abstract Log toLog(Log log); - - /** Return the base object for this path element, or null for roots. */ - public abstract Object getObject(); - - @Override - public String toString() { - StringBuilderLog log = new StringBuilderLog(); - toLog(log); - return log.getResult(); - } - } - - interface TargetMatcher { - boolean matches(Object obj); - } - - static class ObjectTargetMatcher implements TargetMatcher { - private final Object target; - - ObjectTargetMatcher(Object target) { - this.target = target; - } - - @Override - public boolean matches(Object obj) { - return (obj == target); - } - } - - static class RangeTargetMatcher implements TargetMatcher { - private final Pointer targetBegin; - private final Pointer targetEnd; - - RangeTargetMatcher(Pointer rangeBegin, Pointer rangeEndExclusive) { - this.targetBegin = rangeBegin; - this.targetEnd = rangeEndExclusive; - } - - @Override - public boolean matches(Object obj) { - Pointer objAddr = Word.objectToUntrackedPointer(obj); - return objAddr.aboveOrEqual(targetBegin) && objAddr.belowThan(targetEnd); - } - } - - static class AbstractVisitor { - protected TargetMatcher target; - protected PathEdge result; - - void initialize(TargetMatcher targetMatcher, PathEdge resultPath) { - this.target = targetMatcher; - this.result = resultPath; - } - } - - public static class FrameVisitor extends StackFrameVisitor { - private TargetMatcher target; - private PathEdge result; - - void initialize(TargetMatcher targetMatcher, PathEdge resultPath) { - this.target = targetMatcher; - this.result = resultPath; - } - - void reset() { - initialize(null, null); - } - - @Override - @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting stack frames.") - public boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo) { - frameSlotVisitor.initialize(ip, target, result); - return CodeInfoTable.visitObjectReferences(sp, ip, codeInfo, frameSlotVisitor); - } - - @Override - @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting stack frames.") - protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame) { - /* Support for deoptimized frames is not implemented at the moment. */ - return true; - } - } - - private static class FrameSlotVisitor extends AbstractVisitor implements ObjectReferenceVisitor { - private CodePointer ip; - - FrameSlotVisitor() { - } - - void initialize(CodePointer ipArg, TargetMatcher targetMatcher, PathEdge edge) { - super.initialize(targetMatcher, edge); - ip = ipArg; - } - - @Override - public boolean visitObjectReference(Pointer stackSlot, boolean compressed, Object holderObject) { - Log trace = Log.noopLog(); - if (stackSlot.isNull()) { - return true; - } - Pointer referentPointer = ReferenceAccess.singleton().readObjectAsUntrackedPointer(stackSlot, compressed); - trace.string(" referentPointer: ").zhex(referentPointer); - if (target.matches(referentPointer.toObject())) { - result.fill(new StackElement(stackSlot, ip), new LeafElement(referentPointer.toObject())); - return false; - } - return true; - } - } - - private static class BootImageHeapObjRefVisitor extends AbstractVisitor implements ObjectReferenceVisitor { - private Object container; - - BootImageHeapObjRefVisitor() { - } - - void initialize(Object containerObj, TargetMatcher targetMatcher, PathEdge resultPath) { - super.initialize(targetMatcher, resultPath); - this.container = containerObj; - } - - @Override - public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { - if (objRef.isNull()) { - return true; - } - Object referent = ReferenceAccess.singleton().readObjectAt(objRef, compressed); - if (target.matches(referent)) { - UnsignedWord offset = objRef.subtract(Word.objectToUntrackedPointer(container)); - result.fill(new BootImageHeapElement(container, offset), new LeafElement(referent)); - return false; - } - return true; - } - } - - private static class HeapObjectVisitor extends AbstractVisitor implements ObjectVisitor { - HeapObjectVisitor() { - } - - @Override - public boolean visitObject(Object containerObject) { - Pointer containerPointer = Word.objectToUntrackedPointer(containerObject); - heapObjRefVisitor.initialize(containerPointer, target, result); - return InteriorObjRefWalker.walkObject(containerObject, heapObjRefVisitor); - } - } - - private static class HeapObjRefVisitor extends AbstractVisitor implements ObjectReferenceVisitor { - private Pointer containerPointer; - - HeapObjRefVisitor() { - } - - @NeverInline("Starting a stack walk in the caller frame") - static boolean isInterfering(Object currentObject) { - return currentObject instanceof PathElement || currentObject instanceof FindPathToObjectOperation || currentObject instanceof TargetMatcher; - } - - public void initialize(Pointer container, TargetMatcher targetMatcher, PathEdge edge) { - super.initialize(targetMatcher, edge); - containerPointer = container; - } - - @Override - public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { - if (objRef.isNull()) { - return true; - } - Object containerObject = containerPointer.toObject(); - if (!isInterfering(containerObject)) { - Pointer referentPointer = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); - if (target.matches(referentPointer.toObject())) { - UnsignedWord offset = objRef.subtract(containerPointer); - result.fill(new HeapElement(containerObject, offset), new LeafElement(referentPointer.toObject())); - return false; - } - } - return true; - } - } - - /** Element at the end of a path, the target of the search. */ - public static class LeafElement extends PathElement { - private final Object leaf; - - LeafElement(Object leaf) { - this.leaf = leaf; - } - - @Override - public Object getObject() { - return leaf; - } - - @Override - public Log toLog(Log log) { - log.string("[leaf:"); - log.string(" ").object(leaf); - log.string("]"); - return log; - } - } - - /** A path element for a reference from a Object field. */ - public static class HeapElement extends PathElement { - private final Object base; - private final UnsignedWord offset; - - HeapElement(Object base, UnsignedWord offset) { - this.base = base; - this.offset = offset; - } - - @Override - public Object getObject() { - return base; - } - - @Override - public Log toLog(Log log) { - log.string("[heap:"); - log.string(" base: ").object(base); - log.string(" offset: ").unsigned(offset); - Pointer objPointer = Word.objectToUntrackedPointer(base); - Pointer fieldObjRef = objPointer.add(offset); - Pointer fieldPointer = fieldObjRef.readWord(0); - log.string(" field: ").zhex(fieldPointer); - log.string("]"); - return log; - } - } - - /** A path element for a reference from a stack frame. */ - public static class StackElement extends PathElement { - private final Pointer stackSlot; - private final CodePointer ip; - private final Pointer slotValue; - - StackElement(Pointer stackSlot, CodePointer ip) { - this.stackSlot = stackSlot; - this.ip = ip; - this.slotValue = stackSlot.readWord(0); - } - - @Override - public Object getObject() { - // Stack frames are roots. - return null; - } - - @Override - public Log toLog(Log log) { - log.string("[stack:"); - log.string(" slot: ").zhex(stackSlot); - log.string(" ip: ").zhex(ip); - log.string(" value: ").zhex(slotValue); - log.string("]"); - return log; - } - } - - /** A path element for a reference from the native image heap. */ - public static class BootImageHeapElement extends PathElement { - private final Object base; - private final UnsignedWord offset; - - BootImageHeapElement(Object base, UnsignedWord offset) { - this.base = base; - this.offset = offset; - } - - @Override - public Object getObject() { - // Native image heap objects are roots. - return null; - } - - @Override - public Log toLog(Log log) { - log.string("[native image heap:"); - log.string(" object: ").object(base); - log.string(" offset: ").unsigned(offset); - log.string("]"); - return log; - } - } - - /** A path element for a cyclic reference. */ - public static class CyclicElement extends PathElement { - private final Object previous; - - CyclicElement(Object previous) { - this.previous = previous; - } - - @Override - public Object getObject() { - // Cyclic elements are roots. - return null; - } - - @Override - public Log toLog(Log log) { - log.string("[cyclic:"); - log.string(" previous: ").object(previous); - log.string("]"); - return log; - } - } - - public static final class TestingBackDoor { - private TestingBackDoor() { - } - - public static void findPathToObject(PathExhibitor exhibitor, Object obj) { - PathEdge result = new PathEdge(); - FindPathToObjectOperation op = new FindPathToObjectOperation(exhibitor, obj, result); - op.enqueue(); - } - } - - private static final class FindPathToObjectOperation extends JavaVMOperation { - private final PathExhibitor exhibitor; - private final Object object; - private final PathEdge result; - - FindPathToObjectOperation(PathExhibitor exhibitor, Object object, PathEdge result) { - super(VMOperationInfos.get(FindPathToObjectOperation.class, "Find path to object", SystemEffect.SAFEPOINT)); - this.exhibitor = exhibitor; - this.object = object; - this.result = result; - } - - @Override - @NeverInline("Starting a stack walk.") - protected void operate() { - exhibitor.findPathToRoot(object, result, KnownIntrinsics.readCallerStackPointer()); - } - } - - public static class PathEdge { - private PathElement from; - private PathElement to; - - public PathEdge() { - } - - public boolean isFilled() { - return from != null && to != null; - } - - public PathElement getFrom() { - return from; - } - - public PathElement getTo() { - return to; - } - - public void fill(PathElement fromElem, PathElement toElem) { - from = fromElem; - to = toElem; - } - - public void reset() { - from = null; - to = null; - } - } -} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index 9d61e0684eda..4a150065783b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -36,6 +36,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.heap.ObjectHeader; @@ -120,7 +121,7 @@ private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { // promoted object. return; } - Object refObject = referentAddr.toObject(); + Object refObject = referentAddr.toObjectNonNull(); if (willSurviveThisCollection(refObject)) { // Either an object that got promoted without being moved or an object in the old gen. RememberedSet.get().dirtyCardIfNecessary(dr, refObject); @@ -136,7 +137,8 @@ private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { if (elapsed.belowThan(maxSoftRefAccessIntervalMs)) { // Important: we need to pass the reference object as holder so that the remembered // set can be updated accordingly! - refVisitor.visitObjectReference(ReferenceInternals.getReferentFieldAddress(dr), true, dr); + int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); + refVisitor.visitObjectReferences(ReferenceInternals.getReferentFieldAddress(dr), true, referenceSize, dr, 1); return; // referent will survive } } @@ -212,7 +214,7 @@ private static boolean processRememberedRef(Reference dr) { if (maybeUpdateForwardedReference(dr, refPointer)) { return true; } - Object refObject = refPointer.toObject(); + Object refObject = refPointer.toObjectNonNull(); if (willSurviveThisCollection(refObject)) { RememberedSet.get().dirtyCardIfNecessary(dr, refObject); return true; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheReachabilityAnalyzer.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheReachabilityAnalyzer.java index 59c853d01f63..05516ebc7fb8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheReachabilityAnalyzer.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheReachabilityAnalyzer.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.genscavenge; +import java.io.Serial; + import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; @@ -36,31 +38,34 @@ import jdk.graal.compiler.word.Word; +/** + * Analyzes if run-time compiled code has any references to otherwise unreachable objects. Throws an + * {@link UnreachableObjectsException} if a reference to an otherwise unreachable object is + * detected. + */ @DuplicatedInNativeCode final class RuntimeCodeCacheReachabilityAnalyzer implements ObjectReferenceVisitor { - private boolean unreachableObjects; + private static final UnreachableObjectsException UNREACHABLE_OBJECTS_EXCEPTION = new UnreachableObjectsException(); @Platforms(Platform.HOSTED_ONLY.class) RuntimeCodeCacheReachabilityAnalyzer() { } - public void initialize() { - this.unreachableObjects = false; - } - - public boolean hasUnreachableObjects() { - return unreachableObjects; + @Override + public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count) { + Pointer pos = firstObjRef; + Pointer end = firstObjRef.add(Word.unsigned(count).multiply(referenceSize)); + while (pos.belowThan(end)) { + visitObjectReference(pos, compressed); + pos = pos.add(referenceSize); + } } - @Override - public boolean visitObjectReference(Pointer ptrPtrToObject, boolean compressed, Object holderObject) { - assert !unreachableObjects; + private static void visitObjectReference(Pointer ptrPtrToObject, boolean compressed) { Pointer ptrToObj = ReferenceAccess.singleton().readObjectAsUntrackedPointer(ptrPtrToObject, compressed); if (ptrToObj.isNonNull() && !isReachable(ptrToObj)) { - unreachableObjects = true; - return false; + throw UNREACHABLE_OBJECTS_EXCEPTION; } - return true; } public static boolean isReachable(Pointer ptrToObj) { @@ -90,11 +95,22 @@ public static boolean isReachable(Pointer ptrToObj) { private static boolean isAssumedReachable(Class clazz) { Class[] classesAssumedReachable = RuntimeCodeCacheCleaner.CLASSES_ASSUMED_REACHABLE; - for (int i = 0; i < classesAssumedReachable.length; i++) { - if (classesAssumedReachable[i].isAssignableFrom(clazz)) { + for (Class aClass : classesAssumedReachable) { + if (aClass.isAssignableFrom(clazz)) { return true; } } return false; } + + static final class UnreachableObjectsException extends RuntimeException { + @Serial private static final long serialVersionUID = 1L; + + @Override + @SuppressWarnings("sync-override") + public Throwable fillInStackTrace() { + /* No stacktrace needed. */ + return this; + } + } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheWalker.java index 1a2c7270fdc3..a681528da6a0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheWalker.java @@ -33,7 +33,7 @@ import com.oracle.svm.core.code.RuntimeCodeCache.CodeInfoVisitor; import com.oracle.svm.core.code.RuntimeCodeInfoAccess; import com.oracle.svm.core.code.UntetheredCodeInfoAccess; -import com.oracle.svm.core.heap.ObjectReferenceVisitor; +import com.oracle.svm.core.genscavenge.RuntimeCodeCacheReachabilityAnalyzer.UnreachableObjectsException; import com.oracle.svm.core.util.DuplicatedInNativeCode; import jdk.graal.compiler.word.Word; @@ -49,19 +49,19 @@ */ final class RuntimeCodeCacheWalker implements CodeInfoVisitor { private final RuntimeCodeCacheReachabilityAnalyzer checkForUnreachableObjectsVisitor; - private final ObjectReferenceVisitor greyToBlackObjectVisitor; + private final GreyToBlackObjRefVisitor greyToBlackObjectVisitor; @Platforms(Platform.HOSTED_ONLY.class) - RuntimeCodeCacheWalker(ObjectReferenceVisitor greyToBlackObjectVisitor) { + RuntimeCodeCacheWalker(GreyToBlackObjRefVisitor greyToBlackObjectVisitor) { this.checkForUnreachableObjectsVisitor = new RuntimeCodeCacheReachabilityAnalyzer(); this.greyToBlackObjectVisitor = greyToBlackObjectVisitor; } @Override @DuplicatedInNativeCode - public boolean visitCode(CodeInfo codeInfo) { + public void visitCode(CodeInfo codeInfo) { if (RuntimeCodeInfoAccess.areAllObjectsOnImageHeap(codeInfo)) { - return true; + return; } /* @@ -84,7 +84,7 @@ public boolean visitCode(CodeInfo codeInfo) { */ RuntimeCodeInfoAccess.walkObjectFields(codeInfo, greyToBlackObjectVisitor); CodeInfoAccess.setState(codeInfo, CodeInfo.STATE_PENDING_FREE); - return true; + return; } /* @@ -98,7 +98,7 @@ public boolean visitCode(CodeInfo codeInfo) { if (state == CodeInfo.STATE_NON_ENTRANT || invalidateCodeThatReferencesUnreachableObjects && state == CodeInfo.STATE_CODE_CONSTANTS_LIVE && hasWeakReferenceToUnreachableObject(codeInfo)) { RuntimeCodeInfoAccess.walkObjectFields(codeInfo, greyToBlackObjectVisitor); CodeInfoAccess.setState(codeInfo, CodeInfo.STATE_PENDING_REMOVAL_FROM_CODE_CACHE); - return true; + return; } } @@ -112,7 +112,6 @@ public boolean visitCode(CodeInfo codeInfo) { */ RuntimeCodeInfoAccess.walkStrongReferences(codeInfo, greyToBlackObjectVisitor); RuntimeCodeInfoAccess.walkWeakReferences(codeInfo, greyToBlackObjectVisitor); - return true; } private static boolean isReachable(Object possiblyForwardedObject) { @@ -120,8 +119,11 @@ private static boolean isReachable(Object possiblyForwardedObject) { } private boolean hasWeakReferenceToUnreachableObject(CodeInfo codeInfo) { - checkForUnreachableObjectsVisitor.initialize(); - RuntimeCodeInfoAccess.walkWeakReferences(codeInfo, checkForUnreachableObjectsVisitor); - return checkForUnreachableObjectsVisitor.hasUnreachableObjects(); + try { + RuntimeCodeInfoAccess.walkWeakReferences(codeInfo, checkForUnreachableObjectsVisitor); + return false; + } catch (UnreachableObjectsException e) { + return true; + } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 68b6aaa26bcb..56115a9752fc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -159,33 +159,25 @@ boolean isToSpace() { return isToSpace; } - public boolean walkObjects(ObjectVisitor visitor) { + public void walkObjects(ObjectVisitor visitor) { AlignedHeapChunk.AlignedHeader aChunk = getFirstAlignedHeapChunk(); while (aChunk.isNonNull()) { - if (!AlignedHeapChunk.walkObjects(aChunk, visitor)) { - return false; - } + AlignedHeapChunk.walkObjects(aChunk, visitor); aChunk = HeapChunk.getNext(aChunk); } UnalignedHeapChunk.UnalignedHeader uChunk = getFirstUnalignedHeapChunk(); while (uChunk.isNonNull()) { - if (!UnalignedHeapChunk.walkObjects(uChunk, visitor)) { - return false; - } + UnalignedHeapChunk.walkObjects(uChunk, visitor); uChunk = HeapChunk.getNext(uChunk); } - return true; } - boolean walkAlignedHeapChunks(AlignedHeapChunk.Visitor visitor) { + void walkAlignedHeapChunks(AlignedHeapChunk.Visitor visitor) { AlignedHeapChunk.AlignedHeader chunk = getFirstAlignedHeapChunk(); while (chunk.isNonNull()) { - if (!visitor.visitChunk(chunk)) { - return false; - } + visitor.visitChunk(chunk); chunk = HeapChunk.getNext(chunk); } - return true; } public void logUsage(Log log, boolean logIfEmpty) { @@ -439,7 +431,7 @@ private Object copyAlignedObject(Object originalObj) { Pointer originalMemory = Word.objectToUntrackedPointer(originalObj); UnmanagedMemoryUtil.copyLongsForward(originalMemory, copyMemory, originalSize); - Object copy = copyMemory.toObject(); + Object copy = copyMemory.toObjectNonNull(); if (probability(SLOW_PATH_PROBABILITY, addIdentityHashField)) { // Must do first: ensures correct object size below and in other places int value = IdentityHashCodeSupport.computeHashCodeFromAddress(originalObj); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/StackVerifier.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/StackVerifier.java index 63642c13846e..dd84c8ff5088 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/StackVerifier.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/StackVerifier.java @@ -42,6 +42,8 @@ import com.oracle.svm.core.stack.StackFrameVisitor; import com.oracle.svm.core.thread.VMThreads; +import jdk.graal.compiler.word.Word; + /** Walk the stack and verify all objects that are referenced from stack frames. */ final class StackVerifier { private static final StackFrameVerificationVisitor STACK_FRAME_VISITOR = new StackFrameVerificationVisitor(); @@ -113,9 +115,17 @@ public void initialize() { } @Override - public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { + public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count) { + Pointer pos = firstObjRef; + Pointer end = firstObjRef.add(Word.unsigned(count).multiply(referenceSize)); + while (pos.belowThan(end)) { + visitObjectReference(pos, compressed, holderObject); + pos = pos.add(referenceSize); + } + } + + private void visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { result &= HeapVerifier.verifyReference(holderObject, objRef, compressed); - return true; } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index b46e32edbf4c..87c6f0bab557 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -33,6 +33,7 @@ import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectVisitor; +import com.oracle.svm.core.heap.UninterruptibleObjectVisitor; import com.oracle.svm.core.util.UnsignedUtils; import jdk.graal.compiler.api.directives.GraalDirectives; @@ -136,14 +137,14 @@ static UnalignedHeader getEnclosingChunkFromObjectPointer(Pointer ptr) { return (UnalignedHeader) chunkPointer; } - public static boolean walkObjects(UnalignedHeader that, ObjectVisitor visitor) { - return HeapChunk.walkObjectsFrom(that, getObjectStart(that), visitor); + public static void walkObjects(UnalignedHeader that, ObjectVisitor visitor) { + HeapChunk.walkObjectsFrom(that, getObjectStart(that), visitor); } @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static boolean walkObjectsInline(UnalignedHeader that, ObjectVisitor visitor) { - return HeapChunk.walkObjectsFromInline(that, getObjectStart(that), visitor); + public static void walkObjectsInline(UnalignedHeader that, UninterruptibleObjectVisitor visitor) { + HeapChunk.walkObjectsFromInline(that, getObjectStart(that), visitor); } @Fold diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index 36f73fb7774d..e8f74da76ef7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -83,22 +83,15 @@ void tearDown() { } @Override - public boolean walkObjects(ObjectVisitor visitor) { + public void walkObjects(ObjectVisitor visitor) { /* Flush the thread-local allocation data. */ ThreadLocalAllocation.disableAndFlushForAllThreads(); - if (!getEden().walkObjects(visitor)) { - return false; - } + getEden().walkObjects(visitor); for (int i = 0; i < maxSurvivorSpaces; i++) { - if (!survivorFromSpaces[i].walkObjects(visitor)) { - return false; - } - if (!survivorToSpaces[i].walkObjects(visitor)) { - return false; - } + survivorFromSpaces[i].walkObjects(visitor); + survivorToSpaces[i].walkObjects(visitor); } - return true; } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectFixupVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectFixupVisitor.java index 6d734001785b..2aebdc876020 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectFixupVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectFixupVisitor.java @@ -30,14 +30,13 @@ import java.lang.ref.Reference; import com.oracle.svm.core.AlwaysInline; -import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.heap.ReferenceInternals; import com.oracle.svm.core.heap.UninterruptibleObjectVisitor; import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.InteriorObjRefWalker; import com.oracle.svm.core.snippets.KnownIntrinsics; -import com.oracle.svm.core.util.VMError; /** Visits surviving objects before compaction to update their references. */ public final class ObjectFixupVisitor implements UninterruptibleObjectVisitor { @@ -50,21 +49,14 @@ public ObjectFixupVisitor(ObjectRefFixupVisitor refFixupVisitor) { @Override @AlwaysInline("GC performance") @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).", callerMustBe = true) - public boolean visitObjectInline(Object obj) { + public void visitObject(Object obj) { DynamicHub hub = KnownIntrinsics.readHub(obj); if (probability(SLOW_PATH_PROBABILITY, hub.isReferenceInstanceClass())) { // update Target_java_lang_ref_Reference.referent Reference dr = (Reference) obj; - refFixupVisitor.visitObjectReferenceInline(ReferenceInternals.getReferentFieldAddress(dr), 0, true, dr); + int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); + refFixupVisitor.visitObjectReferences(ReferenceInternals.getReferentFieldAddress(dr), true, referenceSize, dr, 1); } InteriorObjRefWalker.walkObjectInline(obj, refFixupVisitor); - return true; - } - - @Override - @NeverInline("Non-performance critical version") - @Uninterruptible(reason = "Visitor requires uninterruptible walk.", callerMustBe = true) - public boolean visitObject(Object o) { - throw VMError.shouldNotReachHere("for performance reasons"); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectMoveInfo.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectMoveInfo.java index c4b5325f6937..0c2b79539f1a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectMoveInfo.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectMoveInfo.java @@ -39,7 +39,6 @@ import com.oracle.svm.core.genscavenge.remset.BrickTable; import com.oracle.svm.core.genscavenge.remset.FirstObjectTable; import com.oracle.svm.core.hub.LayoutEncoding; -import com.oracle.svm.core.util.VMError; import jdk.graal.compiler.api.replacements.Fold; import jdk.graal.compiler.word.Word; @@ -180,7 +179,7 @@ public static void walkObjectsForFixup(AlignedHeapChunk.AlignedHeader chunk, Obj assert objSeqEnd.belowOrEqual(HeapChunk.getTopPointer(chunk)); while (p.notEqual(objSeqEnd)) { assert p.belowThan(objSeqEnd); - Object obj = p.toObject(); + Object obj = p.toObjectNonNull(); UnsignedWord objSize = LayoutEncoding.getSizeFromObjectInlineInGC(obj); /* @@ -194,10 +193,7 @@ public static void walkObjectsForFixup(AlignedHeapChunk.AlignedHeader chunk, Obj UnsignedWord offset = newAddress.subtract(AlignedHeapChunk.getObjectsStart(objSeqNewChunk)); FirstObjectTable.setTableForObject(AlignedChunkRememberedSet.getFirstObjectTableStart(objSeqNewChunk), offset, offset.add(objSize)); - if (!visitor.visitObjectInline(obj)) { - throw VMError.shouldNotReachHereAtRuntime(); - } - + visitor.visitObject(obj); p = p.add(objSize); } if (nextObjSeq.isNonNull() && chunk.getShouldSweepInsteadOfCompact()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectRefFixupVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectRefFixupVisitor.java index 87d78f4a491e..9ec7fb1cfb24 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectRefFixupVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/ObjectRefFixupVisitor.java @@ -37,40 +37,43 @@ import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.ReferenceAccess; +import jdk.graal.compiler.word.Word; + /** * Updates each reference after marking and before compaction to point to the referenced object's * future location. */ public final class ObjectRefFixupVisitor implements ObjectReferenceVisitor { @Override - public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { - return visitObjectReferenceInline(objRef, 0, compressed, holderObject); + @AlwaysInline("GC performance") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count) { + Pointer pos = firstObjRef; + Pointer end = firstObjRef.add(Word.unsigned(count).multiply(referenceSize)); + while (pos.belowThan(end)) { + visitObjectReference(pos, compressed, holderObject); + pos = pos.add(referenceSize); + } } - @Override @AlwaysInline("GC performance") @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { - assert innerOffset == 0; + private static void visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { Pointer p = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); if (p.isNull() || HeapImpl.getHeapImpl().isInImageHeap(p)) { - return true; + return; } - Object obj; - Object original = p.toObject(); + Object original = p.toObjectNonNull(); if (ObjectHeaderImpl.isAlignedObject(original)) { Pointer newLocation = ObjectMoveInfo.getNewObjectAddress(p); assert newLocation.isNonNull() // || holderObject == null // references from CodeInfo, invalidated or weak || holderObject instanceof Reference; // cleared referent - obj = newLocation.toObject(); + Object obj = newLocation.toObjectNonNull(); ReferenceAccess.singleton().writeObjectAt(objRef, obj, compressed); - } else { - obj = original; } // Note that image heap cards have already been cleaned and re-marked during the scan - return true; } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/PlanningVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/PlanningVisitor.java index 9599d45ed318..cfc32de4cb38 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/PlanningVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/PlanningVisitor.java @@ -93,11 +93,11 @@ public boolean visitChunk(AlignedHeapChunk.AlignedHeader chunk) { Object forwardedObj = ObjectHeaderImpl.getObjectHeaderImpl().getForwardedObject(p, header); objSize = LayoutEncoding.getSizeFromObjectWithoutOptionalIdHashFieldInGC(forwardedObj); } else { - objSize = LayoutEncoding.getSizeFromObjectInlineInGC(p.toObject()); + objSize = LayoutEncoding.getSizeFromObjectInlineInGC(p.toObjectNonNull()); } if (ObjectHeaderImpl.isMarkedHeader(header)) { - ObjectHeaderImpl.unsetMarkedAndKeepRememberedSetBit(p.toObject()); + ObjectHeaderImpl.unsetMarkedAndKeepRememberedSetBit(p.toObjectNonNull()); /* * Adding the optional identity hash field would increase an object's size, so we diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/RuntimeCodeCacheFixupWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/RuntimeCodeCacheFixupWalker.java index ae1b72d8685e..e15bd362c257 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/RuntimeCodeCacheFixupWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/RuntimeCodeCacheFixupWalker.java @@ -43,9 +43,9 @@ public RuntimeCodeCacheFixupWalker(ObjectRefFixupVisitor visitor) { } @Override - public boolean visitCode(CodeInfo codeInfo) { + public void visitCode(CodeInfo codeInfo) { if (RuntimeCodeInfoAccess.areAllObjectsOnImageHeap(codeInfo)) { - return true; + return; } /* @@ -54,6 +54,5 @@ public boolean visitCode(CodeInfo codeInfo) { */ RuntimeCodeInfoAccess.walkStrongReferences(codeInfo, visitor); RuntimeCodeInfoAccess.walkWeakReferences(codeInfo, visitor); - return true; } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java index 4b279660c240..bf9738fc5427 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java @@ -116,7 +116,7 @@ static void enableRememberedSet(AlignedHeader chunk) { Pointer offset = AlignedHeapChunk.getObjectsStart(chunk); Pointer top = HeapChunk.getTopPointer(chunk); while (offset.belowThan(top)) { - Object obj = offset.toObject(); + Object obj = offset.toObjectNonNull(); UnsignedWord objSize = LayoutEncoding.getSizeFromObjectInGC(obj); enableRememberedSetForObject(chunk, obj, objSize); offset = offset.add(objSize); @@ -206,8 +206,8 @@ private static void walkObjects(AlignedHeader chunk, Pointer start, Pointer end, UnsignedWord index = CardTable.memoryOffsetToIndex(start.subtract(objectsStart)); Pointer ptr = FirstObjectTable.getFirstObjectImprecise(fotStart, objectsStart, index); while (ptr.belowThan(end)) { - Object obj = ptr.toObject(); - visitor.visitObjectInline(obj); + Object obj = ptr.toObjectNonNull(); + visitor.visitObject(obj); ptr = LayoutEncoding.getObjectEndInlineInGC(obj); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java index 48eaa6c642f9..fb5d7c943949 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java @@ -158,7 +158,7 @@ public static boolean verify(Pointer cardTableStart, Pointer cardTableEnd, Point while (curPtr.belowThan(objectsLimit)) { // As we only use imprecise card marking at the moment, only the card at the address // of the object may be dirty. - Object obj = curPtr.toObject(); + Object obj = curPtr.toObjectNonNull(); UnsignedWord cardTableIndex = memoryOffsetToIndex(curPtr.subtract(objectsStart)); if (isClean(cardTableStart, cardTableIndex)) { CARD_TABLE_VERIFICATION_VISITOR.initialize(obj, cardTableStart, objectsStart); @@ -243,11 +243,19 @@ public void reset() { } @Override + public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count) { + Pointer pos = firstObjRef; + Pointer end = firstObjRef.add(Word.unsigned(count).multiply(referenceSize)); + while (pos.belowThan(end)) { + visitObjectReference(pos, compressed); + pos = pos.add(referenceSize); + } + } + @SuppressFBWarnings(value = {"NS_DANGEROUS_NON_SHORT_CIRCUIT"}, justification = "Non-short circuit logic is used on purpose here.") - public boolean visitObjectReference(Pointer reference, boolean compressed, Object holderObject) { + private void visitObjectReference(Pointer reference, boolean compressed) { Pointer referencedObject = ReferenceAccess.singleton().readObjectAsUntrackedPointer(reference, compressed); success &= verifyReference(parentObject, cardTableStart, objectsStart, reference, referencedObject); - return true; } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java index 204baa696015..ef697f9f0e49 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java @@ -244,7 +244,7 @@ static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer objectsStart, Pointer indexedMemoryStart = objectsStart.add(indexToMemoryOffset(index)); // If the object starts before the memory for this index, skip over it. if (firstObject.belowThan(indexedMemoryStart)) { - Object crossingObject = firstObject.toObject(); + Object crossingObject = firstObject.toObjectNonNull(); result = LayoutEncoding.getObjectEndInGC(crossingObject); } else { assert firstObject.equal(indexedMemoryStart) : "preciseFirstPointer.equal(indexedMemoryStart)"; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java index dbfb87402f2f..f0f2c1c33df7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java @@ -66,7 +66,7 @@ public static void enableRememberedSet(UnalignedHeader chunk) { CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); // Unaligned chunks don't have a first object table. - Object obj = UnalignedHeapChunk.getObjectStart(chunk).toObject(); + Object obj = UnalignedHeapChunk.getObjectStart(chunk).toObjectNonNull(); ObjectHeaderImpl.setRememberedSetBit(obj); } @@ -101,8 +101,8 @@ public static void walkDirtyObjects(UnalignedHeader chunk, UninterruptibleObject } Pointer objectsStart = UnalignedHeapChunk.getObjectStart(chunk); - Object obj = objectsStart.toObject(); - visitor.visitObjectInline(obj); + Object obj = objectsStart.toObjectNonNull(); + visitor.visitObject(obj); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java index 27d3c61a4fa5..72a66ac44215 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java @@ -34,7 +34,7 @@ public final class MemoryWalker { public interface ImageHeapRegionVisitor { /** Visit a region from the native image heap. */ @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting memory.") - boolean visitNativeImageHeapRegion(T region, MemoryWalker.NativeImageHeapRegionAccess access); + void visitNativeImageHeapRegion(T region, MemoryWalker.NativeImageHeapRegionAccess access); } /** A set of access methods for visiting regions of the native image heap. */ @@ -52,7 +52,7 @@ public interface NativeImageHeapRegionAccess { boolean consistsOfHugeObjects(T region); - boolean visitObjects(T region, ObjectVisitor visitor); + void visitObjects(T region, ObjectVisitor visitor); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java index fe20441cf3e2..518ee303fa10 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java @@ -444,36 +444,29 @@ public static void setObject(NonmovableObjectArray array, int index, T va * Visits all array elements with the provided {@link ObjectReferenceVisitor}. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static boolean walkUnmanagedObjectArray(NonmovableObjectArray array, ObjectReferenceVisitor visitor) { + public static void walkUnmanagedObjectArray(NonmovableObjectArray array, ObjectReferenceVisitor visitor) { if (array.isNonNull()) { - return walkUnmanagedObjectArray(array, visitor, 0, lengthOf(array)); + walkUnmanagedObjectArray(array, visitor, 0, lengthOf(array)); } - return true; } /** * Visits all array elements with the provided {@link ObjectReferenceVisitor}. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static boolean walkUnmanagedObjectArray(NonmovableObjectArray array, ObjectReferenceVisitor visitor, int startIndex, int count) { + public static void walkUnmanagedObjectArray(NonmovableObjectArray array, ObjectReferenceVisitor visitor, int startIndex, int count) { if (array.isNonNull()) { assert startIndex >= 0 && count <= lengthOf(array) - startIndex; int refSize = ConfigurationValues.getObjectLayout().getReferenceSize(); assert refSize == (1 << readElementShift(array)); - Pointer p = ((Pointer) array).add(readArrayBase(array)).add(startIndex * refSize); - for (int i = 0; i < count; i++) { - if (!callVisitor(visitor, p)) { - return false; - } - p = p.add(refSize); - } + Pointer firstObjRef = ((Pointer) array).add(readArrayBase(array)).add(startIndex * refSize); + callVisitor(visitor, firstObjRef, refSize, count); } - return true; } @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) - private static boolean callVisitor(ObjectReferenceVisitor visitor, Pointer p) { - return visitor.visitObjectReference(p, true, null); + private static void callVisitor(ObjectReferenceVisitor visitor, Pointer firstObjRef, int referenceSize, int count) { + visitor.visitObjectReferences(firstObjRef, true, referenceSize, null, count); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java index e8a5da95df49..72ba9839f536 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java @@ -689,8 +689,10 @@ void verifyMethod(SharedMethod method, CompilationResult compilation, int compil assert offset < compilationSize : infopoint; CodeInfoAccess.lookupCodeInfo(info, offset + compilationOffset, queryResult, constantAccess); - CollectingObjectReferenceVisitor visitor = new CollectingObjectReferenceVisitor(); - CodeReferenceMapDecoder.walkOffsetsFromPointer(Word.zero(), CodeInfoAccess.getStackReferenceMapEncoding(info), queryResult.getReferenceMapIndex(), visitor, null); + /* Use a non-zero base to avoid negative addresses. */ + Pointer base = Word.pointer(1024L * 1024L * 1024L); + CollectingObjectReferenceVisitor visitor = new CollectingObjectReferenceVisitor(base); + CodeReferenceMapDecoder.walkOffsetsFromPointer(base, CodeInfoAccess.getStackReferenceMapEncoding(info), queryResult.getReferenceMapIndex(), visitor, null); ReferenceMapEncoder.Input expected = (ReferenceMapEncoder.Input) infopoint.debugInfo.getReferenceMap(); visitor.result.verify(); assert expected.equals(visitor.result) : infopoint; @@ -926,17 +928,25 @@ public static Object createFromLoader(ImageSingletonLoader loader) { } class CollectingObjectReferenceVisitor implements ObjectReferenceVisitor { + private final Pointer base; protected final SubstrateReferenceMap result = new SubstrateReferenceMap(); - @Override - public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { - return visitObjectReferenceInline(objRef, 0, compressed, holderObject); + CollectingObjectReferenceVisitor(Pointer base) { + this.base = base; } @Override - public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { - int derivedOffset = NumUtil.safeToInt(objRef.rawValue()); - result.markReferenceAtOffset(derivedOffset, derivedOffset - innerOffset, compressed); - return true; + public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count) { + Pointer pos = firstObjRef; + Pointer end = firstObjRef.add(Word.unsigned(count).multiply(referenceSize)); + while (pos.belowThan(end)) { + visitObjectReference(pos, compressed); + pos = pos.add(referenceSize); + } + } + + private void visitObjectReference(Pointer objRef, boolean compressed) { + int offset = NumUtil.safeToInt(objRef.subtract(base).rawValue()); + result.markReferenceAtOffset(offset, compressed); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java index f8085444015b..d22be88ec99b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java @@ -151,7 +151,7 @@ public static CodeInfoQueryResult lookupDeoptimizationEntrypoint(CodeInfo info, } /** Note that this method is only called for regular frames but not for deoptimized frames. */ - public static boolean visitObjectReferences(Pointer sp, CodePointer ip, CodeInfo info, ObjectReferenceVisitor visitor) { + public static void visitObjectReferences(Pointer sp, CodePointer ip, CodeInfo info, ObjectReferenceVisitor visitor) { counters().visitObjectReferencesCount.inc(); /* @@ -169,7 +169,7 @@ public static boolean visitObjectReferences(Pointer sp, CodePointer ip, CodeInfo if (referenceMapIndex == ReferenceMapIndex.NO_REFERENCE_MAP) { throw fatalErrorNoReferenceMap(sp, ip, info); } - return CodeReferenceMapDecoder.walkOffsetsFromPointer(sp, referenceMapEncoding, referenceMapIndex, visitor, null); + CodeReferenceMapDecoder.walkOffsetsFromPointer(sp, referenceMapEncoding, referenceMapIndex, visitor, null); } @Uninterruptible(reason = "Not really uninterruptible, but we are about to fail.", calleeMustBe = false) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java index ca229b632ecf..50d8e64a6b99 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java @@ -345,6 +345,6 @@ public interface CodeInfoVisitor { * continue, else false. */ @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting code.") - boolean visitCode(CodeInfo codeInfo); + void visitCode(CodeInfo codeInfo); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java index 693a33614d11..1dd3a57fded1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.code; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; @@ -48,6 +47,8 @@ import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.word.Word; + /** * This class contains methods that only make sense for runtime compiled code. */ @@ -153,8 +154,8 @@ public static boolean areAllObjectsOnImageHeap(CodeInfo info) { * Walks all strong references in a {@link CodeInfo} object. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static boolean walkStrongReferences(CodeInfo info, ObjectReferenceVisitor visitor) { - return NonmovableArrays.walkUnmanagedObjectArray(cast(info).getObjectFields(), visitor, CodeInfoImpl.FIRST_STRONGLY_REFERENCED_OBJFIELD, CodeInfoImpl.STRONGLY_REFERENCED_OBJFIELD_COUNT); + public static void walkStrongReferences(CodeInfo info, ObjectReferenceVisitor visitor) { + NonmovableArrays.walkUnmanagedObjectArray(cast(info).getObjectFields(), visitor, CodeInfoImpl.FIRST_STRONGLY_REFERENCED_OBJFIELD, CodeInfoImpl.STRONGLY_REFERENCED_OBJFIELD_COUNT); } /** @@ -162,21 +163,17 @@ public static boolean walkStrongReferences(CodeInfo info, ObjectReferenceVisitor */ @DuplicatedInNativeCode @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static boolean walkWeakReferences(CodeInfo info, ObjectReferenceVisitor visitor) { + public static void walkWeakReferences(CodeInfo info, ObjectReferenceVisitor visitor) { CodeInfoImpl impl = cast(info); - boolean continueVisiting = true; - continueVisiting = continueVisiting && - NonmovableArrays.walkUnmanagedObjectArray(impl.getObjectFields(), visitor, CodeInfoImpl.FIRST_WEAKLY_REFERENCED_OBJFIELD, CodeInfoImpl.WEAKLY_REFERENCED_OBJFIELD_COUNT); + NonmovableArrays.walkUnmanagedObjectArray(impl.getObjectFields(), visitor, CodeInfoImpl.FIRST_WEAKLY_REFERENCED_OBJFIELD, CodeInfoImpl.WEAKLY_REFERENCED_OBJFIELD_COUNT); if (CodeInfoAccess.isAliveState(impl.getState())) { - continueVisiting = continueVisiting && CodeReferenceMapDecoder.walkOffsetsFromPointer(impl.getCodeStart(), - impl.getCodeConstantsReferenceMapEncoding(), impl.getCodeConstantsReferenceMapIndex(), visitor, null); + CodeReferenceMapDecoder.walkOffsetsFromPointer(impl.getCodeStart(), impl.getCodeConstantsReferenceMapEncoding(), impl.getCodeConstantsReferenceMapIndex(), visitor, null); } - continueVisiting = continueVisiting && NonmovableArrays.walkUnmanagedObjectArray(impl.getObjectConstants(), visitor); - continueVisiting = continueVisiting && NonmovableArrays.walkUnmanagedObjectArray(impl.getClasses(), visitor); - continueVisiting = continueVisiting && NonmovableArrays.walkUnmanagedObjectArray(impl.getMemberNames(), visitor); - continueVisiting = continueVisiting && NonmovableArrays.walkUnmanagedObjectArray(impl.getOtherStrings(), visitor); - continueVisiting = continueVisiting && NonmovableArrays.walkUnmanagedObjectArray(impl.getDeoptimizationObjectConstants(), visitor); - return continueVisiting; + NonmovableArrays.walkUnmanagedObjectArray(impl.getObjectConstants(), visitor); + NonmovableArrays.walkUnmanagedObjectArray(impl.getClasses(), visitor); + NonmovableArrays.walkUnmanagedObjectArray(impl.getMemberNames(), visitor); + NonmovableArrays.walkUnmanagedObjectArray(impl.getOtherStrings(), visitor); + NonmovableArrays.walkUnmanagedObjectArray(impl.getDeoptimizationObjectConstants(), visitor); } /** @@ -184,22 +181,16 @@ public static boolean walkWeakReferences(CodeInfo info, ObjectReferenceVisitor v * and/or {@link #walkWeakReferences} instead. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static boolean walkTether(CodeInfo info, ObjectReferenceVisitor visitor) { - Pointer address = NonmovableArrays.addressOf(cast(info).getObjectFields(), CodeInfoImpl.TETHER_OBJFIELD); - return callVisitor(visitor, address); - } - - @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) - private static boolean callVisitor(ObjectReferenceVisitor visitor, Pointer address) { - return visitor.visitObjectReference(address, true, null); + public static void walkTether(CodeInfo info, ObjectReferenceVisitor visitor) { + NonmovableArrays.walkUnmanagedObjectArray(cast(info).getObjectFields(), visitor, CodeInfoImpl.TETHER_OBJFIELD, 1); } /** * This method only visits a very specific subset of all the references, so you typically want * to use {@link #walkStrongReferences} and/or {@link #walkWeakReferences} instead. */ - public static boolean walkObjectFields(CodeInfo info, ObjectReferenceVisitor visitor) { - return NonmovableArrays.walkUnmanagedObjectArray(cast(info).getObjectFields(), visitor); + public static void walkObjectFields(CodeInfo info, ObjectReferenceVisitor visitor) { + NonmovableArrays.walkUnmanagedObjectArray(cast(info).getObjectFields(), visitor); } public static CodeInfo allocateMethodInfo() { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java index 41fa4efb8ec5..9e7f3eb1bc37 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java @@ -54,7 +54,6 @@ import com.oracle.svm.core.util.BasedOnJDKFile; import com.oracle.svm.core.util.UnsignedUtils; import com.oracle.svm.core.util.VMError; -import com.oracle.svm.core.util.coder.NativeCoder; import jdk.graal.compiler.api.replacements.Snippet; import jdk.graal.compiler.core.common.spi.ForeignCallDescriptor; @@ -149,7 +148,7 @@ private static Object doClone(Object original) throws CloneNotSupportedException int objectOffset = refMapPos.readInt(0); refMapPos = refMapPos.add(4); - long count = NativeCoder.readU4(refMapPos); + long count = refMapPos.readInt(0); refMapPos = refMapPos.add(4); /* Copy non-object data. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/CodeReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/CodeReferenceMapDecoder.java index f527aeaae498..f32a75e8494d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/CodeReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/CodeReferenceMapDecoder.java @@ -24,12 +24,8 @@ */ package com.oracle.svm.core.heap; -import jdk.graal.compiler.core.common.util.AbstractTypeReader; -import jdk.graal.compiler.core.common.util.UnsafeArrayTypeWriter; -import jdk.graal.compiler.word.Word; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; -import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.FrameAccess; @@ -39,27 +35,31 @@ import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.NonmovableByteArrayReader; +import jdk.graal.compiler.core.common.util.AbstractTypeReader; +import jdk.graal.compiler.core.common.util.UnsafeArrayTypeWriter; +import jdk.graal.compiler.word.Word; + @DuplicatedInNativeCode public class CodeReferenceMapDecoder { /** * Walk the reference map encoding from a Pointer, applying a visitor to each Object reference. * - * @param baseAddress A Pointer to a collections of primitives and Object references. + * @param baseAddress A Pointer to a collection of primitives and Object references. * @param referenceMapEncoding The encoding for the Object references in the collection. * @param referenceMapIndex The start index for the particular reference map in the encoding. * @param visitor The visitor to be applied to each Object reference. * @param holderObject The object containing the reference, or {@code null} if the reference is * not part of an object (which it is in case of a stack reference). - * @return false if any of the visits returned false, true otherwise. */ @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static boolean walkOffsetsFromPointer(PointerBase baseAddress, NonmovableArray referenceMapEncoding, long referenceMapIndex, ObjectReferenceVisitor visitor, Object holderObject) { + public static void walkOffsetsFromPointer(PointerBase baseAddress, NonmovableArray referenceMapEncoding, long referenceMapIndex, ObjectReferenceVisitor visitor, + Object holderObject) { assert referenceMapIndex != ReferenceMapIndex.NO_REFERENCE_MAP; assert referenceMapEncoding.isNonNull(); - UnsignedWord uncompressedSize = Word.unsigned(FrameAccess.uncompressedReferenceSize()); - UnsignedWord compressedSize = Word.unsigned(ConfigurationValues.getObjectLayout().getReferenceSize()); + int uncompressedSize = FrameAccess.uncompressedReferenceSize(); + int compressedSize = ConfigurationValues.getObjectLayout().getReferenceSize(); Pointer objRef = (Pointer) baseAddress; long idx = referenceMapIndex; @@ -117,21 +117,11 @@ public static boolean walkOffsetsFromPointer(PointerBase baseAddress, Nonmovable objRef = objRef.add(Word.unsigned(gap)); boolean compressed = (count < 0); - UnsignedWord refSize = compressed ? compressedSize : uncompressedSize; + int refSize = compressed ? compressedSize : uncompressedSize; count = (count < 0) ? -count : count; if (derived) { - /* - * To correctly relocate a derived pointer, we need to know the value pointed to by - * the base reference and the derived reference before either one is relocated. This - * allows us to compute the inner offset, i.e. how much into the actual object does - * the derived reference point to. - */ - Pointer basePtr = baseAddress.isNull() ? objRef : objRef.readWord(0); - - if (!callVisitObjectReferenceInline(visitor, objRef, 0, compressed, holderObject)) { - return false; - } + callVisitObjectReferencesInline(visitor, objRef, compressed, refSize, holderObject, 1); /* count in this case is the number of derived references for this base pointer */ for (long d = 0; d < count; d++) { @@ -156,27 +146,15 @@ public static boolean walkOffsetsFromPointer(PointerBase baseAddress, Nonmovable } else { derivedRef = objRef.subtract(Word.unsigned(-refOffset).multiply(refSize)); } - - Pointer derivedPtr = baseAddress.isNull() ? derivedRef : derivedRef.readWord(0); - long innerOffsetRaw = derivedPtr.subtract(basePtr).rawValue(); - int innerOffset = (int) innerOffsetRaw; - assert innerOffset == innerOffsetRaw; - - if (!callVisitObjectReferenceInline(visitor, derivedRef, innerOffset, compressed, holderObject)) { - return false; - } + callVisitDerivedReferenceInline(visitor, objRef, derivedRef, holderObject); } objRef = objRef.add(refSize); } else { - for (long c = 0; c < count; c += 1) { - if (!callVisitObjectReferenceInline(visitor, objRef, 0, compressed, holderObject)) { - return false; - } - objRef = objRef.add(refSize); - } + assert (int) count == count; + callVisitObjectReferencesInline(visitor, objRef, compressed, refSize, holderObject, (int) count); + objRef = objRef.add(Word.unsigned(count).multiply(refSize)); } } - return true; } /** Uninterruptible duplicate of {@link AbstractTypeReader#decodeSign}. */ @@ -187,7 +165,13 @@ public static long decodeSign(long value) { @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) - private static boolean callVisitObjectReferenceInline(ObjectReferenceVisitor visitor, Pointer derivedRef, int innerOffset, boolean compressed, Object holderObject) { - return visitor.visitObjectReferenceInline(derivedRef, innerOffset, compressed, holderObject); + private static void callVisitObjectReferencesInline(ObjectReferenceVisitor visitor, Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count) { + visitor.visitObjectReferences(firstObjRef, compressed, referenceSize, holderObject, count); + } + + @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static void callVisitDerivedReferenceInline(ObjectReferenceVisitor visitor, Pointer baseObjRef, Pointer derivedObjRef, Object holderObject) { + visitor.visitDerivedReference(baseObjRef, derivedObjRef, holderObject); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java index 826e56ee1c76..b911f1919ad0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java @@ -92,19 +92,19 @@ protected Heap() { * Walk all the objects in the heap. Must only be executed as part of a VM operation that causes * a safepoint. */ - public abstract boolean walkObjects(ObjectVisitor visitor); + public abstract void walkObjects(ObjectVisitor visitor); /** * Walk all native image heap objects. Must only be executed as part of a VM operation that * causes a safepoint. */ - public abstract boolean walkImageHeapObjects(ObjectVisitor visitor); + public abstract void walkImageHeapObjects(ObjectVisitor visitor); /** * Walk all heap objects except the native image heap objects. Must only be executed as part of * a VM operation that causes a safepoint. */ - public abstract boolean walkCollectedHeapObjects(ObjectVisitor visitor); + public abstract void walkCollectedHeapObjects(ObjectVisitor visitor); /** Returns the number of classes in the heap (initialized as well as uninitialized). */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java index 37a422bbce2e..0c0de56a7830 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java @@ -31,28 +31,35 @@ import org.graalvm.word.WordBase; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.graal.jdk.SubstrateObjectCloneSnippets; import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.NonmovableByteArrayReader; -import com.oracle.svm.core.util.coder.NativeCoder; import jdk.graal.compiler.word.Word; @DuplicatedInNativeCode public class InstanceReferenceMapDecoder { + @NeverInline("Non-performance critical version.") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE) + public static void walkReferences(Pointer baseAddress, InstanceReferenceMap referenceMap, ObjectReferenceVisitor visitor, Object holderObject) { + walkReferencesInline(baseAddress, referenceMap, visitor, holderObject); + } + /** This code is duplicated in {@link SubstrateObjectCloneSnippets}. */ @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public static boolean walkOffsetsFromPointer(Pointer baseAddress, InstanceReferenceMap referenceMap, ObjectReferenceVisitor visitor, Object holderObject) { + public static void walkReferencesInline(Pointer baseAddress, InstanceReferenceMap referenceMap, ObjectReferenceVisitor visitor, Object holderObject) { Pointer position = (Pointer) referenceMap; int entryCount = position.readInt(0); - position = position.add(4); + if (entryCount == 0) { + return; + } - int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); - boolean compressed = ReferenceAccess.singleton().haveCompressedReferences(); + position = position.add(4); assert entryCount >= 0; UnsignedWord sizeOfEntries = Word.unsigned(InstanceReferenceMapEncoder.MAP_ENTRY_SIZE).multiply(entryCount); @@ -61,19 +68,12 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, InstanceRefere int offset = position.readInt(0); position = position.add(4); - long count = NativeCoder.readU4(position); + int count = position.readInt(0); position = position.add(4); Pointer objRef = baseAddress.add(offset); - for (int c = 0; c < count; c++) { - final boolean visitResult = callVisitor(visitor, holderObject, compressed, objRef); - if (!visitResult) { - return false; - } - objRef = objRef.add(referenceSize); - } + callVisitor(visitor, holderObject, objRef, count); } - return true; } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) @@ -90,8 +90,9 @@ public static boolean isEmpty(InstanceReferenceMap referenceMap) { @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) - private static boolean callVisitor(ObjectReferenceVisitor visitor, Object holderObject, boolean compressed, Pointer objRef) { - return visitor.visitObjectReferenceInline(objRef, 0, compressed, holderObject); + private static void callVisitor(ObjectReferenceVisitor visitor, Object holderObject, Pointer objRef, int count) { + int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); + visitor.visitObjectReferences(objRef, true, referenceSize, holderObject, count); } public interface InstanceReferenceMap extends WordBase { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapEncoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapEncoder.java index 6a156b1400f8..62e14bf2c652 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapEncoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapEncoder.java @@ -40,7 +40,7 @@ *
  • entryCount entries with the following format: *
      *
    • int offset - the offset where a reference is located
      • - *
    • uint referenceCount - the number of adjacent references that are located at the offset
      • + *
    • int referenceCount - the number of adjacent references that are located at the offset
      • *
    *
    • * @@ -112,6 +112,6 @@ private long encode(ReferenceMapEncoder.OffsetIterator offsets) { private void encodeRun(int offset, int refsCount) { assert offset >= 0 && refsCount >= 0; writeBuffer.putS4(offset); - writeBuffer.putU4(refsCount); + writeBuffer.putS4(refsCount); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java index f51f41d7a382..b3d14c33220a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectReferenceVisitor.java @@ -24,34 +24,48 @@ */ package com.oracle.svm.core.heap; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import org.graalvm.word.Pointer; -import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.util.VMError; -/** Visitor for object references. */ +/** + * Visitor for object references in Java heap objects, Java stack frames, or off-heap data + * structures. + */ public interface ObjectReferenceVisitor { /** - * Visit an object reference. + * Visits a sequence of object references. Implementors of this method must loop over the + * references, which involves some boilerplate code. This code duplication is intentional as it + * reduces the number of virtual dispatches. * - * @param objRef Address of object reference to visit (not address of the referenced object). - * @param compressed True if the reference is in compressed form, false otherwise. + * @param firstObjRef Address where the first object reference is stored. + * @param compressed true if the references are regular Java references (like an instance + * field), false if they are absolute word-sized pointers (like an uncompressed + * pointer on the stack). + * @param referenceSize size in bytes of one reference * @param holderObject The object containing the reference, or {@code null} if the reference is - * not part of an object. - * @return {@code true} if visiting should continue, {@code false} if visiting should stop. + * not part of a Java object (e.g., the reference is on the stack or in a data + * structure that is located in native memory). + * @param count The number of object references. */ @RestrictHeapAccess(access = RestrictHeapAccess.Access.UNRESTRICTED, reason = "Some implementations allocate.") - boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject); + void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count); /** - * @param innerOffset If the reference is a {@linkplain CodeReferenceMapDecoder derived - * reference}, a positive integer that must be subtracted from the address to which - * the object reference points in order to get the start of the referenced object. + * Visits a derived reference. Derived references can only be on the stack or in a + * {@link StoredContinuation}. + * + * @param baseObjRef Address where the base reference is stored. + * @param derivedObjRef Address where the derived reference is stored. + * @param holderObject The object containing the reference, or {@code null} if the reference is + * not part of a Java object (e.g., the reference is on the stack or in a data + * structure that is located in native memory). */ - @AlwaysInline("GC performance") - @RestrictHeapAccess(access = RestrictHeapAccess.Access.UNRESTRICTED, reason = "Some implementations allocate.") - default boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { - VMError.guarantee(innerOffset == 0, "visitor does not support derived references"); - return visitObjectReference(objRef, compressed, holderObject); + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + default void visitDerivedReference(@SuppressWarnings("unused") Pointer baseObjRef, @SuppressWarnings("unused") Pointer derivedObjRef, @SuppressWarnings("unused") Object holderObject) { + throw VMError.shouldNotReachHere("Derived references are not supported by this visitor."); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java index ac706669506e..30fa9873a03a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectVisitor.java @@ -33,14 +33,7 @@ public interface ObjectVisitor { * Visit an Object. * * @param o The Object to be visited. - * @return true if visiting should continue, false if visiting should stop. */ @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting the heap.") - boolean visitObject(Object o); - - /** Like visitObject(Object), but inlined for performance. */ - @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while visiting the heap.") - default boolean visitObjectInline(Object o) { - return visitObject(o); - } + void visitObject(Object o); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java index 888520e14bcc..f19622256759 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java @@ -52,9 +52,8 @@ public final class PodReferenceMapDecoder { @DuplicatedInNativeCode @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEncoding, ObjectReferenceVisitor visitor, Object obj) { + public static void walkOffsetsFromPointer(Pointer baseAddress, int layoutEncoding, ObjectReferenceVisitor visitor, Object obj) { int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); - boolean isCompressed = ReferenceAccess.singleton().haveCompressedReferences(); UnsignedWord refOffset = LayoutEncoding.getArrayBaseOffset(layoutEncoding); UnsignedWord mapOffset = getReferenceMapOffset(obj, layoutEncoding); @@ -66,22 +65,17 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEnco gap = toUnsignedInt(baseAddress.readByte(mapOffset)); nrefs = toUnsignedInt(baseAddress.readByte(mapOffset.add(1))); - for (int i = 0; i < nrefs; i++) { - if (!callVisitor(baseAddress, visitor, obj, isCompressed, refOffset)) { - return false; - } - refOffset = refOffset.add(referenceSize); - } - refOffset = refOffset.add(referenceSize * gap); + Pointer firstObjRef = baseAddress.add(refOffset); + callVisitor(firstObjRef, visitor, obj, nrefs); + refOffset = refOffset.add(referenceSize * (nrefs + gap)); } while (gap != 0 || nrefs == 0xff); - - return true; } @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) - private static boolean callVisitor(Pointer baseAddress, ObjectReferenceVisitor visitor, Object obj, boolean isCompressed, UnsignedWord refOffset) { - return visitor.visitObjectReferenceInline(baseAddress.add(refOffset), 0, isCompressed, obj); + private static void callVisitor(Pointer firstObjRef, ObjectReferenceVisitor visitor, Object obj, int count) { + int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); + visitor.visitObjectReferences(firstObjRef, true, referenceSize, obj, count); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/RuntimeCodeCacheCleaner.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/RuntimeCodeCacheCleaner.java index 246cb1b7e27d..da0730f35281 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/RuntimeCodeCacheCleaner.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/RuntimeCodeCacheCleaner.java @@ -61,9 +61,9 @@ public RuntimeCodeCacheCleaner() { } @Override - public boolean visitCode(CodeInfo codeInfo) { + public void visitCode(CodeInfo codeInfo) { if (RuntimeCodeInfoAccess.areAllObjectsOnImageHeap(codeInfo)) { - return true; + return; } int state = CodeInfoAccess.getState(codeInfo); @@ -75,7 +75,6 @@ public boolean visitCode(CodeInfo codeInfo) { assert CodeInfoAccess.getState(codeInfo) == CodeInfo.STATE_REMOVED_FROM_CODE_CACHE; freeMemory(codeInfo); } - return true; } private static void freeMemory(CodeInfo codeInfo) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java index 1f3a0a2a633d..9a6e59eb4a54 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java @@ -185,9 +185,9 @@ public static boolean shouldWalkContinuation(StoredContinuation s) { @AlwaysInline("De-virtualize calls to ObjectReferenceVisitor") @Uninterruptible(reason = "StoredContinuation must not move.", callerMustBe = true) - public static boolean walkReferences(StoredContinuation s, ObjectReferenceVisitor visitor) { + public static void walkReferences(StoredContinuation s, ObjectReferenceVisitor visitor) { if (!shouldWalkContinuation(s)) { - return true; + return; } JavaStackWalk walk = StackValue.get(JavaStackWalker.sizeOfJavaStackWalk()); @@ -208,8 +208,6 @@ public static boolean walkReferences(StoredContinuation s, ObjectReferenceVisito CodeInfoAccess.releaseTether(untetheredCodeInfo, tether); } } - - return true; } @AlwaysInline("De-virtualize calls to visitor.") diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/UninterruptibleObjectVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/UninterruptibleObjectVisitor.java index 2b75b670ce13..8fd56edda19d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/UninterruptibleObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/UninterruptibleObjectVisitor.java @@ -24,16 +24,14 @@ */ package com.oracle.svm.core.heap; -import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.Uninterruptible; +/** + * Only the GC may have subclasses and only one subclass may be in an image at a time. Otherwise, we + * could end up with virtual calls in performance-critical code. + */ public interface UninterruptibleObjectVisitor extends ObjectVisitor { @Override @Uninterruptible(reason = "Implementations require uninterruptible walk.", callerMustBe = true) - boolean visitObject(Object o); - - @Override - @AlwaysInline("GC performance") - @Uninterruptible(reason = "Implementations require uninterruptible walk.", callerMustBe = true) - boolean visitObjectInline(Object o); + void visitObject(Object o); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpMetadata.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpMetadata.java index 465402c7f2ef..9e3a7e7b2428 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpMetadata.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpMetadata.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.heap.dump; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -53,6 +52,7 @@ import com.oracle.svm.core.util.coder.Pack200Coder; import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.word.Word; /** * Provides access to the encoded heap dump metadata that was prepared at image build-time. @@ -434,7 +434,7 @@ public void initialize() { } @Override - public boolean visitObject(Object o) { + public void visitObject(Object o) { if (o instanceof DynamicHub hub) { ClassInfo classInfo = HeapDumpMetadata.singleton().getClassInfo(hub.getTypeID()); assert classInfo.getHub() == null; @@ -442,7 +442,6 @@ public boolean visitObject(Object o) { classInfo.setSerialNum(++classSerialNum); classInfo.setInstanceFieldsDumpSize(-1); } - return true; } } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java index c019150f8342..fcc9316a2a41 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java @@ -1259,7 +1259,17 @@ private void markStackValuesAsGCRoots(Pointer sp, CodePointer ip, CodeInfo codeI @Override @RestrictHeapAccess(access = NO_ALLOCATION, reason = "Heap dumping must not allocate.") - public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { + public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count) { + Pointer pos = firstObjRef; + Pointer end = firstObjRef.add(Word.unsigned(count).multiply(referenceSize)); + while (pos.belowThan(end)) { + visitObjectReference(pos, compressed); + pos = pos.add(referenceSize); + } + } + + /** Derived references are not relevant for heap dumping, so we ignore innerOffset. */ + private void visitObjectReference(Pointer objRef, boolean compressed) { assert markGCRoots; Object obj = ReferenceAccess.singleton().readObjectAt(objRef, compressed); @@ -1271,7 +1281,6 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h /* Position of the stack frame in the stack trace. */ writeInt(getWrittenFrames()); } - return true; } private void visitFrame(FrameInfoQueryResult frame) { @@ -1355,7 +1364,7 @@ public void initialize(GrowableWordArray largeObjects) { @Override @RestrictHeapAccess(access = NO_ALLOCATION, reason = "Heap dumping must not allocate.") - public boolean visitObject(Object obj) { + public void visitObject(Object obj) { if (!isFillerObject(obj)) { if (isLarge(obj)) { boolean added = GrowableWordArrayAccess.add(largeObjects, Word.objectToUntrackedPointer(obj), NmtCategory.HeapDump); @@ -1366,7 +1375,6 @@ public boolean visitObject(Object obj) { writeObject(obj); } } - return true; } private static boolean isFillerObject(Object obj) { @@ -1402,19 +1410,27 @@ private class CodeMetadataVisitor implements RuntimeCodeCache.CodeInfoVisitor, O } @Override - public boolean visitCode(CodeInfo info) { + @RestrictHeapAccess(access = NO_ALLOCATION, reason = "Heap dumping must not allocate.") + public void visitCode(CodeInfo info) { RuntimeCodeInfoAccess.walkObjectFields(info, this); - return true; } @Override @RestrictHeapAccess(access = NO_ALLOCATION, reason = "Heap dumping must not allocate.") - public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { + public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count) { + Pointer pos = firstObjRef; + Pointer end = firstObjRef.add(Word.unsigned(count).multiply(referenceSize)); + while (pos.belowThan(end)) { + visitObjectReference(pos, compressed); + pos = pos.add(referenceSize); + } + } + + private void visitObjectReference(Pointer objRef, boolean compressed) { Object obj = ReferenceAccess.singleton().readObjectAt(objRef, compressed); if (obj != null) { markAsJniGlobalGCRoot(obj); } - return true; } } @@ -1432,12 +1448,20 @@ public void initialize(int threadSerialNum) { @Override @RestrictHeapAccess(access = NO_ALLOCATION, reason = "Heap dumping must not allocate.") - public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { + public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count) { + Pointer pos = firstObjRef; + Pointer end = firstObjRef.add(Word.unsigned(count).multiply(referenceSize)); + while (pos.belowThan(end)) { + visitObjectReference(pos, compressed); + pos = pos.add(referenceSize); + } + } + + private void visitObjectReference(Pointer objRef, boolean compressed) { Object obj = ReferenceAccess.singleton().readObjectAt(objRef, compressed); if (obj != null) { markThreadLocalAsGCRoot(obj); } - return true; } private void markThreadLocalAsGCRoot(Object obj) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index 33224c27de07..9463610bcef9 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -34,148 +34,149 @@ import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.heap.InstanceReferenceMapDecoder; import com.oracle.svm.core.heap.InstanceReferenceMapDecoder.InstanceReferenceMap; -import com.oracle.svm.core.heap.ObjectHeader; import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.heap.Pod; import com.oracle.svm.core.heap.PodReferenceMapDecoder; -import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.heap.ReferenceInternals; import com.oracle.svm.core.heap.StoredContinuation; import com.oracle.svm.core.heap.StoredContinuationAccess; +import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.thread.ContinuationSupport; import com.oracle.svm.core.util.VMError; import jdk.graal.compiler.nodes.java.ArrayLengthNode; import jdk.graal.compiler.word.Word; -/** - * The vanilla walkObject and walkOffsetsFromPointer methods are not inlined, but there are - * walkObjectInline and walkOffsetsFromPointerInline methods available for performance critical - * code. - */ - public class InteriorObjRefWalker { - /** * Walk a possibly-hybrid Object, consisting of both an array and some fixed fields. * * @param obj The Object to be walked. * @param visitor The visitor to be applied to each Object reference in the Object. - * @return True if the walk was successful, or false otherwise. */ @NeverInline("Non-performance critical version") @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).") - public static boolean walkObject(final Object obj, final ObjectReferenceVisitor visitor) { - return walkObjectInline(obj, visitor); + public static void walkObject(Object obj, ObjectReferenceVisitor visitor) { + walkObjectInline(obj, visitor); } - @AlwaysInline("Performance critical version") + /** + * Same as {@link #walkObject} but force-inlined. Performance-critical code should call this + * method instead of {@link #walkObject}. However, be aware that this increases code size. + */ + @AlwaysInline("GC performance") @Uninterruptible(reason = "Forced inlining (StoredContinuation objects must not move).", callerMustBe = true) - public static boolean walkObjectInline(final Object obj, final ObjectReferenceVisitor visitor) { - final DynamicHub objHub = ObjectHeader.readDynamicHubFromObject(obj); - final Pointer objPointer = Word.objectToUntrackedPointer(obj); + public static void walkObjectInline(Object obj, ObjectReferenceVisitor visitor) { + DynamicHub objHub = KnownIntrinsics.readHub(obj); + + int hubType = objHub.getHubType(); + if (HubType.isInstance(hubType)) { + /* Combine all instance cases to reduce the code size. */ + walkInstanceInline(obj, visitor, objHub); + } - switch (objHub.getHubType()) { + switch (hubType) { case HubType.INSTANCE: - return walkInstance(obj, visitor, objHub, objPointer); + case HubType.PRIMITIVE_ARRAY: + /* Nothing (more) to do. */ + return; case HubType.REFERENCE_INSTANCE: - return walkReferenceInstance(obj, visitor, objHub, objPointer); + walkReferenceSpecificFieldsInline(obj, visitor); + return; case HubType.POD_INSTANCE: - return walkPod(obj, visitor, objHub, objPointer); + walkPodArrayPartInline(obj, visitor, objHub); + return; case HubType.STORED_CONTINUATION_INSTANCE: - return walkStoredContinuation(obj, visitor); - case HubType.OTHER: - return walkOther(); - case HubType.PRIMITIVE_ARRAY: - return true; + walkStoredContinuationInline(obj, visitor); + return; case HubType.OBJECT_ARRAY: - return walkObjectArray(obj, visitor, objHub, objPointer); + walkObjectArrayInline(obj, visitor, objHub); + return; + case HubType.OTHER: + default: + throw VMError.shouldNotReachHere("Object with invalid hub type."); } - - throw VMError.shouldNotReachHere("Object with invalid hub type."); } - public static boolean walkInstanceReferenceOffsets(DynamicHub objHub, IntConsumer offsetConsumer) { + public static void walkInstanceReferenceOffsets(DynamicHub objHub, IntConsumer offsetConsumer) { if (objHub.getHubType() != HubType.INSTANCE && objHub.getHubType() != HubType.REFERENCE_INSTANCE) { throw new IllegalArgumentException("Unsupported hub type: " + objHub.getHubType()); } InstanceReferenceMap referenceMap = DynamicHubSupport.getInstanceReferenceMap(objHub); - return InstanceReferenceMapDecoder.walkOffsetsFromPointer(Word.zero(), referenceMap, new ObjectReferenceVisitor() { + InstanceReferenceMapDecoder.walkReferences(Word.zero(), referenceMap, new ObjectReferenceVisitor() { @Override - public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { + public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count) { + Pointer pos = firstObjRef; + Pointer end = firstObjRef.add(Word.unsigned(count).multiply(referenceSize)); + while (pos.belowThan(end)) { + visitObjectReference(pos); + pos = pos.add(referenceSize); + } + } + + private void visitObjectReference(Pointer objRef) { offsetConsumer.accept((int) objRef.rawValue()); - return true; } }, null); } - @AlwaysInline("Performance critical version") + @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static boolean walkInstance(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { + private static void walkInstanceInline(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub) { // Visit Object reference in the fields of the Object. + Pointer objPointer = Word.objectToUntrackedPointer(obj); InstanceReferenceMap referenceMap = DynamicHubSupport.getInstanceReferenceMap(objHub); - return InstanceReferenceMapDecoder.walkOffsetsFromPointer(objPointer, referenceMap, visitor, obj); + InstanceReferenceMapDecoder.walkReferencesInline(objPointer, referenceMap, visitor, obj); } - @AlwaysInline("Performance critical version") + @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static boolean walkReferenceInstance(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { + private static void walkReferenceSpecificFieldsInline(Object obj, ObjectReferenceVisitor visitor) { + Pointer objPointer = Word.objectToUntrackedPointer(obj); long discoveredOffset = ReferenceInternals.getNextDiscoveredFieldOffset(); Pointer objRef = objPointer.add(Word.unsigned(discoveredOffset)); - // The Object reference at the discovered offset needs to be visited separately as it is not - // part of the reference map. - // Visit Object reference in the fields of the Object. - return callVisitor(obj, visitor, ReferenceAccess.singleton().haveCompressedReferences(), objRef) && walkInstance(obj, visitor, objHub, objPointer); + /* + * The Object reference at the discovered offset needs to be visited separately as it is not + * part of the reference map. + */ + callVisitorInline(obj, visitor, objRef, 1); } - @AlwaysInline("Performance critical version") + @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static boolean walkPod(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { + private static void walkPodArrayPartInline(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub) { if (!Pod.RuntimeSupport.isPresent()) { throw VMError.shouldNotReachHere("Pod objects cannot be in the heap if the pod support is disabled."); } - return walkInstance(obj, visitor, objHub, objPointer) && PodReferenceMapDecoder.walkOffsetsFromPointer(objPointer, objHub.getLayoutEncoding(), visitor, obj); + + Pointer objPointer = Word.objectToUntrackedPointer(obj); + PodReferenceMapDecoder.walkOffsetsFromPointer(objPointer, objHub.getLayoutEncoding(), visitor, obj); } - @AlwaysInline("Performance critical version") + @AlwaysInline("GC performance") @Uninterruptible(reason = "StoredContinuation must not move.", callerMustBe = true) - private static boolean walkStoredContinuation(Object obj, ObjectReferenceVisitor visitor) { + private static void walkStoredContinuationInline(Object obj, ObjectReferenceVisitor visitor) { if (!ContinuationSupport.isSupported()) { throw VMError.shouldNotReachHere("Stored continuation objects cannot be in the heap if the continuation support is disabled."); } - return StoredContinuationAccess.walkReferences((StoredContinuation) obj, visitor); + StoredContinuationAccess.walkReferences((StoredContinuation) obj, visitor); } - @AlwaysInline("Performance critical version") + @AlwaysInline("GC performance") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static boolean walkOther() { - throw VMError.shouldNotReachHere("Unexpected object with hub type 'other' in the heap."); - } - - @AlwaysInline("Performance critical version") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static boolean walkObjectArray(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { + private static void walkObjectArrayInline(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub) { + Pointer objPointer = Word.objectToUntrackedPointer(obj); int length = ArrayLengthNode.arrayLength(obj); - int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); - boolean isCompressed = ReferenceAccess.singleton().haveCompressedReferences(); - - Pointer pos = objPointer.add(LayoutEncoding.getArrayBaseOffset(objHub.getLayoutEncoding())); - Pointer end = pos.add(Word.unsigned(referenceSize).multiply(length)); - while (pos.belowThan(end)) { - final boolean visitResult = callVisitor(obj, visitor, isCompressed, pos); - if (!visitResult) { - return false; - } - pos = pos.add(referenceSize); - } - return true; + Pointer firstObjRef = objPointer.add(LayoutEncoding.getArrayBaseOffset(objHub.getLayoutEncoding())); + callVisitorInline(obj, visitor, firstObjRef, length); } @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) - private static boolean callVisitor(Object obj, ObjectReferenceVisitor visitor, boolean isCompressed, Pointer pos) { - return visitor.visitObjectReferenceInline(pos, 0, isCompressed, obj); + private static void callVisitorInline(Object obj, ObjectReferenceVisitor visitor, Pointer firstObjRef, int count) { + int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); + visitor.visitObjectReferences(firstObjRef, true, referenceSize, obj, count); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/interpreter/InterpreterSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/interpreter/InterpreterSupport.java index 66155a429585..6cb7ff976d10 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/interpreter/InterpreterSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/interpreter/InterpreterSupport.java @@ -24,27 +24,27 @@ */ package com.oracle.svm.core.interpreter; -import com.oracle.svm.core.AlwaysInline; -import com.oracle.svm.core.BuildPhaseProvider; -import com.oracle.svm.core.FrameAccess; -import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.heap.ObjectReferenceVisitor; -import com.oracle.svm.core.heap.UnknownPrimitiveField; -import jdk.vm.ci.meta.ResolvedJavaMethod; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.function.CFunctionPointer; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; +import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.BuildPhaseProvider; +import com.oracle.svm.core.FrameAccess; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.code.FrameInfoQueryResult; import com.oracle.svm.core.code.FrameSourceInfo; +import com.oracle.svm.core.heap.ObjectReferenceVisitor; +import com.oracle.svm.core.heap.UnknownPrimitiveField; import jdk.graal.compiler.api.replacements.Fold; -import org.graalvm.word.UnsignedWord; - -import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; +import jdk.vm.ci.meta.ResolvedJavaMethod; /* Enables unoptimized execution of AOT compiled methods with an interpreter. The SVM * constraints apply, e.g. this itself does not enable class loading. */ @@ -128,7 +128,7 @@ public static void walkInterpreterLeaveStubFrame(ObjectReferenceVisitor visitor, /* Constant offset due to "deopt slot" */ int baseOffset = wordSize; Pointer objRef = sp.add(baseOffset + wordSize * referenceIndex); - callVisitor(visitor, objRef); + callVisitor(visitor, objRef, 1); referenceIndex++; gcReferenceMap >>= 1; @@ -136,8 +136,8 @@ public static void walkInterpreterLeaveStubFrame(ObjectReferenceVisitor visitor, } @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) - private static boolean callVisitor(ObjectReferenceVisitor visitor, Pointer address) { - return visitor.visitObjectReference(address, false, null); + private static void callVisitor(ObjectReferenceVisitor visitor, Pointer firstObjRef, int count) { + visitor.visitObjectReferences(firstObjRef, false, FrameAccess.uncompressedReferenceSize(), null, count); } /** diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalSupport.java index 26e79f8d3874..095968f2ab42 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalSupport.java @@ -78,7 +78,7 @@ public int getThreadLocalsReferenceMapIndex() { public void walk(IsolateThread isolateThread, ObjectReferenceVisitor referenceVisitor) { NonmovableArray threadRefMapEncoding = NonmovableArrays.fromImageHeap(vmThreadReferenceMapEncoding); InstanceReferenceMap referenceMap = InstanceReferenceMapDecoder.getReferenceMap(threadRefMapEncoding, vmThreadReferenceMapIndex); - InstanceReferenceMapDecoder.walkOffsetsFromPointer((Pointer) isolateThread, referenceMap, referenceVisitor, null); + InstanceReferenceMapDecoder.walkReferences((Pointer) isolateThread, referenceMap, referenceVisitor, null); } @Override diff --git a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmAllocation.java b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmAllocation.java index 45fd7d32fd18..ee90d6d836bf 100644 --- a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmAllocation.java +++ b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmAllocation.java @@ -809,21 +809,18 @@ public static void coalesceAt(Pointer blockBase, BlockHeader header) { } } - public static boolean walkObjects(ObjectVisitor visitor) { + public static void walkObjects(ObjectVisitor visitor) { Pointer currentBlock = MemoryLayout.getAllocatorBase(); while (currentBlock.belowThan(MemoryLayout.getAllocatorTop())) { BlockHeader header = StackValue.get(BlockHeader.class); readBlockHeader(currentBlock, header); if (header.getIsObject()) { - if (!visitor.visitObject(getInnerPointer(currentBlock).toObjectNonNull())) { - return false; - } + visitor.visitObject(getInnerPointer(currentBlock).toObjectNonNull()); } currentBlock = getNextBlock(currentBlock, header); } - return true; } /** diff --git a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmHeap.java b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmHeap.java index b313cab5fb21..f88d6fc6e5ff 100644 --- a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmHeap.java +++ b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmHeap.java @@ -138,30 +138,34 @@ public RuntimeCodeInfoGCSupport getRuntimeCodeInfoGCSupport() { } @Override - public boolean walkObjects(ObjectVisitor visitor) { + public void walkObjects(ObjectVisitor visitor) { VMOperation.guaranteeInProgressAtSafepoint("must only be executed at a safepoint"); - return walkImageHeapObjects(visitor) && walkCollectedHeapObjects(visitor); + walkImageHeapObjects(visitor); + walkCollectedHeapObjects(visitor); } @Override - public boolean walkImageHeapObjects(ObjectVisitor visitor) { + public void walkImageHeapObjects(ObjectVisitor visitor) { VMOperation.guaranteeInProgressAtSafepoint("Must only be called at a safepoint"); if (visitor != null) { - return ImageHeapWalker.walkImageHeapObjects(imageHeapInfo, visitor) && - (!AuxiliaryImageHeap.isPresent() || AuxiliaryImageHeap.singleton().walkObjects(visitor)); + ImageHeapWalker.walkImageHeapObjects(imageHeapInfo, visitor); + if (AuxiliaryImageHeap.isPresent()) { + AuxiliaryImageHeap.singleton().walkObjects(visitor); + } } - return true; } @Override - public boolean walkCollectedHeapObjects(ObjectVisitor visitor) { + public void walkCollectedHeapObjects(ObjectVisitor visitor) { VMOperation.guaranteeInProgressAtSafepoint("Must only be called at a safepoint"); - return WasmAllocation.walkObjects(visitor); + WasmAllocation.walkObjects(visitor); } - public boolean walkNativeImageHeapRegions(MemoryWalker.ImageHeapRegionVisitor visitor) { - return ImageHeapWalker.walkRegions(imageHeapInfo, visitor) && - (!AuxiliaryImageHeap.isPresent() || AuxiliaryImageHeap.singleton().walkRegions(visitor)); + public void walkNativeImageHeapRegions(MemoryWalker.ImageHeapRegionVisitor visitor) { + ImageHeapWalker.walkRegions(imageHeapInfo, visitor); + if (AuxiliaryImageHeap.isPresent()) { + AuxiliaryImageHeap.singleton().walkRegions(visitor); + } } @Override @@ -206,20 +210,18 @@ private ClassListBuilderVisitor(List> list) { } @Override - public boolean visitNativeImageHeapRegion(T region, MemoryWalker.NativeImageHeapRegionAccess access) { + public void visitNativeImageHeapRegion(T region, MemoryWalker.NativeImageHeapRegionAccess access) { if (!access.isWritable(region) && !access.consistsOfHugeObjects(region)) { access.visitObjects(region, this); } - return true; } @Override @RestrictHeapAccess(access = RestrictHeapAccess.Access.UNRESTRICTED, reason = "Allocation is fine: this method traverses only the image heap.") - public boolean visitObject(Object o) { + public void visitObject(Object o) { if (o instanceof Class) { list.add((Class) o); } - return true; } } diff --git a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmHeapVerifier.java b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmHeapVerifier.java index 8ae1f2f374a4..bfd608ee28e2 100644 --- a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmHeapVerifier.java +++ b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmHeapVerifier.java @@ -216,9 +216,8 @@ void initialize() { } @Override - public boolean visitObject(Object object) { + public void visitObject(Object object) { result &= verifyObject(object); - return true; } } @@ -234,20 +233,19 @@ private static class ImageHeapRegionVerifier extends ObjectVerifier implements M } @Override - public boolean visitNativeImageHeapRegion(T region, MemoryWalker.NativeImageHeapRegionAccess access) { + public void visitNativeImageHeapRegion(T region, MemoryWalker.NativeImageHeapRegionAccess access) { access.visitObjects(region, this); - return true; } @Override - public boolean visitObject(Object object) { + public void visitObject(Object object) { Word pointer = Word.objectToUntrackedPointer(object); if (!Heap.getHeap().isInImageHeap(object)) { Log.log().string("Image heap object ").zhex(pointer).string(" is not considered as part of the image heap.").newline(); result = false; } - return super.visitObject(object); + super.visitObject(object); } } @@ -266,9 +264,17 @@ public void initialize() { } @Override - public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { + public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count) { + Pointer pos = firstObjRef; + Pointer end = firstObjRef.add(Word.unsigned(count).multiply(referenceSize)); + while (pos.belowThan(end)) { + visitObjectReference(pos, compressed, holderObject); + pos = pos.add(referenceSize); + } + } + + private void visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { result &= verifyReference(holderObject, objRef, compressed); - return true; } } } diff --git a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmLMGC.java b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmLMGC.java index 4bdbf3a5f5c5..d4df07c456f4 100644 --- a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmLMGC.java +++ b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmLMGC.java @@ -48,6 +48,7 @@ import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.annotate.Substitute; import com.oracle.svm.core.annotate.TargetClass; +import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.HeapVerifier; import com.oracle.svm.core.heap.GC; import com.oracle.svm.core.heap.GCCause; @@ -167,9 +168,7 @@ public boolean visitFrame(Pointer sp, CodePointer ip) { // All objects reachable from the stack have to first be marked gray. grayToBlackObjectVisitor.promoteToGray(o); - if (!grayToBlackObjectVisitor.visitObjectInline(o)) { - return false; - } + grayToBlackObjectVisitor.visitObject(o); } } @@ -371,8 +370,6 @@ private static void releaseSpace() { VMError.guarantee(WasmObjectHeader.isBlackObject(o), "Found gray object after mark phase"); WasmObjectHeader.markWhite(o); } - - return true; }); WasmAllocation.coalesce(); @@ -474,11 +471,10 @@ final class BlackenImageHeapRootsVisitor implements MemoryWalker.ImageHeapRegion } @Override - public boolean visitNativeImageHeapRegion(T region, MemoryWalker.NativeImageHeapRegionAccess access) { + public void visitNativeImageHeapRegion(T region, MemoryWalker.NativeImageHeapRegionAccess access) { if (access.isWritable(region)) { access.visitObjects(region, visitor); } - return true; } } @@ -547,11 +543,6 @@ private static boolean isGray(Object o) { return WasmObjectHeader.isGrayObject(o); } - @Override - public boolean visitObject(Object o) { - return visitObjectInline(o); - } - /** * Tries to mark the given objects as well as all objects reachable from it as black. *

    @@ -561,9 +552,9 @@ public boolean visitObject(Object o) { */ @Override @AlwaysInline("GC performance") - public boolean visitObjectInline(Object o) { + public void visitObject(Object o) { if (!isGray(o)) { - return true; + return; } assert worklist.isEmpty() : "Worklist must be empty before every visit"; @@ -581,14 +572,12 @@ public boolean visitObjectInline(Object o) { if (probability(SLOW_PATH_PROBABILITY, KnownIntrinsics.readHub(obj).isReferenceInstanceClass())) { discoverReference(obj, grayReferenceVisitor); } - boolean shouldContinue = InteriorObjRefWalker.walkObject(obj, grayReferenceVisitor); - assert shouldContinue : "Walk of an object was not successful"; + InteriorObjRefWalker.walkObject(obj, grayReferenceVisitor); // The object no longer has any references to white objects, it can be marked black promoteToBlack(obj); } - return true; } /** @@ -614,7 +603,8 @@ private static void discoverReference(Object obj, ObjectReferenceVisitor referen // Always mark referent as reachable (no support for soft or weak references yet) // TODO GR-43486 allow for soft and weak referenced referents to be potentially freed by the // GC - referenceVisitor.visitObjectReferenceInline(ReferenceInternals.getReferentFieldAddress(dr), 0, false, dr); + int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); + referenceVisitor.visitObjectReferences(ReferenceInternals.getReferentFieldAddress(dr), false, referenceSize, dr, 1); } /** @@ -622,26 +612,26 @@ private static void discoverReference(Object obj, ObjectReferenceVisitor referen */ private final class GrayHeapVisitor implements ObjectReferenceVisitor { @Override - public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { - return visitObjectReferenceInline(objRef, 0, compressed, holderObject); + @AlwaysInline("GC performance") + public void visitObjectReferences(Pointer firstObjRef, boolean compressed, int referenceSize, Object holderObject, int count) { + Pointer pos = firstObjRef; + Pointer end = firstObjRef.add(Word.unsigned(count).multiply(referenceSize)); + while (pos.belowThan(end)) { + visitObjectReference(pos, compressed); + pos = pos.add(referenceSize); + } } - @Override - @AlwaysInline("GC performance") - public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { - assert innerOffset >= 0 : innerOffset; + public void visitObjectReference(Pointer objRef, boolean compressed) { assert !objRef.isNull() : "Tried to visit object references of null object"; - Pointer offsetP = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); - assert offsetP.isNonNull() || innerOffset == 0 : innerOffset; - - Pointer p = offsetP.subtract(innerOffset); + Pointer p = ReferenceAccess.singleton().readObjectAsUntrackedPointer(objRef, compressed); if (p.isNull()) { - return true; + return; } if (WasmHeap.getHeapImpl().isInImageHeap(p)) { - return true; + return; } ObjectHeader oh = Heap.getHeap().getObjectHeader(); @@ -652,7 +642,7 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole * processing. */ if (!WasmObjectHeader.isWhiteHeader(header)) { - return true; + return; } Object obj = p.toObjectNonNull(); @@ -666,7 +656,6 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole worklist.push(obj); } } - return true; } } } diff --git a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmStackVerifier.java b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmStackVerifier.java index b33f48612072..f4c5b0ea84b6 100644 --- a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmStackVerifier.java +++ b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmStackVerifier.java @@ -31,6 +31,7 @@ import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.word.Pointer; +import com.oracle.svm.core.FrameAccess; import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.snippets.KnownIntrinsics; @@ -79,7 +80,7 @@ public boolean visitFrame(Pointer currentSP, CodePointer currentIP) { WebImageWasmStackWalker.getCodeInfo(currentIP, queryResult); for (int offset : queryResult.getOffsets()) { - verifyFrameReferencesVisitor.visitObjectReference(currentSP.add(offset), false, null); + verifyFrameReferencesVisitor.visitObjectReferences(currentSP.add(offset), false, FrameAccess.uncompressedReferenceSize(), null, 1); } return true; diff --git a/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSHeap.java b/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSHeap.java index 1a722708322b..937671723e7f 100644 --- a/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSHeap.java +++ b/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSHeap.java @@ -94,18 +94,15 @@ public RuntimeCodeInfoGCSupport getRuntimeCodeInfoGCSupport() { } @Override - public boolean walkObjects(ObjectVisitor visitor) { - return false; + public void walkObjects(ObjectVisitor visitor) { } @Override - public boolean walkImageHeapObjects(ObjectVisitor visitor) { - return false; + public void walkImageHeapObjects(ObjectVisitor visitor) { } @Override - public boolean walkCollectedHeapObjects(ObjectVisitor visitor) { - return false; + public void walkCollectedHeapObjects(ObjectVisitor visitor) { } @Override