diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index cc98db104231..32ec5998afeb 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -75,6 +75,7 @@ import com.oracle.svm.core.nodes.CFunctionEpilogueNode; import com.oracle.svm.core.nodes.CFunctionPrologueNode; import com.oracle.svm.core.option.RuntimeOptionKey; +import com.oracle.svm.core.os.ImageHeapProvider; import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.thread.PlatformThreads; import com.oracle.svm.core.thread.ThreadStatus; @@ -461,6 +462,17 @@ public static boolean isImageHeapAligned() { return SubstrateOptions.SpawnIsolates.getValue(); } + @Override + public UnsignedWord getImageHeapReservedBytes() { + return ImageHeapProvider.get().getImageHeapAddressSpaceSize(); + } + + @Override + public UnsignedWord getImageHeapCommittedBytes() { + int imageHeapOffset = HeapImpl.getHeapImpl().getImageHeapOffsetInAddressSpace(); + return ImageHeapProvider.get().getImageHeapAddressSpaceSize().subtract(imageHeapOffset); + } + @Override public boolean walkImageHeapObjects(ObjectVisitor visitor) { VMOperation.guaranteeInProgressAtSafepoint("Must only be called at a safepoint"); diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java index f8daf0193386..a294ca73f99b 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java @@ -147,12 +147,11 @@ private static UnsignedWord getLayeredImageHeapAddressSpaceSize() { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected UnsignedWord getImageHeapAddressSpaceSize() { + public UnsignedWord getImageHeapAddressSpaceSize() { if (ImageLayerBuildingSupport.buildingImageLayer()) { return getLayeredImageHeapAddressSpaceSize(); - } else { - return super.getImageHeapAddressSpaceSize(); } + return super.getImageHeapAddressSpaceSize(); } @Uninterruptible(reason = "Called during isolate initialization.") diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java index 093925e3a9d3..d239471b23a0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java @@ -231,6 +231,10 @@ public Pointer getImageHeapStart() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public abstract UnsignedWord getUsedMemoryAfterLastGC(); + public abstract UnsignedWord getImageHeapReservedBytes(); + + public abstract UnsignedWord getImageHeapCommittedBytes(); + /** Consider all references in the given object as needing remembered set entries. */ @Uninterruptible(reason = "Ensure that no GC can occur between modification of the object and this call.", callerMustBe = true) public abstract void dirtyAllReferencesOf(Object obj); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/events/EveryChunkNativePeriodicEvents.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/events/EveryChunkNativePeriodicEvents.java index 73e40876004c..90e0300ce3db 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/events/EveryChunkNativePeriodicEvents.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/events/EveryChunkNativePeriodicEvents.java @@ -124,50 +124,67 @@ private static void emitNativeMemoryTrackingEvents() { private static void emitNmtPeakEvents() { NativeMemoryUsageTotalPeakEvent nmtTotalPeakEvent = new NativeMemoryUsageTotalPeakEvent(); - long totalPeakUsed = NativeMemoryTracking.singleton().getPeakTotalUsedMemory(); - nmtTotalPeakEvent.peakCommitted = totalPeakUsed; - nmtTotalPeakEvent.peakReserved = totalPeakUsed; - nmtTotalPeakEvent.countAtPeak = NativeMemoryTracking.singleton().getCountAtTotalPeakUsage(); + long totalPeakMalloc = NativeMemoryTracking.singleton().getPeakTotalMallocMemory(); + long totalPeakCommittedVM = NativeMemoryTracking.singleton().getPeakTotalCommittedVirtualMemory(); + long totalPeakReservedVM = NativeMemoryTracking.singleton().getPeakTotalReservedVirtualMemory(); + + nmtTotalPeakEvent.peakCommitted = totalPeakCommittedVM + totalPeakMalloc; + nmtTotalPeakEvent.peakReserved = totalPeakReservedVM + totalPeakMalloc; + nmtTotalPeakEvent.countAtPeak = NativeMemoryTracking.singleton().getCountAtPeakTotalMallocMemory(); nmtTotalPeakEvent.commit(); for (NmtCategory nmtCategory : NmtCategory.values()) { NativeMemoryUsagePeakEvent nmtPeakEvent = new NativeMemoryUsagePeakEvent(); nmtPeakEvent.type = nmtCategory.getName(); - long peakUsed = NativeMemoryTracking.singleton().getPeakUsedMemory(nmtCategory); - nmtPeakEvent.peakCommitted = peakUsed; - nmtPeakEvent.peakReserved = peakUsed; - nmtPeakEvent.countAtPeak = NativeMemoryTracking.singleton().getCountAtPeakUsage(nmtCategory); + long peakMalloc = NativeMemoryTracking.singleton().getPeakMallocMemory(nmtCategory); + long peakCommittedVM = NativeMemoryTracking.singleton().getPeakCommittedVirtualMemory(nmtCategory); + long peakReservedVM = NativeMemoryTracking.singleton().getPeakReservedVirtualMemory(nmtCategory); + nmtPeakEvent.peakCommitted = peakCommittedVM + peakMalloc; + nmtPeakEvent.peakReserved = peakReservedVM + peakMalloc; + nmtPeakEvent.countAtPeak = NativeMemoryTracking.singleton().getCountAtPeakMallocMemory(nmtCategory); nmtPeakEvent.commit(); } } @Uninterruptible(reason = "Accesses a JFR buffer.") private static void emitJdkNmtEvents(NmtCategory[] nmtCategories) { - long timestamp = JfrTicks.elapsedTicks(); + boolean emitNativeMemoryUsageEvent = JfrEvent.NativeMemoryUsage.shouldEmit(); + boolean emitNativeMemoryUsageTotalEvent = JfrEvent.NativeMemoryUsageTotal.shouldEmit(); + if (!emitNativeMemoryUsageEvent && !emitNativeMemoryUsageTotalEvent) { + return; + } + JfrNativeEventWriterData data = StackValue.get(JfrNativeEventWriterData.class); JfrNativeEventWriterDataAccess.initializeThreadLocalNativeBuffer(data); + long timestamp = JfrTicks.elapsedTicks(); - if (JfrEvent.NativeMemoryUsage.shouldEmit()) { + if (emitNativeMemoryUsageEvent) { for (NmtCategory nmtCategory : nmtCategories) { - long usedMemory = NativeMemoryTracking.singleton().getUsedMemory(nmtCategory); + NativeMemoryTracking nmt = NativeMemoryTracking.singleton(); + long mallocMemory = nmt.getMallocMemory(nmtCategory); + long committedVM = nmt.getCommittedVirtualMemory(nmtCategory); + long reservedVM = nmt.getReservedVirtualMemory(nmtCategory); JfrNativeEventWriter.beginSmallEvent(data, JfrEvent.NativeMemoryUsage); JfrNativeEventWriter.putLong(data, timestamp); JfrNativeEventWriter.putLong(data, nmtCategory.ordinal()); - JfrNativeEventWriter.putLong(data, usedMemory); // reserved - JfrNativeEventWriter.putLong(data, usedMemory); // committed + JfrNativeEventWriter.putLong(data, mallocMemory + reservedVM); + JfrNativeEventWriter.putLong(data, mallocMemory + committedVM); JfrNativeEventWriter.endSmallEvent(data); } } - if (JfrEvent.NativeMemoryUsageTotal.shouldEmit()) { - long totalUsedMemory = NativeMemoryTracking.singleton().getTotalUsedMemory(); + if (emitNativeMemoryUsageTotalEvent) { + NativeMemoryTracking nmt = NativeMemoryTracking.singleton(); + long totalMallocMemory = nmt.getTotalMallocMemory(); + long totalCommittedVM = nmt.getTotalCommittedVirtualMemory(); + long totalReservedVM = nmt.getTotalReservedVirtualMemory(); JfrNativeEventWriter.beginSmallEvent(data, JfrEvent.NativeMemoryUsageTotal); JfrNativeEventWriter.putLong(data, timestamp); - JfrNativeEventWriter.putLong(data, totalUsedMemory); // reserved - JfrNativeEventWriter.putLong(data, totalUsedMemory); // committed + JfrNativeEventWriter.putLong(data, totalMallocMemory + totalReservedVM); + JfrNativeEventWriter.putLong(data, totalMallocMemory + totalCommittedVM); JfrNativeEventWriter.endSmallEvent(data); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java index 149d897ede1a..74d416a33aa8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java @@ -1,6 +1,6 @@ /* - * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2024, Red Hat Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -39,6 +39,7 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.VMInspectionOptions; +import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.jdk.RuntimeSupport; import com.oracle.svm.core.memory.NativeMemory; import com.oracle.svm.core.util.UnsignedUtils; @@ -53,20 +54,35 @@ * For tracking memory allocations, we have an internal API (see {@link NativeMemory}) that adds a * custom {@link NmtMallocHeader header} to each allocation if NMT is enabled. This header stores * data that is needed to properly untrack the memory when it is freed. + * + * Virtual memory tracking makes the assumption that commits within a reserved region happen neatly. + * There will never be overlapping commits and the size requested to be committed/uncommitted is + * exactly the size committed/uncommitted. In Hotspot, this assumption is not made, and an internal + * model of virtual memory is maintained. */ public class NativeMemoryTracking { private static final UnsignedWord ALIGNMENT = WordFactory.unsigned(16); private static final int MAGIC = 0xF0F1F2F3; + private static final long KB = 1024; - private final NmtMallocMemoryInfo[] categories; - private final NmtMallocMemoryInfo total; + private final NmtMallocMemoryInfo[] mallocCategories; + private final NmtVirtualMemoryInfo[] virtualMemCategories; + private final NmtMallocMemoryInfo mallocTotal; + private final NmtVirtualMemoryInfo virtualMemTotal; @Platforms(Platform.HOSTED_ONLY.class) public NativeMemoryTracking() { - total = new NmtMallocMemoryInfo(); - categories = new NmtMallocMemoryInfo[NmtCategory.values().length]; - for (int i = 0; i < categories.length; i++) { - categories[i] = new NmtMallocMemoryInfo(); + mallocTotal = new NmtMallocMemoryInfo(); + virtualMemTotal = new NmtVirtualMemoryInfo(); + + mallocCategories = new NmtMallocMemoryInfo[NmtCategory.values().length]; + for (int i = 0; i < mallocCategories.length; i++) { + mallocCategories[i] = new NmtMallocMemoryInfo(); + } + + virtualMemCategories = new NmtVirtualMemoryInfo[NmtCategory.values().length]; + for (int i = 0; i < virtualMemCategories.length; i++) { + virtualMemCategories[i] = new NmtVirtualMemoryInfo(); } } @@ -116,9 +132,9 @@ public void track(PointerBase innerPtr) { UnsignedWord allocationSize = header.getAllocationSize(); UnsignedWord totalSize = allocationSize.add(nmtHeaderSize); - getInfo(header.getCategory()).track(allocationSize); - getInfo(NmtCategory.NMT).track(nmtHeaderSize); - total.track(totalSize); + getMallocInfo(header.getCategory()).track(allocationSize); + getMallocInfo(NmtCategory.NMT).track(nmtHeaderSize); + mallocTotal.track(totalSize); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) @@ -134,9 +150,9 @@ public PointerBase untrack(PointerBase innerPtr) { @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public void untrack(UnsignedWord size, int category) { - getInfo(category).untrack(size); - getInfo(NmtCategory.NMT).untrack(sizeOfNmtHeader()); - total.untrack(size.add(sizeOfNmtHeader())); + getMallocInfo(category).untrack(size); + getMallocInfo(NmtCategory.NMT).untrack(sizeOfNmtHeader()); + mallocTotal.untrack(size.add(sizeOfNmtHeader())); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) @@ -152,73 +168,201 @@ private static Pointer getInnerPointer(NmtMallocHeader mallocHeader) { } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public long getUsedMemory(NmtCategory category) { - return getInfo(category).getUsed(); + public void trackReserve(UnsignedWord size, NmtCategory category) { + trackReserve(size.rawValue(), category); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void trackReserve(long size, NmtCategory category) { + getVirtualInfo(category).trackReserved(size); + virtualMemTotal.trackReserved(size); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void trackCommit(UnsignedWord size, NmtCategory category) { + trackCommit(size.rawValue(), category); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void trackCommit(long size, NmtCategory category) { + getVirtualInfo(category).trackCommitted(size); + virtualMemTotal.trackCommitted(size); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void trackUncommit(UnsignedWord size, NmtCategory category) { + trackUncommit(size.rawValue(), category); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void trackUncommit(long size, NmtCategory category) { + getVirtualInfo(category).trackUncommit(size); + virtualMemTotal.trackUncommit(size); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void trackFree(UnsignedWord size, NmtCategory category) { + trackFree(size.rawValue(), category); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void trackFree(long size, NmtCategory category) { + getVirtualInfo(category).trackFree(size); + virtualMemTotal.trackFree(size); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getMallocMemory(NmtCategory category) { + return getMallocInfo(category).getUsed(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getMallocCount(NmtCategory category) { + return getMallocInfo(category).getCount(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getPeakMallocMemory(NmtCategory category) { + return getMallocInfo(category).getPeakUsed(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public long getPeakUsedMemory(NmtCategory category) { - return getInfo(category).getPeakUsed(); + public long getCountAtPeakMallocMemory(NmtCategory category) { + return getMallocInfo(category).getCountAtPeakUsage(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public long getCountAtPeakUsage(NmtCategory category) { - return getInfo(category).getCountAtPeakUsage(); + public long getTotalMallocCount() { + return mallocTotal.getCount(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public long getTotalCount() { - return total.getCount(); + public long getTotalMallocMemory() { + return mallocTotal.getUsed(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public long getTotalUsedMemory() { - return total.getUsed(); + public long getPeakTotalMallocMemory() { + return mallocTotal.getPeakUsed(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public long getPeakTotalUsedMemory() { - return total.getPeakUsed(); + public long getCountAtPeakTotalMallocMemory() { + return mallocTotal.getCountAtPeakUsage(); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public long getCountAtTotalPeakUsage() { - return total.getCountAtPeakUsage(); + public long getReservedVirtualMemory(NmtCategory category) { + return getVirtualInfo(category).getReservedSize(); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private NmtMallocMemoryInfo getInfo(NmtCategory category) { - return getInfo(category.ordinal()); + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getCommittedVirtualMemory(NmtCategory category) { + return getVirtualInfo(category).getCommittedSize(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getPeakReservedVirtualMemory(NmtCategory category) { + return getVirtualInfo(category).getPeakReservedSize(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getPeakCommittedVirtualMemory(NmtCategory category) { + return getVirtualInfo(category).getPeakCommittedSize(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getTotalReservedVirtualMemory() { + return virtualMemTotal.getReservedSize(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getTotalCommittedVirtualMemory() { + return virtualMemTotal.getCommittedSize(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getPeakTotalReservedVirtualMemory() { + return virtualMemTotal.getPeakReservedSize(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public long getPeakTotalCommittedVirtualMemory() { + return virtualMemTotal.getPeakCommittedSize(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private NmtMallocMemoryInfo getMallocInfo(int category) { + return mallocCategories[category]; } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private NmtMallocMemoryInfo getInfo(int category) { - assert category < categories.length; - return categories[category]; + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private NmtMallocMemoryInfo getMallocInfo(NmtCategory category) { + return getMallocInfo(category.ordinal()); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private NmtVirtualMemoryInfo getVirtualInfo(NmtCategory category) { + return getVirtualInfo(category.ordinal()); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private NmtVirtualMemoryInfo getVirtualInfo(int category) { + return virtualMemCategories[category]; + } + + public static RuntimeSupport.Hook initializationHook() { + return isFirstIsolate -> { + // Track the image heap virtual memory usage. + NativeMemoryTracking.singleton().trackReserve(Heap.getHeap().getImageHeapReservedBytes(), NmtCategory.ImageHeap); + NativeMemoryTracking.singleton().trackCommit(Heap.getHeap().getImageHeapCommittedBytes(), NmtCategory.ImageHeap); + }; } public static RuntimeSupport.Hook shutdownHook() { - return isFirstIsolate -> NativeMemoryTracking.singleton().printStatistics(); + return isFirstIsolate -> { + NativeMemoryTracking.singleton().printStatistics(); + }; } private void printStatistics() { if (VMInspectionOptions.PrintNMTStatistics.getValue()) { - System.out.println(); - System.out.println("Native memory tracking"); - System.out.println(" Peak total used memory: " + getPeakTotalUsedMemory() + " bytes"); - System.out.println(" Total alive allocations at peak usage: " + getCountAtTotalPeakUsage()); - System.out.println(" Total used memory: " + getTotalUsedMemory() + " bytes"); - System.out.println(" Total alive allocations: " + getTotalCount()); - - for (int i = 0; i < NmtCategory.values().length; i++) { - String name = NmtCategory.values()[i].getName(); - NmtMallocMemoryInfo info = getInfo(i); - - System.out.println(" " + name + " peak used memory: " + info.getPeakUsed() + " bytes"); - System.out.println(" " + name + " alive allocations at peak: " + info.getCountAtPeakUsage()); - System.out.println(" " + name + " currently used memory: " + info.getUsed() + " bytes"); - System.out.println(" " + name + " currently alive allocations: " + info.getCount()); - } + System.out.println(generateReportString()); + } + } + + public String generateReportString() { + String lineBreak = System.lineSeparator(); + + StringBuilder result = new StringBuilder(3000); + result.append(lineBreak); + result.append("Native memory tracking").append(lineBreak).append(lineBreak); + + result.append("Total").append(lineBreak); + long reservedTotal = (getTotalReservedVirtualMemory() + getTotalMallocMemory()) / KB; + long committedTotal = (getTotalCommittedVirtualMemory() + getTotalMallocMemory()) / KB; + result.append(" ").append("(reserved=").append(reservedTotal).append("KB, committed=").append(committedTotal).append("KB)").append(lineBreak); + result.append(" ").append("(malloc=").append(getTotalMallocMemory() / KB).append("KB, count=").append(getTotalMallocCount()).append(")").append(lineBreak); + result.append(" ").append("(peak malloc=").append(getPeakTotalMallocMemory() / KB).append("KB, count at peak=").append(getCountAtPeakTotalMallocMemory()).append(")").append(lineBreak); + result.append(" ").append("(mmap: reserved=").append(getTotalReservedVirtualMemory() / KB).append("KB, committed=").append(getTotalCommittedVirtualMemory() / KB).append("KB)") + .append(lineBreak); + result.append(" ").append("(mmap: peak reserved=").append(getPeakTotalReservedVirtualMemory() / KB).append("KB, peak committed=").append(getPeakTotalCommittedVirtualMemory() / KB) + .append("KB)").append(lineBreak); + + for (int i = 0; i < NmtCategory.values().length; i++) { + NmtCategory category = NmtCategory.values()[i]; + result.append(category.getName()).append(lineBreak); + long reserved = (getReservedVirtualMemory(category) + getMallocMemory(category)) / KB; + long committed = (getCommittedVirtualMemory(category) + getMallocMemory(category)) / KB; + result.append(" ").append("(reserved=").append(reserved).append("KB, committed=").append(committed).append("KB)").append(lineBreak); + result.append(" ").append("(malloc=").append(getMallocMemory(category) / KB).append("KB, count=").append(getMallocCount(category)).append(")").append(lineBreak); + result.append(" ").append("(peak malloc=").append(getPeakMallocMemory(category) / KB).append("KB, count at peak=").append(getCountAtPeakMallocMemory(category)).append(")") + .append(lineBreak); + result.append(" ").append("(mmap: reserved=").append(getReservedVirtualMemory(category) / KB).append("KB, committed=").append(getCommittedVirtualMemory(category) / KB) + .append("KB)").append(lineBreak); + result.append(" ").append("(mmap: peak reserved=").append(getPeakReservedVirtualMemory(category) / KB).append("KB, peak committed=") + .append(getPeakCommittedVirtualMemory(category) / KB).append("KB)").append(lineBreak); } + return result.toString(); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java index ff71cdfabaed..4a629e1f5bb9 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java @@ -37,6 +37,10 @@ public enum NmtCategory { GC("GC"), /** Heap dumping infrastructure. */ HeapDump("Heap Dump"), + /** Image heap (may include GC-specific data). */ + ImageHeap("Image Heap"), + /** Collected Java heap (may include GC-specific data). */ + JavaHeap("Java Heap"), /** Java Flight Recorder. */ JFR("JFR"), /** Java Native Interface. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java index 9b7786681fe4..120317ca8292 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java @@ -1,6 +1,6 @@ /* - * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2024, Red Hat Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -47,6 +47,7 @@ public void afterRegistration(AfterRegistrationAccess access) { @Override public void beforeAnalysis(BeforeAnalysisAccess access) { + RuntimeSupport.getRuntimeSupport().addInitializationHook(NativeMemoryTracking.initializationHook()); RuntimeSupport.getRuntimeSupport().addShutdownHook(NativeMemoryTracking.shutdownHook()); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryInfo.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryInfo.java new file mode 100644 index 000000000000..8da50ecf4895 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryInfo.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2024, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicLong; + +class NmtVirtualMemoryInfo { + + private final AtomicLong peakReservedSize = new AtomicLong(0); + private final AtomicLong peakCommittedSize = new AtomicLong(0); + private final AtomicLong reservedSize = new AtomicLong(0); + private final AtomicLong committedSize = new AtomicLong(0); + + @Platforms(Platform.HOSTED_ONLY.class) + NmtVirtualMemoryInfo() { + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + void trackReserved(long size) { + long newReservedSize = reservedSize.addAndGet(size); + updatePeak(newReservedSize, peakReservedSize); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + void trackCommitted(long size) { + long newCommittedSize = committedSize.addAndGet(size); + updatePeak(newCommittedSize, peakCommittedSize); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + void trackUncommit(long size) { + long lastSize = committedSize.addAndGet(-size); + assert lastSize >= 0; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + void trackFree(long size) { + long lastSize = reservedSize.addAndGet(-size); + assert lastSize >= 0; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static void updatePeak(long newSize, AtomicLong peakToUpdate) { + long oldPeak; + do { + oldPeak = peakToUpdate.get(); + } while (newSize > oldPeak && !peakToUpdate.compareAndSet(oldPeak, newSize)); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + long getReservedSize() { + return reservedSize.get(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + long getCommittedSize() { + return committedSize.get(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + long getPeakReservedSize() { + return peakReservedSize.get(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + long getPeakCommittedSize() { + return peakCommittedSize.get(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java index f3a89590aede..905d5db2135d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java @@ -37,9 +37,13 @@ import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.VMInspectionOptions; import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.nmt.NativeMemoryTracking; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.util.UnsignedUtils; +import com.oracle.svm.core.util.VMError; public abstract class AbstractCommittedMemoryProvider implements CommittedMemoryProvider { @Uninterruptible(reason = "Still being initialized.") @@ -71,11 +75,11 @@ protected static int protectSingleIsolateImageHeap() { @Override public Pointer allocateExecutableMemory(UnsignedWord nbytes, UnsignedWord alignment) { - return allocate(nbytes, alignment, true); + return allocate(nbytes, alignment, true, NmtCategory.Code); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected Pointer allocate(UnsignedWord size, UnsignedWord alignment, boolean executable) { + protected Pointer allocate(UnsignedWord size, UnsignedWord alignment, boolean executable, NmtCategory nmtCategory) { Pointer reserved = WordFactory.nullPointer(); if (!UnsignedUtils.isAMultiple(getGranularity(), alignment)) { reserved = VirtualMemoryProvider.get().reserve(size, alignment, executable); @@ -95,41 +99,28 @@ protected Pointer allocate(UnsignedWord size, UnsignedWord alignment, boolean ex return nullPointer(); } assert reserved.isNull() || reserved.equal(committed); - tracker.track(size); + + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + NativeMemoryTracking.singleton().trackReserve(size, nmtCategory); + NativeMemoryTracking.singleton().trackCommit(size, nmtCategory); + } return committed; } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void freeExecutableMemory(PointerBase start, UnsignedWord nbytes, UnsignedWord alignment) { - free(start, nbytes); + free(start, nbytes, NmtCategory.Code); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected void free(PointerBase start, UnsignedWord nbytes) { - if (VirtualMemoryProvider.get().free(start, nbytes) == 0) { - tracker.untrack(nbytes); + protected static void free(PointerBase start, UnsignedWord nbytes, NmtCategory nmtCategory) { + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + NativeMemoryTracking.singleton().trackUncommit(nbytes, nmtCategory); + NativeMemoryTracking.singleton().trackFree(nbytes, nmtCategory); } - } - - private final VirtualMemoryTracker tracker = new VirtualMemoryTracker(); - - public static class VirtualMemoryTracker { - - private UnsignedWord totalAllocated; - public VirtualMemoryTracker() { - this.totalAllocated = WordFactory.zero(); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void track(UnsignedWord size) { - totalAllocated = totalAllocated.add(size); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void untrack(UnsignedWord size) { - totalAllocated = totalAllocated.subtract(size); - } + int result = VirtualMemoryProvider.get().free(start, nbytes); + VMError.guarantee(result == 0, "Error while freeing virtual memory."); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractImageHeapProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractImageHeapProvider.java index d8fd7bcc5eea..2cc534f98ba4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractImageHeapProvider.java @@ -50,13 +50,14 @@ protected UnsignedWord getTotalRequiredAddressSpaceSize() { return size; } + @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected UnsignedWord getImageHeapAddressSpaceSize() { + public UnsignedWord getImageHeapAddressSpaceSize() { UnsignedWord pageSize = VirtualMemoryProvider.get().getGranularity(); int imageHeapOffset = Heap.getHeap().getImageHeapOffsetInAddressSpace(); assert imageHeapOffset >= 0; UnsignedWord size = WordFactory.unsigned(imageHeapOffset); - size = size.add(getImageHeapSizeInFile(IMAGE_HEAP_BEGIN.get(), IMAGE_HEAP_END.get())); + size = size.add(getImageHeapSizeInFile()); size = UnsignedUtils.roundUp(size, pageSize); return size; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ChunkBasedCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ChunkBasedCommittedMemoryProvider.java index 369792e0e8bf..8e44e8ec7919 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ChunkBasedCommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ChunkBasedCommittedMemoryProvider.java @@ -33,6 +33,7 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.heap.RestrictHeapAccess; +import com.oracle.svm.core.nmt.NmtCategory; import jdk.graal.compiler.api.replacements.Fold; @@ -44,11 +45,11 @@ public static ChunkBasedCommittedMemoryProvider get() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer allocateAlignedChunk(UnsignedWord nbytes, UnsignedWord alignment) { - return allocate(nbytes, alignment, false); + return allocate(nbytes, alignment, false, NmtCategory.JavaHeap); } public Pointer allocateUnalignedChunk(UnsignedWord nbytes) { - return allocate(nbytes, getAlignmentForUnalignedChunks(), false); + return allocate(nbytes, getAlignmentForUnalignedChunks(), false, NmtCategory.JavaHeap); } /** @@ -61,12 +62,12 @@ public boolean areUnalignedChunksZeroed() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void freeAlignedChunk(PointerBase start, UnsignedWord nbytes, @SuppressWarnings("unused") UnsignedWord alignment) { - free(start, nbytes); + free(start, nbytes, NmtCategory.JavaHeap); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void freeUnalignedChunk(PointerBase start, UnsignedWord nbytes) { - free(start, nbytes); + free(start, nbytes, NmtCategory.JavaHeap); } /** diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java index 118ea194d590..4351118bde9f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java @@ -24,12 +24,15 @@ */ package com.oracle.svm.core.os; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.heap.Heap; @@ -71,11 +74,16 @@ static ImageHeapProvider get() { * written. May be null if this value is not required. * @return a result code from {@link CEntryPointErrors}. */ + @Uninterruptible(reason = "Still being initialized.") int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer); /** * Disposes an instance of the image heap that was created with this provider. This method must * only be called if the image heap memory was allocated by the {@link ImageHeapProvider}. */ + @Uninterruptible(reason = "Called during isolate tear-down.") int freeImageHeap(PointerBase heapBase); + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + UnsignedWord getImageHeapAddressSpaceSize(); } diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java index 6f633651ae44..9a4272ed8331 100644 --- a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java @@ -87,11 +87,11 @@ public void testPeakTracking() { assertEquals(0, getUsedMemory()); Pointer ptr1 = NativeMemory.malloc(M, NmtCategory.Code); - long peakUsed = NativeMemoryTracking.singleton().getPeakUsedMemory(NmtCategory.Code); + long peakUsed = NativeMemoryTracking.singleton().getPeakMallocMemory(NmtCategory.Code); assertEquals(M, peakUsed); Pointer ptr2 = NativeMemory.malloc(M, NmtCategory.Code); - peakUsed = NativeMemoryTracking.singleton().getPeakUsedMemory(NmtCategory.Code); + peakUsed = NativeMemoryTracking.singleton().getPeakMallocMemory(NmtCategory.Code); assertEquals(2 * M, peakUsed); NativeMemory.free(ptr1); @@ -101,20 +101,54 @@ public void testPeakTracking() { ptr2 = WordFactory.nullPointer(); assertEquals(0, getUsedMemory()); - assertEquals(2 * M, NativeMemoryTracking.singleton().getPeakUsedMemory(NmtCategory.Code)); + assertEquals(2 * M, NativeMemoryTracking.singleton().getPeakMallocMemory(NmtCategory.Code)); Pointer ptr3 = NativeMemory.malloc(3 * M, NmtCategory.Code); - peakUsed = NativeMemoryTracking.singleton().getPeakUsedMemory(NmtCategory.Code); + peakUsed = NativeMemoryTracking.singleton().getPeakMallocMemory(NmtCategory.Code); assertEquals(3 * M, peakUsed); NativeMemory.free(ptr3); ptr3 = WordFactory.nullPointer(); assertEquals(0, getUsedMemory()); - assertEquals(3 * M, NativeMemoryTracking.singleton().getPeakUsedMemory(NmtCategory.Code)); + assertEquals(3 * M, NativeMemoryTracking.singleton().getPeakMallocMemory(NmtCategory.Code)); } private static long getUsedMemory() { - return NativeMemoryTracking.singleton().getUsedMemory(NmtCategory.Code); + return NativeMemoryTracking.singleton().getMallocMemory(NmtCategory.Code); + } + + @Test + public void testVirtualMemoryTracking() { + // The application should already be using some virtual memory for the heap. + assertTrue(NativeMemoryTracking.singleton().getReservedVirtualMemory(NmtCategory.ImageHeap) > 0); + assertTrue(NativeMemoryTracking.singleton().getReservedVirtualMemory(NmtCategory.JavaHeap) > 0); + + assertTrue(NativeMemoryTracking.singleton().getCommittedVirtualMemory(NmtCategory.JavaHeap) > 0); + assertTrue(NativeMemoryTracking.singleton().getCommittedVirtualMemory(NmtCategory.ImageHeap) > 0); + + assertTrue(NativeMemoryTracking.singleton().getPeakCommittedVirtualMemory(NmtCategory.JavaHeap) > 0); + assertTrue(NativeMemoryTracking.singleton().getPeakCommittedVirtualMemory(NmtCategory.ImageHeap) > 0); + + assertTrue(NativeMemoryTracking.singleton().getPeakReservedVirtualMemory(NmtCategory.JavaHeap) > 0); + assertTrue(NativeMemoryTracking.singleton().getPeakReservedVirtualMemory(NmtCategory.ImageHeap) > 0); + + // Ensure we have a zero baseline + assertTrue(NativeMemoryTracking.singleton().getReservedVirtualMemory(NmtCategory.Code) == 0); + assertTrue(NativeMemoryTracking.singleton().getCommittedVirtualMemory(NmtCategory.Code) == 0); + assertTrue(NativeMemoryTracking.singleton().getPeakReservedVirtualMemory(NmtCategory.Code) == 0); + assertTrue(NativeMemoryTracking.singleton().getPeakCommittedVirtualMemory(NmtCategory.Code) == 0); + + // Use some memory + NativeMemoryTracking.singleton().trackReserve(1024, NmtCategory.Code); + NativeMemoryTracking.singleton().trackCommit(512, NmtCategory.Code); + assertTrue(NativeMemoryTracking.singleton().getReservedVirtualMemory(NmtCategory.Code) == 1024); + assertTrue(NativeMemoryTracking.singleton().getCommittedVirtualMemory(NmtCategory.Code) == 512); + + // Uncommit and check peaks + NativeMemoryTracking.singleton().trackUncommit(512, NmtCategory.Code); + NativeMemoryTracking.singleton().trackFree(1024, NmtCategory.Code); + assertTrue(NativeMemoryTracking.singleton().getPeakReservedVirtualMemory(NmtCategory.Code) == 1024); + assertTrue(NativeMemoryTracking.singleton().getPeakCommittedVirtualMemory(NmtCategory.Code) == 512); } }