[GR-41037] Compute native image heap graph

PullRequest: graal/13030

Author: spaske00

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/image/ImageHeap.java

@@ -29,7 +29,7 @@
 public interface ImageHeap {
     Collection<? extends ImageHeapObject> getObjects();
 
-    ImageHeapObject addLateToImageHeap(Object object, String reason);
+    ImageHeapObject addLateToImageHeap(Object object, Object reason);
 
     ImageHeapObject addFillerObject(int size);
 

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ByteFormattingUtil.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.hosted;
+
+public class ByteFormattingUtil {
+    private static final double BYTES_TO_KiB = 1024d;
+    private static final double BYTES_TO_MiB = 1024d * 1024d;
+    private static final double BYTES_TO_GiB = 1024d * 1024d * 1024d;
+
+    public static String bytesToHuman(long bytes) {
+        return bytesToHuman("%4.2f", bytes);
+    }
+
+    public static String bytesToHuman(String format, long bytes) {
+        if (bytes < BYTES_TO_KiB) {
+            return String.format(format, (double) bytes) + "B";
+        } else if (bytes < BYTES_TO_MiB) {
+            return String.format(format, bytesToKiB(bytes)) + "KB";
+        } else if (bytes < BYTES_TO_GiB) {
+            return String.format(format, bytesToMiB(bytes)) + "MB";
+        } else {
+            return String.format(format, bytesToGiB(bytes)) + "GB";
+        }
+    }
+
+    static double bytesToKiB(long bytes) {
+        return bytes / BYTES_TO_KiB;
+    }
+
+    static double bytesToGiB(long bytes) {
+        return bytes / BYTES_TO_GiB;
+    }
+
+    static double bytesToMiB(long bytes) {
+        return bytes / BYTES_TO_MiB;
+    }
+}

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java

@@ -267,7 +267,7 @@ public void printInitializeEnd() {
         String gcName = Heap.getHeap().getGC().getName();
         recordJsonMetric(GeneralInfo.GC, gcName);
         long maxHeapSize = SubstrateGCOptions.MaxHeapSize.getValue();
-        String maxHeapValue = maxHeapSize == 0 ? "unlimited" : Utils.bytesToHuman(maxHeapSize);
+        String maxHeapValue = maxHeapSize == 0 ? "unlimited" : ByteFormattingUtil.bytesToHuman(maxHeapSize);
         l().a(" ").doclink("Garbage collector", "#glossary-gc").a(": ").a(gcName).a(" (").doclink("max heap size", "#glossary-gc-max-heap-size").a(": ").a(maxHeapValue).a(")").println();
     }
 
@@ -414,16 +414,17 @@ public void printCreationEnd(int imageFileSize, int numHeapObjects, long imageHe
         stagePrinter.end(imageTimer.getTotalTime() + writeTimer.getTotalTime());
         creationStageEndCompleted = true;
         String format = "%9s (%5.2f%%) for ";
-        l().a(format, Utils.bytesToHuman(codeAreaSize), codeAreaSize / (double) imageFileSize * 100)
+        l().a(format, ByteFormattingUtil.bytesToHuman(codeAreaSize), codeAreaSize / (double) imageFileSize * 100)
                         .doclink("code area", "#glossary-code-area").a(":%,10d compilation units", numCompilations).println();
         EconomicMap<Pair<String, String>, ResourceStorageEntry> resources = Resources.singleton().resources();
         int numResources = resources.size();
         recordJsonMetric(ImageDetailKey.IMAGE_HEAP_RESOURCE_COUNT, numResources);
-        l().a(format, Utils.bytesToHuman(imageHeapSize), imageHeapSize / (double) imageFileSize * 100)
+        l().a(format, ByteFormattingUtil.bytesToHuman(imageHeapSize), imageHeapSize / (double) imageFileSize * 100)
                         .doclink("image heap", "#glossary-image-heap").a(":%,9d objects and %,d resources", numHeapObjects, numResources).println();
         if (debugInfoSize > 0) {
             recordJsonMetric(ImageDetailKey.DEBUG_INFO_SIZE, debugInfoSize); // Optional metric
-            DirectPrinter l = l().a(format, Utils.bytesToHuman(debugInfoSize), debugInfoSize / (double) imageFileSize * 100)
+            DirectPrinter l = l().a(format, ByteFormattingUtil.bytesToHuman(debugInfoSize), debugInfoSize / (double) imageFileSize * 100)
+
                             .doclink("debug info", "#glossary-debug-info");
             if (debugInfoTimer != null) {
                 l.a(" generated in %.1fs", Utils.millisToSeconds(debugInfoTimer.getTotalTime()));
@@ -435,9 +436,9 @@ public void printCreationEnd(int imageFileSize, int numHeapObjects, long imageHe
         recordJsonMetric(ImageDetailKey.TOTAL_SIZE, imageFileSize);
         recordJsonMetric(ImageDetailKey.CODE_AREA_SIZE, codeAreaSize);
         recordJsonMetric(ImageDetailKey.NUM_COMP_UNITS, numCompilations);
-        l().a(format, Utils.bytesToHuman(otherBytes), otherBytes / (double) imageFileSize * 100)
+        l().a(format, ByteFormattingUtil.bytesToHuman(otherBytes), otherBytes / (double) imageFileSize * 100)
                         .doclink("other data", "#glossary-other-data").println();
-        l().a("%9s in total", Utils.bytesToHuman(imageFileSize)).println();
+        l().a("%9s in total", ByteFormattingUtil.bytesToHuman(imageFileSize)).println();
         printBreakdowns();
     }
 
@@ -568,7 +569,7 @@ private void printBreakdowns() {
             if (packagesBySize.hasNext()) {
                 Entry<String, Long> e = packagesBySize.next();
                 String className = Utils.truncateClassOrPackageName(e.getKey());
-                codeSizePart = String.format("%9s %s", Utils.bytesToHuman(e.getValue()), className);
+                codeSizePart = String.format("%9s %s", ByteFormattingUtil.bytesToHuman(e.getValue()), className);
                 printedCodeBytes += e.getValue();
                 printedCodeItems++;
             }
@@ -581,7 +582,7 @@ private void printBreakdowns() {
                 if (!className.startsWith(BREAKDOWN_BYTE_ARRAY_PREFIX)) {
                     className = Utils.truncateClassOrPackageName(className);
                 }
-                heapSizePart = String.format("%9s %s", Utils.bytesToHuman(e.getValue()), className);
+                heapSizePart = String.format("%9s %s", ByteFormattingUtil.bytesToHuman(e.getValue()), className);
                 printedHeapBytes += e.getValue();
                 printedHeapItems++;
             }
@@ -595,9 +596,10 @@ private void printBreakdowns() {
         int numHeapItems = heapBreakdown.size();
         long totalCodeBytes = codeBreakdown.values().stream().collect(Collectors.summingLong(Long::longValue));
         long totalHeapBytes = heapBreakdown.values().stream().collect(Collectors.summingLong(Long::longValue));
-        p.l().a(String.format("%9s for %s more ", Utils.bytesToHuman(totalCodeBytes - printedCodeBytes), numCodeItems - printedCodeItems)).doclink("origins", "#glossary-code-area-origins")
+
+        p.l().a(String.format("%9s for %s more packages", ByteFormattingUtil.bytesToHuman(totalCodeBytes - printedCodeBytes), numCodeItems - printedCodeItems))
                         .jumpToMiddle()
-                        .a(String.format("%9s for %s more object types", Utils.bytesToHuman(totalHeapBytes - printedHeapBytes), numHeapItems - printedHeapItems)).flushln();
+                        .a(String.format("%9s for %s more object types", ByteFormattingUtil.bytesToHuman(totalHeapBytes - printedHeapBytes), numHeapItems - printedHeapItems)).flushln();
     }
 
     public void printEpilog(Optional<String> optionalImageName, Optional<NativeImageGenerator> optionalGenerator, ImageClassLoader classLoader, Optional<Throwable> optionalError,
@@ -702,11 +704,11 @@ private static Path reportImageBuildStatistics(String imageName, BigBang bb) {
         String description = "image build statistics";
         if (ImageBuildStatistics.Options.ImageBuildStatisticsFile.hasBeenSet(bb.getOptions())) {
             final File file = new File(ImageBuildStatistics.Options.ImageBuildStatisticsFile.getValue(bb.getOptions()));
-            return ReportUtils.report(description, file.getAbsoluteFile().toPath(), statsReporter, false);
+            return com.oracle.graal.pointsto.reports.ReportUtils.report(description, file.getAbsoluteFile().toPath(), statsReporter, false);
         } else {
-            String name = "image_build_statistics_" + ReportUtils.extractImageName(imageName);
+            String name = "image_build_statistics_" + com.oracle.graal.pointsto.reports.ReportUtils.extractImageName(imageName);
             String path = SubstrateOptions.Path.getValue() + File.separatorChar + "reports";
-            return ReportUtils.report(description, path, name, "json", statsReporter, false);
+            return com.oracle.graal.pointsto.reports.ReportUtils.report(description, path, name, "json", statsReporter, false);
         }
     }
 
@@ -721,7 +723,7 @@ private void printResourceStatistics() {
                         .doclink("GCs", "#glossary-garbage-collections");
         long peakRSS = ProgressReporterCHelper.getPeakRSS();
         if (peakRSS >= 0) {
-            p.a(" | ").doclink("Peak RSS", "#glossary-peak-rss").a(": ").a("%.2fGB", Utils.bytesToGiB(peakRSS));
+            p.a(" | ").doclink("Peak RSS", "#glossary-peak-rss").a(": ").a("%.2fGB", ByteFormattingUtil.bytesToGiB(peakRSS));
         }
         recordJsonMetric(ResourceUsageKey.PEAK_RSS, (peakRSS >= 0 ? peakRSS : UNAVAILABLE_METRIC));
         OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean();
@@ -747,7 +749,7 @@ private void checkForExcessiveGarbageCollection() {
                             .a(": %.1fs spent in %d GCs during the last stage, taking up %.2f%% of the time.",
                                             Utils.millisToSeconds(gcTimeDeltaMillis), currentGCStats.totalCount - lastGCStats.totalCount, ratio * 100)
                             .println();
-            l().a("            Please ensure more than %.2fGB of memory is available for Native Image", Utils.bytesToGiB(ProgressReporterCHelper.getPeakRSS())).println();
+            l().a("            Please ensure more than %.2fGB of memory is available for Native Image", ByteFormattingUtil.bytesToGiB(ProgressReporterCHelper.getPeakRSS())).println();
             l().a("            to reduce GC overhead and improve image build time.").println();
         }
         lastGCStats = currentGCStats;
@@ -770,40 +772,8 @@ private static Timer getTimer(TimerCollection.Registry type) {
     private static class Utils {
         private static final double MILLIS_TO_SECONDS = 1000d;
         private static final double NANOS_TO_SECONDS = 1000d * 1000d * 1000d;
-        private static final double BYTES_TO_KiB = 1024d;
-        private static final double BYTES_TO_MiB = 1024d * 1024d;
-        private static final double BYTES_TO_GiB = 1024d * 1024d * 1024d;
-
         private static final Field STRING_VALUE = ReflectionUtil.lookupField(String.class, "value");
 
-        private static String bytesToHuman(long bytes) {
-            return bytesToHuman("%4.2f", bytes);
-        }
-
-        private static String bytesToHuman(String format, long bytes) {
-            if (bytes < BYTES_TO_KiB) {
-                return String.format(format, (double) bytes) + "B";
-            } else if (bytes < BYTES_TO_MiB) {
-                return String.format(format, bytesToKiB(bytes)) + "KB";
-            } else if (bytes < BYTES_TO_GiB) {
-                return String.format(format, bytesToMiB(bytes)) + "MB";
-            } else {
-                return String.format(format, bytesToGiB(bytes)) + "GB";
-            }
-        }
-
-        private static double bytesToKiB(long bytes) {
-            return bytes / BYTES_TO_KiB;
-        }
-
-        private static double bytesToGiB(long bytes) {
-            return bytes / BYTES_TO_GiB;
-        }
-
-        private static double bytesToMiB(long bytes) {
-            return bytes / BYTES_TO_MiB;
-        }
-
         private static double millisToSeconds(double millis) {
             return millis / MILLIS_TO_SECONDS;
         }
@@ -821,7 +791,7 @@ private static int getInternalByteArrayLength(String string) {
         }
 
         private static double getUsedMemory() {
-            return bytesToGiB(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory());
+            return ByteFormattingUtil.bytesToGiB(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory());
         }
 
         private static String stringFilledWith(int size, String fill) {
@@ -862,6 +832,10 @@ private static String truncateClassOrPackageName(String classOrPackageName) {
         }
     }
 
+    private static void resetANSIMode() {
+        NativeImageSystemIOWrappers.singleton().getOut().print(ANSI.RESET);
+    }
+
     private static class GCStats {
         private final long totalCount;
         private final long totalTimeMillis;

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/HeapHistogram.java

@@ -24,6 +24,7 @@
  */
 package com.oracle.svm.hosted.image;
 
+import java.io.PrintWriter;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -39,6 +40,7 @@ public class HeapHistogram {
     protected static boolean PrintStrings = false;
 
     private final Map<HostedClass, HistogramEntry> data = new HashMap<>();
+    private final PrintWriter out;
 
     static class HistogramEntry {
         protected final HostedClass clazz;
@@ -50,6 +52,14 @@ static class HistogramEntry {
         }
     }
 
+    public HeapHistogram() {
+        this.out = new PrintWriter(System.out);
+    }
+
+    public HeapHistogram(PrintWriter out) {
+        this.out = out;
+    }
+
     private static final Comparator<HistogramEntry> SIZE_COMPARATOR = (o1, o2) -> {
         // Larger sizes first
         int result = Long.compare(o2.size, o1.size);
@@ -77,18 +87,18 @@ public void add(ObjectInfo objectInfo, long size) {
         entry.size += size;
 
         if (PrintStrings && objectInfo.getObject() instanceof String) {
-            String reason = String.valueOf(objectInfo.reason);
+            String reason = String.valueOf(objectInfo.getMainReason());
             String value = ((String) objectInfo.getObject()).replace("\n", "");
             if (!reason.startsWith("com.oracle.svm.core.hub.DynamicHub")) {
-                System.out.format("%120s ::: %s\n", value, reason);
+                out.format("%120s ::: %s\n", value, reason);
             }
         }
     }
 
     public void printHeadings(final String title) {
         assert NativeImageOptions.PrintHeapHistogram.getValue();
-        System.out.format("\n%s\n", title);
-        System.out.format(headerFormat, "Count", "Size", "Size%", "Cum%", "Class");
+        out.format("%s\n", title);
+        out.format(headerFormat, "Count", "Size", "Size%", "Cum%", "Class");
     }
 
     public void print() {
@@ -101,7 +111,7 @@ public void print() {
         long printedSize = 0;
         for (HistogramEntry entry : entries) {
             printedSize += entry.size;
-            System.out.format(entryFormat, entry.count, entry.size, entry.size * 100d / totalSize, printedSize * 100d / totalSize, entry.clazz.toJavaName());
+            out.format(entryFormat, entry.count, entry.size, entry.size * 100d / totalSize, printedSize * 100d / totalSize, entry.clazz.toJavaName());
         }
     }
 

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/ImageHeapConnectedComponentsFeature.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.hosted.image;
+
+import com.oracle.graal.pointsto.reports.ReportUtils;
+import com.oracle.svm.core.SubstrateOptions;
+import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature;
+import com.oracle.svm.core.feature.InternalFeature;
+import com.oracle.svm.core.option.HostedOptionKey;
+import com.oracle.svm.hosted.FeatureImpl;
+import org.graalvm.compiler.options.Option;
+import org.graalvm.compiler.options.OptionType;
+
+import java.io.File;
+import java.io.PrintWriter;
+import java.nio.file.Path;
+import java.util.function.Consumer;
+
+@AutomaticallyRegisteredFeature
+public class ImageHeapConnectedComponentsFeature implements InternalFeature {
+    public static class Options {
+        @Option(help = {"file:doc-files/PrintImageHeapConnectedComponents.md"}, type = OptionType.Debug)//
+        static final HostedOptionKey<Boolean> PrintImageHeapConnectedComponents = new HostedOptionKey<>(false);
+    }
+
+    private AbstractImage image;
+    private NativeImageHeap heap;
+
+    @Override
+    public boolean isInConfiguration(IsInConfigurationAccess access) {
+        return Options.PrintImageHeapConnectedComponents.getValue();
+    }
+
+    @Override
+    public void afterHeapLayout(AfterHeapLayoutAccess a) {
+        FeatureImpl.AfterHeapLayoutAccessImpl access = (FeatureImpl.AfterHeapLayoutAccessImpl) a;
+        this.heap = access.getHeap();
+    }
+
+    @Override
+    public void beforeImageWrite(BeforeImageWriteAccess access) {
+        this.image = ((FeatureImpl.BeforeImageWriteAccessImpl) access).getImage();
+    }
+
+    @Override
+    public void afterImageWrite(AfterImageWriteAccess a) {
+        FeatureImpl.AfterImageWriteAccessImpl access = (FeatureImpl.AfterImageWriteAccessImpl) a;
+        Path imagePath = access.getImagePath().getFileName();
+        String imageName = imagePath != null ? imagePath.toString() : "native-image";
+        ImageHeapConnectedComponentsPrinter printer = new ImageHeapConnectedComponentsPrinter(heap, access.getUniverse().getBigBang(), image, imageName);
+        printReport("connected_components_" + imageName, "txt", printer::printConnectedComponentsObjectHistogramReport);
+        printReport("summary_info_for_every_object_in_connected_components_" + imageName, "json", printer::printSummaryInfoForEveryObjectInConnectedComponents);
+        printReport("access_points_for_connected_components_" + imageName, "json", printer::printAccessPointsForConnectedComponents);
+        heap.objectReachabilityInfo.clear();
+    }
+
+    private static void printReport(String reportName, String extension, Consumer<PrintWriter> writer) {
+        File file = ReportUtils.reportFile(SubstrateOptions.reportsPath(), reportName, extension);
+        ReportUtils.report(reportName, file.toPath(), writer);
+    }
+}