Adjust initial and max memory usage of builder.

Author: fniephaus

substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.driver;
+
+import java.lang.management.ManagementFactory;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.oracle.svm.core.OS;
+
+class MemoryUtil {
+    private static final double MAX_RAM_MIN_PERCENTAGE = 50.0;
+    private static final double MAX_RAM_MAX_PERCENTAGE = 80.0;
+    private static final double MAX_RAM_HEADROOM_PERCENTAGE = 10.0;
+
+    private static final Pattern MEMINFO_AVAILABLE_REGEX = Pattern.compile("^MemAvailable:\\s+(\\d+) kB");
+
+    public static double determineMaxRAMPercentage() {
+        double availableRAMPercentage = Math.floor(getAvailableRAMPercentage() - MAX_RAM_HEADROOM_PERCENTAGE);
+        return Math.max(MAX_RAM_MIN_PERCENTAGE, Math.min(availableRAMPercentage, MAX_RAM_MAX_PERCENTAGE));
+    }
+
+    private static double getAvailableRAMPercentage() {
+        var osBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
+        double usableMemory = osBean.getFreeMemorySize();
+        // TODO: drop debug code
+        System.out.println("Free memory percentage: " + (usableMemory / osBean.getTotalMemorySize()));
+        if (OS.LINUX.isCurrent()) {
+            // Increase value if more memory is available on Linux.
+            usableMemory = Math.max(usableMemory, getLinuxAvailableMemory());
+            // TODO: drop debug code
+            System.out.println("Available memory percentage: " + (usableMemory / osBean.getTotalMemorySize()));
+        }
+        return (usableMemory / osBean.getTotalMemorySize()) * 100;
+    }
+
+    private static long getLinuxAvailableMemory() {
+        try {
+            String memAvailableLine = Files.readAllLines(Paths.get("/proc/meminfo")).stream().filter(l -> l.startsWith("MemAvailable")).findFirst().orElse("");
+            Matcher m = MEMINFO_AVAILABLE_REGEX.matcher(memAvailableLine);
+            if (m.matches()) {
+                return Long.parseLong(m.group(1)) * 1024 /* kB to B */;
+            }
+        } catch (Exception e) {
+        }
+        return -1;
+    }
+}

substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java

@@ -29,8 +29,6 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.lang.management.ManagementFactory;
-import java.lang.management.OperatingSystemMXBean;
 import java.lang.reflect.Method;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -58,7 +56,6 @@
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
-import java.util.function.Supplier;
 import java.util.jar.Attributes;
 import java.util.jar.JarFile;
 import java.util.jar.Manifest;
@@ -808,11 +805,12 @@ static void ensureDirectoryExists(Path dir) {
 
     private void prepareImageBuildArgs() {
         addImageBuilderJavaArgs("-Xss10m");
-        addImageBuilderJavaArgs(oXms + getXmsValue());
-        String xmxVal = getXmxValue(1);
-        if (!"0".equals(xmxVal)) {
-            addImageBuilderJavaArgs(oXmx + xmxVal);
-        }
+
+        /* Builder needs at least 2GB of memory. */
+        addImageBuilderJavaArgs("-Xms2g");
+
+        /* Adjust max memory usage depending on availability. */
+        addImageBuilderJavaArgs("-XX:MaxRAMPercentage=" + MemoryUtil.determineMaxRAMPercentage());
 
         /* Let builder exit on first OutOfMemoryError. */
         addImageBuilderJavaArgs("-XX:+ExitOnOutOfMemoryError");
@@ -863,27 +861,6 @@ protected static boolean replaceArg(Collection<String> args, String argPrefix, S
         return elementsRemoved;
     }
 
-    private static <T> T consolidateArgs(Collection<String> args, String argPrefix,
-                    Function<String, T> fromSuffix, Function<T, String> toSuffix,
-                    Supplier<T> init, BiFunction<T, T, T> combiner) {
-        T consolidatedValue = null;
-        boolean needsConsolidate = false;
-        for (String arg : args) {
-            if (arg.startsWith(argPrefix)) {
-                if (consolidatedValue == null) {
-                    consolidatedValue = init.get();
-                } else {
-                    needsConsolidate = true;
-                }
-                consolidatedValue = combiner.apply(consolidatedValue, fromSuffix.apply(arg.substring(argPrefix.length())));
-            }
-        }
-        if (consolidatedValue != null && needsConsolidate) {
-            replaceArg(args, argPrefix, toSuffix.apply(consolidatedValue));
-        }
-        return consolidatedValue;
-    }
-
     private static LinkedHashSet<String> collectListArgs(Collection<String> args, String argPrefix, String delimiter) {
         LinkedHashSet<String> allEntries = new LinkedHashSet<>();
         for (String arg : args) {
@@ -1052,14 +1029,7 @@ private int completeImageBuild() {
                 }
             }
         }
-        /* Perform JavaArgs consolidation - take the maximum of -Xmx, minimum of -Xms */
-        Long xmxValue = consolidateArgs(imageBuilderJavaArgs, oXmx, SubstrateOptionsParser::parseLong, String::valueOf, () -> 0L, Math::max);
-        Long xmsValue = consolidateArgs(imageBuilderJavaArgs, oXms, SubstrateOptionsParser::parseLong, String::valueOf, () -> SubstrateOptionsParser.parseLong(getXmsValue()), Math::max);
-        if (xmxValue != null) {
-            if (Long.compareUnsigned(xmsValue, xmxValue) > 0) {
-                replaceArg(imageBuilderJavaArgs, oXms, Long.toUnsignedString(xmxValue));
-            }
-        }
+
         addImageBuilderJavaArgs(customJavaArgs.toArray(new String[0]));
 
         /* Perform option consolidation of imageBuilderArgs */
@@ -2088,26 +2058,6 @@ List<String> getNativeImageArgs() {
         return Stream.concat(getDefaultNativeImageArgs().stream(), config.getBuildArgs().stream()).toList();
     }
 
-    protected String getXmsValue() {
-        return "1g";
-    }
-
-    @SuppressWarnings("deprecation") // getTotalPhysicalMemorySize is deprecated after JDK 11
-    private static long getPhysicalMemorySize() {
-        OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean();
-        long totalPhysicalMemorySize = ((com.sun.management.OperatingSystemMXBean) osMXBean).getTotalPhysicalMemorySize();
-        return totalPhysicalMemorySize;
-    }
-
-    protected String getXmxValue(int maxInstances) {
-        Long memMax = Long.divideUnsigned(Long.divideUnsigned(getPhysicalMemorySize(), 10) * 8, maxInstances);
-        String maxXmx = "14g";
-        if (Long.compareUnsigned(memMax, SubstrateOptionsParser.parseLong(maxXmx)) >= 0) {
-            return maxXmx;
-        }
-        return Long.toUnsignedString(memMax);
-    }
-
     private static boolean isDumbTerm() {
         String term = System.getenv().getOrDefault("TERM", "");
         return term.isEmpty() || term.equals("dumb") || term.equals("unknown");