From b08529036bd67aee4a9b20231a6164d237fc1e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 2 Sep 2021 14:15:22 +0200 Subject: [PATCH 01/15] Default to addImageClasspath even with image-builder on modulepath --- .../src/com/oracle/svm/driver/MacroOptionHandler.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java index ed8c60b12df8..f53010eba43c 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java @@ -91,11 +91,7 @@ private void applyEnabled(MacroOption.EnabledOption enabledOption, String argume boolean explicitImageClasspath = enabledOption.forEachPropertyValue( config, "ImageClasspath", entry -> nativeImage.addImageClasspath(ClasspathUtils.stringToClasspath(entry)), PATH_SEPARATOR_REGEX); if (!explicitImageModulePath && !explicitImageClasspath) { - if (NativeImage.USE_NI_JPMS) { - NativeImage.getJars(imageJarsDirectory).forEach(nativeImage::addImageModulePath); - } else { - NativeImage.getJars(imageJarsDirectory).forEach(nativeImage::addImageClasspath); - } + NativeImage.getJars(imageJarsDirectory).forEach(nativeImage::addImageClasspath); } String imageName = enabledOption.getProperty(config, "ImageName"); From ba8eb8d419e85a57dd940ea304e64d1227ab9b14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 1 Sep 2021 16:09:42 +0200 Subject: [PATCH 02/15] Remove detection of modules that are already in bootlayer of image-builder from driver --- .../com/oracle/svm/driver/NativeImage.java | 67 ------------------- 1 file changed, 67 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index b73b7045d57b..6a9f49ed24c2 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -53,7 +53,6 @@ import java.util.Map; import java.util.Optional; import java.util.Properties; -import java.util.Scanner; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Consumer; @@ -1631,47 +1630,6 @@ void addImageClasspath(Path classpath) { LinkedHashSet builderModuleNames = null; - LinkedHashSet getBuilderModuleNames() { - if (builderModuleNames == null) { - ProcessBuilder pb = new ProcessBuilder(); - List command = pb.command(); - command.add(config.getJavaExecutable().toString()); - command.add("--module-path"); - command.add(Stream.concat(config.getBuilderModulePath().stream(), config.getImageProvidedModulePath().stream()) - .map(this::canonicalize).map(Path::toString) - .collect(Collectors.joining(File.pathSeparator))); - command.add("--list-modules"); - pb.redirectErrorStream(); - Process process = null; - int exitValue = -1; - LinkedHashSet modules = new LinkedHashSet<>(); - String message = "Unable to determine image builder modules"; - try { - process = pb.start(); - try (Scanner scanner = new Scanner(process.getInputStream())) { - while (scanner.hasNextLine()) { - String token = scanner.next(); - modules.add(token.split("@", 2)[0]); - scanner.nextLine(); - } - } - exitValue = process.waitFor(); - } catch (IOException | InterruptedException e) { - throw showError(message, e); - } finally { - if (process != null) { - process.destroy(); - } - if (exitValue != 0) { - throw showError(message + " (nonzero exit value)"); - } - } - - builderModuleNames = modules; - } - return builderModuleNames; - } - void addImageModulePath(Path modulePathEntry) { addImageModulePath(modulePathEntry, true); } @@ -1697,36 +1655,11 @@ void addImageModulePath(Path modulePathEntry, boolean strict) { /* Duplicate entries are silently ignored like with the java command */ return; } - List moduleNames = getModuleNames(mpEntry); - if (moduleNames.size() == 1) { - String moduleName = moduleNames.get(0); - if (getBuilderModuleNames().contains(moduleName)) { - /* Modules provided by the image-builder are not allowed on the -imagemp */ - String modulePathEntryStr = "module " + moduleName + " from " + modulePathEntry; - showWarning("Ignoring " + modulePathEntryStr + " (implicitly provided by builder)"); - return; - } - } imageModulePath.add(mpEntry); processClasspathNativeImageMetaInf(mpEntry); } - @SuppressWarnings("unchecked") - List getModuleNames(Path... modulePathEntries) { - try { - Class moduleAccess = Class.forName("com.oracle.svm.driver.jdk11.ModuleAccess"); - Method getModuleNames = moduleAccess.getMethod("getModuleNames", Path[].class); - return (List) getModuleNames.invoke(null, (Object) modulePathEntries); - } catch (ClassNotFoundException e) { - throw showError("com.oracle.svm.driver.jdk11.ModuleAccess.getModuleNames only available on Java > 8"); - } catch (ReflectiveOperationException e) { - String entries = Arrays.stream(modulePathEntries) - .map(Path::toString).collect(Collectors.joining(", ", "[", "]")); - throw showError("Failed to get module-names for " + entries, e); - } - } - /** * For adding classpath elements that are put *explicitly* on the image-classpath (i.e. when * specified as -cp/-classpath/--class-path entry). This method handles invalid classpath From aa4a56a6f4343857ad8fb8061604ada58d4cec90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 2 Sep 2021 11:40:14 +0200 Subject: [PATCH 03/15] Ensure macro generation excludes builder-jars from classpath/modulepath --- sdk/mx.sdk/mx_sdk_vm_impl.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/sdk/mx.sdk/mx_sdk_vm_impl.py b/sdk/mx.sdk/mx_sdk_vm_impl.py index 79bcabb6ba4d..c0677f42d852 100644 --- a/sdk/mx.sdk/mx_sdk_vm_impl.py +++ b/sdk/mx.sdk/mx_sdk_vm_impl.py @@ -1183,6 +1183,9 @@ def getBuildTask(self, args): class NativePropertiesBuildTask(mx.ProjectBuildTask): + + implicit_excludes = ['substratevm:LIBRARY_SUPPORT'] + def __init__(self, subject, args): """ :type subject: GraalVmNativeProperties @@ -1204,12 +1207,14 @@ def _get_location_classpath(self): graalvm_dist = get_final_graalvm_distribution() image_config = self.subject.image_config graalvm_location = dirname(graalvm_dist.find_single_source_location('dependency:' + self.subject.name)) - self._location_classpath = NativePropertiesBuildTask.get_launcher_classpath(graalvm_dist, graalvm_location, image_config, self.subject.component) + self._location_classpath = NativePropertiesBuildTask.get_launcher_classpath(graalvm_dist, graalvm_location, image_config, self.subject.component, exclude_implicit=True) return self._location_classpath @staticmethod - def get_launcher_classpath(graalvm_dist, start, image_config, component): - location_cp = graalvm_home_relative_classpath(image_config.jar_distributions, start, graal_vm=graalvm_dist) + def get_launcher_classpath(graalvm_dist, start, image_config, component, exclude_implicit=False): + with_substratevm = 'substratevm' in [s.name for s in mx.suites()] + exclude_names = NativePropertiesBuildTask.implicit_excludes if with_substratevm and exclude_implicit else None + location_cp = graalvm_home_relative_classpath(image_config.jar_distributions, start, graal_vm=graalvm_dist, exclude_names=exclude_names) location_classpath = location_cp.split(os.pathsep) if location_cp else [] if image_config.dir_jars: if not component: @@ -1268,7 +1273,7 @@ def contents(self): if isinstance(image_config, mx_sdk.LauncherConfig): if image_config.is_sdk_launcher: - launcher_classpath = NativePropertiesBuildTask.get_launcher_classpath(graalvm_dist, graalvm_home, image_config, self.subject.component) + launcher_classpath = NativePropertiesBuildTask.get_launcher_classpath(graalvm_dist, graalvm_home, image_config, self.subject.component, exclude_implicit=True) build_args += [ '-H:-ParseRuntimeOptions', '-Dorg.graalvm.launcher.classpath=' + os.pathsep.join(launcher_classpath), @@ -1825,7 +1830,7 @@ def polyglot_config_contents(self): assert self.with_polyglot_config() graalvm_dist = self.subject.get_containing_graalvm() graalvm_location = dirname(graalvm_dist.find_single_source_location('dependency:{}/polyglot.config'.format(self.subject.name))) - classpath = NativePropertiesBuildTask.get_launcher_classpath(graalvm_dist, graalvm_location, image_config, self.subject.component) + classpath = NativePropertiesBuildTask.get_launcher_classpath(graalvm_dist, graalvm_location, image_config, self.subject.component, exclude_implicit=True) main_class = image_config.main_class return u"|".join((u":".join(classpath), main_class)) return self._polyglot_config_contents @@ -1978,8 +1983,7 @@ def _get_graalvm_archive_path(jdk_path, graal_vm=None): 'JDK_TOOLS', } - -def graalvm_home_relative_classpath(dependencies, start=None, with_boot_jars=False, graal_vm=None): +def graalvm_home_relative_classpath(dependencies, start=None, with_boot_jars=False, graal_vm=None, exclude_names=None): if graal_vm is None: graal_vm = get_final_graalvm_distribution() start = start or _get_graalvm_archive_path('', graal_vm=graal_vm) @@ -1993,7 +1997,14 @@ def graalvm_home_relative_classpath(dependencies, start=None, with_boot_jars=Fal jimage = mx.project('graalvm-jimage', fatalIfMissing=False) jimage_deps = jimage.deps if jimage else None mx.logv("Composing classpath for " + str(dependencies) + ". Entries:\n" + '\n'.join(('- {}:{}'.format(d.suite, d.name) for d in mx.classpath_entries(dependencies)))) - for _cp_entry in mx.classpath_entries(dependencies): + cp_entries = mx.classpath_entries(dependencies) + + if exclude_names: + for exclude_entry in mx.classpath_entries(names=exclude_names): + if exclude_entry in cp_entries: + cp_entries.remove(exclude_entry) + + for _cp_entry in cp_entries: if jimage_deps and _jlink_libraries() and _cp_entry in jimage_deps: continue if _cp_entry.isJdkLibrary() or _cp_entry.isJreLibrary(): From 01ed57fbc5e58b70906280748b73378da18c94ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 2 Sep 2021 11:41:49 +0200 Subject: [PATCH 04/15] Add missing module-info to make --macro:native-image-diagnostics-agent-library --macro:native-image-agent-library --macro:native-image-configure-launcher work again --- substratevm/mx.substratevm/suite.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index 1e7b60819843..488b9400ebe4 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -1174,6 +1174,11 @@ "com.oracle.svm.jvmtiagentbase", "com.oracle.svm.jvmtiagentbase.jvmti", ], + "requires" : [ + "static com.oracle.mxtool.junit", + "static junit", + "static hamcrest", + ], }, }, @@ -1323,6 +1328,11 @@ "exports" : [ "com.oracle.svm.agent", ], + "requires" : [ + "static com.oracle.mxtool.junit", + "static junit", + "static hamcrest", + ], "requiresConcealed" : { "jdk.internal.vm.ci" : [ "jdk.vm.ci.meta", @@ -1347,6 +1357,11 @@ "exports" : [ "com.oracle.svm.diagnosticsagent", ], + "requires" : [ + "static com.oracle.mxtool.junit", + "static junit", + "static hamcrest", + ], }, }, @@ -1367,6 +1382,11 @@ "* to org.graalvm.nativeimage.agent.tracing", "com.oracle.svm.configure", ], + "requires" : [ + "static com.oracle.mxtool.junit", + "static junit", + "static hamcrest", + ], }, }, From cc11ae1d1fad550424b1fdc0a6cf7e9d6815874a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Fri, 3 Sep 2021 17:18:27 +0200 Subject: [PATCH 05/15] Allow switching to builder-on-module-path on demand --- .../compiler/options/ModuleSupport.java | 2 +- substratevm/mx.substratevm/mx_substratevm.py | 36 +++--- .../com/oracle/svm/driver/NativeImage.java | 118 ++++++++++-------- 3 files changed, 84 insertions(+), 72 deletions(-) diff --git a/compiler/src/org.graalvm.compiler.options.jdk11/src/org/graalvm/compiler/options/ModuleSupport.java b/compiler/src/org.graalvm.compiler.options.jdk11/src/org/graalvm/compiler/options/ModuleSupport.java index 5e952e899372..04a67c8a9616 100644 --- a/compiler/src/org.graalvm.compiler.options.jdk11/src/org/graalvm/compiler/options/ModuleSupport.java +++ b/compiler/src/org.graalvm.compiler.options.jdk11/src/org/graalvm/compiler/options/ModuleSupport.java @@ -28,7 +28,7 @@ public class ModuleSupport { - public static final boolean USE_NI_JPMS = System.getenv().getOrDefault("USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM", "false").toLowerCase().equals("true"); + public static final boolean USE_NI_JPMS = Boolean.parseBoolean(System.getenv().get("USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM")); static Iterable getOptionsLoader() { /* diff --git a/substratevm/mx.substratevm/mx_substratevm.py b/substratevm/mx.substratevm/mx_substratevm.py index 76ced4e52a25..b6757f58acb2 100644 --- a/substratevm/mx.substratevm/mx_substratevm.py +++ b/substratevm/mx.substratevm/mx_substratevm.py @@ -60,8 +60,6 @@ else: from io import StringIO -USE_NI_JPMS = os.environ.get('USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM', 'false').lower() == 'true' - suite = mx.suite('substratevm') svmSuites = [suite] @@ -71,13 +69,19 @@ def svm_java_compliance(): def svm_java8(): return svm_java_compliance() <= mx.JavaCompliance('1.8') -def graal_compiler_flags(all_unnamed=True): +def graal_compiler_flags(): version_tag = svm_java_compliance().value - compiler_flags = mx.dependency('substratevm:svm-compiler-flags-builder').compute_graal_compiler_flags_map(all_unnamed=all_unnamed) + compiler_flags = mx.dependency('substratevm:svm-compiler-flags-builder').compute_graal_compiler_flags_map() if version_tag not in compiler_flags: missing_flags_message = 'Missing graal-compiler-flags for {0}.\n Did you forget to run "mx build"?' mx.abort(missing_flags_message.format(version_tag)) - return compiler_flags[version_tag] + def adjusted_exports(line): + if line.startswith('--add-exports='): + before, sep, _ = line.rpartition('=') + return before + sep + 'ALL-UNNAMED' + else: + return line + return [adjusted_exports(line) for line in compiler_flags[version_tag]] def svm_unittest_config_participant(config): vmArgs, mainClass, mainClassArgs = config @@ -839,7 +843,7 @@ def _native_image_launcher_extra_jvm_args(): support_distributions=['substratevm:NATIVE_IMAGE_GRAALVM_SUPPORT'], launcher_configs=[ mx_sdk_vm.LauncherConfig( - use_modules='image' if USE_NI_JPMS else 'launcher' if not svm_java8() else None, + use_modules='image' if not svm_java8() else None, main_module="org.graalvm.nativeimage.driver", destination="bin/", jar_distributions=["substratevm:SVM_DRIVER"], @@ -850,7 +854,7 @@ def _native_image_launcher_extra_jvm_args(): ], library_configs=[ mx_sdk_vm.LibraryConfig( - use_modules='image' if USE_NI_JPMS else 'launcher' if not svm_java8() else None, + use_modules='image' if not svm_java8() else None, destination="", jvm_library=True, jar_distributions=[ @@ -864,7 +868,7 @@ def _native_image_launcher_extra_jvm_args(): ], ), mx_sdk_vm.LibraryConfig( - use_modules='image' if USE_NI_JPMS else 'launcher' if not svm_java8() else None, + use_modules='image' if not svm_java8() else None, destination="", jvm_library=True, jar_distributions=[ @@ -1017,7 +1021,7 @@ def _native_image_configure_extra_jvm_args(): support_distributions=[], launcher_configs=[ mx_sdk_vm.LauncherConfig( - use_modules='image' if USE_NI_JPMS else 'launcher' if not svm_java8() else None, + use_modules='image' if not svm_java8() else None, main_module="org.graalvm.nativeimage.configure", destination="bin/", jar_distributions=["substratevm:SVM_CONFIGURE"], @@ -1088,8 +1092,6 @@ def hellomodule(args): """ if svm_java8(): mx.abort('Experimental module support requires Java 11+') - if os.environ.get('USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM', 'false') != 'true': - mx.abort('Experimental module support requires USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM=true for "mx build" and "mx hellomodule"') # Build a helloworld Java module with maven module_path = [] @@ -1440,7 +1442,7 @@ def config_file_update(self, file_path, lines, file_paths): # If renaming or moving this method, please update the error message in # com.oracle.svm.driver.NativeImage.BuildConfiguration.getBuilderJavaArgs(). - def compute_graal_compiler_flags_map(self, all_unnamed=not USE_NI_JPMS): + def compute_graal_compiler_flags_map(self): graal_compiler_flags_map = dict() graal_compiler_flags_map[8] = [ '-d64', @@ -1457,15 +1459,9 @@ def compute_graal_compiler_flags_map(self, all_unnamed=not USE_NI_JPMS): distributions_transitive = mx.classpath_entries(self.deps) jdk = mx.get_jdk(tag='default') required_exports = mx_javamodules.requiredExports(distributions_transitive, jdk) - target_module = 'ALL-UNNAMED' if all_unnamed else None - exports_flags = mx_sdk_vm.AbstractNativeImageConfig.get_add_exports_list(required_exports, target_module) + exports_flags = mx_sdk_vm.AbstractNativeImageConfig.get_add_exports_list(required_exports) graal_compiler_flags_map[11].extend(exports_flags) - - # Currently JDK 13, 14, 15 and JDK 11 have the same flags - graal_compiler_flags_map[13] = graal_compiler_flags_map[11] - graal_compiler_flags_map[14] = graal_compiler_flags_map[11] - graal_compiler_flags_map[15] = graal_compiler_flags_map[11] - graal_compiler_flags_map[16] = graal_compiler_flags_map[11] + # Currently JDK 17 and JDK 11 have the same flags graal_compiler_flags_map[17] = graal_compiler_flags_map[11] # DO NOT ADD ANY NEW ADD-OPENS OR ADD-EXPORTS HERE! # diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 6a9f49ed24c2..5c65f55913da 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -93,8 +93,8 @@ public class NativeImage { - /* Used to enable native-image JPMS support (WIP) - will become default once completed */ - static final boolean USE_NI_JPMS = System.getenv().getOrDefault("USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM", "false").equalsIgnoreCase("true"); + private static final String ENV_VAR_USE_MODULE_SYSTEM = "USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM"; + static boolean modulePathBuild = Boolean.parseBoolean(System.getenv().get(ENV_VAR_USE_MODULE_SYSTEM)); private static final String DEFAULT_GENERATOR_CLASS_NAME = NativeImageGeneratorRunner.class.getName(); private static final String DEFAULT_GENERATOR_MODULE_NAME = ModuleSupport.getModuleName(NativeImageGeneratorRunner.class); @@ -366,20 +366,7 @@ default List getBuilderBootClasspath() { * @return additional arguments for JVM that runs image builder */ default List getBuilderJavaArgs() { - String javaVersion = String.valueOf(JavaVersionUtil.JAVA_SPEC); - String[] flagsForVersion = graalCompilerFlags.get(javaVersion); - if (flagsForVersion == null) { - String suffix = ""; - if (System.getProperty("java.home").contains("-dev")) { - suffix = " Update SubstrateCompilerFlagsBuilder.compute_graal_compiler_flags_map() in " + - "mx_substratevm.py to add a configuration for a new Java version."; - } - showError(String.format("Image building not supported for Java version %s in %s with VM configuration \"%s\".%s", - System.getProperty("java.version"), - System.getProperty("java.home"), - System.getProperty("java.vm.name"), - suffix)); - } + ArrayList builderJavaArgs = new ArrayList<>(); if (useJVMCINativeLibrary == null) { useJVMCINativeLibrary = false; @@ -413,13 +400,35 @@ default List getBuilderJavaArgs() { } } - ArrayList builderJavaArgs = new ArrayList<>(); - builderJavaArgs.addAll(Arrays.asList(flagsForVersion)); if (useJVMCINativeLibrary) { builderJavaArgs.add("-XX:+UseJVMCINativeLibrary"); } else { builderJavaArgs.add("-XX:-UseJVMCICompiler"); } + + String javaVersion = String.valueOf(JavaVersionUtil.JAVA_SPEC); + String[] flagsForVersion = graalCompilerFlags.get(javaVersion); + if (flagsForVersion == null) { + String suffix = ""; + if (System.getProperty("java.home").contains("-dev")) { + suffix = " Update SubstrateCompilerFlagsBuilder.compute_graal_compiler_flags_map() in " + + "mx_substratevm.py to add a configuration for a new Java version."; + } + showError(String.format("Image building not supported for Java version %s in %s with VM configuration \"%s\".%s", + System.getProperty("java.version"), + System.getProperty("java.home"), + System.getProperty("java.vm.name"), + suffix)); + } + + for (String line : flagsForVersion) { + if (!modulePathBuild && line.startsWith("--add-exports=")) { + builderJavaArgs.add(line.substring(0, line.lastIndexOf('=') + 1) + "ALL-UNNAMED"); + } else { + builderJavaArgs.add(line); + } + } + return builderJavaArgs; } @@ -545,7 +554,7 @@ public Path getJavaExecutable() { @Override public List getBuilderClasspath() { - if (USE_NI_JPMS) { + if (modulePathBuild) { return Collections.emptyList(); } List result = new ArrayList<>(); @@ -604,7 +613,7 @@ public List getBuilderBootClasspath() { @Override public List getBuilderModulePath() { List result = new ArrayList<>(); - if (USE_NI_JPMS) { + if (modulePathBuild) { result.addAll(getJars(rootDir.resolve(Paths.get("lib", "svm", "builder")))); } else { result.addAll(getJars(rootDir.resolve(Paths.get("lib", "jvmci")), "graal-sdk", "enterprise-graal")); @@ -858,7 +867,6 @@ protected static void ensureDirectoryExists(Path dir) { } private void prepareImageBuildArgs() { - config.getBuilderJavaArgs().forEach(this::addImageBuilderJavaArgs); addImageBuilderJavaArgs("-Xss10m"); addImageBuilderJavaArgs(oXms + getXmsValue()); String xmxVal = getXmxValue(1); @@ -887,32 +895,6 @@ private void prepareImageBuildArgs() { * non-system class and and triggers a warning. */ addImageBuilderJavaArgs("-Xshare:off"); - config.getBuilderClasspath().forEach(this::addImageBuilderClasspath); - - if (config.getBuilderInspectServerPath() != null) { - addPlainImageBuilderArg(oHInspectServerContentPath + config.getBuilderInspectServerPath()); - } - - if (config.useJavaModules()) { - config.getBuilderModulePath().forEach(this::addImageBuilderModulePath); - String upgradeModulePath = config.getBuilderUpgradeModulePath().stream() - .map(p -> canonicalize(p).toString()) - .collect(Collectors.joining(File.pathSeparator)); - if (!upgradeModulePath.isEmpty()) { - addImageBuilderJavaArgs(Arrays.asList("--upgrade-module-path", upgradeModulePath)); - } - } else { - config.getBuilderJVMCIClasspath().forEach((Consumer) this::addImageBuilderClasspath); - if (!config.getBuilderJVMCIClasspathAppend().isEmpty()) { - String builderJavaArg = config.getBuilderJVMCIClasspathAppend() - .stream().map(path -> canonicalize(path).toString()) - .collect(Collectors.joining(File.pathSeparator, "-Djvmci.class.path.append=", "")); - addImageBuilderJavaArgs(builderJavaArg); - } - - config.getBuilderBootClasspath().forEach((Consumer) this::addImageBuilderBootClasspath); - config.getResourcesJar().ifPresent(this::addImageBuilderClasspath); - } config.getImageClasspath().forEach(this::addCustomImageClasspath); } @@ -1089,6 +1071,33 @@ void handleClassPathAttribute(Path jarFilePath, Attributes mainAttributes) { private int completeImageBuild() { List leftoverArgs = processNativeImageArgs(); + config.getBuilderClasspath().forEach(this::addImageBuilderClasspath); + + if (config.getBuilderInspectServerPath() != null) { + addPlainImageBuilderArg(oHInspectServerContentPath + config.getBuilderInspectServerPath()); + } + + if (config.useJavaModules()) { + config.getBuilderModulePath().forEach(this::addImageBuilderModulePath); + String upgradeModulePath = config.getBuilderUpgradeModulePath().stream() + .map(p -> canonicalize(p).toString()) + .collect(Collectors.joining(File.pathSeparator)); + if (!upgradeModulePath.isEmpty()) { + addImageBuilderJavaArgs(Arrays.asList("--upgrade-module-path", upgradeModulePath)); + } + } else { + config.getBuilderJVMCIClasspath().forEach((Consumer) this::addImageBuilderClasspath); + if (!config.getBuilderJVMCIClasspathAppend().isEmpty()) { + String builderJavaArg = config.getBuilderJVMCIClasspathAppend() + .stream().map(path -> canonicalize(path).toString()) + .collect(Collectors.joining(File.pathSeparator, "-Djvmci.class.path.append=", "")); + addImageBuilderJavaArgs(builderJavaArg); + } + + config.getBuilderBootClasspath().forEach((Consumer) this::addImageBuilderBootClasspath); + config.getResourcesJar().ifPresent(this::addImageBuilderClasspath); + } + completeOptionArgs(); addTargetArguments(); @@ -1168,7 +1177,7 @@ private int completeImageBuild() { boolean hasMainClass = mainClass != null && !mainClass.isEmpty(); if (extraImageArgs.isEmpty()) { if (buildExecutable && !hasMainClassModule && !hasMainClass) { - String moduleMsg = USE_NI_JPMS ? " (or /)" : ""; + String moduleMsg = modulePathBuild ? " (or /)" : ""; showError("Please specify class" + moduleMsg + " containing the main entry point method. (see --help)"); } } else if (!moduleOptionMode) { @@ -1222,7 +1231,7 @@ private int completeImageBuild() { finalImageClasspath.addAll(imageClasspath); List imageProvidedJars; - if (USE_NI_JPMS) { + if (modulePathBuild) { imageProvidedJars = config.getImageProvidedModulePath(); finalImageModulePath.addAll(imageProvidedJars); } else { @@ -1235,7 +1244,9 @@ private int completeImageBuild() { /* Bypass regular build and proceed with fallback image building */ return 2; } - return buildImage(imageBuilderJavaArgs, imageBuilderBootClasspath, imageBuilderClasspath, imageBuilderModulePath, imageBuilderArgs, finalImageClasspath, finalImageModulePath); + + List finalImageBuilderJavaArgs = Stream.concat(config.getBuilderJavaArgs().stream(), imageBuilderJavaArgs.stream()).collect(Collectors.toList()); + return buildImage(finalImageBuilderJavaArgs, imageBuilderBootClasspath, imageBuilderClasspath, imageBuilderModulePath, imageBuilderArgs, finalImageClasspath, finalImageModulePath); } private static String getLocationAgnosticArgPrefix(String argPrefix) { @@ -1411,7 +1422,7 @@ protected int buildImage(List javaArgs, LinkedHashSet bcp, LinkedH arguments.addAll(strings); } - if (USE_NI_JPMS) { + if (modulePathBuild) { arguments.addAll(Arrays.asList("--module", DEFAULT_GENERATOR_MODULE_NAME + "/" + DEFAULT_GENERATOR_CLASS_NAME)); } else { arguments.add(config.getGeneratorMainClass()); @@ -1457,6 +1468,9 @@ protected int buildImage(List javaArgs, LinkedHashSet bcp, LinkedH try { ProcessBuilder pb = new ProcessBuilder(); pb.command(command); + if (modulePathBuild) { + pb.environment().put(ENV_VAR_USE_MODULE_SYSTEM, Boolean.toString(true)); + } p = pb.inheritIO().start(); exitStatus = p.waitFor(); } catch (IOException | InterruptedException e) { @@ -1656,6 +1670,7 @@ void addImageModulePath(Path modulePathEntry, boolean strict) { return; } + modulePathBuild = true; imageModulePath.add(mpEntry); processClasspathNativeImageMetaInf(mpEntry); } @@ -1728,6 +1743,7 @@ void setJarOptionMode(boolean val) { void setModuleOptionMode(boolean val) { moduleOptionMode = val; + modulePathBuild = true; } boolean isVerbose() { From 9832f311dd86f4ede7ff4f59d7f820ece021a87a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 6 Sep 2021 11:43:38 +0200 Subject: [PATCH 06/15] Remove obsolete use of envvar USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM --- .github/workflows/main.yml | 1 - substratevm/ci_includes/gate.hocon | 3 --- 2 files changed, 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fd6ceead2c57..b5dce652c33e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -84,7 +84,6 @@ jobs: JDK: "labsjdk-ce-11" GATE: "hellomodule" PRIMARY: "substratevm" - USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM: true - env: JDK: "labsjdk-ce-11" GATE: "style,fullbuild" diff --git a/substratevm/ci_includes/gate.hocon b/substratevm/ci_includes/gate.hocon index e5943932e121..fe41f3535dbf 100644 --- a/substratevm/ci_includes/gate.hocon +++ b/substratevm/ci_includes/gate.hocon @@ -45,9 +45,6 @@ builds += [ } ${labsjdk-ee-11} ${svm-common-linux-gate} ${linux-deploy} { name: "gate-svm-modules-basic" - environment : { - USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM : "true" - } run: [ ${svm-cmd-gate} ["build,hellomodule,test"] ] From e718d0d2a281c25ab0fffb9068bdbcd572850768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 6 Sep 2021 18:23:12 +0200 Subject: [PATCH 07/15] Allow adding extra_deps to required_exports --- substratevm/mx.substratevm/mx_substratevm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/substratevm/mx.substratevm/mx_substratevm.py b/substratevm/mx.substratevm/mx_substratevm.py index b6757f58acb2..49a21d6a3fd4 100644 --- a/substratevm/mx.substratevm/mx_substratevm.py +++ b/substratevm/mx.substratevm/mx_substratevm.py @@ -1387,6 +1387,8 @@ def __str__(self): class SubstrateCompilerFlagsBuilder(mx.ArchivableProject): + extra_deps = [] + def config_file(self, ver): return 'graal-compiler-flags-' + str(ver) + '.config' @@ -1456,7 +1458,7 @@ def compute_graal_compiler_flags_map(self): ] # Packages to add-export - distributions_transitive = mx.classpath_entries(self.deps) + distributions_transitive = mx.classpath_entries(self.deps + SubstrateCompilerFlagsBuilder.extra_deps) jdk = mx.get_jdk(tag='default') required_exports = mx_javamodules.requiredExports(distributions_transitive, jdk) exports_flags = mx_sdk_vm.AbstractNativeImageConfig.get_add_exports_list(required_exports) From 1b1a439a363de0a9eeed7c112709b841a1b984de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 7 Sep 2021 10:59:52 +0200 Subject: [PATCH 08/15] Refactor NativeImage.BuildConfiguration,DefaultBuildConfiguration --- .../com/oracle/svm/driver/NativeImage.java | 396 +++++++----------- 1 file changed, 155 insertions(+), 241 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 5c65f55913da..7dc8b4f2c7f8 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -31,9 +31,6 @@ import java.io.InputStreamReader; import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; -import java.lang.reflect.InvocationHandler; -import java.lang.reflect.Method; -import java.lang.reflect.Proxy; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.InvalidPathException; @@ -94,7 +91,6 @@ public class NativeImage { private static final String ENV_VAR_USE_MODULE_SYSTEM = "USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM"; - static boolean modulePathBuild = Boolean.parseBoolean(System.getenv().get(ENV_VAR_USE_MODULE_SYSTEM)); private static final String DEFAULT_GENERATOR_CLASS_NAME = NativeImageGeneratorRunner.class.getName(); private static final String DEFAULT_GENERATOR_MODULE_NAME = ModuleSupport.getModuleName(NativeImageGeneratorRunner.class); @@ -262,11 +258,59 @@ private static String oR(OptionKey option) { private final List excludedConfigs = new ArrayList<>(); - public interface BuildConfiguration { + static class BuildConfiguration { + + boolean modulePathBuild = Boolean.parseBoolean(System.getenv().get(ENV_VAR_USE_MODULE_SYSTEM)); + + protected final Path workDir; + protected final Path rootDir; + protected final List args; + + BuildConfiguration(BuildConfiguration original) { + modulePathBuild = original.modulePathBuild; + workDir = original.workDir; + rootDir = original.rootDir; + args = new ArrayList<>(original.args); + } + + BuildConfiguration(List args) { + this(null, null, args); + } + + @SuppressWarnings("deprecation") + BuildConfiguration(Path rootDir, Path workDir, List args) { + this.args = args; + this.workDir = workDir != null ? workDir : Paths.get(".").toAbsolutePath().normalize(); + if (rootDir != null) { + this.rootDir = rootDir; + } else { + if (IS_AOT) { + Path executablePath = Paths.get(ProcessProperties.getExecutableName()); + assert executablePath != null; + Path binDir = executablePath.getParent(); + Path rootDirCandidate = binDir.getParent(); + if (rootDirCandidate.endsWith(platform)) { + rootDirCandidate = rootDirCandidate.getParent(); + } + if (rootDirCandidate.endsWith(Paths.get("lib", "svm"))) { + rootDirCandidate = rootDirCandidate.getParent().getParent(); + } + this.rootDir = rootDirCandidate; + } else { + String rootDirProperty = "native-image.root"; + String rootDirString = System.getProperty(rootDirProperty); + if (rootDirString == null) { + rootDirString = System.getProperty("java.home"); + } + this.rootDir = Paths.get(rootDirString); + } + } + } + /** * @return the name of the image generator main class. */ - default String getGeneratorMainClass() { + public String getGeneratorMainClass() { String generatorClassName = DEFAULT_GENERATOR_CLASS_NAME; if (useJavaModules()) { generatorClassName += DEFAULT_GENERATOR_9PLUS_SUFFIX; @@ -278,26 +322,44 @@ default String getGeneratorMainClass() { * @return relative path usage get resolved against this path (also default path for image * building) */ - Path getWorkingDirectory(); + public Path getWorkingDirectory() { + return workDir; + } /** * @return java.home that is associated with this BuildConfiguration */ - default Path getJavaHome() { - throw VMError.unimplemented(); + public Path getJavaHome() { + return useJavaModules() ? rootDir : rootDir.getParent(); } /** * @return path to Java executable */ - default Path getJavaExecutable() { - throw VMError.unimplemented(); + public Path getJavaExecutable() { + Path binJava = Paths.get("bin", OS.getCurrent() == OS.WINDOWS ? "java.exe" : "java"); + if (Files.isExecutable(rootDir.resolve(binJava))) { + return rootDir.resolve(binJava); + } + + String javaHome = System.getenv("JAVA_HOME"); + if (javaHome == null) { + throw showError("Environment variable JAVA_HOME is not set"); + } + Path javaHomeDir = Paths.get(javaHome); + if (!Files.isDirectory(javaHomeDir)) { + throw showError("Environment variable JAVA_HOME does not refer to a directory"); + } + if (!Files.isExecutable(javaHomeDir.resolve(binJava))) { + throw showError("Environment variable JAVA_HOME does not refer to a directory with a " + binJava + " executable"); + } + return javaHomeDir.resolve(binJava); } /** * @return true if Java modules system should be used */ - default boolean useJavaModules() { + public boolean useJavaModules() { try { Class.forName("java.lang.Module"); } catch (ClassNotFoundException e) { @@ -309,63 +371,81 @@ default boolean useJavaModules() { /** * @return classpath for SubstrateVM image builder components */ - default List getBuilderClasspath() { - throw VMError.unimplemented(); + public List getBuilderClasspath() { + if (modulePathBuild) { + return Collections.emptyList(); + } + List result = new ArrayList<>(); + if (useJavaModules()) { + result.addAll(getJars(rootDir.resolve(Paths.get("lib", "jvmci")), "graal-sdk", "graal", "enterprise-graal")); + } + result.addAll(getJars(rootDir.resolve(Paths.get("lib", "svm", "builder")))); + return result; } /** * @return base clibrary paths needed for general image building */ - default List getBuilderCLibrariesPaths() { - throw VMError.unimplemented(); + public List getBuilderCLibrariesPaths() { + return Collections.singletonList(rootDir.resolve(Paths.get("lib", "svm", "clibraries"))); } /** * @return path to content of the inspect web server (points-to analysis debugging) */ - default Path getBuilderInspectServerPath() { - throw VMError.unimplemented(); + public Path getBuilderInspectServerPath() { + Path inspectPath = rootDir.resolve(Paths.get("lib", "svm", "inspect")); + if (Files.isDirectory(inspectPath)) { + return inspectPath; + } + return null; } /** * @return base image classpath needed for every image (e.g. LIBRARY_SUPPORT) */ - default List getImageProvidedClasspath() { - throw VMError.unimplemented(); + public List getImageProvidedClasspath() { + return getImageProvidedJars(); } /** * @return base image module-path needed for every image (e.g. LIBRARY_SUPPORT) */ - default List getImageProvidedModulePath() { - throw VMError.unimplemented(); + public List getImageProvidedModulePath() { + return getImageProvidedJars(); + } + + private List getImageProvidedJars() { + return getJars(rootDir.resolve(Paths.get("lib", "svm"))); } /** * @return JVMCI API classpath for image builder (jvmci + graal jars) */ - default List getBuilderJVMCIClasspath() { - throw VMError.unimplemented(); + public List getBuilderJVMCIClasspath() { + return getJars(rootDir.resolve(Paths.get("lib", "jvmci"))); } /** * @return entries for jvmci.class.path.append system property (if needed) */ - default List getBuilderJVMCIClasspathAppend() { - throw VMError.unimplemented(); + public List getBuilderJVMCIClasspathAppend() { + return getBuilderJVMCIClasspath().stream() + .filter(f -> f.getFileName().toString().toLowerCase().endsWith("graal.jar")) + .collect(Collectors.toList()); } /** * @return boot-classpath for image builder (graal-sdk.jar) */ - default List getBuilderBootClasspath() { - throw VMError.unimplemented(); + public List getBuilderBootClasspath() { + return getJars(rootDir.resolve(Paths.get("lib", "boot"))); } /** * @return additional arguments for JVM that runs image builder */ - default List getBuilderJavaArgs() { + public List getBuilderJavaArgs() { ArrayList builderJavaArgs = new ArrayList<>(); if (useJVMCINativeLibrary == null) { @@ -435,182 +515,6 @@ default List getBuilderJavaArgs() { /** * @return entries for the --module-path of the image builder */ - default List getBuilderModulePath() { - throw VMError.unimplemented(); - } - - /** - * @return entries for the --upgrade-module-path of the image builder - */ - default List getBuilderUpgradeModulePath() { - throw VMError.unimplemented(); - } - - /** - * @return classpath for image (the classes the user wants to build an image from) - */ - default List getImageClasspath() { - throw VMError.unimplemented(); - } - - /** - * @return native-image (i.e. image build) arguments - */ - default List getBuildArgs() { - throw VMError.unimplemented(); - } - - /** - * @return true for fallback image building - */ - default boolean buildFallbackImage() { - return false; - } - - default Path getAgentJAR() { - return null; - } - - /** - * ResourcesJar packs resources files needed for some jdk services such as xml - * serialization. - * - * @return the path to the resources.jar file - */ - default Optional getResourcesJar() { - return Optional.empty(); - } - } - - protected static class DefaultBuildConfiguration implements BuildConfiguration { - protected final Path workDir; - protected final Path rootDir; - protected final List args; - - protected DefaultBuildConfiguration(List args) { - this(null, null, args); - } - - @SuppressWarnings("deprecation") - DefaultBuildConfiguration(Path rootDir, Path workDir, List args) { - this.args = args; - this.workDir = workDir != null ? workDir : Paths.get(".").toAbsolutePath().normalize(); - if (rootDir != null) { - this.rootDir = rootDir; - } else { - if (IS_AOT) { - Path executablePath = Paths.get(ProcessProperties.getExecutableName()); - assert executablePath != null; - Path binDir = executablePath.getParent(); - Path rootDirCandidate = binDir.getParent(); - if (rootDirCandidate.endsWith(platform)) { - rootDirCandidate = rootDirCandidate.getParent(); - } - if (rootDirCandidate.endsWith(Paths.get("lib", "svm"))) { - rootDirCandidate = rootDirCandidate.getParent().getParent(); - } - this.rootDir = rootDirCandidate; - } else { - String rootDirProperty = "native-image.root"; - String rootDirString = System.getProperty(rootDirProperty); - if (rootDirString == null) { - rootDirString = System.getProperty("java.home"); - } - this.rootDir = Paths.get(rootDirString); - } - } - } - - @Override - public Path getWorkingDirectory() { - return workDir; - } - - @Override - public Path getJavaHome() { - return useJavaModules() ? rootDir : rootDir.getParent(); - } - - @Override - public Path getJavaExecutable() { - Path binJava = Paths.get("bin", OS.getCurrent() == OS.WINDOWS ? "java.exe" : "java"); - if (Files.isExecutable(rootDir.resolve(binJava))) { - return rootDir.resolve(binJava); - } - - String javaHome = System.getenv("JAVA_HOME"); - if (javaHome == null) { - throw showError("Environment variable JAVA_HOME is not set"); - } - Path javaHomeDir = Paths.get(javaHome); - if (!Files.isDirectory(javaHomeDir)) { - throw showError("Environment variable JAVA_HOME does not refer to a directory"); - } - if (!Files.isExecutable(javaHomeDir.resolve(binJava))) { - throw showError("Environment variable JAVA_HOME does not refer to a directory with a " + binJava + " executable"); - } - return javaHomeDir.resolve(binJava); - } - - @Override - public List getBuilderClasspath() { - if (modulePathBuild) { - return Collections.emptyList(); - } - List result = new ArrayList<>(); - if (useJavaModules()) { - result.addAll(getJars(rootDir.resolve(Paths.get("lib", "jvmci")), "graal-sdk", "graal", "enterprise-graal")); - } - result.addAll(getJars(rootDir.resolve(Paths.get("lib", "svm", "builder")))); - return result; - } - - @Override - public List getBuilderCLibrariesPaths() { - return Collections.singletonList(rootDir.resolve(Paths.get("lib", "svm", "clibraries"))); - } - - @Override - public List getImageProvidedClasspath() { - return getImageProvidedJars(); - } - - @Override - public List getImageProvidedModulePath() { - return getImageProvidedJars(); - } - - private List getImageProvidedJars() { - return getJars(rootDir.resolve(Paths.get("lib", "svm"))); - } - - @Override - public Path getBuilderInspectServerPath() { - Path inspectPath = rootDir.resolve(Paths.get("lib", "svm", "inspect")); - if (Files.isDirectory(inspectPath)) { - return inspectPath; - } - return null; - } - - @Override - public List getBuilderJVMCIClasspath() { - return getJars(rootDir.resolve(Paths.get("lib", "jvmci"))); - } - - @Override - public List getBuilderJVMCIClasspathAppend() { - return getBuilderJVMCIClasspath().stream() - .filter(f -> f.getFileName().toString().toLowerCase().endsWith("graal.jar")) - .collect(Collectors.toList()); - } - - @Override - public List getBuilderBootClasspath() { - return getJars(rootDir.resolve(Paths.get("lib", "boot"))); - } - - @Override public List getBuilderModulePath() { List result = new ArrayList<>(); if (modulePathBuild) { @@ -622,17 +526,23 @@ public List getBuilderModulePath() { return result; } - @Override + /** + * @return entries for the --upgrade-module-path of the image builder + */ public List getBuilderUpgradeModulePath() { return getJars(rootDir.resolve(Paths.get("lib", "jvmci")), "graal", "graal-management"); } - @Override + /** + * @return classpath for image (the classes the user wants to build an image from) + */ public List getImageClasspath() { return Collections.emptyList(); } - @Override + /** + * @return native-image (i.e. image build) arguments + */ public List getBuildArgs() { if (args.isEmpty()) { return Collections.emptyList(); @@ -644,12 +554,23 @@ public List getBuildArgs() { return buildArgs; } - @Override + /** + * @return true for fallback image building + */ + public boolean buildFallbackImage() { + return false; + } + public Path getAgentJAR() { return rootDir.resolve(Paths.get("lib", "svm", "builder", "svm.jar")); } - @Override + /** + * ResourcesJar packs resources files needed for some jdk services such as xml + * serialization. + * + * @return the path to the resources.jar file + */ public Optional getResourcesJar() { return Optional.of(rootDir.resolve(Paths.get("lib", "resources.jar"))); } @@ -769,35 +690,28 @@ private ArrayList createFallbackBuildArgs() { return buildArgs; } - private static final class FallbackBuildConfiguration implements InvocationHandler { - private final NativeImage original; - private final List buildArgs; + private static final class FallbackBuildConfiguration extends BuildConfiguration { + + private final List fallbackBuildArgs; private FallbackBuildConfiguration(NativeImage original) { - this.original = original; - this.buildArgs = original.createFallbackBuildArgs(); + super(original.config); + fallbackBuildArgs = original.createFallbackBuildArgs(); + } + + @Override + public List getImageClasspath() { + return Collections.emptyList(); } - static BuildConfiguration create(NativeImage imageName) { - FallbackBuildConfiguration handler = new FallbackBuildConfiguration(imageName); - BuildConfiguration fallback = (BuildConfiguration) Proxy.newProxyInstance(BuildConfiguration.class.getClassLoader(), - new Class[]{BuildConfiguration.class}, - handler); - return fallback; + @Override + public List getBuildArgs() { + return fallbackBuildArgs; } @Override - public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { - switch (method.getName()) { - case "getImageClasspath": - return Collections.emptyList(); - case "getBuildArgs": - return buildArgs; - case "buildFallbackImage": - return true; - default: - return method.invoke(original.config, args); - } + public boolean buildFallbackImage() { + return true; } } @@ -1177,7 +1091,7 @@ private int completeImageBuild() { boolean hasMainClass = mainClass != null && !mainClass.isEmpty(); if (extraImageArgs.isEmpty()) { if (buildExecutable && !hasMainClassModule && !hasMainClass) { - String moduleMsg = modulePathBuild ? " (or /)" : ""; + String moduleMsg = config.modulePathBuild ? " (or /)" : ""; showError("Please specify class" + moduleMsg + " containing the main entry point method. (see --help)"); } } else if (!moduleOptionMode) { @@ -1231,7 +1145,7 @@ private int completeImageBuild() { finalImageClasspath.addAll(imageClasspath); List imageProvidedJars; - if (modulePathBuild) { + if (config.modulePathBuild) { imageProvidedJars = config.getImageProvidedModulePath(); finalImageModulePath.addAll(imageProvidedJars); } else { @@ -1422,7 +1336,7 @@ protected int buildImage(List javaArgs, LinkedHashSet bcp, LinkedH arguments.addAll(strings); } - if (modulePathBuild) { + if (config.modulePathBuild) { arguments.addAll(Arrays.asList("--module", DEFAULT_GENERATOR_MODULE_NAME + "/" + DEFAULT_GENERATOR_CLASS_NAME)); } else { arguments.add(config.getGeneratorMainClass()); @@ -1468,7 +1382,7 @@ protected int buildImage(List javaArgs, LinkedHashSet bcp, LinkedH try { ProcessBuilder pb = new ProcessBuilder(); pb.command(command); - if (modulePathBuild) { + if (config.modulePathBuild) { pb.environment().put(ENV_VAR_USE_MODULE_SYSTEM, Boolean.toString(true)); } p = pb.inheritIO().start(); @@ -1486,7 +1400,7 @@ protected int buildImage(List javaArgs, LinkedHashSet bcp, LinkedH private static final Function defaultNativeImageProvider = config -> new NativeImage(config); public static void main(String[] args) { - performBuild(new DefaultBuildConfiguration(Arrays.asList(args)), defaultNativeImageProvider); + performBuild(new BuildConfiguration(Arrays.asList(args)), defaultNativeImageProvider); } public static void build(BuildConfiguration config) { @@ -1494,7 +1408,7 @@ public static void build(BuildConfiguration config) { } public static void agentBuild(Path javaHome, Path workDir, List buildArgs) { - performBuild(new DefaultBuildConfiguration(javaHome, workDir, buildArgs), NativeImage::new); + performBuild(new BuildConfiguration(javaHome, workDir, buildArgs), NativeImage::new); } private static void performBuild(BuildConfiguration config, Function nativeImageProvider) { @@ -1532,7 +1446,7 @@ protected static void build(BuildConfiguration config, Function Date: Tue, 7 Sep 2021 12:25:57 +0200 Subject: [PATCH 09/15] Add missing com.oracle.objectfile.macho export for module org.graalvm.nativeimage.objectfile --- substratevm/mx.substratevm/suite.py | 1 + 1 file changed, 1 insertion(+) diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index 488b9400ebe4..e89f066b06cc 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -1225,6 +1225,7 @@ "com.oracle.objectfile", "com.oracle.objectfile.io", "com.oracle.objectfile.debuginfo", + "com.oracle.objectfile.macho", ], "requires" : ["jdk.unsupported"], From e37e9688e95f18d7c39ca1b7047199bbe459abab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 7 Sep 2021 18:53:35 +0200 Subject: [PATCH 10/15] Dependencies of svm-compiler-flags-builder are buildDependencies --- substratevm/mx.substratevm/suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index e89f066b06cc..a2eb353650ab 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -626,7 +626,7 @@ "svm-compiler-flags-builder": { "class" : "SubstrateCompilerFlagsBuilder", - "dependencies" : [ + "buildDependencies" : [ "SVM", ], }, From a7f6b34f157091341a44de44c563c2cd85a66f30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 8 Sep 2021 14:29:36 +0200 Subject: [PATCH 11/15] Generate svm-compiler-flags-builder project via mx_register_dynamic_suite_constituents --- substratevm/mx.substratevm/mx_substratevm.py | 13 +++++++++++-- substratevm/mx.substratevm/suite.py | 7 ------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/substratevm/mx.substratevm/mx_substratevm.py b/substratevm/mx.substratevm/mx_substratevm.py index 49a21d6a3fd4..829f7abd9ad4 100644 --- a/substratevm/mx.substratevm/mx_substratevm.py +++ b/substratevm/mx.substratevm/mx_substratevm.py @@ -1385,9 +1385,18 @@ def clean(self, forBuild=False): def __str__(self): return 'JvmFuncsFallbacksBuildTask {}'.format(self.subject) +def mx_register_dynamic_suite_constituents(register_project, _): + register_project(SubstrateCompilerFlagsBuilder()) + class SubstrateCompilerFlagsBuilder(mx.ArchivableProject): - extra_deps = [] + flags_build_dependencies = [ + 'substratevm:SVM' + ] + + def __init__(self): + mx.ArchivableProject.__init__(self, suite, 'svm-compiler-flags-builder', [], None, None) + self.buildDependencies = list(SubstrateCompilerFlagsBuilder.flags_build_dependencies) def config_file(self, ver): return 'graal-compiler-flags-' + str(ver) + '.config' @@ -1458,7 +1467,7 @@ def compute_graal_compiler_flags_map(self): ] # Packages to add-export - distributions_transitive = mx.classpath_entries(self.deps + SubstrateCompilerFlagsBuilder.extra_deps) + distributions_transitive = mx.classpath_entries(self.buildDependencies) jdk = mx.get_jdk(tag='default') required_exports = mx_javamodules.requiredExports(distributions_transitive, jdk) exports_flags = mx_sdk_vm.AbstractNativeImageConfig.get_add_exports_list(required_exports) diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index a2eb353650ab..b823d090a035 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -624,13 +624,6 @@ "spotbugs": "false", }, - "svm-compiler-flags-builder": { - "class" : "SubstrateCompilerFlagsBuilder", - "buildDependencies" : [ - "SVM", - ], - }, - "com.oracle.svm.junit": { "subDir": "src", "sourceDirs": ["src"], From 0e56563f0872520ddece8ab29d73e3248a83ce6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 9 Sep 2021 10:05:12 +0200 Subject: [PATCH 12/15] Put the field initialization into constructor --- .../src/com/oracle/svm/driver/NativeImage.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 7dc8b4f2c7f8..781d15ad4a07 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -260,7 +260,7 @@ private static String oR(OptionKey option) { static class BuildConfiguration { - boolean modulePathBuild = Boolean.parseBoolean(System.getenv().get(ENV_VAR_USE_MODULE_SYSTEM)); + boolean modulePathBuild; protected final Path workDir; protected final Path rootDir; @@ -279,6 +279,7 @@ static class BuildConfiguration { @SuppressWarnings("deprecation") BuildConfiguration(Path rootDir, Path workDir, List args) { + modulePathBuild = Boolean.parseBoolean(System.getenv().get(ENV_VAR_USE_MODULE_SYSTEM)); this.args = args; this.workDir = workDir != null ? workDir : Paths.get(".").toAbsolutePath().normalize(); if (rootDir != null) { From aeff080cb389608a810bf33317df8e38984cf0e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 9 Sep 2021 10:24:19 +0200 Subject: [PATCH 13/15] Comment add-exports transformation --- .../src/com/oracle/svm/driver/NativeImage.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 781d15ad4a07..22ac4521e448 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -504,6 +504,12 @@ public List getBuilderJavaArgs() { for (String line : flagsForVersion) { if (!modulePathBuild && line.startsWith("--add-exports=")) { + /*- + * Turns e.g. + * --add-exports=jdk.internal.vm.ci/jdk.vm.ci.code.stack=jdk.internal.vm.compiler,org.graalvm.nativeimage.builder + * into: + * --add-exports=jdk.internal.vm.ci/jdk.vm.ci.code.stack=ALL-UNNAMED + */ builderJavaArgs.add(line.substring(0, line.lastIndexOf('=') + 1) + "ALL-UNNAMED"); } else { builderJavaArgs.add(line); From ed713b955e72ab60759f694614c17ac77bd38a07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 9 Sep 2021 10:29:11 +0200 Subject: [PATCH 14/15] Comment graal_compiler_flags.adjusted_exports --- substratevm/mx.substratevm/mx_substratevm.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/substratevm/mx.substratevm/mx_substratevm.py b/substratevm/mx.substratevm/mx_substratevm.py index 829f7abd9ad4..10559f76de6b 100644 --- a/substratevm/mx.substratevm/mx_substratevm.py +++ b/substratevm/mx.substratevm/mx_substratevm.py @@ -76,6 +76,12 @@ def graal_compiler_flags(): missing_flags_message = 'Missing graal-compiler-flags for {0}.\n Did you forget to run "mx build"?' mx.abort(missing_flags_message.format(version_tag)) def adjusted_exports(line): + """ + Turns e.g. + --add-exports=jdk.internal.vm.ci/jdk.vm.ci.code.stack=jdk.internal.vm.compiler,org.graalvm.nativeimage.builder + into: + --add-exports=jdk.internal.vm.ci/jdk.vm.ci.code.stack=ALL-UNNAMED + """ if line.startswith('--add-exports='): before, sep, _ = line.rpartition('=') return before + sep + 'ALL-UNNAMED' From db63ba5500fe42f3547fb3995e8de0e4b2fa8ee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 9 Sep 2021 10:34:16 +0200 Subject: [PATCH 15/15] Add comment on set-difference computation --- sdk/mx.sdk/mx_sdk_vm_impl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/mx.sdk/mx_sdk_vm_impl.py b/sdk/mx.sdk/mx_sdk_vm_impl.py index c0677f42d852..0acb219f353e 100644 --- a/sdk/mx.sdk/mx_sdk_vm_impl.py +++ b/sdk/mx.sdk/mx_sdk_vm_impl.py @@ -1999,6 +1999,7 @@ def graalvm_home_relative_classpath(dependencies, start=None, with_boot_jars=Fal mx.logv("Composing classpath for " + str(dependencies) + ". Entries:\n" + '\n'.join(('- {}:{}'.format(d.suite, d.name) for d in mx.classpath_entries(dependencies)))) cp_entries = mx.classpath_entries(dependencies) + # Compute the set-difference of the transitive dependencies of `dependencies` and the transitive dependencies of `exclude_names` if exclude_names: for exclude_entry in mx.classpath_entries(names=exclude_names): if exclude_entry in cp_entries: