From ed3c0034019cd00fcd69dd94001690c9f2c938d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 29 Nov 2022 21:37:33 +0100 Subject: [PATCH 01/46] First draft of replay-bundle support --- .../com/oracle/svm/core/SubstrateOptions.java | 2 +- .../oracle/svm/core/c/ProjectHeaderFile.java | 4 +- .../com/oracle/svm/core/option/APIOption.java | 8 +- .../oracle/svm/driver/APIOptionHandler.java | 84 ++++-- .../svm/driver/CmdLineOptionHandler.java | 29 ++ .../com/oracle/svm/driver/NativeImage.java | 79 ++++-- .../com/oracle/svm/driver/ReplaySupport.java | 264 ++++++++++++++++++ .../oracle/svm/hosted/c/NativeLibraries.java | 2 +- 8 files changed, 408 insertions(+), 64 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index b4811a2be9c7..d15756c23f61 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -271,7 +271,7 @@ public static void setDebugInfoValueUpdateHandler(ValueUpdateHandler up public static final HostedOptionKey IncludeNodeSourcePositions = new HostedOptionKey<>(false); @Option(help = "Search path for C libraries passed to the linker (list of comma-separated directories)")// - public static final HostedOptionKey CLibraryPath = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey CLibraryPath = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.commaSeparated()); @Option(help = "Path passed to the linker as the -rpath (list of comma-separated directories)")// public static final HostedOptionKey LinkerRPath = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/ProjectHeaderFile.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/ProjectHeaderFile.java index ff9c346f426d..8e7f4082504d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/ProjectHeaderFile.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/ProjectHeaderFile.java @@ -169,8 +169,8 @@ static class MainHeaderResolver implements HeaderResolver { @Override public HeaderSearchResult resolveHeader(String projectName, String headerFile) { List locations = new ArrayList<>(); - for (String clibPathComponent : SubstrateOptions.CLibraryPath.getValue().values()) { - Path clibPathHeaderFile = Paths.get(clibPathComponent).resolve(headerFile).normalize().toAbsolutePath(); + for (Path clibPathComponent : SubstrateOptions.CLibraryPath.getValue().values()) { + Path clibPathHeaderFile = clibPathComponent.resolve(headerFile).normalize().toAbsolutePath(); locations.add(clibPathHeaderFile.toString()); if (Files.exists(clibPathHeaderFile)) { return new HeaderSearchResult(Optional.of("\"" + clibPathHeaderFile + "\""), locations); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/APIOption.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/APIOption.java index 69ad9049159d..21200fd882fe 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/APIOption.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/APIOption.java @@ -122,13 +122,7 @@ enum APIOptionKind { * -{H,R}:-<OptionDescriptor#name>. For other options using * {@code Negated} is not allowed. */ - Negated, - /** - * Denotes that the annotated {@code String} option represents one or more file system - * paths, separated by ','. Relative paths will be resolved against the current working - * directory in which the native image tool is executed. - */ - Paths + Negated } class Utils { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java index b73cf00f9262..e2a7a4fa1694 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java @@ -25,6 +25,7 @@ package com.oracle.svm.driver; import java.lang.reflect.Field; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; @@ -46,7 +47,7 @@ import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; -import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.common.option.MultiOptionValue; import com.oracle.svm.core.option.APIOption; import com.oracle.svm.core.option.APIOption.APIOptionKind; import com.oracle.svm.core.option.APIOptionGroup; @@ -60,6 +61,7 @@ import com.oracle.svm.util.ModuleSupport; import com.oracle.svm.util.ReflectionUtil; import com.oracle.svm.util.ReflectionUtil.ReflectionUtilError; +import com.oracle.svm.util.StringUtil; class APIOptionHandler extends NativeImage.OptionHandler { @@ -69,7 +71,6 @@ static final class OptionInfo { final String builderOption; final String defaultValue; final String helpText; - final boolean hasPathArguments; final boolean defaultFinal; final String deprecationWarning; final boolean extra; @@ -77,14 +78,13 @@ static final class OptionInfo { final List> valueTransformers; final APIOptionGroup group; - OptionInfo(String[] variants, char[] valueSeparator, String builderOption, String defaultValue, String helpText, boolean hasPathArguments, boolean defaultFinal, String deprecationWarning, + OptionInfo(String[] variants, char[] valueSeparator, String builderOption, String defaultValue, String helpText, boolean defaultFinal, String deprecationWarning, List> valueTransformers, APIOptionGroup group, boolean extra) { this.variants = variants; this.valueSeparator = valueSeparator; this.builderOption = builderOption; this.defaultValue = defaultValue; this.helpText = helpText; - this.hasPathArguments = hasPathArguments; this.defaultFinal = defaultFinal; this.deprecationWarning = deprecationWarning; this.valueTransformers = valueTransformers; @@ -99,6 +99,7 @@ boolean isDeprecated() { private final SortedMap apiOptions; private final Map groupInfos; + final Map pathOptions; APIOptionHandler(NativeImage nativeImage) { super(nativeImage); @@ -106,19 +107,24 @@ boolean isDeprecated() { APIOptionSupport support = ImageSingletons.lookup(APIOptionSupport.class); groupInfos = support.groupInfos; apiOptions = support.options; + pathOptions = support.pathOptions; } else { groupInfos = new HashMap<>(); - apiOptions = extractOptions(ServiceLoader.load(OptionDescriptors.class, nativeImage.getClass().getClassLoader()), groupInfos); + pathOptions = new HashMap<>(); + apiOptions = extractOptions(ServiceLoader.load(OptionDescriptors.class, nativeImage.getClass().getClassLoader()), groupInfos, pathOptions); } } - static SortedMap extractOptions(ServiceLoader optionDescriptors, Map groupInfos) { + static SortedMap extractOptions(ServiceLoader optionDescriptors, Map groupInfos, Map pathOptions) { EconomicMap hostedOptions = EconomicMap.create(); EconomicMap runtimeOptions = EconomicMap.create(); HostedOptionParser.collectOptions(optionDescriptors, hostedOptions, runtimeOptions); SortedMap apiOptions = new TreeMap<>(); Map, APIOptionGroup> groupInstances = new HashMap<>(); - hostedOptions.getValues().forEach(o -> extractOption(NativeImage.oH, o, apiOptions, groupInfos, groupInstances)); + hostedOptions.getValues().forEach(o -> { + extractOption(NativeImage.oH, o, apiOptions, groupInfos, groupInstances); + extractPathOption(NativeImage.oH, o, pathOptions); + }); runtimeOptions.getValues().forEach(o -> extractOption(NativeImage.oR, o, apiOptions, groupInfos, groupInstances)); groupInfos.forEach((groupName, groupInfo) -> { if (groupInfo.defaultValues.size() > 1) { @@ -178,9 +184,6 @@ private static void extractOption(String optionPrefix, OptionDescriptor optionDe "Class specified as group for @APIOption " + apiOptionName + " cannot be loaded or instantiated: " + apiAnnotation.group().getTypeName(), ex.getCause()); } } - if (apiAnnotation.kind().equals(APIOptionKind.Paths)) { - VMError.shouldNotReachHere(String.format("Boolean APIOption %s(%s) cannot use APIOptionKind.Paths", apiOptionName, rawOptionName)); - } if (apiAnnotation.defaultValue().length > 0) { VMError.shouldNotReachHere(String.format("Boolean APIOption %s(%s) cannot use APIOption.defaultValue", apiOptionName, rawOptionName)); } @@ -258,7 +261,6 @@ private static void extractOption(String optionPrefix, OptionDescriptor optionDe boolean defaultFinal = booleanOption || hasFixedValue; apiOptions.put(apiOptionName, new APIOptionHandler.OptionInfo(apiAnnotation.name(), apiAnnotation.valueSeparator(), builderOption, defaultValue, helpText, - apiAnnotation.kind().equals(APIOptionKind.Paths), defaultFinal, apiAnnotation.deprecated(), valueTransformers, group, apiAnnotation.extra())); } } catch (NoSuchFieldException e) { @@ -266,6 +268,18 @@ private static void extractOption(String optionPrefix, OptionDescriptor optionDe } } + private static void extractPathOption(String optionPrefix, OptionDescriptor optionDescriptor, Map pathOptions) { + Object defaultValue = optionDescriptor.getOptionKey().getDefaultValue(); + if (defaultValue instanceof MultiOptionValue) { + var multiOptionDefaultValue = ((MultiOptionValue) defaultValue); + if (Path.class.isAssignableFrom(multiOptionDefaultValue.getValueType())) { + String rawOptionName = optionDescriptor.getName(); + String builderOption = optionPrefix + rawOptionName; + pathOptions.put(builderOption, multiOptionDefaultValue.getDelimiter()); + } + } + } + private static String startLowerCase(String str) { return str.substring(0, 1).toLowerCase() + str.substring(1); } @@ -349,12 +363,6 @@ String translateOption(ArgumentQueue argQueue) { optionValue = optionNameAndOptionValue[1]; } if (optionValue != null) { - if (option.hasPathArguments) { - optionValue = Arrays.stream(SubstrateUtil.split(optionValue, ",")) - .filter(s -> !s.isEmpty()) - .map(this::tryCanonicalize) - .collect(Collectors.joining(",")); - } Object transformed = optionValue; for (Function transformer : option.valueTransformers) { transformed = transformer.apply(transformed); @@ -368,12 +376,39 @@ String translateOption(ArgumentQueue argQueue) { return null; } - private String tryCanonicalize(String path) { + String transformBuilderArgument(String builderArgument, Function replayFunction) { + String[] nameAndValue = StringUtil.split(builderArgument, "=", 2); + if (nameAndValue.length != 2) { + return builderArgument; + } + String optionName = StringUtil.split(nameAndValue[0], "@", 2)[0]; + String optionValue = nameAndValue[1]; + String pathOptionSeparator = pathOptions.get(optionName); + if (pathOptionSeparator == null) { + return builderArgument; + } + List rawEntries; + if (pathOptionSeparator.isEmpty()) { + rawEntries = List.of(optionValue); + } else { + rawEntries = List.of(StringUtil.split(optionValue, pathOptionSeparator)); + } + String transformedOptionValue = rawEntries.stream() + .filter(s -> !s.isEmpty()) + .map(this::tryCanonicalize) + .map(replayFunction) + .map(Path::toString) + .collect(Collectors.joining(pathOptionSeparator)); + return optionName + "=" + transformedOptionValue; + } + + private Path tryCanonicalize(String path) { + Path origPath = Paths.get(path); try { - return nativeImage.canonicalize(Paths.get(path)).toString(); + return nativeImage.canonicalize(origPath); } catch (NativeImage.NativeImageError e) { /* Allow features to handle the path string. */ - return path; + return origPath; } } @@ -470,10 +505,12 @@ final class APIOptionSupport { final Map groupInfos; final SortedMap options; + final Map pathOptions; - APIOptionSupport(Map groupInfos, SortedMap options) { + APIOptionSupport(Map groupInfos, SortedMap options, Map pathOptions) { this.groupInfos = groupInfos; this.options = options; + this.pathOptions = pathOptions; } } @@ -489,8 +526,9 @@ public void afterRegistration(AfterRegistrationAccess access) { public void duringSetup(DuringSetupAccess access) { FeatureImpl.DuringSetupAccessImpl accessImpl = (FeatureImpl.DuringSetupAccessImpl) access; Map groupInfos = new HashMap<>(); + Map pathOptions = new HashMap<>(); ServiceLoader optionDescriptors = ServiceLoader.load(OptionDescriptors.class, accessImpl.getImageClassLoader().getClassLoader()); - SortedMap options = APIOptionHandler.extractOptions(optionDescriptors, groupInfos); - ImageSingletons.add(APIOptionSupport.class, new APIOptionSupport(groupInfos, options)); + SortedMap options = APIOptionHandler.extractOptions(optionDescriptors, groupInfos, pathOptions); + ImageSingletons.add(APIOptionSupport.class, new APIOptionSupport(groupInfos, options, pathOptions)); } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java index 0edb545a979c..5e8b87f53d76 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java @@ -26,7 +26,9 @@ import java.io.File; import java.nio.file.Paths; +import java.util.Arrays; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.graalvm.compiler.options.OptionType; @@ -45,6 +47,7 @@ class CmdLineOptionHandler extends NativeImage.OptionHandler { private static final String serverOptionPrefix = "--server-"; public static final String DEBUG_ATTACH_OPTION = "--debug-attach"; + public static final String REPLAY_OPTION = "--replay"; private static final String javaRuntimeVersion = System.getProperty("java.runtime.version"); @@ -151,6 +154,32 @@ private boolean consume(ArgumentQueue args, String headArg) { return true; } + if (headArg.startsWith(REPLAY_OPTION)) { + String replayArg = args.poll(); + ReplaySupport.ReplayStatus replayStatus; + if (replayArg.equals(REPLAY_OPTION)) { + /* Handle short form of --replay-apply */ + replayStatus = ReplaySupport.ReplayStatus.apply; + } else { + String replayVariant = replayArg.substring(REPLAY_OPTION.length() + 1); + try { + replayStatus = ReplaySupport.ReplayStatus.valueOf(replayVariant); + } catch (IllegalArgumentException e) { + String suggestedVariants = Arrays.stream(ReplaySupport.ReplayStatus.values()) + .filter(ReplaySupport.ReplayStatus::show) + .map(v -> REPLAY_OPTION + "-" + v) + .collect(Collectors.joining(", ")); + throw NativeImage.showError("Unknown option " + replayArg + ". Valid variants are: " + suggestedVariants + "."); + } + } + if (replayStatus.loadBundle) { + nativeImage.replaySupport = new ReplaySupport(nativeImage, replayStatus, args.poll()); + } else { + nativeImage.replaySupport = new ReplaySupport(nativeImage, replayStatus); + } + return true; + } + if (headArg.startsWith(DEBUG_ATTACH_OPTION)) { if (useDebugAttach) { throw NativeImage.showError("The " + DEBUG_ATTACH_OPTION + " option can only be used once."); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index d2350107d0b8..87a7a6a6261f 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -263,6 +263,14 @@ private static String oR(OptionKey option) { private long imageBuilderPid = -1; + ReplaySupport replaySupport; + + void replaySupportShutdown() { + if (replaySupport != null) { + replaySupport.shutdown(); + } + } + protected static class BuildConfiguration { /* @@ -1253,11 +1261,11 @@ private void addTargetArguments() { } } - private String mainClass; - private String imageName; - private String imagePath; + String mainClass; + String imageName; + String imagePath; - protected static List createImageBuilderArgs(ArrayList imageArgs, LinkedHashSet imagecp, LinkedHashSet imagemp) { + protected static List createImageBuilderArgs(List imageArgs, List imagecp, List imagemp) { List result = new ArrayList<>(); if (!imagecp.isEmpty()) { result.add(SubstrateOptions.IMAGE_CLASSPATH_PREFIX); @@ -1332,7 +1340,16 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa */ arguments.addAll(Arrays.asList(SubstrateOptions.WATCHPID_PREFIX, "" + ProcessProperties.getProcessID())); } - List finalImageBuilderArgs = createImageBuilderArgs(imageArgs, imagecp, imagemp); + + boolean useReplay = replaySupport != null; + Function substituteAuxiliaryPath = useReplay ? replaySupport::substituteAuxiliaryPath : Function.identity(); + Function imageArgsTransformer = rawArg -> apiOptionHandler.transformBuilderArgument(rawArg, substituteAuxiliaryPath); + List finalImageArgs = imageArgs.stream().map(imageArgsTransformer).collect(Collectors.toList()); + Function substituteClassPath = useReplay ? replaySupport::substituteClassPath : Function.identity(); + List finalImageClassPath = imagecp.stream().map(substituteClassPath).collect(Collectors.toList()); + Function substituteModulePath = useReplay ? replaySupport::substituteModulePath : Function.identity(); + List finalImageModulePath = imagemp.stream().map(substituteModulePath).collect(Collectors.toList()); + List finalImageBuilderArgs = createImageBuilderArgs(finalImageArgs, finalImageClassPath, finalImageModulePath); /* Construct ProcessBuilder command from final arguments */ List command = new ArrayList<>(); @@ -1352,12 +1369,10 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa showVerboseMessage(isVerbose() || dryRun, "]"); } - if (dryRun) { + if (dryRun || useReplay && replaySupport.status == ReplaySupport.ReplayStatus.prepare) { return ExitStatus.OK.getValue(); } - int exitStatus = ExitStatus.DRIVER_TO_BUILDER_ERROR.getValue(); - Process p = null; try { ProcessBuilder pb = new ProcessBuilder(); @@ -1366,7 +1381,7 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa sanitizeJVMEnvironment(pb.environment()); p = pb.inheritIO().start(); imageBuilderPid = p.pid(); - exitStatus = p.waitFor(); + return p.waitFor(); } catch (IOException | InterruptedException e) { throw showError(e.getMessage()); } finally { @@ -1374,7 +1389,6 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa p.destroy(); } } - return exitStatus; } private static void sanitizeJVMEnvironment(Map environment) { @@ -1443,26 +1457,31 @@ protected static void build(BuildConfiguration config, Function pathSubstitutions = new HashMap<>(); + + private static final String replayTempDirPrefix = "replayRoot-"; + + ReplaySupport(NativeImage nativeImage, ReplayStatus status) { + assert !status.loadBundle; + + this.nativeImage = nativeImage; + this.status = status; + try { + replayRootDir = Files.createTempDirectory(replayTempDirPrefix); + Path inputDir = replayRootDir.resolve("input"); + stageDir = Files.createDirectories(inputDir.resolve("stage")); + auxiliaryDir = Files.createDirectories(inputDir.resolve("auxiliary")); + Path classesDir = inputDir.resolve("classes"); + classPathDir = Files.createDirectories(classesDir.resolve("cp")); + modulePathDir = Files.createDirectories(classesDir.resolve("p")); + } catch (IOException e) { + throw NativeImage.showError("Unable to create replay-bundle directory layout", e); + } + } + + ReplaySupport(NativeImage nativeImage, ReplayStatus status, String replayBundleFilename) { + assert status.loadBundle; + + this.nativeImage = nativeImage; + this.status = status; + + Path replayBundlePath = Path.of(replayBundleFilename); + if (!Files.isReadable(replayBundlePath)) { + throw NativeImage.showError("The given replay-bundle file " + replayBundleFilename + " cannot be read"); + } + + if (Files.isDirectory(replayBundlePath)) { + replayRootDir = replayBundlePath; + } else { + try { + replayRootDir = Files.createTempDirectory(replayTempDirPrefix); + try (JarFile archive = new JarFile(replayBundlePath.toFile())) { + archive.stream().forEach(jarEntry -> { + Path replayBundleFile = replayRootDir.resolve(jarEntry.getName()); + try { + Path replayBundleFileParent = replayBundleFile.getParent(); + if (replayBundleFileParent != null) { + Files.createDirectories(replayBundleFileParent); + } + Files.copy(archive.getInputStream(jarEntry), replayBundleFile); + } catch (IOException e) { + throw NativeImage.showError("Unable to copy " + jarEntry.getName() + " from replay-bundle " + replayBundlePath + " to " + replayBundleFile, e); + } + }); + } + } catch (IOException e) { + throw NativeImage.showError("Unable to create replay-bundle directory layout from replay-file " + replayBundlePath, e); + } + } + + Path inputDir = replayRootDir.resolve("input"); + stageDir = inputDir.resolve("stage"); + auxiliaryDir = inputDir.resolve("auxiliary"); + Path classesDir = inputDir.resolve("classes"); + classPathDir = classesDir.resolve("cp"); + modulePathDir = classesDir.resolve("p"); + + Path pathSubstitutionsFile = stageDir.resolve("path_substitutions.json"); + try (Reader reader = Files.newBufferedReader(pathSubstitutionsFile)) { + new PathSubstitutionsMapParser(true).parseAndRegister(reader); + } catch (IOException e) { + throw NativeImage.showError("Failed to read bundle-file " + pathSubstitutionsFile, e); + } + } + + Path substituteAuxiliaryPath(Path origPath) { + return substitutePath(origPath, auxiliaryDir); + } + + Path substituteClassPath(Path origPath) { + return substitutePath(origPath, classPathDir); + } + + Path substituteModulePath(Path origPath) { + return substitutePath(origPath, modulePathDir); + } + + @SuppressWarnings("try") + private Path substitutePath(Path origPath, Path destinationDir) { + assert destinationDir.startsWith(replayRootDir); + + Path alreadySubstitutedPath = pathSubstitutions.get(origPath); + if (alreadySubstitutedPath != null) { + return replayRootDir.resolve(alreadySubstitutedPath); + } + + if (origPath.startsWith(nativeImage.config.getJavaHome())) { + /* If origPath comes from native-image itself, substituting is not needed. */ + return origPath; + } + + if (!Files.isReadable(origPath)) { + /* Prevent subsequent retries to substitute invalid paths */ + pathSubstitutions.put(origPath, origPath); + return origPath; + } + + String[] baseNamePlusExtension = StringUtil.split(origPath.getFileName().toString(), ".", 2); + String baseName = baseNamePlusExtension[0]; + String extension = baseNamePlusExtension.length == 2 ? "." + baseNamePlusExtension[1] : ""; + String substitutedPathFilename = baseName + "_" + SubstrateUtil.digest(origPath.toString()) + extension; + Path substitutedPath = destinationDir.resolve(substitutedPathFilename); + if (Files.exists(substitutedPath)) { + /* If we ever see this, we have to implement substitutedPath collision-handling */ + throw NativeImage.showError("Failed to create a unique path-name in " + destinationDir + ". " + substitutedPath + " already exists"); + } + + if (Files.isDirectory(origPath)) { + try (Stream walk = Files.walk(origPath)) { + walk.forEach(sourcePath -> copyFile(sourcePath, substitutedPath.resolve(origPath.relativize(sourcePath)))); + } catch (IOException e) { + throw NativeImage.showError("Failed to iterate through directory " + origPath, e); + } + } else { + copyFile(origPath, substitutedPath); + } + pathSubstitutions.put(origPath, replayRootDir.relativize(substitutedPath)); + return substitutedPath; + } + + private void copyFile(Path source, Path target) { + try { + System.out.println("> Copy " + nativeImage.config.workDir.relativize(source) + " to " + target); + Files.copy(source, target); + } catch (IOException e) { + throw NativeImage.showError("Failed to copy " + source + " to " + target, e); + } + } + + void shutdown() { + if (!status.loadBundle) { + writeBundle(); + } + + nativeImage.deleteAllFiles(replayRootDir); + } + + void writeBundle() { + Path pathSubstitutionsFile = stageDir.resolve("path_substitutions.json"); + try (JsonWriter writer = new JsonWriter(pathSubstitutionsFile)) { + /* Printing as list with defined sort-order ensures useful diffs are possible */ + JsonPrinter.printCollection(writer, pathSubstitutions.entrySet(), Map.Entry.comparingByKey(), ReplaySupport::printPathSubstitution); + } catch (IOException e) { + throw NativeImage.showError("Failed to write bundle-file " + pathSubstitutionsFile, e); + } + + byte[] bundleEntryDataBuffer = new byte[16 * 1024]; + Path bundleFile = Path.of(nativeImage.imagePath).resolve(nativeImage.imageName + ".replay.jar"); + try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(bundleFile), new Manifest())) { + try (Stream walk = Files.walk(replayRootDir)) { + walk.forEach(bundleEntry -> { + if (Files.isDirectory(bundleEntry)) { + return; + } + String jarEntryName = replayRootDir.relativize(bundleEntry).toString(); + JarEntry entry = new JarEntry(jarEntryName.replace(File.separator, "/")); + try { + entry.setTime(Files.getLastModifiedTime(bundleEntry).toMillis()); + jarOutStream.putNextEntry(entry); + Files.copy(bundleEntry, jarOutStream); + jarOutStream.closeEntry(); + } catch (IOException e) { + throw NativeImage.showError("Failed to copy " + bundleEntry + " into replay-bundle file " + bundleFile, e); + } + }); + } + } catch (IOException e) { + throw NativeImage.showError("Failed to create replay-bundle file " + bundleFile, e); + } + } + + private static final String substitutionMapSrcField = "src"; + private static final String substitutionMapDstField = "dst"; + + private static void printPathSubstitution(Map.Entry entry, JsonWriter w) throws IOException { + w.append('{').quote(substitutionMapSrcField).append(" : ").quote(entry.getKey()); + w.append(", ").quote(substitutionMapDstField).append(" : ").quote(entry.getValue()); + w.append('}'); + } + + private class PathSubstitutionsMapParser extends ConfigurationParser { + + private PathSubstitutionsMapParser(boolean strictConfiguration) { + super(strictConfiguration); + } + + @Override + public void parseAndRegister(Object json, URI origin) throws IOException { + for (var rawEntry : asList(json, "Expected a list of path substitution objects")) { + var entry = asMap(rawEntry, "Expected a substitution object"); + Object srcPathString = entry.get(substitutionMapSrcField); + if (srcPathString == null) { + throw new JSONParserException("Expected " + substitutionMapSrcField + "-field in substitution object"); + } + Object dstPathString = entry.get(substitutionMapDstField); + if (dstPathString == null) { + throw new JSONParserException("Expected " + substitutionMapDstField + "-field in substitution object"); + } + pathSubstitutions.put(Path.of(srcPathString.toString()), Path.of(dstPathString.toString())); + } + } + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/NativeLibraries.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/NativeLibraries.java index 909dfb8e01e2..b47315d6921d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/NativeLibraries.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/NativeLibraries.java @@ -552,7 +552,7 @@ private Class getDirectives(ResolvedJavaType type } public void finish() { - libraryPaths.addAll(SubstrateOptions.CLibraryPath.getValue().values()); + libraryPaths.addAll(SubstrateOptions.CLibraryPath.getValue().values().stream().map(Path::toString).collect(Collectors.toList())); for (NativeCodeContext context : compilationUnitToContext.values()) { if (context.isInConfiguration()) { libraries.addAll(context.getDirectives().getLibraries()); From e698a919b09343b5d7af114751869e7294a710e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 29 Nov 2022 21:45:58 +0100 Subject: [PATCH 02/46] Add missing copyright header --- .../com/oracle/svm/driver/ReplaySupport.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java index 4950bfcbf408..b565391c5c97 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ package com.oracle.svm.driver; import java.io.File; From 5d7c8ea0525eba37d0031cccdd254b6d572bf030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 1 Dec 2022 09:26:25 +0100 Subject: [PATCH 03/46] Implement path_canonicalizations.json and build.json --- .../svm/driver/CmdLineOptionHandler.java | 31 ++++- .../com/oracle/svm/driver/NativeImage.java | 14 ++- .../com/oracle/svm/driver/ReplaySupport.java | 117 +++++++++++++++--- 3 files changed, 144 insertions(+), 18 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java index 5e8b87f53d76..46b1999a9662 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java @@ -26,7 +26,9 @@ import java.io.File; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -173,9 +175,34 @@ private boolean consume(ArgumentQueue args, String headArg) { } } if (replayStatus.loadBundle) { - nativeImage.replaySupport = new ReplaySupport(nativeImage, replayStatus, args.poll()); + String replayBundleFilename = args.poll(); + nativeImage.replaySupport = new ReplaySupport(nativeImage, replayStatus, replayBundleFilename); + List buildArgs = nativeImage.replaySupport.getBuildArgs(); + for (int i = buildArgs.size() - 1; i >= 0; i--) { + args.push(buildArgs.get(i)); + } } else { - nativeImage.replaySupport = new ReplaySupport(nativeImage, replayStatus); + List filteredBuildArgs = new ArrayList<>(); + boolean skipNext = false; + for (String arg : nativeImage.config.getBuildArgs()) { + if (skipNext) { + skipNext = false; + continue; + } + if (arg.startsWith(REPLAY_OPTION)) { + continue; + } + if (arg.startsWith("-Dllvm.bin.dir=")) { + continue; + } + if (arg.equals("--configurations-path")) { + // FIXME Provide proper --configurations-path support + skipNext = true; + continue; + } + filteredBuildArgs.add(arg); + } + nativeImage.replaySupport = new ReplaySupport(nativeImage, replayStatus, filteredBuildArgs); } return true; } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 87a7a6a6261f..967418cc0adb 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -160,6 +160,10 @@ public String poll() { return queue.poll(); } + public void push(String arg) { + queue.push(arg); + } + public String peek() { return queue.peek(); } @@ -1491,9 +1495,15 @@ Path canonicalize(Path path) { } Path canonicalize(Path path, boolean strict) { + if (replaySupport != null) { + Path prev = replaySupport.restoreCanonicalization(path); + if (prev != null) { + return prev; + } + } Path absolutePath = path.isAbsolute() ? path : config.getWorkingDirectory().resolve(path); if (!strict) { - return absolutePath; + return replaySupport != null ? replaySupport.recordCanonicalization(path, absolutePath) : absolutePath; } boolean hasWildcard = absolutePath.endsWith(ClasspathUtils.cpWildcardSubstitute); if (hasWildcard) { @@ -1510,7 +1520,7 @@ Path canonicalize(Path path, boolean strict) { } realPath = realPath.resolve(ClasspathUtils.cpWildcardSubstitute); } - return realPath; + return replaySupport != null ? replaySupport.recordCanonicalization(path, realPath) : realPath; } catch (IOException e) { throw showError("Invalid Path entry " + ClasspathUtils.classpathToString(path), e); } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java index b565391c5c97..7d1e1ac9e810 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java @@ -30,7 +30,10 @@ import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -76,11 +79,14 @@ boolean show() { final Path modulePathDir; final Path auxiliaryDir; + Map pathCanonicalizations = new HashMap<>(); Map pathSubstitutions = new HashMap<>(); + private final List buildArgs; + private static final String replayTempDirPrefix = "replayRoot-"; - ReplaySupport(NativeImage nativeImage, ReplayStatus status) { + ReplaySupport(NativeImage nativeImage, ReplayStatus status, List buildArgs) { assert !status.loadBundle; this.nativeImage = nativeImage; @@ -96,6 +102,8 @@ boolean show() { } catch (IOException e) { throw NativeImage.showError("Unable to create replay-bundle directory layout", e); } + + this.buildArgs = Collections.unmodifiableList(buildArgs); } ReplaySupport(NativeImage nativeImage, ReplayStatus status, String replayBundleFilename) { @@ -140,14 +148,49 @@ boolean show() { classPathDir = classesDir.resolve("cp"); modulePathDir = classesDir.resolve("p"); + Path pathCanonicalizationsFile = stageDir.resolve("path_canonicalizations.json"); + try (Reader reader = Files.newBufferedReader(pathCanonicalizationsFile)) { + new PathMapParser(pathCanonicalizations).parseAndRegister(reader); + } catch (IOException e) { + throw NativeImage.showError("Failed to read bundle-file " + pathCanonicalizationsFile, e); + } Path pathSubstitutionsFile = stageDir.resolve("path_substitutions.json"); try (Reader reader = Files.newBufferedReader(pathSubstitutionsFile)) { - new PathSubstitutionsMapParser(true).parseAndRegister(reader); + new PathMapParser(pathSubstitutions).parseAndRegister(reader); + } catch (IOException e) { + throw NativeImage.showError("Failed to read bundle-file " + pathSubstitutionsFile, e); + } + Path buildArgsFile = stageDir.resolve("build.json"); + try (Reader reader = Files.newBufferedReader(buildArgsFile)) { + List buildArgsFromFile = new ArrayList<>(); + new BuildArgsParser(buildArgsFromFile).parseAndRegister(reader); + buildArgs = Collections.unmodifiableList(buildArgsFromFile); } catch (IOException e) { throw NativeImage.showError("Failed to read bundle-file " + pathSubstitutionsFile, e); } } + public List getBuildArgs() { + return buildArgs; + } + + Path recordCanonicalization(Path before, Path after) { + if (after.startsWith(nativeImage.config.getJavaHome())) { + return after; + } + System.out.println("RecordCanonicalization src: " + before + ", dst: " + after); + pathCanonicalizations.put(before, after); + return after; + } + + Path restoreCanonicalization(Path before) { + Path after = pathCanonicalizations.get(before); + if (after != null) { + System.out.println("RestoreCanonicalization src: " + before + ", dst: " + after); + } + return after; + } + Path substituteAuxiliaryPath(Path origPath) { return substitutePath(origPath, auxiliaryDir); } @@ -164,9 +207,10 @@ Path substituteModulePath(Path origPath) { private Path substitutePath(Path origPath, Path destinationDir) { assert destinationDir.startsWith(replayRootDir); - Path alreadySubstitutedPath = pathSubstitutions.get(origPath); - if (alreadySubstitutedPath != null) { - return replayRootDir.resolve(alreadySubstitutedPath); + Path previousRelativeSubstitutedPath = pathSubstitutions.get(origPath); + if (previousRelativeSubstitutedPath != null) { + System.out.println("RestoreSubstitution src: " + origPath + ", dst: " + previousRelativeSubstitutedPath); + return replayRootDir.resolve(previousRelativeSubstitutedPath); } if (origPath.startsWith(nativeImage.config.getJavaHome())) { @@ -180,6 +224,9 @@ private Path substitutePath(Path origPath, Path destinationDir) { return origPath; } + // TODO Report error if user tries to use funky paths like / or c: + // TODO Report error if overlapping dir-trees are passed in + // TODO add .endsWith(ClasspathUtils.cpWildcardSubstitute) handling (copy whole directory) String[] baseNamePlusExtension = StringUtil.split(origPath.getFileName().toString(), ".", 2); String baseName = baseNamePlusExtension[0]; String extension = baseNamePlusExtension.length == 2 ? "." + baseNamePlusExtension[1] : ""; @@ -199,13 +246,17 @@ private Path substitutePath(Path origPath, Path destinationDir) { } else { copyFile(origPath, substitutedPath); } - pathSubstitutions.put(origPath, replayRootDir.relativize(substitutedPath)); + Path relativeSubstitutedPath = replayRootDir.relativize(substitutedPath); + System.out.println("RecordSubstitution src: " + origPath + ", dst: " + relativeSubstitutedPath); + pathSubstitutions.put(origPath, relativeSubstitutedPath); return substitutedPath; } private void copyFile(Path source, Path target) { try { - System.out.println("> Copy " + nativeImage.config.workDir.relativize(source) + " to " + target); + if (nativeImage.isVerbose()) { + System.out.println("> Copy to bundle: " + nativeImage.config.workDir.relativize(source)); + } Files.copy(source, target); } catch (IOException e) { throw NativeImage.showError("Failed to copy " + source + " to " + target, e); @@ -221,15 +272,29 @@ void shutdown() { } void writeBundle() { + Path pathCanonicalizationsFile = stageDir.resolve("path_canonicalizations.json"); + try (JsonWriter writer = new JsonWriter(pathCanonicalizationsFile)) { + /* Printing as list with defined sort-order ensures useful diffs are possible */ + JsonPrinter.printCollection(writer, pathCanonicalizations.entrySet(), Map.Entry.comparingByKey(), ReplaySupport::printPathMapping); + } catch (IOException e) { + throw NativeImage.showError("Failed to write bundle-file " + pathCanonicalizationsFile, e); + } Path pathSubstitutionsFile = stageDir.resolve("path_substitutions.json"); try (JsonWriter writer = new JsonWriter(pathSubstitutionsFile)) { /* Printing as list with defined sort-order ensures useful diffs are possible */ - JsonPrinter.printCollection(writer, pathSubstitutions.entrySet(), Map.Entry.comparingByKey(), ReplaySupport::printPathSubstitution); + JsonPrinter.printCollection(writer, pathSubstitutions.entrySet(), Map.Entry.comparingByKey(), ReplaySupport::printPathMapping); + } catch (IOException e) { + throw NativeImage.showError("Failed to write bundle-file " + pathSubstitutionsFile, e); + } + + Path buildArgsFile = stageDir.resolve("build.json"); + try (JsonWriter writer = new JsonWriter(buildArgsFile)) { + /* Printing as list with defined sort-order ensures useful diffs are possible */ + JsonPrinter.printCollection(writer, buildArgs, null, ReplaySupport::printBuildArg); } catch (IOException e) { throw NativeImage.showError("Failed to write bundle-file " + pathSubstitutionsFile, e); } - byte[] bundleEntryDataBuffer = new byte[16 * 1024]; Path bundleFile = Path.of(nativeImage.imagePath).resolve(nativeImage.imageName + ".replay.jar"); try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(bundleFile), new Manifest())) { try (Stream walk = Files.walk(replayRootDir)) { @@ -257,16 +322,23 @@ void writeBundle() { private static final String substitutionMapSrcField = "src"; private static final String substitutionMapDstField = "dst"; - private static void printPathSubstitution(Map.Entry entry, JsonWriter w) throws IOException { + private static void printPathMapping(Map.Entry entry, JsonWriter w) throws IOException { w.append('{').quote(substitutionMapSrcField).append(" : ").quote(entry.getKey()); w.append(", ").quote(substitutionMapDstField).append(" : ").quote(entry.getValue()); w.append('}'); } - private class PathSubstitutionsMapParser extends ConfigurationParser { + private static void printBuildArg(String entry, JsonWriter w) throws IOException { + w.quote(entry); + } + + private static class PathMapParser extends ConfigurationParser { - private PathSubstitutionsMapParser(boolean strictConfiguration) { - super(strictConfiguration); + private final Map pathMap; + + private PathMapParser(Map pathMap) { + super(true); + this.pathMap = pathMap; } @Override @@ -281,7 +353,24 @@ public void parseAndRegister(Object json, URI origin) throws IOException { if (dstPathString == null) { throw new JSONParserException("Expected " + substitutionMapDstField + "-field in substitution object"); } - pathSubstitutions.put(Path.of(srcPathString.toString()), Path.of(dstPathString.toString())); + pathMap.put(Path.of(srcPathString.toString()), Path.of(dstPathString.toString())); + } + } + } + + private static class BuildArgsParser extends ConfigurationParser { + + private final List args; + + private BuildArgsParser(List args) { + super(true); + this.args = args; + } + + @Override + public void parseAndRegister(Object json, URI origin) throws IOException { + for (var arg : asList(json, "Expected a list of arguments")) { + args.add(arg.toString()); } } } From 24b30eabff43a85188749fb360ccb36791460f6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 1 Dec 2022 10:47:41 +0100 Subject: [PATCH 04/46] Ensure driver has access to org.graalvm.util.json --- compiler/mx.compiler/suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/mx.compiler/suite.py b/compiler/mx.compiler/suite.py index eadc0cbe98fb..d3e4b9f5d98a 100644 --- a/compiler/mx.compiler/suite.py +++ b/compiler/mx.compiler/suite.py @@ -2039,7 +2039,7 @@ "org.graalvm.compiler.truffle.jfr to jdk.internal.vm.compiler.truffle.jfr", "org.graalvm.libgraal to jdk.internal.vm.compiler.management", "org.graalvm.util to jdk.internal.vm.compiler.management", - "org.graalvm.util.json to org.graalvm.nativeimage.librarysupport,org.graalvm.nativeimage.agent.tracing,org.graalvm.nativeimage.configure", + "org.graalvm.util.json to org.graalvm.nativeimage.librarysupport,org.graalvm.nativeimage.agent.tracing,org.graalvm.nativeimage.configure,org.graalvm.nativeimage.driver", ], "uses" : [ "com.oracle.truffle.api.impl.TruffleLocator", From 051c4001c912e8a6a6b9e54d856e1f21ef656285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 5 Dec 2022 16:16:01 +0100 Subject: [PATCH 05/46] Provide error handling in reply bundle path preparation --- .../oracle/svm/driver/APIOptionHandler.java | 23 +++++--- .../com/oracle/svm/driver/ReplaySupport.java | 54 +++++++++++++++++-- .../svm/hosted/ClassLoaderSupportImpl.java | 5 +- .../hosted/NativeImageClassLoaderSupport.java | 11 +--- .../src/com/oracle/svm/util/ClassUtil.java | 15 ++++++ 5 files changed, 84 insertions(+), 24 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java index e2a7a4fa1694..e1cbacd8db06 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java @@ -381,25 +381,32 @@ String transformBuilderArgument(String builderArgument, Function rep if (nameAndValue.length != 2) { return builderArgument; } - String optionName = StringUtil.split(nameAndValue[0], "@", 2)[0]; String optionValue = nameAndValue[1]; + + String[] nameAndOrigin = StringUtil.split(nameAndValue[0], "@", 2); + String optionName = nameAndOrigin[0]; String pathOptionSeparator = pathOptions.get(optionName); if (pathOptionSeparator == null) { return builderArgument; } + OptionOrigin optionOrigin = OptionOrigin.from(nameAndOrigin.length == 2 ? nameAndOrigin[1] : null); List rawEntries; if (pathOptionSeparator.isEmpty()) { rawEntries = List.of(optionValue); } else { rawEntries = List.of(StringUtil.split(optionValue, pathOptionSeparator)); } - String transformedOptionValue = rawEntries.stream() - .filter(s -> !s.isEmpty()) - .map(this::tryCanonicalize) - .map(replayFunction) - .map(Path::toString) - .collect(Collectors.joining(pathOptionSeparator)); - return optionName + "=" + transformedOptionValue; + try { + String transformedOptionValue = rawEntries.stream() + .filter(s -> !s.isEmpty()) + .map(this::tryCanonicalize) + .map(replayFunction) + .map(Path::toString) + .collect(Collectors.joining(pathOptionSeparator)); + return optionName + "=" + transformedOptionValue; + } catch (ReplaySupport.ReplayPathSubstitutionError error) { + throw NativeImage.showError("Failed to prepare path entry '" + error.origPath + "' of option " + optionName + " from " + optionOrigin + " for replay bundle inclusion.", error); + } } private Path tryCanonicalize(String path) { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java index 7d1e1ac9e810..501588317277 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.io.Reader; import java.net.URI; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -43,10 +44,12 @@ import org.graalvm.util.json.JSONParserException; +import com.oracle.svm.core.OS; import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.configure.ConfigurationParser; import com.oracle.svm.core.util.json.JsonPrinter; import com.oracle.svm.core.util.json.JsonWriter; +import com.oracle.svm.util.ClassUtil; import com.oracle.svm.util.StringUtil; class ReplaySupport { @@ -196,11 +199,34 @@ Path substituteAuxiliaryPath(Path origPath) { } Path substituteClassPath(Path origPath) { - return substitutePath(origPath, classPathDir); + try { + return substitutePath(origPath, classPathDir); + } catch (ReplayPathSubstitutionError error) { + throw NativeImage.showError("Failed to prepare class-path entry '" + error.origPath + "' for replay bundle inclusion.", error); + } } Path substituteModulePath(Path origPath) { - return substitutePath(origPath, modulePathDir); + try { + return substitutePath(origPath, modulePathDir); + } catch (ReplayPathSubstitutionError error) { + throw NativeImage.showError("Failed to prepare module-path entry '" + error.origPath + "' for replay bundle inclusion.", error); + } + } + + @SuppressWarnings("serial") + static final class ReplayPathSubstitutionError extends Error { + public final Path origPath; + + public ReplayPathSubstitutionError(String message, Path origPath) { + super(message); + this.origPath = origPath; + } + + public ReplayPathSubstitutionError(String message, Path origPath, Throwable cause) { + super(message, cause); + this.origPath = origPath; + } } @SuppressWarnings("try") @@ -218,13 +244,31 @@ private Path substitutePath(Path origPath, Path destinationDir) { return origPath; } + boolean forbiddenPath = false; + if (!OS.WINDOWS.isCurrent()) { + for (Path path : ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES) { + if (origPath.startsWith(path)) { + forbiddenPath = true; + break; + } + } + } + for (Path rootDirectory : FileSystems.getDefault().getRootDirectories()) { + /* Refuse /, C:, D:, ... */ + if (origPath.equals(rootDirectory)) { + forbiddenPath = true; + } + } + if (forbiddenPath) { + throw new ReplayPathSubstitutionError("Replay bundles do not allow inclusion of directory " + origPath, origPath); + } + if (!Files.isReadable(origPath)) { /* Prevent subsequent retries to substitute invalid paths */ pathSubstitutions.put(origPath, origPath); return origPath; } - // TODO Report error if user tries to use funky paths like / or c: // TODO Report error if overlapping dir-trees are passed in // TODO add .endsWith(ClasspathUtils.cpWildcardSubstitute) handling (copy whole directory) String[] baseNamePlusExtension = StringUtil.split(origPath.getFileName().toString(), ".", 2); @@ -234,14 +278,14 @@ private Path substitutePath(Path origPath, Path destinationDir) { Path substitutedPath = destinationDir.resolve(substitutedPathFilename); if (Files.exists(substitutedPath)) { /* If we ever see this, we have to implement substitutedPath collision-handling */ - throw NativeImage.showError("Failed to create a unique path-name in " + destinationDir + ". " + substitutedPath + " already exists"); + throw new ReplayPathSubstitutionError("Failed to create a unique path-name in " + destinationDir + ". " + substitutedPath + " already exists", origPath); } if (Files.isDirectory(origPath)) { try (Stream walk = Files.walk(origPath)) { walk.forEach(sourcePath -> copyFile(sourcePath, substitutedPath.resolve(origPath.relativize(sourcePath)))); } catch (IOException e) { - throw NativeImage.showError("Failed to iterate through directory " + origPath, e); + throw new ReplayPathSubstitutionError("Failed to iterate through directory " + origPath, origPath, e); } } else { copyFile(origPath, substitutedPath); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java index 3009ce09cf21..69d1d8cfb305 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java @@ -60,6 +60,7 @@ import com.oracle.svm.core.util.UserError; import com.oracle.svm.core.util.VMError; +import com.oracle.svm.util.ClassUtil; import jdk.internal.module.Modules; public class ClassLoaderSupportImpl extends ClassLoaderSupport { @@ -166,8 +167,8 @@ private static void scanDirectory(Path root, ResourceCollector collector, Native } try (Stream pathStream = Files.list(entry)) { Stream filtered = pathStream; - if (support.excludeDirectoriesRoot.equals(entry)) { - filtered = filtered.filter(Predicate.not(support.excludeDirectories::contains)); + if (ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES_ROOT.equals(entry)) { + filtered = filtered.filter(Predicate.not(ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES::contains)); } filtered.forEach(queue::push); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java index f02f9823bf3c..05287c453e77 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java @@ -71,6 +71,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.oracle.svm.util.ClassUtil; import org.graalvm.collections.EconomicMap; import org.graalvm.collections.EconomicSet; import org.graalvm.collections.MapCursor; @@ -583,14 +584,6 @@ Optional getMainClassFromModule(Object module) { return ((Module) module).getDescriptor().mainClass(); } - final Path excludeDirectoriesRoot = Paths.get("/"); - final Set excludeDirectories = getExcludeDirectories(); - - private Set getExcludeDirectories() { - return Stream.of("dev", "sys", "proc", "etc", "var", "tmp", "boot", "lost+found") - .map(excludeDirectoriesRoot::resolve).collect(Collectors.toUnmodifiableSet()); - } - private final class LoadClassHandler { private final ForkJoinPool executor; @@ -690,7 +683,7 @@ private void loadClassesFromPath(Path path) { } } else { URI container = path.toUri(); - loadClassesFromPath(container, path, excludeDirectoriesRoot, excludeDirectories); + loadClassesFromPath(container, path, ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES_ROOT, ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES); } } diff --git a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ClassUtil.java b/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ClassUtil.java index 3bda6db8b949..108fa8d3d9f9 100644 --- a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ClassUtil.java +++ b/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ClassUtil.java @@ -24,7 +24,22 @@ */ package com.oracle.svm.util; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + public final class ClassUtil { + + public static final Path CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES_ROOT = Paths.get("/"); + public static final Set CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES = getClassModulePathExcludeDirectories(); + + static private Set getClassModulePathExcludeDirectories() { + return Stream.of("dev", "sys", "proc", "etc", "var", "tmp", "boot", "lost+found") + .map(CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES_ROOT::resolve).collect(Collectors.toUnmodifiableSet()); + } + /** * Alternative to {@link Class#getSimpleName} that does not probe an enclosing class or method, * which can fail when they cannot be loaded. From 5cbcb7c2befdf5032f6943b76ca630df8489d49d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 6 Dec 2022 15:28:56 +0100 Subject: [PATCH 06/46] Major replay-bundle refactoring --- .../svm/driver/CmdLineOptionHandler.java | 56 +--------- .../svm/driver/DefaultOptionHandler.java | 28 +++-- .../com/oracle/svm/driver/NativeImage.java | 61 ++++++----- .../com/oracle/svm/driver/ReplaySupport.java | 103 +++++++++++++++--- 4 files changed, 145 insertions(+), 103 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java index 46b1999a9662..dd0a0d0e2b9b 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java @@ -26,11 +26,7 @@ import java.io.File; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; import java.util.regex.Pattern; -import java.util.stream.Collectors; import org.graalvm.compiler.options.OptionType; @@ -49,7 +45,6 @@ class CmdLineOptionHandler extends NativeImage.OptionHandler { private static final String serverOptionPrefix = "--server-"; public static final String DEBUG_ATTACH_OPTION = "--debug-attach"; - public static final String REPLAY_OPTION = "--replay"; private static final String javaRuntimeVersion = System.getProperty("java.runtime.version"); @@ -114,7 +109,7 @@ private boolean consume(ArgumentQueue args, String headArg) { NativeImage.showError(headArg + " requires a " + File.pathSeparator + " separated list of directories"); } for (String configDir : configPath.split(File.pathSeparator)) { - nativeImage.addMacroOptionRoot(nativeImage.canonicalize(Paths.get(configDir))); + nativeImage.addMacroOptionRoot(Paths.get(configDir)); } return true; case "--exclude-config": @@ -156,54 +151,9 @@ private boolean consume(ArgumentQueue args, String headArg) { return true; } - if (headArg.startsWith(REPLAY_OPTION)) { + if (headArg.startsWith(ReplaySupport.REPLAY_OPTION)) { String replayArg = args.poll(); - ReplaySupport.ReplayStatus replayStatus; - if (replayArg.equals(REPLAY_OPTION)) { - /* Handle short form of --replay-apply */ - replayStatus = ReplaySupport.ReplayStatus.apply; - } else { - String replayVariant = replayArg.substring(REPLAY_OPTION.length() + 1); - try { - replayStatus = ReplaySupport.ReplayStatus.valueOf(replayVariant); - } catch (IllegalArgumentException e) { - String suggestedVariants = Arrays.stream(ReplaySupport.ReplayStatus.values()) - .filter(ReplaySupport.ReplayStatus::show) - .map(v -> REPLAY_OPTION + "-" + v) - .collect(Collectors.joining(", ")); - throw NativeImage.showError("Unknown option " + replayArg + ". Valid variants are: " + suggestedVariants + "."); - } - } - if (replayStatus.loadBundle) { - String replayBundleFilename = args.poll(); - nativeImage.replaySupport = new ReplaySupport(nativeImage, replayStatus, replayBundleFilename); - List buildArgs = nativeImage.replaySupport.getBuildArgs(); - for (int i = buildArgs.size() - 1; i >= 0; i--) { - args.push(buildArgs.get(i)); - } - } else { - List filteredBuildArgs = new ArrayList<>(); - boolean skipNext = false; - for (String arg : nativeImage.config.getBuildArgs()) { - if (skipNext) { - skipNext = false; - continue; - } - if (arg.startsWith(REPLAY_OPTION)) { - continue; - } - if (arg.startsWith("-Dllvm.bin.dir=")) { - continue; - } - if (arg.equals("--configurations-path")) { - // FIXME Provide proper --configurations-path support - skipNext = true; - continue; - } - filteredBuildArgs.add(arg); - } - nativeImage.replaySupport = new ReplaySupport(nativeImage, replayStatus, filteredBuildArgs); - } + nativeImage.replaySupport = ReplaySupport.create(nativeImage, replayArg, args); return true; } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java index 1062c12038d3..1c2eb26d669d 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java @@ -188,7 +188,8 @@ public boolean consume(ArgumentQueue args) { if (headArg.startsWith("@") && !disableAtFiles) { args.poll(); headArg = headArg.substring(1); - Path argFile = Paths.get(headArg); + Path origArgFile = Paths.get(headArg); + Path argFile = nativeImage.replaySupport != null ? nativeImage.replaySupport.substituteAuxiliaryPath(origArgFile) : origArgFile; NativeImage.NativeImageArgsProcessor processor = nativeImage.new NativeImageArgsProcessor(OptionOrigin.argFilePrefix + argFile); readArgFile(argFile).forEach(processor::accept); List leftoverArgs = processor.apply(false); @@ -413,14 +414,27 @@ private void processModulePathArgs(String mpArgs) { } } - private void handleJarFileArg(Path filePath) { - if (Files.isDirectory(filePath)) { - NativeImage.showError(filePath + " is a directory. (" + requireValidJarFileMessage + ")"); + private void handleJarFileArg(Path jarFilePath) { + if (Files.isDirectory(jarFilePath)) { + NativeImage.showError(jarFilePath + " is a directory. (" + requireValidJarFileMessage + ")"); } - if (!NativeImage.processJarManifestMainAttributes(filePath, nativeImage::handleMainClassAttribute)) { - NativeImage.showError("No manifest in " + filePath); + String jarFileName = jarFilePath.getFileName().toString(); + String jarSuffix = ".jar"; + String jarFileNameBase; + if (jarFileName.endsWith(jarSuffix)) { + jarFileNameBase = jarFileName.substring(0, jarFileName.length() - jarSuffix.length()); + } else { + jarFileNameBase = jarFileName; } - nativeImage.addCustomImageClasspath(filePath); + if (!jarFileNameBase.isEmpty()) { + String origin = "manifest from " + jarFilePath.toUri(); + nativeImage.addPlainImageBuilderArg(NativeImage.injectHostedOptionOrigin(nativeImage.oHName + jarFileNameBase, origin)); + } + Path finalFilePath = nativeImage.replaySupport != null ? nativeImage.replaySupport.substituteClassPath(jarFilePath) : jarFilePath; + if (!NativeImage.processJarManifestMainAttributes(finalFilePath, nativeImage::handleMainClassAttribute)) { + NativeImage.showError("No manifest in " + finalFilePath); + } + nativeImage.addCustomImageClasspath(finalFilePath); } @Override diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 967418cc0adb..c1a72dc8658e 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -247,7 +247,7 @@ private static String oR(OptionKey option) { protected final BuildConfiguration config; - private final Map userConfigProperties = new HashMap<>(); + final Map userConfigProperties = new HashMap<>(); private final Map propertyFileSubstitutionValues = new HashMap<>(); private boolean verbose = Boolean.valueOf(System.getenv("VERBOSE_GRAALVM_LAUNCHERS")); @@ -554,14 +554,7 @@ public List getImageClasspath() { * @return native-image (i.e. image build) arguments */ public List getBuildArgs() { - if (args.isEmpty()) { - return Collections.emptyList(); - } - List buildArgs = new ArrayList<>(); - buildArgs.addAll(Arrays.asList("--configurations-path", rootDir.toString())); - buildArgs.addAll(Arrays.asList("--configurations-path", rootDir.resolve(Paths.get("lib", "svm")).toString())); - buildArgs.addAll(args); - return buildArgs; + return args; } /** @@ -724,17 +717,18 @@ public boolean buildFallbackImage() { private final DriverMetaInfProcessor metaInfProcessor; + static final String CONFIG_FILE_ENV_VAR_KEY = "NATIVE_IMAGE_CONFIG_FILE"; + protected NativeImage(BuildConfiguration config) { this.config = config; this.metaInfProcessor = new DriverMetaInfProcessor(); - String configFileEnvVarKey = "NATIVE_IMAGE_CONFIG_FILE"; - String configFile = System.getenv(configFileEnvVarKey); + String configFile = System.getenv(CONFIG_FILE_ENV_VAR_KEY); if (configFile != null && !configFile.isEmpty()) { try { userConfigProperties.putAll(loadProperties(canonicalize(Paths.get(configFile)))); } catch (NativeImageError | Exception e) { - showError("Invalid environment variable " + configFileEnvVarKey, e); + showError("Invalid environment variable " + CONFIG_FILE_ENV_VAR_KEY, e); } } @@ -743,6 +737,8 @@ protected NativeImage(BuildConfiguration config) { /* Discover supported MacroOptions */ optionRegistry = new MacroOption.Registry(); + optionRegistry.addMacroOptionRoot(config.rootDir); + optionRegistry.addMacroOptionRoot(config.rootDir.resolve(Paths.get("lib", "svm"))); cmdLineOptionHandler = new CmdLineOptionHandler(this); @@ -755,7 +751,9 @@ protected NativeImage(BuildConfiguration config) { } void addMacroOptionRoot(Path configDir) { - optionRegistry.addMacroOptionRoot(canonicalize(configDir)); + Path origRootDir = canonicalize(configDir); + Path rootDir = replaySupport != null ? replaySupport.substituteClassPath(origRootDir) : origRootDir; + optionRegistry.addMacroOptionRoot(rootDir); } protected void registerOptionHandler(OptionHandler handler) { @@ -956,17 +954,6 @@ void handleMainClassAttribute(Path jarFilePath, Attributes mainAttributes) { } String origin = "manifest from " + jarFilePath.toUri(); addPlainImageBuilderArg(NativeImage.injectHostedOptionOrigin(oHClass + mainClassValue, origin)); - String jarFileName = jarFilePath.getFileName().toString(); - String jarSuffix = ".jar"; - String jarFileNameBase; - if (jarFileName.endsWith(jarSuffix)) { - jarFileNameBase = jarFileName.substring(0, jarFileName.length() - jarSuffix.length()); - } else { - jarFileNameBase = jarFileName; - } - if (!jarFileNameBase.isEmpty()) { - addPlainImageBuilderArg(NativeImage.injectHostedOptionOrigin(oHName + jarFileNameBase, origin)); - } } void handleClassPathAttribute(LinkedHashSet destination, Path jarFilePath, Attributes mainAttributes) { @@ -1346,6 +1333,18 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa } boolean useReplay = replaySupport != null; + if (useReplay) { + Path imageNamePath = Path.of(imageName); + if (imageNamePath.getParent() != null) { + throw NativeImage.showError("Replay-bundle support can only be used with simple output image-name. Use '" + imageNamePath.getFileName() + "' instead of '" + imageNamePath + "'."); + } + if (!Path.of(imagePath).equals(config.getWorkingDirectory())) { + /* Allow bundle write to succeed in working directory */ + imagePath = config.getWorkingDirectory().toString(); + + throw NativeImage.showError("Replay-bundle support can only be used with default image output directory. Ensure '" + oHPath + "' is not explicitly set."); + } + } Function substituteAuxiliaryPath = useReplay ? replaySupport::substituteAuxiliaryPath : Function.identity(); Function imageArgsTransformer = rawArg -> apiOptionHandler.transformBuilderArgument(rawArg, substituteAuxiliaryPath); List finalImageArgs = imageArgs.stream().map(imageArgsTransformer).collect(Collectors.toList()); @@ -1636,8 +1635,9 @@ void addImageModulePath(Path modulePathEntry, boolean strict) { return; } - imageModulePath.add(mpEntry); - processClasspathNativeImageMetaInf(mpEntry); + Path mpEntryFinal = replaySupport != null ? replaySupport.substituteModulePath(mpEntry) : mpEntry; + imageModulePath.add(mpEntryFinal); + processClasspathNativeImageMetaInf(mpEntryFinal); } /** @@ -1674,10 +1674,11 @@ private void addImageClasspathEntry(LinkedHashSet destination, Path classp return; } - if (!imageClasspath.contains(classpathEntry) && !customImageClasspath.contains(classpathEntry)) { - destination.add(classpathEntry); - processManifestMainAttributes(classpathEntry, (jarFilePath, attributes) -> handleClassPathAttribute(destination, jarFilePath, attributes)); - processClasspathNativeImageMetaInf(classpathEntry); + Path classpathEntryFinal = replaySupport != null ? replaySupport.substituteModulePath(classpathEntry) : classpathEntry; + if (!imageClasspath.contains(classpathEntryFinal) && !customImageClasspath.contains(classpathEntryFinal)) { + destination.add(classpathEntryFinal); + processManifestMainAttributes(classpathEntryFinal, (jarFilePath, attributes) -> handleClassPathAttribute(destination, jarFilePath, attributes)); + processClasspathNativeImageMetaInf(classpathEntryFinal); } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java index 501588317277..d3892d407ba7 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java @@ -32,14 +32,17 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.jar.JarOutputStream; import java.util.jar.Manifest; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.graalvm.util.json.JSONParserException; @@ -50,10 +53,11 @@ import com.oracle.svm.core.util.json.JsonPrinter; import com.oracle.svm.core.util.json.JsonWriter; import com.oracle.svm.util.ClassUtil; -import com.oracle.svm.util.StringUtil; class ReplaySupport { + static final String REPLAY_OPTION = "--replay"; + enum ReplayStatus { prepare(false, false), create(false, false), @@ -89,7 +93,54 @@ boolean show() { private static final String replayTempDirPrefix = "replayRoot-"; - ReplaySupport(NativeImage nativeImage, ReplayStatus status, List buildArgs) { + static ReplaySupport create(NativeImage nativeImage, String replayArg, NativeImage.ArgumentQueue args) { + if (!nativeImage.userConfigProperties.isEmpty()) { + throw NativeImage.showError("Replay-bundle support cannot be combined with " + NativeImage.CONFIG_FILE_ENV_VAR_KEY + " environment variable use."); + } + + ReplaySupport.ReplayStatus replayStatus; + if (replayArg.equals(REPLAY_OPTION)) { + /* Handle short form of --replay-apply */ + replayStatus = ReplaySupport.ReplayStatus.apply; + } else { + String replayVariant = replayArg.substring(REPLAY_OPTION.length() + 1); + try { + replayStatus = ReplaySupport.ReplayStatus.valueOf(replayVariant); + } catch (IllegalArgumentException e) { + String suggestedVariants = Arrays.stream(ReplaySupport.ReplayStatus.values()) + .filter(ReplaySupport.ReplayStatus::show) + .map(v -> REPLAY_OPTION + "-" + v) + .collect(Collectors.joining(", ")); + throw NativeImage.showError("Unknown option " + replayArg + ". Valid variants are: " + suggestedVariants + "."); + } + } + ReplaySupport replaySupport; + if (replayStatus.loadBundle) { + String replayBundleFilename = args.poll(); + replaySupport = new ReplaySupport(nativeImage, replayStatus, replayBundleFilename); + List buildArgs = replaySupport.getBuildArgs(); + for (int i = buildArgs.size() - 1; i >= 0; i--) { + String buildArg = buildArgs.get(i); + if (buildArg.startsWith(REPLAY_OPTION)) { + assert !ReplayStatus.valueOf(buildArg.substring(REPLAY_OPTION.length() + 1)).loadBundle; + continue; + } + if (buildArg.startsWith("-Dllvm.bin.dir=")) { + Optional existing = nativeImage.config.getBuildArgs().stream().filter(arg -> arg.startsWith("-Dllvm.bin.dir=")).findFirst(); + if (existing.isPresent() && !existing.get().equals(buildArg)) { + throw NativeImage.showError("Replay-bundle native-image argument '" + buildArg + "' conflicts with existing '" + existing.get() + "'."); + } + continue; + } + args.push(buildArg); + } + } else { + replaySupport = new ReplaySupport(nativeImage, replayStatus); + } + return replaySupport; + } + + private ReplaySupport(NativeImage nativeImage, ReplayStatus status) { assert !status.loadBundle; this.nativeImage = nativeImage; @@ -105,11 +156,10 @@ boolean show() { } catch (IOException e) { throw NativeImage.showError("Unable to create replay-bundle directory layout", e); } - - this.buildArgs = Collections.unmodifiableList(buildArgs); + this.buildArgs = Collections.unmodifiableList(nativeImage.config.getBuildArgs()); } - ReplaySupport(NativeImage nativeImage, ReplayStatus status, String replayBundleFilename) { + private ReplaySupport(NativeImage nativeImage, ReplayStatus status, String replayBundleFilename) { assert status.loadBundle; this.nativeImage = nativeImage; @@ -178,17 +228,25 @@ public List getBuildArgs() { } Path recordCanonicalization(Path before, Path after) { + if (before.startsWith(replayRootDir)) { + if (nativeImage.isVerbose()) { + System.out.println(("RecordCanonicalization Skip: " + before)); + } + return before; + } if (after.startsWith(nativeImage.config.getJavaHome())) { return after; } - System.out.println("RecordCanonicalization src: " + before + ", dst: " + after); + if (nativeImage.isVerbose()) { + System.out.println("RecordCanonicalization src: " + before + ", dst: " + after); + } pathCanonicalizations.put(before, after); return after; } Path restoreCanonicalization(Path before) { Path after = pathCanonicalizations.get(before); - if (after != null) { + if (after != null && nativeImage.isVerbose()) { System.out.println("RestoreCanonicalization src: " + before + ", dst: " + after); } return after; @@ -233,9 +291,18 @@ public ReplayPathSubstitutionError(String message, Path origPath, Throwable caus private Path substitutePath(Path origPath, Path destinationDir) { assert destinationDir.startsWith(replayRootDir); + if (origPath.startsWith(replayRootDir)) { + if (nativeImage.isVerbose()) { + System.out.println(("RecordSubstitution/RestoreSubstitution Skip: " + origPath)); + } + return origPath; + } + Path previousRelativeSubstitutedPath = pathSubstitutions.get(origPath); if (previousRelativeSubstitutedPath != null) { - System.out.println("RestoreSubstitution src: " + origPath + ", dst: " + previousRelativeSubstitutedPath); + if (nativeImage.isVerbose()) { + System.out.println("RestoreSubstitution src: " + origPath + ", dst: " + previousRelativeSubstitutedPath); + } return replayRootDir.resolve(previousRelativeSubstitutedPath); } @@ -271,9 +338,17 @@ private Path substitutePath(Path origPath, Path destinationDir) { // TODO Report error if overlapping dir-trees are passed in // TODO add .endsWith(ClasspathUtils.cpWildcardSubstitute) handling (copy whole directory) - String[] baseNamePlusExtension = StringUtil.split(origPath.getFileName().toString(), ".", 2); - String baseName = baseNamePlusExtension[0]; - String extension = baseNamePlusExtension.length == 2 ? "." + baseNamePlusExtension[1] : ""; + String origFileName = origPath.getFileName().toString(); + int extensionPos = origFileName.lastIndexOf('.'); + String baseName; + String extension; + if (extensionPos > 0) { + baseName = origFileName.substring(0, extensionPos); + extension = origFileName.substring(extensionPos); + } else { + baseName = origFileName; + extension = ""; + } String substitutedPathFilename = baseName + "_" + SubstrateUtil.digest(origPath.toString()) + extension; Path substitutedPath = destinationDir.resolve(substitutedPathFilename); if (Files.exists(substitutedPath)) { @@ -291,7 +366,9 @@ private Path substitutePath(Path origPath, Path destinationDir) { copyFile(origPath, substitutedPath); } Path relativeSubstitutedPath = replayRootDir.relativize(substitutedPath); - System.out.println("RecordSubstitution src: " + origPath + ", dst: " + relativeSubstitutedPath); + if (nativeImage.isVerbose()) { + System.out.println("RecordSubstitution src: " + origPath + ", dst: " + relativeSubstitutedPath); + } pathSubstitutions.put(origPath, relativeSubstitutedPath); return substitutedPath; } @@ -339,7 +416,7 @@ void writeBundle() { throw NativeImage.showError("Failed to write bundle-file " + pathSubstitutionsFile, e); } - Path bundleFile = Path.of(nativeImage.imagePath).resolve(nativeImage.imageName + ".replay.jar"); + Path bundleFile = Path.of(nativeImage.imagePath).resolve(nativeImage.imageName + ".replay"); try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(bundleFile), new Manifest())) { try (Stream walk = Files.walk(replayRootDir)) { walk.forEach(bundleEntry -> { From bb4329665f1186be46ba6519e63a0efe1f3d80c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 6 Dec 2022 17:49:10 +0100 Subject: [PATCH 07/46] Style fixes --- .../src/com/oracle/svm/driver/ReplaySupport.java | 12 ++++++------ .../oracle/svm/hosted/ClassLoaderSupportImpl.java | 6 +++--- .../src/com/oracle/svm/util/ClassUtil.java | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java index d3892d407ba7..a296cf812e67 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -54,7 +54,7 @@ import com.oracle.svm.core.util.json.JsonWriter; import com.oracle.svm.util.ClassUtil; -class ReplaySupport { +final class ReplaySupport { static final String REPLAY_OPTION = "--replay"; @@ -276,12 +276,12 @@ Path substituteModulePath(Path origPath) { static final class ReplayPathSubstitutionError extends Error { public final Path origPath; - public ReplayPathSubstitutionError(String message, Path origPath) { + ReplayPathSubstitutionError(String message, Path origPath) { super(message); this.origPath = origPath; } - public ReplayPathSubstitutionError(String message, Path origPath, Throwable cause) { + ReplayPathSubstitutionError(String message, Path origPath, Throwable cause) { super(message, cause); this.origPath = origPath; } @@ -453,7 +453,7 @@ private static void printBuildArg(String entry, JsonWriter w) throws IOException w.quote(entry); } - private static class PathMapParser extends ConfigurationParser { + private static final class PathMapParser extends ConfigurationParser { private final Map pathMap; @@ -479,7 +479,7 @@ public void parseAndRegister(Object json, URI origin) throws IOException { } } - private static class BuildArgsParser extends ConfigurationParser { + private static final class BuildArgsParser extends ConfigurationParser { private final List args; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java index 69d1d8cfb305..322aa5bac90b 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java @@ -59,8 +59,8 @@ import com.oracle.svm.core.util.ClasspathUtils; import com.oracle.svm.core.util.UserError; import com.oracle.svm.core.util.VMError; - import com.oracle.svm.util.ClassUtil; + import jdk.internal.module.Modules; public class ClassLoaderSupportImpl extends ClassLoaderSupport { @@ -111,7 +111,7 @@ public void collectResources(ResourceCollector resourceCollector) { for (Path classpathFile : classLoaderSupport.classpath()) { try { if (Files.isDirectory(classpathFile)) { - scanDirectory(classpathFile, resourceCollector, classLoaderSupport); + scanDirectory(classpathFile, resourceCollector); } else if (ClasspathUtils.isJar(classpathFile)) { scanJar(classpathFile, resourceCollector); } @@ -143,7 +143,7 @@ private static void collectResourceFromModule(ResourceCollector resourceCollecto } } - private static void scanDirectory(Path root, ResourceCollector collector, NativeImageClassLoaderSupport support) throws IOException { + private static void scanDirectory(Path root, ResourceCollector collector) throws IOException { Map> matchedDirectoryResources = new HashMap<>(); Set allEntries = new HashSet<>(); diff --git a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ClassUtil.java b/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ClassUtil.java index 108fa8d3d9f9..d3653553658d 100644 --- a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ClassUtil.java +++ b/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ClassUtil.java @@ -35,7 +35,7 @@ public final class ClassUtil { public static final Path CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES_ROOT = Paths.get("/"); public static final Set CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES = getClassModulePathExcludeDirectories(); - static private Set getClassModulePathExcludeDirectories() { + private static Set getClassModulePathExcludeDirectories() { return Stream.of("dev", "sys", "proc", "etc", "var", "tmp", "boot", "lost+found") .map(CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES_ROOT::resolve).collect(Collectors.toUnmodifiableSet()); } From 3e409bc2e3d9ea8a878273806a5670ffde8ca139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 20 Dec 2022 10:34:25 +0100 Subject: [PATCH 08/46] Rename replay-bundle support to bundle support --- .../oracle/svm/driver/APIOptionHandler.java | 8 +- ...{ReplaySupport.java => BundleSupport.java} | 146 +++++++++--------- .../svm/driver/CmdLineOptionHandler.java | 5 +- .../svm/driver/DefaultOptionHandler.java | 4 +- .../com/oracle/svm/driver/NativeImage.java | 30 ++-- 5 files changed, 96 insertions(+), 97 deletions(-) rename substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/{ReplaySupport.java => BundleSupport.java} (76%) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java index e1cbacd8db06..b9f05b23c549 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java @@ -376,7 +376,7 @@ String translateOption(ArgumentQueue argQueue) { return null; } - String transformBuilderArgument(String builderArgument, Function replayFunction) { + String transformBuilderArgument(String builderArgument, Function transformFunction) { String[] nameAndValue = StringUtil.split(builderArgument, "=", 2); if (nameAndValue.length != 2) { return builderArgument; @@ -400,12 +400,12 @@ String transformBuilderArgument(String builderArgument, Function rep String transformedOptionValue = rawEntries.stream() .filter(s -> !s.isEmpty()) .map(this::tryCanonicalize) - .map(replayFunction) + .map(transformFunction) .map(Path::toString) .collect(Collectors.joining(pathOptionSeparator)); return optionName + "=" + transformedOptionValue; - } catch (ReplaySupport.ReplayPathSubstitutionError error) { - throw NativeImage.showError("Failed to prepare path entry '" + error.origPath + "' of option " + optionName + " from " + optionOrigin + " for replay bundle inclusion.", error); + } catch (BundleSupport.BundlePathSubstitutionError error) { + throw NativeImage.showError("Failed to prepare path entry '" + error.origPath + "' of option " + optionName + " from " + optionOrigin + " for bundle inclusion.", error); } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java similarity index 76% rename from substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java rename to substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index a296cf812e67..ea20ac6e9e78 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/ReplaySupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -54,11 +54,11 @@ import com.oracle.svm.core.util.json.JsonWriter; import com.oracle.svm.util.ClassUtil; -final class ReplaySupport { +final class BundleSupport { - static final String REPLAY_OPTION = "--replay"; + static final String BUNDLE_OPTION = "--bundle"; - enum ReplayStatus { + enum BundleStatus { prepare(false, false), create(false, false), apply(true, true); @@ -66,7 +66,7 @@ enum ReplayStatus { final boolean hidden; final boolean loadBundle; - ReplayStatus(boolean hidden, boolean loadBundle) { + BundleStatus(boolean hidden, boolean loadBundle) { this.hidden = hidden; this.loadBundle = loadBundle; } @@ -78,9 +78,9 @@ boolean show() { final NativeImage nativeImage; - final ReplayStatus status; + final BundleStatus status; - final Path replayRootDir; + final Path rootDir; final Path stageDir; final Path classPathDir; final Path modulePathDir; @@ -91,110 +91,110 @@ boolean show() { private final List buildArgs; - private static final String replayTempDirPrefix = "replayRoot-"; + private static final String bundleTempDirPrefix = "bundleRoot-"; - static ReplaySupport create(NativeImage nativeImage, String replayArg, NativeImage.ArgumentQueue args) { + static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeImage.ArgumentQueue args) { if (!nativeImage.userConfigProperties.isEmpty()) { - throw NativeImage.showError("Replay-bundle support cannot be combined with " + NativeImage.CONFIG_FILE_ENV_VAR_KEY + " environment variable use."); + throw NativeImage.showError("Bundle support cannot be combined with " + NativeImage.CONFIG_FILE_ENV_VAR_KEY + " environment variable use."); } - ReplaySupport.ReplayStatus replayStatus; - if (replayArg.equals(REPLAY_OPTION)) { - /* Handle short form of --replay-apply */ - replayStatus = ReplaySupport.ReplayStatus.apply; + BundleStatus bundleStatus; + if (bundleArg.equals(BUNDLE_OPTION)) { + /* Handle short form of --bundle-apply */ + bundleStatus = BundleStatus.apply; } else { - String replayVariant = replayArg.substring(REPLAY_OPTION.length() + 1); + String bundleVariant = bundleArg.substring(BUNDLE_OPTION.length() + 1); try { - replayStatus = ReplaySupport.ReplayStatus.valueOf(replayVariant); + bundleStatus = BundleStatus.valueOf(bundleVariant); } catch (IllegalArgumentException e) { - String suggestedVariants = Arrays.stream(ReplaySupport.ReplayStatus.values()) - .filter(ReplaySupport.ReplayStatus::show) - .map(v -> REPLAY_OPTION + "-" + v) + String suggestedVariants = Arrays.stream(BundleStatus.values()) + .filter(BundleStatus::show) + .map(v -> BUNDLE_OPTION + "-" + v) .collect(Collectors.joining(", ")); - throw NativeImage.showError("Unknown option " + replayArg + ". Valid variants are: " + suggestedVariants + "."); + throw NativeImage.showError("Unknown option " + bundleArg + ". Valid variants are: " + suggestedVariants + "."); } } - ReplaySupport replaySupport; - if (replayStatus.loadBundle) { - String replayBundleFilename = args.poll(); - replaySupport = new ReplaySupport(nativeImage, replayStatus, replayBundleFilename); - List buildArgs = replaySupport.getBuildArgs(); + BundleSupport bundleSupport; + if (bundleStatus.loadBundle) { + String bundleFilename = args.poll(); + bundleSupport = new BundleSupport(nativeImage, bundleStatus, bundleFilename); + List buildArgs = bundleSupport.getBuildArgs(); for (int i = buildArgs.size() - 1; i >= 0; i--) { String buildArg = buildArgs.get(i); - if (buildArg.startsWith(REPLAY_OPTION)) { - assert !ReplayStatus.valueOf(buildArg.substring(REPLAY_OPTION.length() + 1)).loadBundle; + if (buildArg.startsWith(BUNDLE_OPTION)) { + assert !BundleStatus.valueOf(buildArg.substring(BUNDLE_OPTION.length() + 1)).loadBundle; continue; } if (buildArg.startsWith("-Dllvm.bin.dir=")) { Optional existing = nativeImage.config.getBuildArgs().stream().filter(arg -> arg.startsWith("-Dllvm.bin.dir=")).findFirst(); if (existing.isPresent() && !existing.get().equals(buildArg)) { - throw NativeImage.showError("Replay-bundle native-image argument '" + buildArg + "' conflicts with existing '" + existing.get() + "'."); + throw NativeImage.showError("Bundle native-image argument '" + buildArg + "' conflicts with existing '" + existing.get() + "'."); } continue; } args.push(buildArg); } } else { - replaySupport = new ReplaySupport(nativeImage, replayStatus); + bundleSupport = new BundleSupport(nativeImage, bundleStatus); } - return replaySupport; + return bundleSupport; } - private ReplaySupport(NativeImage nativeImage, ReplayStatus status) { + private BundleSupport(NativeImage nativeImage, BundleStatus status) { assert !status.loadBundle; this.nativeImage = nativeImage; this.status = status; try { - replayRootDir = Files.createTempDirectory(replayTempDirPrefix); - Path inputDir = replayRootDir.resolve("input"); + rootDir = Files.createTempDirectory(bundleTempDirPrefix); + Path inputDir = rootDir.resolve("input"); stageDir = Files.createDirectories(inputDir.resolve("stage")); auxiliaryDir = Files.createDirectories(inputDir.resolve("auxiliary")); Path classesDir = inputDir.resolve("classes"); classPathDir = Files.createDirectories(classesDir.resolve("cp")); modulePathDir = Files.createDirectories(classesDir.resolve("p")); } catch (IOException e) { - throw NativeImage.showError("Unable to create replay-bundle directory layout", e); + throw NativeImage.showError("Unable to create bundle directory layout", e); } this.buildArgs = Collections.unmodifiableList(nativeImage.config.getBuildArgs()); } - private ReplaySupport(NativeImage nativeImage, ReplayStatus status, String replayBundleFilename) { + private BundleSupport(NativeImage nativeImage, BundleStatus status, String bundleFilename) { assert status.loadBundle; this.nativeImage = nativeImage; this.status = status; - Path replayBundlePath = Path.of(replayBundleFilename); - if (!Files.isReadable(replayBundlePath)) { - throw NativeImage.showError("The given replay-bundle file " + replayBundleFilename + " cannot be read"); + Path bundlePath = Path.of(bundleFilename); + if (!Files.isReadable(bundlePath)) { + throw NativeImage.showError("The given bundle file " + bundleFilename + " cannot be read"); } - if (Files.isDirectory(replayBundlePath)) { - replayRootDir = replayBundlePath; + if (Files.isDirectory(bundlePath)) { + rootDir = bundlePath; } else { try { - replayRootDir = Files.createTempDirectory(replayTempDirPrefix); - try (JarFile archive = new JarFile(replayBundlePath.toFile())) { + rootDir = Files.createTempDirectory(bundleTempDirPrefix); + try (JarFile archive = new JarFile(bundlePath.toFile())) { archive.stream().forEach(jarEntry -> { - Path replayBundleFile = replayRootDir.resolve(jarEntry.getName()); + Path bundleFile = rootDir.resolve(jarEntry.getName()); try { - Path replayBundleFileParent = replayBundleFile.getParent(); - if (replayBundleFileParent != null) { - Files.createDirectories(replayBundleFileParent); + Path bundleFileParent = bundleFile.getParent(); + if (bundleFileParent != null) { + Files.createDirectories(bundleFileParent); } - Files.copy(archive.getInputStream(jarEntry), replayBundleFile); + Files.copy(archive.getInputStream(jarEntry), bundleFile); } catch (IOException e) { - throw NativeImage.showError("Unable to copy " + jarEntry.getName() + " from replay-bundle " + replayBundlePath + " to " + replayBundleFile, e); + throw NativeImage.showError("Unable to copy " + jarEntry.getName() + " from bundle " + bundlePath + " to " + bundleFile, e); } }); } } catch (IOException e) { - throw NativeImage.showError("Unable to create replay-bundle directory layout from replay-file " + replayBundlePath, e); + throw NativeImage.showError("Unable to create bundle directory layout from file " + bundlePath, e); } } - Path inputDir = replayRootDir.resolve("input"); + Path inputDir = rootDir.resolve("input"); stageDir = inputDir.resolve("stage"); auxiliaryDir = inputDir.resolve("auxiliary"); Path classesDir = inputDir.resolve("classes"); @@ -228,7 +228,7 @@ public List getBuildArgs() { } Path recordCanonicalization(Path before, Path after) { - if (before.startsWith(replayRootDir)) { + if (before.startsWith(rootDir)) { if (nativeImage.isVerbose()) { System.out.println(("RecordCanonicalization Skip: " + before)); } @@ -259,29 +259,29 @@ Path substituteAuxiliaryPath(Path origPath) { Path substituteClassPath(Path origPath) { try { return substitutePath(origPath, classPathDir); - } catch (ReplayPathSubstitutionError error) { - throw NativeImage.showError("Failed to prepare class-path entry '" + error.origPath + "' for replay bundle inclusion.", error); + } catch (BundlePathSubstitutionError error) { + throw NativeImage.showError("Failed to prepare class-path entry '" + error.origPath + "' for bundle inclusion.", error); } } Path substituteModulePath(Path origPath) { try { return substitutePath(origPath, modulePathDir); - } catch (ReplayPathSubstitutionError error) { - throw NativeImage.showError("Failed to prepare module-path entry '" + error.origPath + "' for replay bundle inclusion.", error); + } catch (BundlePathSubstitutionError error) { + throw NativeImage.showError("Failed to prepare module-path entry '" + error.origPath + "' for bundle inclusion.", error); } } @SuppressWarnings("serial") - static final class ReplayPathSubstitutionError extends Error { + static final class BundlePathSubstitutionError extends Error { public final Path origPath; - ReplayPathSubstitutionError(String message, Path origPath) { + BundlePathSubstitutionError(String message, Path origPath) { super(message); this.origPath = origPath; } - ReplayPathSubstitutionError(String message, Path origPath, Throwable cause) { + BundlePathSubstitutionError(String message, Path origPath, Throwable cause) { super(message, cause); this.origPath = origPath; } @@ -289,9 +289,9 @@ static final class ReplayPathSubstitutionError extends Error { @SuppressWarnings("try") private Path substitutePath(Path origPath, Path destinationDir) { - assert destinationDir.startsWith(replayRootDir); + assert destinationDir.startsWith(rootDir); - if (origPath.startsWith(replayRootDir)) { + if (origPath.startsWith(rootDir)) { if (nativeImage.isVerbose()) { System.out.println(("RecordSubstitution/RestoreSubstitution Skip: " + origPath)); } @@ -303,7 +303,7 @@ private Path substitutePath(Path origPath, Path destinationDir) { if (nativeImage.isVerbose()) { System.out.println("RestoreSubstitution src: " + origPath + ", dst: " + previousRelativeSubstitutedPath); } - return replayRootDir.resolve(previousRelativeSubstitutedPath); + return rootDir.resolve(previousRelativeSubstitutedPath); } if (origPath.startsWith(nativeImage.config.getJavaHome())) { @@ -327,7 +327,7 @@ private Path substitutePath(Path origPath, Path destinationDir) { } } if (forbiddenPath) { - throw new ReplayPathSubstitutionError("Replay bundles do not allow inclusion of directory " + origPath, origPath); + throw new BundlePathSubstitutionError("Bundles do not allow inclusion of directory " + origPath, origPath); } if (!Files.isReadable(origPath)) { @@ -353,19 +353,19 @@ private Path substitutePath(Path origPath, Path destinationDir) { Path substitutedPath = destinationDir.resolve(substitutedPathFilename); if (Files.exists(substitutedPath)) { /* If we ever see this, we have to implement substitutedPath collision-handling */ - throw new ReplayPathSubstitutionError("Failed to create a unique path-name in " + destinationDir + ". " + substitutedPath + " already exists", origPath); + throw new BundlePathSubstitutionError("Failed to create a unique path-name in " + destinationDir + ". " + substitutedPath + " already exists", origPath); } if (Files.isDirectory(origPath)) { try (Stream walk = Files.walk(origPath)) { walk.forEach(sourcePath -> copyFile(sourcePath, substitutedPath.resolve(origPath.relativize(sourcePath)))); } catch (IOException e) { - throw new ReplayPathSubstitutionError("Failed to iterate through directory " + origPath, origPath, e); + throw new BundlePathSubstitutionError("Failed to iterate through directory " + origPath, origPath, e); } } else { copyFile(origPath, substitutedPath); } - Path relativeSubstitutedPath = replayRootDir.relativize(substitutedPath); + Path relativeSubstitutedPath = rootDir.relativize(substitutedPath); if (nativeImage.isVerbose()) { System.out.println("RecordSubstitution src: " + origPath + ", dst: " + relativeSubstitutedPath); } @@ -389,21 +389,21 @@ void shutdown() { writeBundle(); } - nativeImage.deleteAllFiles(replayRootDir); + nativeImage.deleteAllFiles(rootDir); } void writeBundle() { Path pathCanonicalizationsFile = stageDir.resolve("path_canonicalizations.json"); try (JsonWriter writer = new JsonWriter(pathCanonicalizationsFile)) { /* Printing as list with defined sort-order ensures useful diffs are possible */ - JsonPrinter.printCollection(writer, pathCanonicalizations.entrySet(), Map.Entry.comparingByKey(), ReplaySupport::printPathMapping); + JsonPrinter.printCollection(writer, pathCanonicalizations.entrySet(), Map.Entry.comparingByKey(), BundleSupport::printPathMapping); } catch (IOException e) { throw NativeImage.showError("Failed to write bundle-file " + pathCanonicalizationsFile, e); } Path pathSubstitutionsFile = stageDir.resolve("path_substitutions.json"); try (JsonWriter writer = new JsonWriter(pathSubstitutionsFile)) { /* Printing as list with defined sort-order ensures useful diffs are possible */ - JsonPrinter.printCollection(writer, pathSubstitutions.entrySet(), Map.Entry.comparingByKey(), ReplaySupport::printPathMapping); + JsonPrinter.printCollection(writer, pathSubstitutions.entrySet(), Map.Entry.comparingByKey(), BundleSupport::printPathMapping); } catch (IOException e) { throw NativeImage.showError("Failed to write bundle-file " + pathSubstitutionsFile, e); } @@ -411,19 +411,19 @@ void writeBundle() { Path buildArgsFile = stageDir.resolve("build.json"); try (JsonWriter writer = new JsonWriter(buildArgsFile)) { /* Printing as list with defined sort-order ensures useful diffs are possible */ - JsonPrinter.printCollection(writer, buildArgs, null, ReplaySupport::printBuildArg); + JsonPrinter.printCollection(writer, buildArgs, null, BundleSupport::printBuildArg); } catch (IOException e) { throw NativeImage.showError("Failed to write bundle-file " + pathSubstitutionsFile, e); } - Path bundleFile = Path.of(nativeImage.imagePath).resolve(nativeImage.imageName + ".replay"); + Path bundleFile = Path.of(nativeImage.imagePath).resolve(nativeImage.imageName + ".nib"); try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(bundleFile), new Manifest())) { - try (Stream walk = Files.walk(replayRootDir)) { + try (Stream walk = Files.walk(rootDir)) { walk.forEach(bundleEntry -> { if (Files.isDirectory(bundleEntry)) { return; } - String jarEntryName = replayRootDir.relativize(bundleEntry).toString(); + String jarEntryName = rootDir.relativize(bundleEntry).toString(); JarEntry entry = new JarEntry(jarEntryName.replace(File.separator, "/")); try { entry.setTime(Files.getLastModifiedTime(bundleEntry).toMillis()); @@ -431,12 +431,12 @@ void writeBundle() { Files.copy(bundleEntry, jarOutStream); jarOutStream.closeEntry(); } catch (IOException e) { - throw NativeImage.showError("Failed to copy " + bundleEntry + " into replay-bundle file " + bundleFile, e); + throw NativeImage.showError("Failed to copy " + bundleEntry + " into bundle file " + bundleFile, e); } }); } } catch (IOException e) { - throw NativeImage.showError("Failed to create replay-bundle file " + bundleFile, e); + throw NativeImage.showError("Failed to create bundle file " + bundleFile, e); } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java index dd0a0d0e2b9b..3a2001dcb407 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java @@ -151,9 +151,8 @@ private boolean consume(ArgumentQueue args, String headArg) { return true; } - if (headArg.startsWith(ReplaySupport.REPLAY_OPTION)) { - String replayArg = args.poll(); - nativeImage.replaySupport = ReplaySupport.create(nativeImage, replayArg, args); + if (headArg.startsWith(BundleSupport.BUNDLE_OPTION)) { + nativeImage.bundleSupport = BundleSupport.create(nativeImage, args.poll(), args); return true; } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java index 1c2eb26d669d..14bed7b9ed95 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java @@ -189,7 +189,7 @@ public boolean consume(ArgumentQueue args) { args.poll(); headArg = headArg.substring(1); Path origArgFile = Paths.get(headArg); - Path argFile = nativeImage.replaySupport != null ? nativeImage.replaySupport.substituteAuxiliaryPath(origArgFile) : origArgFile; + Path argFile = nativeImage.bundleSupport != null ? nativeImage.bundleSupport.substituteAuxiliaryPath(origArgFile) : origArgFile; NativeImage.NativeImageArgsProcessor processor = nativeImage.new NativeImageArgsProcessor(OptionOrigin.argFilePrefix + argFile); readArgFile(argFile).forEach(processor::accept); List leftoverArgs = processor.apply(false); @@ -430,7 +430,7 @@ private void handleJarFileArg(Path jarFilePath) { String origin = "manifest from " + jarFilePath.toUri(); nativeImage.addPlainImageBuilderArg(NativeImage.injectHostedOptionOrigin(nativeImage.oHName + jarFileNameBase, origin)); } - Path finalFilePath = nativeImage.replaySupport != null ? nativeImage.replaySupport.substituteClassPath(jarFilePath) : jarFilePath; + Path finalFilePath = nativeImage.bundleSupport != null ? nativeImage.bundleSupport.substituteClassPath(jarFilePath) : jarFilePath; if (!NativeImage.processJarManifestMainAttributes(finalFilePath, nativeImage::handleMainClassAttribute)) { NativeImage.showError("No manifest in " + finalFilePath); } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index c1a72dc8658e..b2308fcb8cb5 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -267,11 +267,11 @@ private static String oR(OptionKey option) { private long imageBuilderPid = -1; - ReplaySupport replaySupport; + BundleSupport bundleSupport; void replaySupportShutdown() { - if (replaySupport != null) { - replaySupport.shutdown(); + if (bundleSupport != null) { + bundleSupport.shutdown(); } } @@ -752,7 +752,7 @@ protected NativeImage(BuildConfiguration config) { void addMacroOptionRoot(Path configDir) { Path origRootDir = canonicalize(configDir); - Path rootDir = replaySupport != null ? replaySupport.substituteClassPath(origRootDir) : origRootDir; + Path rootDir = bundleSupport != null ? bundleSupport.substituteClassPath(origRootDir) : origRootDir; optionRegistry.addMacroOptionRoot(rootDir); } @@ -1332,7 +1332,7 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa arguments.addAll(Arrays.asList(SubstrateOptions.WATCHPID_PREFIX, "" + ProcessProperties.getProcessID())); } - boolean useReplay = replaySupport != null; + boolean useReplay = bundleSupport != null; if (useReplay) { Path imageNamePath = Path.of(imageName); if (imageNamePath.getParent() != null) { @@ -1345,12 +1345,12 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa throw NativeImage.showError("Replay-bundle support can only be used with default image output directory. Ensure '" + oHPath + "' is not explicitly set."); } } - Function substituteAuxiliaryPath = useReplay ? replaySupport::substituteAuxiliaryPath : Function.identity(); + Function substituteAuxiliaryPath = useReplay ? bundleSupport::substituteAuxiliaryPath : Function.identity(); Function imageArgsTransformer = rawArg -> apiOptionHandler.transformBuilderArgument(rawArg, substituteAuxiliaryPath); List finalImageArgs = imageArgs.stream().map(imageArgsTransformer).collect(Collectors.toList()); - Function substituteClassPath = useReplay ? replaySupport::substituteClassPath : Function.identity(); + Function substituteClassPath = useReplay ? bundleSupport::substituteClassPath : Function.identity(); List finalImageClassPath = imagecp.stream().map(substituteClassPath).collect(Collectors.toList()); - Function substituteModulePath = useReplay ? replaySupport::substituteModulePath : Function.identity(); + Function substituteModulePath = useReplay ? bundleSupport::substituteModulePath : Function.identity(); List finalImageModulePath = imagemp.stream().map(substituteModulePath).collect(Collectors.toList()); List finalImageBuilderArgs = createImageBuilderArgs(finalImageArgs, finalImageClassPath, finalImageModulePath); @@ -1372,7 +1372,7 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa showVerboseMessage(isVerbose() || dryRun, "]"); } - if (dryRun || useReplay && replaySupport.status == ReplaySupport.ReplayStatus.prepare) { + if (dryRun || useReplay && bundleSupport.status == BundleSupport.BundleStatus.prepare) { return ExitStatus.OK.getValue(); } @@ -1494,15 +1494,15 @@ Path canonicalize(Path path) { } Path canonicalize(Path path, boolean strict) { - if (replaySupport != null) { - Path prev = replaySupport.restoreCanonicalization(path); + if (bundleSupport != null) { + Path prev = bundleSupport.restoreCanonicalization(path); if (prev != null) { return prev; } } Path absolutePath = path.isAbsolute() ? path : config.getWorkingDirectory().resolve(path); if (!strict) { - return replaySupport != null ? replaySupport.recordCanonicalization(path, absolutePath) : absolutePath; + return bundleSupport != null ? bundleSupport.recordCanonicalization(path, absolutePath) : absolutePath; } boolean hasWildcard = absolutePath.endsWith(ClasspathUtils.cpWildcardSubstitute); if (hasWildcard) { @@ -1519,7 +1519,7 @@ Path canonicalize(Path path, boolean strict) { } realPath = realPath.resolve(ClasspathUtils.cpWildcardSubstitute); } - return replaySupport != null ? replaySupport.recordCanonicalization(path, realPath) : realPath; + return bundleSupport != null ? bundleSupport.recordCanonicalization(path, realPath) : realPath; } catch (IOException e) { throw showError("Invalid Path entry " + ClasspathUtils.classpathToString(path), e); } @@ -1635,7 +1635,7 @@ void addImageModulePath(Path modulePathEntry, boolean strict) { return; } - Path mpEntryFinal = replaySupport != null ? replaySupport.substituteModulePath(mpEntry) : mpEntry; + Path mpEntryFinal = bundleSupport != null ? bundleSupport.substituteModulePath(mpEntry) : mpEntry; imageModulePath.add(mpEntryFinal); processClasspathNativeImageMetaInf(mpEntryFinal); } @@ -1674,7 +1674,7 @@ private void addImageClasspathEntry(LinkedHashSet destination, Path classp return; } - Path classpathEntryFinal = replaySupport != null ? replaySupport.substituteModulePath(classpathEntry) : classpathEntry; + Path classpathEntryFinal = bundleSupport != null ? bundleSupport.substituteModulePath(classpathEntry) : classpathEntry; if (!imageClasspath.contains(classpathEntryFinal) && !customImageClasspath.contains(classpathEntryFinal)) { destination.add(classpathEntryFinal); processManifestMainAttributes(classpathEntryFinal, (jarFilePath, attributes) -> handleClassPathAttribute(destination, jarFilePath, attributes)); From 5b030bd6f3868f5b3289378dbc2d33a1cc963ddf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 20 Dec 2022 11:34:45 +0100 Subject: [PATCH 09/46] Add help text for bundle options --- .../src/com.oracle.svm.driver/resources/HelpExtra.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt b/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt index 3690ce0be354..693eb7d28c92 100644 --- a/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt +++ b/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt @@ -16,5 +16,16 @@ Non-standard options help: --diagnostics-mode Enables logging of image-build information to a diagnostics folder. --dry-run output the command line that would be used for building + --bundle-prepare create a bundle file (*.nib file) that allows building an image at a + later point from the bundle file alone. + --bundle-create in addition to building a native image a bundle file gets created that + allows rebuilding the image again at a later point from the bundle. + --bundle bundle-file + or + --bundle-apply bundle-file + an image will be built from the given bundle with the exact same + arguments and files that have been passed to native-image originally to + create the bundle. + -V= provide values for placeholders in native-image.properties files From a6ade3ff7ccaf657732a433f8c9db4ac59c92606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 22 Dec 2022 18:20:27 +0100 Subject: [PATCH 10/46] Add bundle support for output paths --- .../com/oracle/svm/core/FallbackExecutor.java | 6 +- .../svm/core/RuntimeAssertionsSupport.java | 2 +- .../com/oracle/svm/core/SubstrateOptions.java | 24 +- .../oracle/svm/core/VMInspectionOptions.java | 2 +- .../core/configure/ConfigurationFiles.java | 42 ++- .../oracle/svm/core/option/BundleMember.java | 18 + .../option/LocatableMultiOptionValue.java | 63 ++-- .../oracle/svm/core/option/OptionOrigin.java | 16 +- .../oracle/svm/driver/APIOptionHandler.java | 332 ++++++++++-------- .../com/oracle/svm/driver/BundleSupport.java | 41 ++- .../svm/driver/DefaultOptionHandler.java | 4 +- .../com/oracle/svm/driver/NativeImage.java | 113 ++++-- .../hosted/RuntimeCompilationFeature.java | 2 +- .../com/oracle/svm/hosted/FeatureHandler.java | 2 +- .../svm/hosted/LinkAtBuildTimeSupport.java | 4 +- .../hosted/NativeImageClassLoaderOptions.java | 6 +- .../oracle/svm/hosted/NativeImageOptions.java | 4 +- .../oracle/svm/hosted/ResourcesFeature.java | 4 +- .../svm/hosted/SecurityServicesFeature.java | 4 +- .../svm/hosted/ServiceLoaderFeature.java | 4 +- .../ClassInitializationOptions.java | 2 +- .../diagnostic/HostedHeapDumpFeature.java | 2 +- .../svm/hosted/image/CCLinkerInvocation.java | 2 +- .../svm/hosted/image/sources/SourceCache.java | 10 +- .../jdk/localization/LocalizationFeature.java | 6 +- .../svm/truffle/tck/PermissionsFeature.java | 24 +- 26 files changed, 434 insertions(+), 305 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/BundleMember.java diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/FallbackExecutor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/FallbackExecutor.java index decc55655884..29c8909ea14c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/FallbackExecutor.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/FallbackExecutor.java @@ -57,15 +57,15 @@ public class FallbackExecutor { public static class Options { @Option(help = "Internal option used to specify system properties for FallbackExecutor.")// - public static final HostedOptionKey FallbackExecutorSystemProperty = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey FallbackExecutorSystemProperty = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); @Option(help = "Internal option used to specify MainClass for FallbackExecutor.")// public static final HostedOptionKey FallbackExecutorMainClass = new HostedOptionKey<>(null); @Option(help = "Internal option used to specify Classpath for FallbackExecutor.")// public static final HostedOptionKey FallbackExecutorClasspath = new HostedOptionKey<>(null); @Option(help = "Internal option used to specify java arguments for FallbackExecutor.")// - public static final HostedOptionKey FallbackExecutorJavaArg = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey FallbackExecutorJavaArg = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); @Option(help = "Internal option used to specify runtime java arguments for FallbackExecutor.")// - public static final RuntimeOptionKey FallbackExecutorRuntimeJavaArg = new RuntimeOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final RuntimeOptionKey FallbackExecutorRuntimeJavaArg = new RuntimeOptionKey<>(LocatableMultiOptionValue.Strings.build()); } public static void main(String[] args) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/RuntimeAssertionsSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/RuntimeAssertionsSupport.java index 842fe64bd4c6..682170c625dd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/RuntimeAssertionsSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/RuntimeAssertionsSupport.java @@ -96,7 +96,7 @@ public static class Options { @APIOption(name = {"-da", "-disableassertions"}, valueSeparator = VALUE_SEPARATOR, valueTransformer = RuntimeAssertionsOptionTransformer.Disable.class, defaultValue = "", // customHelp = "also -da[:[packagename]...|:classname] or -disableassertions[:[packagename]...|:classname]. Disable assertions with specified granularity.")// @Option(help = "Enable or disable Java assert statements at run time") // - public static final HostedOptionKey RuntimeAssertions = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey RuntimeAssertions = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); @APIOption(name = {"-esa", "-enablesystemassertions"}, customHelp = "also -enablesystemassertions. Enables assertions in all system classes.") // @APIOption(name = {"-dsa", "-disablesystemassertions"}, kind = APIOption.APIOptionKind.Negated, customHelp = "also -disablesystemassertions. Disables assertions in all system classes.") // diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index d15756c23f61..2b10a1766632 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -60,6 +60,7 @@ import com.oracle.svm.core.heap.ReferenceHandler; import com.oracle.svm.core.option.APIOption; import com.oracle.svm.core.option.APIOptionGroup; +import com.oracle.svm.core.option.BundleMember; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.option.HostedOptionValues; import com.oracle.svm.core.option.LocatableMultiOptionValue; @@ -81,7 +82,7 @@ public class SubstrateOptions { @Option(help = "Preserve the local variable information for every Java source line to allow line-by-line stepping in the debugger. Allow the lookup of Java-level method information, e.g., in stack traces.")// public static final HostedOptionKey SourceLevelDebug = new HostedOptionKey<>(false); @Option(help = "Constrain debug info generation to the comma-separated list of package prefixes given to this option.")// - public static final HostedOptionKey SourceLevelDebugFilter = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey SourceLevelDebugFilter = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); public static boolean parseOnce() { /* @@ -271,10 +272,11 @@ public static void setDebugInfoValueUpdateHandler(ValueUpdateHandler up public static final HostedOptionKey IncludeNodeSourcePositions = new HostedOptionKey<>(false); @Option(help = "Search path for C libraries passed to the linker (list of comma-separated directories)")// - public static final HostedOptionKey CLibraryPath = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.commaSeparated()); + @BundleMember(role = BundleMember.Role.Input)// + public static final HostedOptionKey CLibraryPath = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); @Option(help = "Path passed to the linker as the -rpath (list of comma-separated directories)")// - public static final HostedOptionKey LinkerRPath = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey LinkerRPath = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "Directory of the image file to be generated", type = OptionType.User)// public static final HostedOptionKey Path = new HostedOptionKey<>(null); @@ -346,11 +348,11 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o @APIOption(name = "trace-class-initialization")// @Option(help = "Comma-separated list of fully-qualified class names that class initialization is traced for.")// - public static final HostedOptionKey TraceClassInitialization = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey TraceClassInitialization = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @APIOption(name = "trace-object-instantiation")// @Option(help = "Comma-separated list of fully-qualified class names that object instantiation is traced for.")// - public static final HostedOptionKey TraceObjectInstantiation = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey TraceObjectInstantiation = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "Trace all native tool invocations as part of image building", type = User)// public static final HostedOptionKey TraceNativeToolUsage = new HostedOptionKey<>(false); @@ -365,10 +367,10 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o @APIOption(name = "enable-https", fixedValue = "https", customHelp = "enable https support in the generated image")// @APIOption(name = "enable-url-protocols")// @Option(help = "List of comma separated URL protocols to enable.")// - public static final HostedOptionKey EnableURLProtocols = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey EnableURLProtocols = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "List of comma separated URL protocols that must never be included.")// - public static final HostedOptionKey DisableURLProtocols = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey DisableURLProtocols = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @SuppressWarnings("unused") // @APIOption(name = "enable-all-security-services")// @@ -485,7 +487,7 @@ public static long getTearDownFailureNanos() { public static final HostedOptionKey AOTTrivialInline = new HostedOptionKey<>(true); @Option(help = "file:doc-files/NeverInlineHelp.txt", type = OptionType.Debug)// - public static final HostedOptionKey NeverInline = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey NeverInline = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); @Option(help = "Maximum number of nodes in a method so that it is considered trivial.")// public static final HostedOptionKey MaxNodesInTrivialMethod = new HostedOptionKey<>(20); @@ -506,7 +508,7 @@ public static long getTearDownFailureNanos() { public static final HostedOptionKey UseCompressedFrameEncodings = new HostedOptionKey<>(true); @Option(help = "Report error if [:{,}] is discovered during analysis (valid values for UsageKind: InHeap, Allocated, Reachable).", type = OptionType.Debug)// - public static final HostedOptionKey ReportAnalysisForbiddenType = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey ReportAnalysisForbiddenType = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); @Option(help = "Backend used by the compiler", type = OptionType.User)// public static final HostedOptionKey CompilerBackend = new HostedOptionKey<>("lir") { @@ -573,7 +575,7 @@ public static boolean useLIRBackend() { public static final HostedOptionKey CCompilerPath = new HostedOptionKey<>(null); @APIOption(name = "native-compiler-options")// @Option(help = "Provide custom C compiler option used for query code compilation.", type = OptionType.User)// - public static final HostedOptionKey CCompilerOption = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey CCompilerOption = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); @Option(help = "Use strict checks when performing query code compilation.", type = OptionType.User)// public static final HostedOptionKey StrictQueryCodeCompilation = new HostedOptionKey<>(true); @@ -635,7 +637,7 @@ public static void defaultDebugInfoValueUpdateHandler(EconomicMap, } @Option(help = "Search path for source files for Application or GraalVM classes (list of comma-separated directories or jar files)")// - public static final HostedOptionKey DebugInfoSourceSearchPath = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey DebugInfoSourceSearchPath = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); @Option(help = "Directory under which to create source file cache for Application or GraalVM classes")// public static final HostedOptionKey DebugInfoSourceCacheRoot = new HostedOptionKey<>("sources"); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java index 7aa9258e1482..4b9c350cc5e6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java @@ -55,7 +55,7 @@ public final class VMInspectionOptions { @APIOption(name = ENABLE_MONITORING_OPTION, defaultValue = MONITORING_DEFAULT_NAME) // @Option(help = "Enable monitoring features that allow the VM to be inspected at run time. Comma-separated list can contain " + MONITORING_ALLOWED_VALUES + ". " + "For example: `--" + ENABLE_MONITORING_OPTION + "=" + MONITORING_HEAPDUMP_NAME + "," + MONITORING_JFR_NAME + "`.", type = OptionType.User) // - public static final HostedOptionKey EnableMonitoringFeatures = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated(), + public static final HostedOptionKey EnableMonitoringFeatures = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter(), VMInspectionOptions::validateEnableMonitoringFeatures); public static void validateEnableMonitoringFeatures(@SuppressWarnings("unused") OptionKey optionKey) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ConfigurationFiles.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ConfigurationFiles.java index 075355ab094a..a81266e5c4ab 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ConfigurationFiles.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ConfigurationFiles.java @@ -38,6 +38,7 @@ import org.graalvm.compiler.options.Option; import org.graalvm.compiler.options.OptionType; +import com.oracle.svm.core.option.BundleMember; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.option.LocatableMultiOptionValue; import com.oracle.svm.core.util.UserError; @@ -50,45 +51,54 @@ public final class ConfigurationFiles { public static final class Options { @Option(help = "Directories directly containing configuration files for dynamic features at runtime.", type = OptionType.User)// - static final HostedOptionKey ConfigurationFileDirectories = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + static final HostedOptionKey ConfigurationFileDirectories = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "Resource path above configuration resources for dynamic features at runtime.", type = OptionType.User)// - public static final HostedOptionKey ConfigurationResourceRoots = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey ConfigurationResourceRoots = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "file:doc-files/ReflectionConfigurationFilesHelp.txt", type = OptionType.User)// - public static final HostedOptionKey ReflectionConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.commaSeparated()); + @BundleMember(role = BundleMember.Role.Input)// + public static final HostedOptionKey ReflectionConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); @Option(help = "Resources describing program elements to be made available for reflection (see ReflectionConfigurationFiles).", type = OptionType.User)// - public static final HostedOptionKey ReflectionConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey ReflectionConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "file:doc-files/ProxyConfigurationFilesHelp.txt", type = OptionType.User)// - public static final HostedOptionKey DynamicProxyConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.commaSeparated()); + @BundleMember(role = BundleMember.Role.Input)// + public static final HostedOptionKey DynamicProxyConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); @Option(help = "Resources describing program elements to be made available for reflection (see ProxyConfigurationFiles).", type = OptionType.User)// - public static final HostedOptionKey DynamicProxyConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey DynamicProxyConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "file:doc-files/SerializationConfigurationFilesHelp.txt", type = OptionType.User)// - public static final HostedOptionKey SerializationConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.commaSeparated()); + @BundleMember(role = BundleMember.Role.Input)// + public static final HostedOptionKey SerializationConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); @Option(help = "Resources describing program elements to be made available for serialization (see SerializationConfigurationFiles).", type = OptionType.User)// - public static final HostedOptionKey SerializationConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey SerializationConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "file:doc-files/SerializationConfigurationFilesHelp.txt", type = OptionType.User)// - public static final HostedOptionKey SerializationDenyConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.commaSeparated()); + @BundleMember(role = BundleMember.Role.Input)// + public static final HostedOptionKey SerializationDenyConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); @Option(help = "Resources describing program elements that must not be made available for serialization.", type = OptionType.User)// - public static final HostedOptionKey SerializationDenyConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey SerializationDenyConfigurationResources = new HostedOptionKey<>( + LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "Files describing Java resources to be included in the image.", type = OptionType.User)// - public static final HostedOptionKey ResourceConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.commaSeparated()); + @BundleMember(role = BundleMember.Role.Input)// + public static final HostedOptionKey ResourceConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); @Option(help = "Resources describing Java resources to be included in the image.", type = OptionType.User)// - public static final HostedOptionKey ResourceConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey ResourceConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "Files describing program elements to be made accessible via JNI (for syntax, see ReflectionConfigurationFiles)", type = OptionType.User)// - public static final HostedOptionKey JNIConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.commaSeparated()); + @BundleMember(role = BundleMember.Role.Input)// + public static final HostedOptionKey JNIConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); @Option(help = "Resources describing program elements to be made accessible via JNI (see JNIConfigurationFiles).", type = OptionType.User)// - public static final HostedOptionKey JNIConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey JNIConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "Files describing predefined classes that can be loaded at runtime.", type = OptionType.User)// - public static final HostedOptionKey PredefinedClassesConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.commaSeparated()); + @BundleMember(role = BundleMember.Role.Input)// + public static final HostedOptionKey PredefinedClassesConfigurationFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); @Option(help = "Resources describing predefined classes that can be loaded at runtime.", type = OptionType.User)// - public static final HostedOptionKey PredefinedClassesConfigurationResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey PredefinedClassesConfigurationResources = new HostedOptionKey<>( + LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "Causes unknown attributes in configuration objects to abort the image build instead of emitting a warning.")// public static final HostedOptionKey StrictConfiguration = new HostedOptionKey<>(false); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/BundleMember.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/BundleMember.java new file mode 100644 index 000000000000..fd356952c8c5 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/BundleMember.java @@ -0,0 +1,18 @@ +package com.oracle.svm.core.option; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD}) +public @interface BundleMember { + Role role(); + + enum Role { + Input, + Output, + Ignore + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/LocatableMultiOptionValue.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/LocatableMultiOptionValue.java index 30a159062f06..d190b2afe9c7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/LocatableMultiOptionValue.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/LocatableMultiOptionValue.java @@ -26,7 +26,6 @@ import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -40,34 +39,22 @@ public abstract class LocatableMultiOptionValue implements MultiOptionValue { - private static final String DEFAULT_DELIMITER = ""; + protected static final String NO_DELIMITER = ""; private final String delimiter; private final Class valueType; private final List> values; - private LocatableMultiOptionValue(Class valueType) { - this(valueType, DEFAULT_DELIMITER); - } - - private LocatableMultiOptionValue(Class valueType, String delimiter) { - this.delimiter = delimiter; + private LocatableMultiOptionValue(Class valueType, String delimiter, List defaults) { this.valueType = valueType; + this.delimiter = delimiter; values = new ArrayList<>(); - } - - private LocatableMultiOptionValue(Class valueType, List defaults) { - this(valueType, defaults, DEFAULT_DELIMITER); - } - - private LocatableMultiOptionValue(Class valueType, List defaults, String delimiter) { - this(valueType, delimiter); values.addAll(defaults.stream().map(val -> Pair.create(val, "default")).collect(Collectors.toList())); } private LocatableMultiOptionValue(LocatableMultiOptionValue other) { - this.delimiter = other.delimiter; this.valueType = other.valueType; + this.delimiter = other.delimiter; this.values = new ArrayList<>(other.values); } @@ -103,7 +90,7 @@ public void valueUpdate(Object value) { @Override public List values() { if (values.isEmpty()) { - return Collections.emptyList(); + return List.of(); } return values.stream().map(Pair::getLeft).collect(Collectors.toList()); } @@ -131,24 +118,20 @@ public MultiOptionValue createCopy() { return new Strings(this); } - public Strings() { - super(String.class); - } - - public Strings(String delimiter) { - super(String.class, delimiter); + private Strings(String delimiter, List defaultStrings) { + super(String.class, delimiter, defaultStrings); } - public Strings(List defaultStrings) { - super(String.class, defaultStrings); + public static Strings build() { + return new Strings(NO_DELIMITER, List.of()); } - public Strings(List defaultStrings, String delimiter) { - super(String.class, defaultStrings, delimiter); + public static Strings buildWithCommaDelimiter() { + return new Strings(",", List.of()); } - public static Strings commaSeparated() { - return new Strings(","); + public static Strings buildWithDefaults(String... defaultStrings) { + return new Strings(NO_DELIMITER, List.of(defaultStrings)); } } @@ -163,24 +146,24 @@ public MultiOptionValue createCopy() { return new Paths(this); } - public Paths() { - super(Path.class); + private Paths(String delimiter, List defaultPaths) { + super(Path.class, delimiter, defaultPaths); } - public Paths(String delimiter) { - super(Path.class, delimiter); + public static Paths build() { + return new Paths(NO_DELIMITER, List.of()); } - public Paths(List defaultPaths) { - super(Path.class, defaultPaths); + public static Paths buildWithCommaDelimiter() { + return new Paths(",", List.of()); } - public Paths(List defaultPaths, String delimiter) { - super(Path.class, defaultPaths, delimiter); + public static Paths buildWithCustomDelimiter(String delimiter) { + return new Paths(delimiter, List.of()); } - public static Paths commaSeparated() { - return new Paths(","); + public static Paths buildWithDefaults(Path... defaultPaths) { + return new Paths(NO_DELIMITER, List.of(defaultPaths)); } } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/OptionOrigin.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/OptionOrigin.java index 54779df61a55..5fe23cc112ef 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/OptionOrigin.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/OptionOrigin.java @@ -68,6 +68,10 @@ public List getRedirectionValues(@SuppressWarnings("unused") Path values } public static OptionOrigin from(String origin) { + return from(origin, true); + } + + public static OptionOrigin from(String origin, boolean strict) { if (origin == null || origin.startsWith(argFilePrefix)) { return commandLineOptionOriginSingleton; @@ -79,19 +83,25 @@ public static OptionOrigin from(String origin) { if (macroOption != null) { return macroOption; } - throw VMError.shouldNotReachHere("Unsupported OptionOrigin: " + origin); + if (strict) { + throw VMError.shouldNotReachHere("Unsupported OptionOrigin: " + origin); + } + return null; } switch (originURI.getScheme()) { case "jar": return new JarOptionOrigin(originURI); case "file": Path originPath = Path.of(originURI); - if (!Files.isReadable(originPath)) { + if (!Files.isReadable(originPath) && strict) { VMError.shouldNotReachHere("Directory origin with path that cannot be read: " + originPath); } return new DirectoryOptionOrigin(originPath); default: - throw VMError.shouldNotReachHere("OptionOrigin of unsupported scheme: " + originURI); + if (strict) { + throw VMError.shouldNotReachHere("OptionOrigin of unsupported scheme: " + originURI); + } + return null; } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java index b9f05b23c549..6e4a6c988949 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java @@ -24,12 +24,12 @@ */ package com.oracle.svm.driver; +import java.lang.annotation.Annotation; import java.lang.reflect.Field; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -37,6 +37,7 @@ import java.util.ServiceLoader; import java.util.SortedMap; import java.util.TreeMap; +import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -51,6 +52,7 @@ import com.oracle.svm.core.option.APIOption; import com.oracle.svm.core.option.APIOption.APIOptionKind; import com.oracle.svm.core.option.APIOptionGroup; +import com.oracle.svm.core.option.BundleMember; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.option.OptionOrigin; import com.oracle.svm.core.option.SubstrateOptionsParser; @@ -97,9 +99,19 @@ boolean isDeprecated() { } } + static final class PathsOptionInfo { + private final String delimiter; + private final BundleMember.Role role; + + PathsOptionInfo(String delimiter, BundleMember.Role role) { + this.delimiter = delimiter; + this.role = role; + } + } + private final SortedMap apiOptions; private final Map groupInfos; - final Map pathOptions; + private final Map pathOptions; APIOptionHandler(NativeImage nativeImage) { super(nativeImage); @@ -115,7 +127,7 @@ boolean isDeprecated() { } } - static SortedMap extractOptions(ServiceLoader optionDescriptors, Map groupInfos, Map pathOptions) { + static SortedMap extractOptions(ServiceLoader optionDescriptors, Map groupInfos, Map pathOptions) { EconomicMap hostedOptions = EconomicMap.create(); EconomicMap runtimeOptions = EconomicMap.create(); HostedOptionParser.collectOptions(optionDescriptors, hostedOptions, runtimeOptions); @@ -137,149 +149,156 @@ static SortedMap extractOptions(ServiceLoader apiOptions, Map groupInfos, Map, APIOptionGroup> groupInstances) { - try { - Field optionField = optionDescriptor.getDeclaringClass().getDeclaredField(optionDescriptor.getFieldName()); - APIOption[] apiAnnotations = optionField.getAnnotationsByType(APIOption.class); - for (APIOption apiAnnotation : apiAnnotations) { - String builderOption = optionPrefix; - if (apiAnnotation.name().length <= 0) { - VMError.shouldNotReachHere(String.format("APIOption for %s does not provide a name entry", optionDescriptor.getLocation())); - } - String apiOptionName = APIOption.Utils.optionName(apiAnnotation.name()[0]); - String rawOptionName = optionDescriptor.getName(); - APIOptionGroup group = null; - String defaultValue = null; - - boolean booleanOption = false; - Class optionValueType = optionDescriptor.getOptionValueType(); - if (optionValueType.isArray()) { - VMError.guarantee(optionDescriptor.getOptionKey() instanceof HostedOptionKey, "Only HostedOptionKeys are allowed to have array type key values."); - optionValueType = optionValueType.getComponentType(); - } - boolean hasFixedValue = apiAnnotation.fixedValue().length > 0; - if (optionValueType.equals(Boolean.class)) { - if (!apiAnnotation.group().equals(APIOption.NullGroup.class)) { - try { - Class groupClass = apiAnnotation.group(); - APIOptionGroup g = group = groupInstances.computeIfAbsent(groupClass, ReflectionUtil::newInstance); - String groupName = APIOption.Utils.groupName(group); - GroupInfo groupInfo = groupInfos.computeIfAbsent(groupName, (n) -> new GroupInfo(g)); - if (group.helpText() == null || group.helpText().isEmpty()) { - VMError.shouldNotReachHere(String.format("APIOptionGroup %s(%s) needs to provide help text", groupClass.getName(), group.name())); - } - String groupMember = apiAnnotation.name()[0]; - groupInfo.supportedValues.add(groupMember); - - apiOptionName = groupName + groupMember; - - Boolean isEnabled = (Boolean) optionDescriptor.getOptionKey().getDefaultValue(); - if (isEnabled) { - groupInfo.defaultValues.add(groupMember); - /* Use OptionInfo.defaultValue to remember group default value */ - defaultValue = groupMember; - } - } catch (ReflectionUtilError ex) { - throw VMError.shouldNotReachHere( - "Class specified as group for @APIOption " + apiOptionName + " cannot be loaded or instantiated: " + apiAnnotation.group().getTypeName(), ex.getCause()); + for (APIOption apiAnnotation : getAnnotationsByType(optionDescriptor, APIOption.class)) { + String builderOption = optionPrefix; + if (apiAnnotation.name().length <= 0) { + VMError.shouldNotReachHere(String.format("APIOption for %s does not provide a name entry", optionDescriptor.getLocation())); + } + String apiOptionName = APIOption.Utils.optionName(apiAnnotation.name()[0]); + String rawOptionName = optionDescriptor.getName(); + APIOptionGroup group = null; + String defaultValue = null; + + boolean booleanOption = false; + Class optionValueType = optionDescriptor.getOptionValueType(); + if (optionValueType.isArray()) { + VMError.guarantee(optionDescriptor.getOptionKey() instanceof HostedOptionKey, "Only HostedOptionKeys are allowed to have array type key values."); + optionValueType = optionValueType.getComponentType(); + } + boolean hasFixedValue = apiAnnotation.fixedValue().length > 0; + if (optionValueType.equals(Boolean.class)) { + if (!apiAnnotation.group().equals(APIOption.NullGroup.class)) { + try { + Class groupClass = apiAnnotation.group(); + APIOptionGroup g = group = groupInstances.computeIfAbsent(groupClass, ReflectionUtil::newInstance); + String groupName = APIOption.Utils.groupName(group); + GroupInfo groupInfo = groupInfos.computeIfAbsent(groupName, (n) -> new GroupInfo(g)); + if (group.helpText() == null || group.helpText().isEmpty()) { + VMError.shouldNotReachHere(String.format("APIOptionGroup %s(%s) needs to provide help text", groupClass.getName(), group.name())); } - } - if (apiAnnotation.defaultValue().length > 0) { - VMError.shouldNotReachHere(String.format("Boolean APIOption %s(%s) cannot use APIOption.defaultValue", apiOptionName, rawOptionName)); - } - if (hasFixedValue) { - VMError.shouldNotReachHere(String.format("Boolean APIOption %s(%s) cannot use APIOption.fixedValue", apiOptionName, rawOptionName)); - } - builderOption += apiAnnotation.kind().equals(APIOptionKind.Negated) ? "-" : "+"; - builderOption += rawOptionName; - booleanOption = true; - } else { - if (!apiAnnotation.group().equals(APIOption.NullGroup.class)) { - VMError.shouldNotReachHere(String.format("Using @APIOption.group not supported for non-boolean APIOption %s(%s)", apiOptionName, rawOptionName)); - } - if (apiAnnotation.kind().equals(APIOptionKind.Negated)) { - VMError.shouldNotReachHere(String.format("Non-boolean APIOption %s(%s) cannot use APIOptionKind.Negated", apiOptionName, rawOptionName)); - } - if (apiAnnotation.defaultValue().length > 1) { - VMError.shouldNotReachHere(String.format("APIOption %s(%s) cannot have more than one APIOption.defaultValue", apiOptionName, rawOptionName)); - } - if (apiAnnotation.fixedValue().length > 1) { - VMError.shouldNotReachHere(String.format("APIOption %s(%s) cannot have more than one APIOption.fixedValue", apiOptionName, rawOptionName)); - } - if (hasFixedValue && apiAnnotation.defaultValue().length > 0) { - VMError.shouldNotReachHere(String.format("APIOption %s(%s) APIOption.defaultValue and APIOption.fixedValue cannot be combined", apiOptionName, rawOptionName)); - } - if (apiAnnotation.defaultValue().length > 0) { - defaultValue = apiAnnotation.defaultValue()[0]; - } - if (hasFixedValue) { - defaultValue = apiAnnotation.fixedValue()[0]; - } + String groupMember = apiAnnotation.name()[0]; + groupInfo.supportedValues.add(groupMember); - builderOption += rawOptionName; - builderOption += "="; - } + apiOptionName = groupName + groupMember; - String helpText = optionDescriptor.getHelp(); - if (!apiAnnotation.customHelp().isEmpty()) { - helpText = apiAnnotation.customHelp(); - } - if (helpText == null || helpText.isEmpty()) { - VMError.shouldNotReachHere(String.format("APIOption %s(%s) needs to provide help text", apiOptionName, rawOptionName)); - } - if (group == null) { - /* Regular help text needs to start with lower-case letter */ - helpText = startLowerCase(helpText); - } - - List> valueTransformers = new ArrayList<>(apiAnnotation.valueTransformer().length); - for (Class> transformerClass : apiAnnotation.valueTransformer()) { - try { - valueTransformers.add(ReflectionUtil.newInstance(transformerClass)); + Boolean isEnabled = (Boolean) optionDescriptor.getOptionKey().getDefaultValue(); + if (isEnabled) { + groupInfo.defaultValues.add(groupMember); + /* Use OptionInfo.defaultValue to remember group default value */ + defaultValue = groupMember; + } } catch (ReflectionUtilError ex) { throw VMError.shouldNotReachHere( - "Class specified as valueTransformer for @APIOption " + apiOptionName + " cannot be loaded or instantiated: " + transformerClass.getTypeName(), ex.getCause()); + "Class specified as group for @APIOption " + apiOptionName + " cannot be loaded or instantiated: " + apiAnnotation.group().getTypeName(), ex.getCause()); } } - if (apiAnnotation.valueSeparator().length == 0) { - throw VMError.shouldNotReachHere(String.format("APIOption %s(%s) does not specify any valueSeparator", apiOptionName, rawOptionName)); + if (apiAnnotation.defaultValue().length > 0) { + VMError.shouldNotReachHere(String.format("Boolean APIOption %s(%s) cannot use APIOption.defaultValue", apiOptionName, rawOptionName)); } - for (char valueSeparator : apiAnnotation.valueSeparator()) { - if (valueSeparator == APIOption.WHITESPACE_SEPARATOR) { - String msgTail = " cannot use APIOption.WHITESPACE_SEPARATOR as value separator"; - if (booleanOption) { - throw VMError.shouldNotReachHere(String.format("Boolean APIOption %s(%s)" + msgTail, apiOptionName, rawOptionName)); - } - if (hasFixedValue) { - VMError.shouldNotReachHere(String.format("APIOption %s(%s) with fixed value" + msgTail, apiOptionName, rawOptionName)); - } - if (defaultValue != null) { - VMError.shouldNotReachHere(String.format("APIOption %s(%s) with default value" + msgTail, apiOptionName, rawOptionName)); - } + if (hasFixedValue) { + VMError.shouldNotReachHere(String.format("Boolean APIOption %s(%s) cannot use APIOption.fixedValue", apiOptionName, rawOptionName)); + } + builderOption += apiAnnotation.kind().equals(APIOptionKind.Negated) ? "-" : "+"; + builderOption += rawOptionName; + booleanOption = true; + } else { + if (!apiAnnotation.group().equals(APIOption.NullGroup.class)) { + VMError.shouldNotReachHere(String.format("Using @APIOption.group not supported for non-boolean APIOption %s(%s)", apiOptionName, rawOptionName)); + } + if (apiAnnotation.kind().equals(APIOptionKind.Negated)) { + VMError.shouldNotReachHere(String.format("Non-boolean APIOption %s(%s) cannot use APIOptionKind.Negated", apiOptionName, rawOptionName)); + } + if (apiAnnotation.defaultValue().length > 1) { + VMError.shouldNotReachHere(String.format("APIOption %s(%s) cannot have more than one APIOption.defaultValue", apiOptionName, rawOptionName)); + } + if (apiAnnotation.fixedValue().length > 1) { + VMError.shouldNotReachHere(String.format("APIOption %s(%s) cannot have more than one APIOption.fixedValue", apiOptionName, rawOptionName)); + } + if (hasFixedValue && apiAnnotation.defaultValue().length > 0) { + VMError.shouldNotReachHere(String.format("APIOption %s(%s) APIOption.defaultValue and APIOption.fixedValue cannot be combined", apiOptionName, rawOptionName)); + } + if (apiAnnotation.defaultValue().length > 0) { + defaultValue = apiAnnotation.defaultValue()[0]; + } + if (hasFixedValue) { + defaultValue = apiAnnotation.fixedValue()[0]; + } + + builderOption += rawOptionName; + builderOption += "="; + } + + String helpText = optionDescriptor.getHelp(); + if (!apiAnnotation.customHelp().isEmpty()) { + helpText = apiAnnotation.customHelp(); + } + if (helpText == null || helpText.isEmpty()) { + VMError.shouldNotReachHere(String.format("APIOption %s(%s) needs to provide help text", apiOptionName, rawOptionName)); + } + if (group == null) { + /* Regular help text needs to start with lower-case letter */ + helpText = startLowerCase(helpText); + } + + List> valueTransformers = new ArrayList<>(apiAnnotation.valueTransformer().length); + for (Class> transformerClass : apiAnnotation.valueTransformer()) { + try { + valueTransformers.add(ReflectionUtil.newInstance(transformerClass)); + } catch (ReflectionUtilError ex) { + throw VMError.shouldNotReachHere( + "Class specified as valueTransformer for @APIOption " + apiOptionName + " cannot be loaded or instantiated: " + transformerClass.getTypeName(), ex.getCause()); + } + } + if (apiAnnotation.valueSeparator().length == 0) { + throw VMError.shouldNotReachHere(String.format("APIOption %s(%s) does not specify any valueSeparator", apiOptionName, rawOptionName)); + } + for (char valueSeparator : apiAnnotation.valueSeparator()) { + if (valueSeparator == APIOption.WHITESPACE_SEPARATOR) { + String msgTail = " cannot use APIOption.WHITESPACE_SEPARATOR as value separator"; + if (booleanOption) { + throw VMError.shouldNotReachHere(String.format("Boolean APIOption %s(%s)" + msgTail, apiOptionName, rawOptionName)); + } + if (hasFixedValue) { + VMError.shouldNotReachHere(String.format("APIOption %s(%s) with fixed value" + msgTail, apiOptionName, rawOptionName)); + } + if (defaultValue != null) { + VMError.shouldNotReachHere(String.format("APIOption %s(%s) with default value" + msgTail, apiOptionName, rawOptionName)); } } - boolean defaultFinal = booleanOption || hasFixedValue; - apiOptions.put(apiOptionName, - new APIOptionHandler.OptionInfo(apiAnnotation.name(), apiAnnotation.valueSeparator(), builderOption, defaultValue, helpText, - defaultFinal, apiAnnotation.deprecated(), valueTransformers, group, apiAnnotation.extra())); } - } catch (NoSuchFieldException e) { - /* Does not qualify as APIOption */ + boolean defaultFinal = booleanOption || hasFixedValue; + apiOptions.put(apiOptionName, + new APIOptionHandler.OptionInfo(apiAnnotation.name(), apiAnnotation.valueSeparator(), builderOption, defaultValue, helpText, + defaultFinal, apiAnnotation.deprecated(), valueTransformers, group, apiAnnotation.extra())); } } - private static void extractPathOption(String optionPrefix, OptionDescriptor optionDescriptor, Map pathOptions) { + private static void extractPathOption(String optionPrefix, OptionDescriptor optionDescriptor, Map pathOptions) { Object defaultValue = optionDescriptor.getOptionKey().getDefaultValue(); if (defaultValue instanceof MultiOptionValue) { var multiOptionDefaultValue = ((MultiOptionValue) defaultValue); if (Path.class.isAssignableFrom(multiOptionDefaultValue.getValueType())) { String rawOptionName = optionDescriptor.getName(); String builderOption = optionPrefix + rawOptionName; - pathOptions.put(builderOption, multiOptionDefaultValue.getDelimiter()); + BundleMember.Role role = BundleMember.Role.Ignore; + for (BundleMember bundleMember : getAnnotationsByType(optionDescriptor, BundleMember.class)) { + role = bundleMember.role(); + } + pathOptions.put(builderOption, new PathsOptionInfo(multiOptionDefaultValue.getDelimiter(), role)); } } } + private static List getAnnotationsByType(OptionDescriptor optionDescriptor, Class annotationClass) { + try { + Field optionField = optionDescriptor.getDeclaringClass().getDeclaredField(optionDescriptor.getFieldName()); + return List.of(optionField.getAnnotationsByType(annotationClass)); + } catch (NoSuchFieldException e) { + return List.of(); + } + } + private static String startLowerCase(String str) { return str.substring(0, 1).toLowerCase() + str.substring(1); } @@ -376,36 +395,65 @@ String translateOption(ArgumentQueue argQueue) { return null; } - String transformBuilderArgument(String builderArgument, Function transformFunction) { - String[] nameAndValue = StringUtil.split(builderArgument, "=", 2); - if (nameAndValue.length != 2) { + String transformBuilderArgument(String builderArgument, BiFunction transformFunction) { + BuilderArgumentParts parts = BuilderArgumentParts.from(builderArgument); + if (parts.optionValue == null) { return builderArgument; } - String optionValue = nameAndValue[1]; - - String[] nameAndOrigin = StringUtil.split(nameAndValue[0], "@", 2); - String optionName = nameAndOrigin[0]; - String pathOptionSeparator = pathOptions.get(optionName); - if (pathOptionSeparator == null) { + PathsOptionInfo pathsOptionInfo = pathOptions.get(parts.optionName); + if (pathsOptionInfo == null || pathsOptionInfo.role == BundleMember.Role.Ignore) { return builderArgument; } - OptionOrigin optionOrigin = OptionOrigin.from(nameAndOrigin.length == 2 ? nameAndOrigin[1] : null); List rawEntries; - if (pathOptionSeparator.isEmpty()) { - rawEntries = List.of(optionValue); + String delimiter = pathsOptionInfo.delimiter; + if (delimiter.isEmpty()) { + rawEntries = List.of(parts.optionValue); } else { - rawEntries = List.of(StringUtil.split(optionValue, pathOptionSeparator)); + rawEntries = List.of(StringUtil.split(parts.optionValue, delimiter)); } try { String transformedOptionValue = rawEntries.stream() .filter(s -> !s.isEmpty()) .map(this::tryCanonicalize) - .map(transformFunction) + .map(src -> transformFunction.apply(src, pathsOptionInfo.role)) .map(Path::toString) - .collect(Collectors.joining(pathOptionSeparator)); - return optionName + "=" + transformedOptionValue; + .collect(Collectors.joining(delimiter)); + parts.optionValue = transformedOptionValue; + return parts.toString(); } catch (BundleSupport.BundlePathSubstitutionError error) { - throw NativeImage.showError("Failed to prepare path entry '" + error.origPath + "' of option " + optionName + " from " + optionOrigin + " for bundle inclusion.", error); + Object optionOrigin = OptionOrigin.from(parts.optionOrigin, false); + if (optionOrigin == null && parts.optionOrigin != null) { + optionOrigin = parts.optionOrigin; + } + String fromPart = optionOrigin != null ? " from '" + optionOrigin + "'" : ""; + throw NativeImage.showError("Failed to prepare path entry '" + error.origPath + "' of option " + parts.optionName + fromPart + " for bundle inclusion.", error); + } + } + + static final class BuilderArgumentParts { + + final String optionName; + final String optionOrigin; + String optionValue; + + private BuilderArgumentParts(String optionName, String optionOrigin, String optionValue) { + this.optionName = optionName; + this.optionOrigin = optionOrigin; + this.optionValue = optionValue; + } + + static BuilderArgumentParts from(String builderArgument) { + String[] nameAndValue = StringUtil.split(builderArgument, "=", 2); + String optionValue = nameAndValue.length != 2 ? null : nameAndValue[1]; + String[] nameAndOrigin = StringUtil.split(nameAndValue[0], "@", 2); + String optionName = nameAndOrigin[0]; + String optionOrigin = nameAndOrigin.length == 2 ? nameAndOrigin[1] : null; + return new BuilderArgumentParts(optionName, optionOrigin, optionValue); + } + + public String toString() { + String nameAndOrigin = optionOrigin == null ? optionName : optionName + "@" + optionOrigin; + return optionValue == null ? nameAndOrigin : nameAndOrigin + "=" + optionValue; } } @@ -436,7 +484,7 @@ void printOptions(Consumer println, boolean extra) { options.add(option); } else { /* Start with space efficient singletonList */ - optionInfo.put(groupOrOptionName, Collections.singletonList(option)); + optionInfo.put(groupOrOptionName, List.of(option)); } }); optionInfo.forEach((optionName, options) -> { @@ -512,9 +560,9 @@ final class APIOptionSupport { final Map groupInfos; final SortedMap options; - final Map pathOptions; + final Map pathOptions; - APIOptionSupport(Map groupInfos, SortedMap options, Map pathOptions) { + APIOptionSupport(Map groupInfos, SortedMap options, Map pathOptions) { this.groupInfos = groupInfos; this.options = options; this.pathOptions = pathOptions; @@ -533,7 +581,7 @@ public void afterRegistration(AfterRegistrationAccess access) { public void duringSetup(DuringSetupAccess access) { FeatureImpl.DuringSetupAccessImpl accessImpl = (FeatureImpl.DuringSetupAccessImpl) access; Map groupInfos = new HashMap<>(); - Map pathOptions = new HashMap<>(); + Map pathOptions = new HashMap<>(); ServiceLoader optionDescriptors = ServiceLoader.load(OptionDescriptors.class, accessImpl.getImageClassLoader().getClassLoader()); SortedMap options = APIOptionHandler.extractOptions(optionDescriptors, groupInfos, pathOptions); ImageSingletons.add(APIOptionSupport.class, new APIOptionSupport(groupInfos, options, pathOptions)); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index ea20ac6e9e78..754dcd27bed1 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -50,6 +50,7 @@ import com.oracle.svm.core.OS; import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.configure.ConfigurationParser; +import com.oracle.svm.core.option.BundleMember; import com.oracle.svm.core.util.json.JsonPrinter; import com.oracle.svm.core.util.json.JsonWriter; import com.oracle.svm.util.ClassUtil; @@ -85,6 +86,7 @@ boolean show() { final Path classPathDir; final Path modulePathDir; final Path auxiliaryDir; + final Path outputDir; Map pathCanonicalizations = new HashMap<>(); Map pathSubstitutions = new HashMap<>(); @@ -93,6 +95,8 @@ boolean show() { private static final String bundleTempDirPrefix = "bundleRoot-"; + Path bundleFile; + static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeImage.ArgumentQueue args) { if (!nativeImage.userConfigProperties.isEmpty()) { throw NativeImage.showError("Bundle support cannot be combined with " + NativeImage.CONFIG_FILE_ENV_VAR_KEY + " environment variable use."); @@ -137,6 +141,7 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } else { bundleSupport = new BundleSupport(nativeImage, bundleStatus); } + bundleSupport.bundleFile = nativeImage.config.getWorkingDirectory().resolve("unnamed.nib"); return bundleSupport; } @@ -153,6 +158,7 @@ private BundleSupport(NativeImage nativeImage, BundleStatus status) { Path classesDir = inputDir.resolve("classes"); classPathDir = Files.createDirectories(classesDir.resolve("cp")); modulePathDir = Files.createDirectories(classesDir.resolve("p")); + outputDir = Files.createDirectories(rootDir.resolve("output")); } catch (IOException e) { throw NativeImage.showError("Unable to create bundle directory layout", e); } @@ -200,6 +206,7 @@ private BundleSupport(NativeImage nativeImage, BundleStatus status, String bundl Path classesDir = inputDir.resolve("classes"); classPathDir = classesDir.resolve("cp"); modulePathDir = classesDir.resolve("p"); + outputDir = rootDir.resolve("output"); Path pathCanonicalizationsFile = stageDir.resolve("path_canonicalizations.json"); try (Reader reader = Files.newBufferedReader(pathCanonicalizationsFile)) { @@ -252,8 +259,19 @@ Path restoreCanonicalization(Path before) { return after; } - Path substituteAuxiliaryPath(Path origPath) { - return substitutePath(origPath, auxiliaryDir); + Path substituteAuxiliaryPath(Path origPath, BundleMember.Role bundleMemberRole) { + Path destinationDir; + switch (bundleMemberRole) { + case Input: + destinationDir = auxiliaryDir; + break; + case Output: + destinationDir = outputDir; + break; + default: + return origPath; + } + return substitutePath(origPath, destinationDir); } Path substituteClassPath(Path origPath) { @@ -356,15 +374,18 @@ private Path substitutePath(Path origPath, Path destinationDir) { throw new BundlePathSubstitutionError("Failed to create a unique path-name in " + destinationDir + ". " + substitutedPath + " already exists", origPath); } - if (Files.isDirectory(origPath)) { - try (Stream walk = Files.walk(origPath)) { - walk.forEach(sourcePath -> copyFile(sourcePath, substitutedPath.resolve(origPath.relativize(sourcePath)))); - } catch (IOException e) { - throw new BundlePathSubstitutionError("Failed to iterate through directory " + origPath, origPath, e); + if (!destinationDir.startsWith(outputDir)) { + if (Files.isDirectory(origPath)) { + try (Stream walk = Files.walk(origPath)) { + walk.forEach(sourcePath -> copyFile(sourcePath, substitutedPath.resolve(origPath.relativize(sourcePath)))); + } catch (IOException e) { + throw new BundlePathSubstitutionError("Failed to iterate through directory " + origPath, origPath, e); + } + } else { + copyFile(origPath, substitutedPath); } - } else { - copyFile(origPath, substitutedPath); } + Path relativeSubstitutedPath = rootDir.relativize(substitutedPath); if (nativeImage.isVerbose()) { System.out.println("RecordSubstitution src: " + origPath + ", dst: " + relativeSubstitutedPath); @@ -388,7 +409,6 @@ void shutdown() { if (!status.loadBundle) { writeBundle(); } - nativeImage.deleteAllFiles(rootDir); } @@ -416,7 +436,6 @@ void writeBundle() { throw NativeImage.showError("Failed to write bundle-file " + pathSubstitutionsFile, e); } - Path bundleFile = Path.of(nativeImage.imagePath).resolve(nativeImage.imageName + ".nib"); try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(bundleFile), new Manifest())) { try (Stream walk = Files.walk(rootDir)) { walk.forEach(bundleEntry -> { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java index 14bed7b9ed95..b7338d8ab48a 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java @@ -33,6 +33,8 @@ import java.util.ArrayList; import java.util.List; +import com.oracle.svm.common.option.MultiOptionValue; +import com.oracle.svm.core.option.BundleMember; import com.oracle.svm.core.option.OptionOrigin; import com.oracle.svm.driver.NativeImage.ArgumentQueue; @@ -189,7 +191,7 @@ public boolean consume(ArgumentQueue args) { args.poll(); headArg = headArg.substring(1); Path origArgFile = Paths.get(headArg); - Path argFile = nativeImage.bundleSupport != null ? nativeImage.bundleSupport.substituteAuxiliaryPath(origArgFile) : origArgFile; + Path argFile = nativeImage.bundleSupport != null ? nativeImage.bundleSupport.substituteAuxiliaryPath(origArgFile, BundleMember.Role.Input) : origArgFile; NativeImage.NativeImageArgsProcessor processor = nativeImage.new NativeImageArgsProcessor(OptionOrigin.argFilePrefix + argFile); readArgFile(argFile).forEach(processor::accept); List leftoverArgs = processor.apply(false); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index b2308fcb8cb5..4a363dc6e7fd 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -77,6 +77,7 @@ import com.oracle.svm.core.OS; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.core.option.BundleMember; import com.oracle.svm.core.option.OptionUtils; import com.oracle.svm.core.option.SubstrateOptionsParser; import com.oracle.svm.core.util.ClasspathUtils; @@ -269,12 +270,6 @@ private static String oR(OptionKey option) { BundleSupport bundleSupport; - void replaySupportShutdown() { - if (bundleSupport != null) { - bundleSupport.shutdown(); - } - } - protected static class BuildConfiguration { /* @@ -651,7 +646,7 @@ private ArrayList createFallbackBuildArgs() { buildArgs.add(oH + "+" + SubstrateOptions.ParseRuntimeOptions.getName()); Path imagePathPath; try { - imagePathPath = canonicalize(Paths.get(imagePath)); + imagePathPath = canonicalize(imagePath); } catch (NativeImage.NativeImageError | InvalidPathException e) { throw showError("The given " + oHPath + imagePath + " argument does not specify a valid path", e); } @@ -1044,7 +1039,6 @@ private int completeImageBuild() { imageBuilderJavaArgs.addAll(getAgentArguments()); mainClass = getHostedOptionFinalArgumentValue(imageBuilderArgs, oHClass); - imagePath = getHostedOptionFinalArgumentValue(imageBuilderArgs, oHPath); boolean buildExecutable = imageBuilderArgs.stream().noneMatch(arg -> arg.contains(oHEnableSharedLibraryFlag)); boolean listModules = imageBuilderArgs.stream().anyMatch(arg -> arg.contains(oH + "+" + "ListModules")); boolean printFlags = imageBuilderArgs.stream().anyMatch(arg -> arg.contains(enablePrintFlags) || arg.contains(enablePrintFlagsWithExtraHelp)); @@ -1111,7 +1105,34 @@ private int completeImageBuild() { } } - imageName = getHostedOptionFinalArgumentValue(imageBuilderArgs, oHName); + ArgumentEntry imageNameEntry = getHostedOptionFinalArgument(imageBuilderArgs, oHName).orElseThrow(); + imageName = imageNameEntry.value; + ArgumentEntry imagePathEntry = getHostedOptionFinalArgument(imageBuilderArgs, oHPath).orElseThrow(); + imagePath = Path.of(imagePathEntry.value); + Path imageNamePath = Path.of(imageName); + Path imageNamePathParent = imageNamePath.getParent(); + if (imageNamePathParent != null) { + /* Readjust imageName & imagePath so that imageName is just a simple fileName */ + imageName = imageNamePath.getFileName().toString(); + if (!imageNamePathParent.isAbsolute()) { + imageNamePathParent = imagePath.resolve(imageNamePathParent); + } + if (!Files.isDirectory(imageNamePathParent)) { + throw NativeImage.showError("Writing image to non-existent directory " + imageNamePathParent + " is not allowed."); + } + if (!Files.isWritable(imageNamePathParent)) { + throw NativeImage.showError("Writing image to directory without write access " + imageNamePathParent + " is not possible."); + } + imagePath = imageNamePathParent; + /* Update arguments passed to builder */ + updateArgumentEntryValue(imageBuilderArgs, imageNameEntry, imageName); + updateArgumentEntryValue(imageBuilderArgs, imagePathEntry, imagePath.toString()); + } + if (useBundle()) { + bundleSupport.bundleFile = imagePath.resolve(imageName + ".nib"); + imagePath = bundleSupport.substituteAuxiliaryPath(imagePath, BundleMember.Role.Output); + updateArgumentEntryValue(imageBuilderArgs, imagePathEntry, imagePath.toString()); + } if (!leftoverArgs.isEmpty()) { String prefix = "Unrecognized option" + (leftoverArgs.size() == 1 ? ": " : "s: "); @@ -1147,30 +1168,51 @@ private int completeImageBuild() { return buildImage(finalImageBuilderJavaArgs, imageBuilderClasspath, imageBuilderModulePath, imageBuilderArgs, finalImageClasspath, finalImageModulePath); } + private static void updateArgumentEntryValue(List argList, ArgumentEntry listEntry, String newValue) { + APIOptionHandler.BuilderArgumentParts argParts = APIOptionHandler.BuilderArgumentParts.from(argList.get(listEntry.index)); + argParts.optionValue = newValue; + argList.set(listEntry.index, argParts.toString()); + } + private static String getLocationAgnosticArgPrefix(String argPrefix) { VMError.guarantee(argPrefix.startsWith(oH) && argPrefix.endsWith("="), "argPrefix has to be a hosted option that ends with \"=\""); return "^" + argPrefix.substring(0, argPrefix.length() - 1) + "(@[^=]*)?" + argPrefix.substring(argPrefix.length() - 1); } - protected static String getHostedOptionFinalArgumentValue(List args, String argPrefix) { - List values = getHostedOptionArgumentValues(args, argPrefix); - return values.isEmpty() ? null : values.get(values.size() - 1); + private static String getHostedOptionFinalArgumentValue(List args, String argPrefix) { + return getHostedOptionFinalArgument(args, argPrefix).map(entry -> entry.value).orElse(null); + } + + private static Optional getHostedOptionFinalArgument(List args, String argPrefix) { + List values = getHostedOptionArgumentValues(args, argPrefix); + return values.isEmpty() ? Optional.empty() : Optional.of(values.get(values.size() - 1)); } - protected static List getHostedOptionArgumentValues(List args, String argPrefix) { - ArrayList values = new ArrayList<>(); + private static List getHostedOptionArgumentValues(List args, String argPrefix) { + ArrayList values = new ArrayList<>(); String locationAgnosticArgPrefix = getLocationAgnosticArgPrefix(argPrefix); Pattern pattern = Pattern.compile(locationAgnosticArgPrefix); - for (String arg : args) { + for (int i = 0; i < args.size(); i++) { + String arg = args.get(i); Matcher matcher = pattern.matcher(arg); if (matcher.find()) { - values.add(arg.substring(matcher.group().length())); + values.add(new ArgumentEntry(i, arg.substring(matcher.group().length()))); } } return values; } + private static final class ArgumentEntry { + private final int index; + private final String value; + + private ArgumentEntry(int index, String value) { + this.index = index; + this.value = value; + } + } + private boolean shouldAddCWDToCP() { if (config.buildFallbackImage() || printFlagsOptionQuery != null || printFlagsWithExtraHelpOptionQuery != null) { return false; @@ -1189,8 +1231,8 @@ private boolean shouldAddCWDToCP() { private List getAgentArguments() { List args = new ArrayList<>(); String agentOptions = ""; - List traceClassInitializationOpts = getHostedOptionArgumentValues(imageBuilderArgs, oHTraceClassInitialization); - List traceObjectInstantiationOpts = getHostedOptionArgumentValues(imageBuilderArgs, oHTraceObjectInstantiation); + List traceClassInitializationOpts = getHostedOptionArgumentValues(imageBuilderArgs, oHTraceClassInitialization); + List traceObjectInstantiationOpts = getHostedOptionArgumentValues(imageBuilderArgs, oHTraceObjectInstantiation); if (!traceClassInitializationOpts.isEmpty()) { agentOptions = getAgentOptions(traceClassInitializationOpts, "c"); } @@ -1212,8 +1254,8 @@ private List getAgentArguments() { return args; } - private static String getAgentOptions(List options, String optionName) { - return options.stream().flatMap(optValue -> Arrays.stream(optValue.split(","))).map(clazz -> optionName + "=" + clazz).collect(Collectors.joining(",")); + private static String getAgentOptions(List options, String optionName) { + return options.stream().flatMap(optValue -> Arrays.stream(optValue.value.split(","))).map(clazz -> optionName + "=" + clazz).collect(Collectors.joining(",")); } private String targetPlatform = null; @@ -1254,7 +1296,7 @@ private void addTargetArguments() { String mainClass; String imageName; - String imagePath; + Path imagePath; protected static List createImageBuilderArgs(List imageArgs, List imagecp, List imagemp) { List result = new ArrayList<>(); @@ -1332,25 +1374,12 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa arguments.addAll(Arrays.asList(SubstrateOptions.WATCHPID_PREFIX, "" + ProcessProperties.getProcessID())); } - boolean useReplay = bundleSupport != null; - if (useReplay) { - Path imageNamePath = Path.of(imageName); - if (imageNamePath.getParent() != null) { - throw NativeImage.showError("Replay-bundle support can only be used with simple output image-name. Use '" + imageNamePath.getFileName() + "' instead of '" + imageNamePath + "'."); - } - if (!Path.of(imagePath).equals(config.getWorkingDirectory())) { - /* Allow bundle write to succeed in working directory */ - imagePath = config.getWorkingDirectory().toString(); - - throw NativeImage.showError("Replay-bundle support can only be used with default image output directory. Ensure '" + oHPath + "' is not explicitly set."); - } - } - Function substituteAuxiliaryPath = useReplay ? bundleSupport::substituteAuxiliaryPath : Function.identity(); + BiFunction substituteAuxiliaryPath = useBundle() ? bundleSupport::substituteAuxiliaryPath : (a, b) -> a; Function imageArgsTransformer = rawArg -> apiOptionHandler.transformBuilderArgument(rawArg, substituteAuxiliaryPath); List finalImageArgs = imageArgs.stream().map(imageArgsTransformer).collect(Collectors.toList()); - Function substituteClassPath = useReplay ? bundleSupport::substituteClassPath : Function.identity(); + Function substituteClassPath = useBundle() ? bundleSupport::substituteClassPath : Function.identity(); List finalImageClassPath = imagecp.stream().map(substituteClassPath).collect(Collectors.toList()); - Function substituteModulePath = useReplay ? bundleSupport::substituteModulePath : Function.identity(); + Function substituteModulePath = useBundle() ? bundleSupport::substituteModulePath : Function.identity(); List finalImageModulePath = imagemp.stream().map(substituteModulePath).collect(Collectors.toList()); List finalImageBuilderArgs = createImageBuilderArgs(finalImageArgs, finalImageClassPath, finalImageModulePath); @@ -1372,7 +1401,7 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa showVerboseMessage(isVerbose() || dryRun, "]"); } - if (dryRun || useReplay && bundleSupport.status == BundleSupport.BundleStatus.prepare) { + if (dryRun || useBundle() && bundleSupport.status == BundleSupport.BundleStatus.prepare) { return ExitStatus.OK.getValue(); } @@ -1394,6 +1423,10 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa } } + boolean useBundle() { + return bundleSupport != null; + } + private static void sanitizeJVMEnvironment(Map environment) { String[] jvmAffectingEnvironmentVariables = {"JAVA_COMPILER", "_JAVA_OPTIONS", "JAVA_TOOL_OPTIONS", "JDK_JAVA_OPTIONS", "CLASSPATH"}; for (String affectingEnvironmentVariable : jvmAffectingEnvironmentVariables) { @@ -1484,7 +1517,9 @@ protected static void build(BuildConfiguration config, Function PrintRuntimeCompilationCallTree = new HostedOptionKey<>(false); @Option(help = "Maximum number of methods allowed for runtime compilation.")// - public static final HostedOptionKey MaxRuntimeCompileMethods = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey MaxRuntimeCompileMethods = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); @Option(help = "Enforce checking of maximum number of methods allowed for runtime compilation. Useful for checking in the gate that the number of methods does not go up without a good reason.")// public static final HostedOptionKey EnforceMaxRuntimeCompileMethods = new HostedOptionKey<>(false); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/FeatureHandler.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/FeatureHandler.java index a181fc315fb4..5292a0bfea9f 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/FeatureHandler.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/FeatureHandler.java @@ -68,7 +68,7 @@ public class FeatureHandler { public static class Options { @APIOption(name = "features") // @Option(help = "A comma-separated list of fully qualified Feature implementation classes")// - public static final HostedOptionKey Features = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey Features = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); private static List userEnabledFeatures() { return Options.Features.getValue().values(); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/LinkAtBuildTimeSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/LinkAtBuildTimeSupport.java index aa8b0d2d6ea7..04c3a8702554 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/LinkAtBuildTimeSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/LinkAtBuildTimeSupport.java @@ -60,11 +60,11 @@ public final class LinkAtBuildTimeSupport { static final class Options { @APIOption(name = "link-at-build-time", defaultValue = "")// @Option(help = "file:doc-files/LinkAtBuildTimeHelp.txt")// - public static final HostedOptionKey LinkAtBuildTime = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey LinkAtBuildTime = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); @APIOption(name = "link-at-build-time-paths")// @Option(help = "file:doc-files/LinkAtBuildTimePathsHelp.txt")// - public static final HostedOptionKey LinkAtBuildTimePaths = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey LinkAtBuildTimePaths = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); } private final String javaIdentifier = "\\p{javaJavaIdentifierStart}\\p{javaJavaIdentifierPart}*"; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderOptions.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderOptions.java index 24a5a3afea53..30067d790e64 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderOptions.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderOptions.java @@ -37,16 +37,16 @@ public class NativeImageClassLoaderOptions { @APIOption(name = "add-exports", extra = true, valueSeparator = {APIOption.WHITESPACE_SEPARATOR, '='})// @Option(help = "Value " + AddExportsAndOpensFormat + " updates to export to , regardless of module declaration." + " can be ALL-UNNAMED to export to all unnamed modules.")// - public static final HostedOptionKey AddExports = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey AddExports = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); @APIOption(name = "add-opens", extra = true, valueSeparator = {APIOption.WHITESPACE_SEPARATOR, '='})// @Option(help = "Value " + AddExportsAndOpensFormat + " updates to open to , regardless of module declaration.")// - public static final HostedOptionKey AddOpens = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey AddOpens = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); @APIOption(name = "add-reads", extra = true, valueSeparator = {APIOption.WHITESPACE_SEPARATOR, '='})// @Option(help = "Value " + AddReadsFormat + " updates to read , regardless of module declaration." + " can be ALL-UNNAMED to read all unnamed modules.")// - public static final HostedOptionKey AddReads = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey AddReads = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); @APIOption(name = "list-modules")// @Option(help = "List observable modules and exit.")// diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java index 9361e116f822..03565b59cb03 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java @@ -57,7 +57,7 @@ public class NativeImageOptions { "environment. Note that enabling features not present within the target environment " + "may result in application crashes. The specific options available are target " + "platform dependent. See --list-cpu-features for feature list.", type = User)// - public static final HostedOptionKey CPUFeatures = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey CPUFeatures = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @APIOption(name = "list-cpu-features")// @Option(help = "Show CPU features specific to the target platform and exit.", type = User)// @@ -72,7 +72,7 @@ public class NativeImageOptions { "option to the empty string. The specific options available are target platform " + "dependent. See --list-cpu-features for feature list. The default values are: " + "AMD64: 'AVX,AVX2'; AArch64: ''", type = User)// - public static final HostedOptionKey RuntimeCheckedCPUFeatures = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey RuntimeCheckedCPUFeatures = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "Overrides CPUFeatures and uses the native architecture, i.e., the architecture of a machine that builds an image. NativeArchitecture takes precedence over CPUFeatures", type = User)// public static final HostedOptionKey NativeArchitecture = new HostedOptionKey<>(false); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java index b4c2921f67fd..c397aaf00f60 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java @@ -106,10 +106,10 @@ public final class ResourcesFeature implements InternalFeature { public static class Options { @Option(help = "Regexp to match names of resources to be included in the image.", type = OptionType.User)// - public static final HostedOptionKey IncludeResources = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey IncludeResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); @Option(help = "Regexp to match names of resources to be excluded from the image.", type = OptionType.User)// - public static final HostedOptionKey ExcludeResources = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey ExcludeResources = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); } private boolean sealed = false; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SecurityServicesFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SecurityServicesFeature.java index 2d7accdf3088..d5c4e9ea3dba 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SecurityServicesFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SecurityServicesFeature.java @@ -130,11 +130,11 @@ public static class Options { public static final HostedOptionKey TraceSecurityServices = new HostedOptionKey<>(false); @Option(help = "Comma-separated list of additional security service types (fully qualified class names) for automatic registration. Note that these must be JCA compliant.")// - public static final HostedOptionKey AdditionalSecurityServiceTypes = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey AdditionalSecurityServiceTypes = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); @Option(help = "Comma-separated list of additional security provider fully qualified class names to mark as used." + "Note that this option is only necessary if you use custom engine classes not available in JCA that are not JCA compliant.")// - public static final HostedOptionKey AdditionalSecurityProviders = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey AdditionalSecurityProviders = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); } /* diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ServiceLoaderFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ServiceLoaderFeature.java index 06b150233980..5ad5eff9bd61 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ServiceLoaderFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ServiceLoaderFeature.java @@ -104,10 +104,10 @@ public static class Options { public static final HostedOptionKey TraceServiceLoaderFeature = new HostedOptionKey<>(false); @Option(help = "Comma-separated list of services that should be excluded", type = OptionType.Expert) // - public static final HostedOptionKey ServiceLoaderFeatureExcludeServices = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey ServiceLoaderFeatureExcludeServices = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "Comma-separated list of service providers that should be excluded", type = OptionType.Expert) // - public static final HostedOptionKey ServiceLoaderFeatureExcludeServiceProviders = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey ServiceLoaderFeatureExcludeServiceProviders = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/classinitialization/ClassInitializationOptions.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/classinitialization/ClassInitializationOptions.java index 496e30429b3c..4f7ed1eabfb9 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/classinitialization/ClassInitializationOptions.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/classinitialization/ClassInitializationOptions.java @@ -89,7 +89,7 @@ private static class InitializationValueEager extends InitializationValueTransfo deprecated = "Currently there is no replacement for this option. Try using --initialize-at-run-time or use the non-API option -H:ClassInitialization directly.", // defaultValue = "", customHelp = "A comma-separated list of classes (and implicitly all of their subclasses) that are initialized both at runtime and during image building") // @Option(help = "A comma-separated list of classes appended with their initialization strategy (':build_time', ':rerun', or ':run_time')", type = OptionType.User)// - public static final HostedOptionKey ClassInitialization = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey ClassInitialization = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); @Option(help = "Instead of abort, only warn if --initialize-at-build-time= is used.", type = OptionType.Debug)// public static final HostedOptionKey AllowDeprecatedInitializeAllClassesAtBuildTime = new HostedOptionKey<>(false); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/diagnostic/HostedHeapDumpFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/diagnostic/HostedHeapDumpFeature.java index 4777ead34685..e2694178360a 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/diagnostic/HostedHeapDumpFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/diagnostic/HostedHeapDumpFeature.java @@ -53,7 +53,7 @@ public class HostedHeapDumpFeature implements InternalFeature { static class Options { @Option(help = "Dump the heap at a specific time during image building." + "The option accepts a list of comma separated phases, any of: during-analysis, after-analysis, before-compilation.")// - public static final HostedOptionKey DumpHeap = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey DumpHeap = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); } enum Phases { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java index ce5e951fc4ee..8b696294e5a3 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java @@ -68,7 +68,7 @@ protected CCLinkerInvocation(AbstractImage.NativeImageKind imageKind, NativeLibr public static class Options { @Option(help = "Pass the provided raw option that will be appended to the linker command to produce the final binary. The possible options are platform specific and passed through without any validation.")// - public static final HostedOptionKey NativeLinkerOption = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey NativeLinkerOption = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); } protected final List additionalPreOptions = new ArrayList<>(); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/sources/SourceCache.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/sources/SourceCache.java index 68413b39c5a4..2893c9b4a8a3 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/sources/SourceCache.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/sources/SourceCache.java @@ -68,7 +68,7 @@ public class SourceCache { /** * A list of all entries in the source search path specified by the user on the command line. */ - protected static final List sourcePathEntries = new ArrayList<>(); + protected static final List sourcePathEntries = new ArrayList<>(); /** * A list of root directories which may contain source files from which this cache can be @@ -132,7 +132,7 @@ private void addGraalSources() { modulePathEntries.stream() .forEach(modulePathEntry -> addGraalSourceRoot(modulePathEntry, true)); sourcePathEntries.stream() - .forEach(sourcePathEntry -> addGraalSourceRoot(Paths.get(sourcePathEntry), false)); + .forEach(sourcePathEntry -> addGraalSourceRoot(sourcePathEntry, false)); } private void addGraalSourceRoot(Path sourcePath, boolean fromClassPath) { @@ -187,7 +187,7 @@ private void addApplicationSources() { modulePathEntries.stream() .forEach(modulePathEntry -> addApplicationSourceRoot(modulePathEntry, true)); sourcePathEntries.stream() - .forEach(sourcePathEntry -> addApplicationSourceRoot(Paths.get(sourcePathEntry), false)); + .forEach(sourcePathEntry -> addApplicationSourceRoot(sourcePathEntry, false)); } protected void addApplicationSourceRoot(Path sourceRoot, boolean fromClassPath) { @@ -524,7 +524,7 @@ static void addModulePathEntry(Path path) { * * @param path The path to add. */ - static void addSourcePathEntry(String path) { + static void addSourcePathEntry(Path path) { sourcePathEntries.add(path); } } @@ -548,7 +548,7 @@ public void afterAnalysis(AfterAnalysisAccess access) { } // also add any necessary source path entries if (SubstrateOptions.DebugInfoSourceSearchPath.getValue() != null) { - for (String searchPathEntry : SubstrateOptions.DebugInfoSourceSearchPath.getValue().values()) { + for (Path searchPathEntry : SubstrateOptions.DebugInfoSourceSearchPath.getValue().values()) { SourceCache.addSourcePathEntry(searchPathEntry); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java index e51510595e50..77ee830af081 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/localization/LocalizationFeature.java @@ -160,7 +160,7 @@ public class LocalizationFeature implements InternalFeature { public static class Options { @Option(help = "Comma separated list of bundles to be included into the image.", type = OptionType.User)// - public static final HostedOptionKey IncludeResourceBundles = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey IncludeResourceBundles = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "Make all hosted charsets available at run time")// public static final HostedOptionKey AddAllCharsets = new HostedOptionKey<>(false); @@ -173,7 +173,7 @@ public static class Options { public static final HostedOptionKey DefaultCharset = new HostedOptionKey<>(Charset.defaultCharset().name()); @Option(help = "Comma separated list of locales to be included into the image. The default locale is included in the list automatically if not present.", type = OptionType.User)// - public static final HostedOptionKey IncludeLocales = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.commaSeparated()); + public static final HostedOptionKey IncludeLocales = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "Make all hosted locales available at run time.", type = OptionType.User)// public static final HostedOptionKey IncludeAllLocales = new HostedOptionKey<>(false); @@ -185,7 +185,7 @@ public static class Options { public static final HostedOptionKey LocalizationSubstituteLoadLookup = new HostedOptionKey<>(true); @Option(help = "Regular expressions matching which bundles should be compressed.", type = OptionType.User)// - public static final HostedOptionKey LocalizationCompressBundles = new HostedOptionKey<>(new LocatableMultiOptionValue.Strings()); + public static final HostedOptionKey LocalizationCompressBundles = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.build()); @Option(help = "Compress the bundles in parallel.", type = OptionType.Expert)// public static final HostedOptionKey LocalizationCompressInParallel = new HostedOptionKey<>(true); diff --git a/substratevm/src/com.oracle.svm.truffle.tck/src/com/oracle/svm/truffle/tck/PermissionsFeature.java b/substratevm/src/com.oracle.svm.truffle.tck/src/com/oracle/svm/truffle/tck/PermissionsFeature.java index 073a2ec49c0a..9e5429a83f5f 100644 --- a/substratevm/src/com.oracle.svm.truffle.tck/src/com/oracle/svm/truffle/tck/PermissionsFeature.java +++ b/substratevm/src/com.oracle.svm.truffle.tck/src/com/oracle/svm/truffle/tck/PermissionsFeature.java @@ -51,8 +51,6 @@ import java.util.function.Predicate; import java.util.stream.Collectors; -import com.oracle.truffle.api.TruffleLanguage; -import jdk.vm.ci.meta.ModifiersProvider; import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.graph.NodeInputList; import org.graalvm.compiler.nodes.Invoke; @@ -74,6 +72,7 @@ import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.annotate.Substitute; import com.oracle.svm.core.annotate.TargetClass; +import com.oracle.svm.core.option.BundleMember; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.option.LocatableMultiOptionValue; import com.oracle.svm.core.util.UserError; @@ -83,8 +82,10 @@ import com.oracle.svm.hosted.SVMHost; import com.oracle.svm.hosted.config.ConfigurationParserUtils; import com.oracle.svm.util.ClassUtil; +import com.oracle.truffle.api.TruffleLanguage; import jdk.vm.ci.common.JVMCIError; +import jdk.vm.ci.meta.ModifiersProvider; import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.meta.ResolvedJavaType; import sun.misc.Unsafe; @@ -106,17 +107,18 @@ public class PermissionsFeature implements Feature { private static final String CONFIG = "truffle-language-permissions-config.json"; public static class Options { - @Option(help = "Path to file where to store report of Truffle language privilege access.") public static final HostedOptionKey TruffleTCKPermissionsReportFile = new HostedOptionKey<>( - null); + @Option(help = "Path to file where to store report of Truffle language privilege access.")// + public static final HostedOptionKey TruffleTCKPermissionsReportFile = new HostedOptionKey<>(null); - @Option(help = "Comma separated list of exclude files.") public static final HostedOptionKey TruffleTCKPermissionsExcludeFiles = new HostedOptionKey<>( - LocatableMultiOptionValue.Paths.commaSeparated()); + @BundleMember(role = BundleMember.Role.Input)// + @Option(help = "Comma separated list of exclude files.")// + public static final HostedOptionKey TruffleTCKPermissionsExcludeFiles = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); - @Option(help = "Maximal depth of a stack trace.", type = OptionType.Expert) public static final HostedOptionKey TruffleTCKPermissionsMaxStackTraceDepth = new HostedOptionKey<>( - -1); + @Option(help = "Maximal depth of a stack trace.", type = OptionType.Expert)// + public static final HostedOptionKey TruffleTCKPermissionsMaxStackTraceDepth = new HostedOptionKey<>(-1); - @Option(help = "Maximum number of erroneous privileged accesses reported.", type = OptionType.Expert) public static final HostedOptionKey TruffleTCKPermissionsMaxErrors = new HostedOptionKey<>( - 100); + @Option(help = "Maximum number of erroneous privileged accesses reported.", type = OptionType.Expert)// + public static final HostedOptionKey TruffleTCKPermissionsMaxErrors = new HostedOptionKey<>(100); } /** @@ -810,7 +812,7 @@ public boolean test(BaseMethodNode methodNode, BaseMethodNode callerNode, Linked private static final class ResourceAsOptionDecorator extends HostedOptionKey { ResourceAsOptionDecorator(String defaultValue) { - super(new LocatableMultiOptionValue.Strings(Collections.singletonList(defaultValue))); + super(LocatableMultiOptionValue.Strings.buildWithDefaults(defaultValue)); } } From e9a386c612ee31d299f3f145310ee8577e8abef9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 11 Jan 2023 17:29:54 +0100 Subject: [PATCH 11/46] Improve bundle support for output paths --- .../svm/common/option/MultiOptionValue.java | 3 + .../com/oracle/svm/core/SubstrateOptions.java | 3 +- .../core/configure/ConfigurationFiles.java | 12 +- .../option/LocatableMultiOptionValue.java | 15 ++- .../oracle/svm/driver/APIOptionHandler.java | 2 +- .../com/oracle/svm/driver/BundleSupport.java | 110 +++++++++++------- .../com/oracle/svm/driver/NativeImage.java | 12 +- .../oracle/svm/hosted/ProgressReporter.java | 5 +- .../hosted/ProgressReporterJsonHelper.java | 8 +- 9 files changed, 111 insertions(+), 59 deletions(-) diff --git a/substratevm/src/com.oracle.svm.common/src/com/oracle/svm/common/option/MultiOptionValue.java b/substratevm/src/com.oracle.svm.common/src/com/oracle/svm/common/option/MultiOptionValue.java index 520e34e2c569..6c08fd60ff9c 100644 --- a/substratevm/src/com.oracle.svm.common/src/com/oracle/svm/common/option/MultiOptionValue.java +++ b/substratevm/src/com.oracle.svm.common/src/com/oracle/svm/common/option/MultiOptionValue.java @@ -26,6 +26,7 @@ package com.oracle.svm.common.option; import java.util.List; +import java.util.Optional; public interface MultiOptionValue { @@ -38,6 +39,8 @@ public interface MultiOptionValue { */ List values(); + Optional lastValue(); + void valueUpdate(Object value); MultiOptionValue createCopy(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 2b10a1766632..3005a9d7ce4b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -419,10 +419,11 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o @Option(help = "Print GC warnings as part of build output", type = OptionType.User)// public static final HostedOptionKey BuildOutputGCWarnings = new HostedOptionKey<>(true); + @BundleMember(role = BundleMember.Role.Output)// @Option(help = "Print build output statistics as JSON to the specified file. " + "The output conforms to the JSON schema located at: " + "docs/reference-manual/native-image/assets/build-output-schema-v0.9.1.json", type = OptionType.User)// - public static final HostedOptionKey BuildOutputJSONFile = new HostedOptionKey<>(""); + public static final HostedOptionKey BuildOutputJSONFile = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.build()); /* * Object and array allocation options. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ConfigurationFiles.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ConfigurationFiles.java index a81266e5c4ab..be86da7c13a3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ConfigurationFiles.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/configure/ConfigurationFiles.java @@ -28,7 +28,6 @@ import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Enumeration; import java.util.List; @@ -51,7 +50,8 @@ public final class ConfigurationFiles { public static final class Options { @Option(help = "Directories directly containing configuration files for dynamic features at runtime.", type = OptionType.User)// - static final HostedOptionKey ConfigurationFileDirectories = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); + @BundleMember(role = BundleMember.Role.Input)// + static final HostedOptionKey ConfigurationFileDirectories = new HostedOptionKey<>(LocatableMultiOptionValue.Paths.buildWithCommaDelimiter()); @Option(help = "Resource path above configuration resources for dynamic features at runtime.", type = OptionType.User)// public static final HostedOptionKey ConfigurationResourceRoots = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @@ -106,11 +106,11 @@ public static final class Options { public static List findConfigurationFiles(String fileName) { List files = new ArrayList<>(); - for (String directory : Options.ConfigurationFileDirectories.getValue().values()) { - if (Files.exists(Paths.get(directory, ConfigurationFile.LOCK_FILE_NAME))) { - throw foundLockFile("Configuration file directory '" + directory + "'"); + for (Path configDir : Options.ConfigurationFileDirectories.getValue().values()) { + if (Files.exists(configDir.resolve(ConfigurationFile.LOCK_FILE_NAME))) { + throw foundLockFile("Configuration file directory '" + configDir + "'"); } - Path path = Paths.get(directory, fileName); + Path path = configDir.resolve(fileName); if (Files.exists(path)) { files.add(path); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/LocatableMultiOptionValue.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/LocatableMultiOptionValue.java index d190b2afe9c7..6036f0a44249 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/LocatableMultiOptionValue.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/LocatableMultiOptionValue.java @@ -27,6 +27,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -89,10 +90,20 @@ public void valueUpdate(Object value) { @Override public List values() { + return getValuesWithOrigins().map(Pair::getLeft).collect(Collectors.toList()); + } + + @Override + public Optional lastValue() { + return lastValueWithOrigin().map(Pair::getLeft); + } + + public Optional> lastValueWithOrigin() { if (values.isEmpty()) { - return List.of(); + return Optional.empty(); } - return values.stream().map(Pair::getLeft).collect(Collectors.toList()); + Pair pair = values.get(values.size() - 1); + return Optional.of(Pair.create(pair.getLeft(), OptionOrigin.from(pair.getRight()))); } public Stream> getValuesWithOrigins() { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java index 6e4a6c988949..189620e0070d 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java @@ -118,8 +118,8 @@ static final class PathsOptionInfo { if (NativeImage.IS_AOT) { APIOptionSupport support = ImageSingletons.lookup(APIOptionSupport.class); groupInfos = support.groupInfos; - apiOptions = support.options; pathOptions = support.pathOptions; + apiOptions = support.options; } else { groupInfos = new HashMap<>(); pathOptions = new HashMap<>(); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 754dcd27bed1..b121367fd5e9 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -48,7 +48,6 @@ import org.graalvm.util.json.JSONParserException; import com.oracle.svm.core.OS; -import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.configure.ConfigurationParser; import com.oracle.svm.core.option.BundleMember; import com.oracle.svm.core.util.json.JsonPrinter; @@ -87,6 +86,8 @@ boolean show() { final Path modulePathDir; final Path auxiliaryDir; final Path outputDir; + final Path imagePathOutputDir; + final Path auxiliaryOutputDir; Map pathCanonicalizations = new HashMap<>(); Map pathSubstitutions = new HashMap<>(); @@ -141,7 +142,6 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } else { bundleSupport = new BundleSupport(nativeImage, bundleStatus); } - bundleSupport.bundleFile = nativeImage.config.getWorkingDirectory().resolve("unnamed.nib"); return bundleSupport; } @@ -159,6 +159,8 @@ private BundleSupport(NativeImage nativeImage, BundleStatus status) { classPathDir = Files.createDirectories(classesDir.resolve("cp")); modulePathDir = Files.createDirectories(classesDir.resolve("p")); outputDir = Files.createDirectories(rootDir.resolve("output")); + imagePathOutputDir = Files.createDirectories(outputDir.resolve("default")); + auxiliaryOutputDir = Files.createDirectories(outputDir.resolve("other")); } catch (IOException e) { throw NativeImage.showError("Unable to create bundle directory layout", e); } @@ -171,32 +173,32 @@ private BundleSupport(NativeImage nativeImage, BundleStatus status, String bundl this.nativeImage = nativeImage; this.status = status; - Path bundlePath = Path.of(bundleFilename); - if (!Files.isReadable(bundlePath)) { + bundleFile = Path.of(bundleFilename); + if (!Files.isReadable(bundleFile)) { throw NativeImage.showError("The given bundle file " + bundleFilename + " cannot be read"); } - if (Files.isDirectory(bundlePath)) { - rootDir = bundlePath; + if (Files.isDirectory(bundleFile)) { + throw NativeImage.showError("The given bundle file " + bundleFilename + " is a directory and not a file"); } else { try { rootDir = Files.createTempDirectory(bundleTempDirPrefix); - try (JarFile archive = new JarFile(bundlePath.toFile())) { + try (JarFile archive = new JarFile(bundleFile.toFile())) { archive.stream().forEach(jarEntry -> { - Path bundleFile = rootDir.resolve(jarEntry.getName()); + Path bundleEntry = rootDir.resolve(jarEntry.getName()); try { - Path bundleFileParent = bundleFile.getParent(); + Path bundleFileParent = bundleEntry.getParent(); if (bundleFileParent != null) { Files.createDirectories(bundleFileParent); } - Files.copy(archive.getInputStream(jarEntry), bundleFile); + Files.copy(archive.getInputStream(jarEntry), bundleEntry); } catch (IOException e) { - throw NativeImage.showError("Unable to copy " + jarEntry.getName() + " from bundle " + bundlePath + " to " + bundleFile, e); + throw NativeImage.showError("Unable to copy " + jarEntry.getName() + " from bundle " + bundleEntry + " to " + bundleEntry, e); } }); } } catch (IOException e) { - throw NativeImage.showError("Unable to create bundle directory layout from file " + bundlePath, e); + throw NativeImage.showError("Unable to create bundle directory layout from file " + bundleFile, e); } } @@ -207,6 +209,8 @@ private BundleSupport(NativeImage nativeImage, BundleStatus status, String bundl classPathDir = classesDir.resolve("cp"); modulePathDir = classesDir.resolve("p"); outputDir = rootDir.resolve("output"); + imagePathOutputDir = outputDir.resolve("default"); + auxiliaryOutputDir = outputDir.resolve("other"); Path pathCanonicalizationsFile = stageDir.resolve("path_canonicalizations.json"); try (Reader reader = Files.newBufferedReader(pathCanonicalizationsFile)) { @@ -230,6 +234,10 @@ private BundleSupport(NativeImage nativeImage, BundleStatus status, String bundl } } + public boolean isBundleCreation() { + return !status.loadBundle; + } + public List getBuildArgs() { return buildArgs; } @@ -266,7 +274,7 @@ Path substituteAuxiliaryPath(Path origPath, BundleMember.Role bundleMemberRole) destinationDir = auxiliaryDir; break; case Output: - destinationDir = outputDir; + destinationDir = auxiliaryOutputDir; break; default: return origPath; @@ -274,6 +282,11 @@ Path substituteAuxiliaryPath(Path origPath, BundleMember.Role bundleMemberRole) return substitutePath(origPath, destinationDir); } + Path substituteImagePath(Path origPath) { + pathSubstitutions.put(origPath, imagePathOutputDir); + return imagePathOutputDir; + } + Path substituteClassPath(Path origPath) { try { return substitutePath(origPath, classPathDir); @@ -298,11 +311,6 @@ static final class BundlePathSubstitutionError extends Error { super(message); this.origPath = origPath; } - - BundlePathSubstitutionError(String message, Path origPath, Throwable cause) { - super(message, cause); - this.origPath = origPath; - } } @SuppressWarnings("try") @@ -367,23 +375,16 @@ private Path substitutePath(Path origPath, Path destinationDir) { baseName = origFileName; extension = ""; } - String substitutedPathFilename = baseName + "_" + SubstrateUtil.digest(origPath.toString()) + extension; - Path substitutedPath = destinationDir.resolve(substitutedPathFilename); - if (Files.exists(substitutedPath)) { - /* If we ever see this, we have to implement substitutedPath collision-handling */ - throw new BundlePathSubstitutionError("Failed to create a unique path-name in " + destinationDir + ". " + substitutedPath + " already exists", origPath); + + Path substitutedPath = destinationDir.resolve(baseName + extension); + int collisionIndex = 0; + while (Files.exists(substitutedPath)) { + collisionIndex += 1; + substitutedPath = destinationDir.resolve(baseName + "_" + collisionIndex + extension); } if (!destinationDir.startsWith(outputDir)) { - if (Files.isDirectory(origPath)) { - try (Stream walk = Files.walk(origPath)) { - walk.forEach(sourcePath -> copyFile(sourcePath, substitutedPath.resolve(origPath.relativize(sourcePath)))); - } catch (IOException e) { - throw new BundlePathSubstitutionError("Failed to iterate through directory " + origPath, origPath, e); - } - } else { - copyFile(origPath, substitutedPath); - } + copyFiles(origPath, substitutedPath); } Path relativeSubstitutedPath = rootDir.relativize(substitutedPath); @@ -394,25 +395,45 @@ private Path substitutePath(Path origPath, Path destinationDir) { return substitutedPath; } - private void copyFile(Path source, Path target) { + private void copyFiles(Path source, Path target) { + if (Files.isDirectory(source)) { + try (Stream walk = Files.walk(source)) { + walk.forEach(sourcePath -> copyFile(sourcePath, target.resolve(source.relativize(sourcePath)))); + } catch (IOException e) { + throw NativeImage.showError("Failed to iterate through directory " + source, e); + } + } else { + copyFile(source, target); + } + } + + private void copyFile(Path sourceFile, Path target) { try { - if (nativeImage.isVerbose()) { - System.out.println("> Copy to bundle: " + nativeImage.config.workDir.relativize(source)); + if (nativeImage.isVerbose() && target.startsWith(rootDir)) { + System.out.println("> Copy to bundle: " + nativeImage.config.workDir.relativize(sourceFile)); } - Files.copy(source, target); + Files.copy(sourceFile, target); } catch (IOException e) { - throw NativeImage.showError("Failed to copy " + source + " to " + target, e); + throw NativeImage.showError("Failed to copy " + sourceFile + " to " + target, e); } } void shutdown() { - if (!status.loadBundle) { - writeBundle(); + Path originalImagePath = bundleFile.getParent(); + copyFiles(outputDir, originalImagePath.resolve(outputDir.getFileName())); + + try { + if (isBundleCreation()) { + writeBundle(); + } + } finally { + nativeImage.deleteAllFiles(rootDir); } - nativeImage.deleteAllFiles(rootDir); } void writeBundle() { + assert isBundleCreation(); + Path pathCanonicalizationsFile = stageDir.resolve("path_canonicalizations.json"); try (JsonWriter writer = new JsonWriter(pathCanonicalizationsFile)) { /* Printing as list with defined sort-order ensures useful diffs are possible */ @@ -436,6 +457,15 @@ void writeBundle() { throw NativeImage.showError("Failed to write bundle-file " + pathSubstitutionsFile, e); } + /* + * Provide a fallback to ensure we even get a bundle if there are errors before we are able + * to determine the final bundle name (see use of BundleSupport.isBundleCreation() in + * NativeImage.completeImageBuild() to know where this happens). + */ + if (bundleFile == null) { + bundleFile = nativeImage.config.getWorkingDirectory().resolve("unnamed.nib"); + } + try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(bundleFile), new Manifest())) { try (Stream walk = Files.walk(rootDir)) { walk.forEach(bundleEntry -> { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 4a363dc6e7fd..d93a3f38d481 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1129,8 +1129,16 @@ private int completeImageBuild() { updateArgumentEntryValue(imageBuilderArgs, imagePathEntry, imagePath.toString()); } if (useBundle()) { - bundleSupport.bundleFile = imagePath.resolve(imageName + ".nib"); - imagePath = bundleSupport.substituteAuxiliaryPath(imagePath, BundleMember.Role.Output); + if (bundleSupport.isBundleCreation()) { + /* + * If we are in bundle creation mode, we are at the point where we know the final + * imagePath and imageName that we can now use to derive the new bundle name from. + */ + bundleSupport.bundleFile = imagePath.resolve(imageName + ".nib"); + } + /* In bundle mode the imagePath has to be redirected to be within the bundle */ + imagePath = bundleSupport.substituteImagePath(imagePath); + /* and we need to adjust the argument that passes the imagePath to the builder */ updateArgumentEntryValue(imageBuilderArgs, imagePathEntry, imagePath.toString()); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index 6e5e3f69fa07..9d9a51d2e2d3 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -195,8 +195,9 @@ public ProgressReporter(OptionValues options) { builderIO = NativeImageSystemIOWrappers.singleton(); } - if (SubstrateOptions.BuildOutputJSONFile.hasBeenSet(options)) { - jsonHelper = new ProgressReporterJsonHelper(SubstrateOptions.BuildOutputJSONFile.getValue(options)); + Optional buildOutputJSONFile = SubstrateOptions.BuildOutputJSONFile.getValue(options).lastValue(); + if (buildOutputJSONFile.isPresent()) { + jsonHelper = new ProgressReporterJsonHelper(buildOutputJSONFile.get()); } else { jsonHelper = null; } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java index 8dad5cf5ad1b..787111c6feae 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterJsonHelper.java @@ -25,7 +25,6 @@ package com.oracle.svm.hosted; -import java.io.File; import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; @@ -45,9 +44,9 @@ class ProgressReporterJsonHelper { private static final String RESOURCE_USAGE_KEY = "resource_usage"; private final Map statsHolder = new HashMap<>(); - private final String jsonOutputFile; + private final Path jsonOutputFile; - ProgressReporterJsonHelper(String outFile) { + ProgressReporterJsonHelper(Path outFile) { this.jsonOutputFile = outFile; } @@ -105,9 +104,8 @@ public void putResourceUsage(ResourceUsageKey key, Object value) { public Path printToFile() { recordSystemFixedValues(); - final File file = new File(jsonOutputFile); String description = "image statistics in json"; - return ReportUtils.report(description, file.getAbsoluteFile().toPath(), out -> { + return ReportUtils.report(description, jsonOutputFile.toAbsolutePath(), out -> { try { new JsonWriter(out).print(statsHolder); } catch (IOException e) { From a2b00e69782d89b5c4eb9572109316b1dc0bfad7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 12 Jan 2023 18:35:31 +0100 Subject: [PATCH 12/46] Allow bundle output position to be specified with -o --- .../com/oracle/svm/driver/BundleSupport.java | 126 +++++++++++------- .../com/oracle/svm/driver/NativeImage.java | 17 ++- 2 files changed, 85 insertions(+), 58 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index b121367fd5e9..2bb4051af909 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -28,9 +28,11 @@ import java.io.IOException; import java.io.Reader; import java.net.URI; +import java.nio.file.CopyOption; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -95,8 +97,11 @@ boolean show() { private final List buildArgs; private static final String bundleTempDirPrefix = "bundleRoot-"; + private static final String bundleFileExtension = ".nib"; + private static final String originalDirExtension = ".orig"; - Path bundleFile; + private Path bundlePath; + private String bundleName; static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeImage.ArgumentQueue args) { if (!nativeImage.userConfigProperties.isEmpty()) { @@ -130,6 +135,12 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma assert !BundleStatus.valueOf(buildArg.substring(BUNDLE_OPTION.length() + 1)).loadBundle; continue; } + if (buildArg.startsWith(nativeImage.oHPath)) { + continue; + } + if (buildArg.equals(DefaultOptionHandler.verboseOption)) { + continue; + } if (buildArg.startsWith("-Dllvm.bin.dir=")) { Optional existing = nativeImage.config.getBuildArgs().stream().filter(arg -> arg.startsWith("-Dllvm.bin.dir=")).findFirst(); if (existing.isPresent() && !existing.get().equals(buildArg)) { @@ -146,7 +157,7 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } private BundleSupport(NativeImage nativeImage, BundleStatus status) { - assert !status.loadBundle; + assert !status.loadBundle : "This constructor is only used when a new bundle gets created"; this.nativeImage = nativeImage; this.status = status; @@ -165,50 +176,66 @@ private BundleSupport(NativeImage nativeImage, BundleStatus status) { throw NativeImage.showError("Unable to create bundle directory layout", e); } this.buildArgs = Collections.unmodifiableList(nativeImage.config.getBuildArgs()); + + setBundleLocation(nativeImage.config.getWorkingDirectory(), "unknown"); } - private BundleSupport(NativeImage nativeImage, BundleStatus status, String bundleFilename) { - assert status.loadBundle; + private BundleSupport(NativeImage nativeImage, BundleStatus status, String bundleFilenameArg) { + assert status.loadBundle : "This constructor is used when a previously created bundle gets applied"; this.nativeImage = nativeImage; this.status = status; - bundleFile = Path.of(bundleFilename); + Path bundleFile = Path.of(bundleFilenameArg).toAbsolutePath(); + String bundleFileName = bundleFile.getFileName().toString(); + if (!bundleFileName.endsWith(bundleFileExtension)) { + throw NativeImage.showError("The given bundle file " + bundleFileName + " does not end with '" + bundleFileExtension + "'"); + } + if (!Files.isReadable(bundleFile)) { - throw NativeImage.showError("The given bundle file " + bundleFilename + " cannot be read"); + throw NativeImage.showError("The given bundle file " + bundleFileName + " cannot be read"); } if (Files.isDirectory(bundleFile)) { - throw NativeImage.showError("The given bundle file " + bundleFilename + " is a directory and not a file"); - } else { - try { - rootDir = Files.createTempDirectory(bundleTempDirPrefix); - try (JarFile archive = new JarFile(bundleFile.toFile())) { - archive.stream().forEach(jarEntry -> { - Path bundleEntry = rootDir.resolve(jarEntry.getName()); - try { - Path bundleFileParent = bundleEntry.getParent(); - if (bundleFileParent != null) { - Files.createDirectories(bundleFileParent); - } - Files.copy(archive.getInputStream(jarEntry), bundleEntry); - } catch (IOException e) { - throw NativeImage.showError("Unable to copy " + jarEntry.getName() + " from bundle " + bundleEntry + " to " + bundleEntry, e); + throw NativeImage.showError("The given bundle file " + bundleFileName + " is a directory and not a file"); + } + + try { + rootDir = Files.createTempDirectory(bundleTempDirPrefix); + outputDir = rootDir.resolve("output"); + String originalOutputDirName = outputDir.getFileName().toString() + originalDirExtension; + + try (JarFile archive = new JarFile(bundleFile.toFile())) { + archive.stream().forEach(jarEntry -> { + Path bundleEntry = rootDir.resolve(jarEntry.getName()); + if (bundleEntry.startsWith(outputDir)) { + /* Extract original output to different path */ + bundleEntry = rootDir.resolve(originalOutputDirName).resolve(outputDir.relativize(bundleEntry)); + } + try { + Path bundleFileParent = bundleEntry.getParent(); + if (bundleFileParent != null) { + Files.createDirectories(bundleFileParent); } - }); - } - } catch (IOException e) { - throw NativeImage.showError("Unable to create bundle directory layout from file " + bundleFile, e); + Files.copy(archive.getInputStream(jarEntry), bundleEntry); + } catch (IOException e) { + throw NativeImage.showError("Unable to copy " + jarEntry.getName() + " from bundle " + bundleEntry + " to " + bundleEntry, e); + } + }); } + } catch (IOException e) { + throw NativeImage.showError("Unable to create bundle directory layout from file " + bundleFileName, e); } + bundlePath = bundleFile.getParent(); + bundleName = bundleFileName; + Path inputDir = rootDir.resolve("input"); stageDir = inputDir.resolve("stage"); auxiliaryDir = inputDir.resolve("auxiliary"); Path classesDir = inputDir.resolve("classes"); classPathDir = classesDir.resolve("cp"); modulePathDir = classesDir.resolve("p"); - outputDir = rootDir.resolve("output"); imagePathOutputDir = outputDir.resolve("default"); auxiliaryOutputDir = outputDir.resolve("other"); @@ -384,7 +411,7 @@ private Path substitutePath(Path origPath, Path destinationDir) { } if (!destinationDir.startsWith(outputDir)) { - copyFiles(origPath, substitutedPath); + copyFiles(origPath, substitutedPath, false); } Path relativeSubstitutedPath = rootDir.relativize(substitutedPath); @@ -395,32 +422,37 @@ private Path substitutePath(Path origPath, Path destinationDir) { return substitutedPath; } - private void copyFiles(Path source, Path target) { + private void copyFiles(Path source, Path target, boolean overwrite) { if (Files.isDirectory(source)) { try (Stream walk = Files.walk(source)) { - walk.forEach(sourcePath -> copyFile(sourcePath, target.resolve(source.relativize(sourcePath)))); + walk.forEach(sourcePath -> copyFile(sourcePath, target.resolve(source.relativize(sourcePath)), overwrite)); } catch (IOException e) { throw NativeImage.showError("Failed to iterate through directory " + source, e); } } else { - copyFile(source, target); + copyFile(source, target, overwrite); } } - private void copyFile(Path sourceFile, Path target) { + private void copyFile(Path sourceFile, Path target, boolean overwrite) { try { - if (nativeImage.isVerbose() && target.startsWith(rootDir)) { - System.out.println("> Copy to bundle: " + nativeImage.config.workDir.relativize(sourceFile)); + if (nativeImage.isVerbose()) { + System.out.println("> Copy " + sourceFile + " to " + target); + } + if (overwrite && Files.isDirectory(sourceFile) && Files.isDirectory(target)) { + return; } - Files.copy(sourceFile, target); + CopyOption[] options = overwrite ? new CopyOption[]{StandardCopyOption.REPLACE_EXISTING} : new CopyOption[0]; + Files.copy(sourceFile, target, options); } catch (IOException e) { throw NativeImage.showError("Failed to copy " + sourceFile + " to " + target, e); } } - void shutdown() { - Path originalImagePath = bundleFile.getParent(); - copyFiles(outputDir, originalImagePath.resolve(outputDir.getFileName())); + void complete() { + if (Files.exists(outputDir)) { + copyFiles(outputDir, bundlePath.resolve(nativeImage.imageName + "." + outputDir.getFileName()), true); + } try { if (isBundleCreation()) { @@ -431,6 +463,11 @@ void shutdown() { } } + public void setBundleLocation(Path imagePath, String imageName) { + bundlePath = imagePath; + bundleName = imageName + bundleFileExtension; + } + void writeBundle() { assert isBundleCreation(); @@ -457,16 +494,7 @@ void writeBundle() { throw NativeImage.showError("Failed to write bundle-file " + pathSubstitutionsFile, e); } - /* - * Provide a fallback to ensure we even get a bundle if there are errors before we are able - * to determine the final bundle name (see use of BundleSupport.isBundleCreation() in - * NativeImage.completeImageBuild() to know where this happens). - */ - if (bundleFile == null) { - bundleFile = nativeImage.config.getWorkingDirectory().resolve("unnamed.nib"); - } - - try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(bundleFile), new Manifest())) { + try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(bundlePath.resolve(bundleName)), new Manifest())) { try (Stream walk = Files.walk(rootDir)) { walk.forEach(bundleEntry -> { if (Files.isDirectory(bundleEntry)) { @@ -480,12 +508,12 @@ void writeBundle() { Files.copy(bundleEntry, jarOutStream); jarOutStream.closeEntry(); } catch (IOException e) { - throw NativeImage.showError("Failed to copy " + bundleEntry + " into bundle file " + bundleFile, e); + throw NativeImage.showError("Failed to copy " + bundleEntry + " into bundle file " + bundleName, e); } }); } } catch (IOException e) { - throw NativeImage.showError("Failed to create bundle file " + bundleFile, e); + throw NativeImage.showError("Failed to create bundle file " + bundleName, e); } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index d93a3f38d481..7c284bfd3375 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1129,14 +1129,13 @@ private int completeImageBuild() { updateArgumentEntryValue(imageBuilderArgs, imagePathEntry, imagePath.toString()); } if (useBundle()) { - if (bundleSupport.isBundleCreation()) { - /* - * If we are in bundle creation mode, we are at the point where we know the final - * imagePath and imageName that we can now use to derive the new bundle name from. - */ - bundleSupport.bundleFile = imagePath.resolve(imageName + ".nib"); - } - /* In bundle mode the imagePath has to be redirected to be within the bundle */ + /* + * In creation-mode, we are at the point where we know the final imagePath and imageName + * that we can now use to derive the new bundle name from. For apply-mode setting + * imagePath determines where to copy the bundle output to. + */ + bundleSupport.setBundleLocation(imagePath, imageName); + /* The imagePath has to be redirected to be within the bundle */ imagePath = bundleSupport.substituteImagePath(imagePath); /* and we need to adjust the argument that passes the imagePath to the builder */ updateArgumentEntryValue(imageBuilderArgs, imagePathEntry, imagePath.toString()); @@ -1526,7 +1525,7 @@ protected static void build(BuildConfiguration config, Function Date: Thu, 12 Jan 2023 23:27:06 +0100 Subject: [PATCH 13/46] Allow support for --bundle-update --- .../resources/HelpExtra.txt | 4 + .../com/oracle/svm/driver/BundleSupport.java | 79 +++++++++++-------- .../com/oracle/svm/driver/NativeImage.java | 4 + 3 files changed, 55 insertions(+), 32 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt b/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt index 693eb7d28c92..8ba4e0c71884 100644 --- a/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt +++ b/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt @@ -26,6 +26,10 @@ Non-standard options help: an image will be built from the given bundle with the exact same arguments and files that have been passed to native-image originally to create the bundle. + --bundle-update bundle-file + same as --bundle-apply but given extra arguments add to the arguments + provided by the bundle. After image build the additional arguments + are incorporated into an updated bundle file. -V= provide values for placeholders in native-image.properties files diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 2bb4051af909..b635d951e1dd 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -35,6 +35,7 @@ import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -61,16 +62,19 @@ final class BundleSupport { static final String BUNDLE_OPTION = "--bundle"; enum BundleStatus { - prepare(false, false), - create(false, false), - apply(true, true); + prepare(false, false, true), + create(false, false, true), + update(false, true, true), + apply(true, true, false); final boolean hidden; final boolean loadBundle; + final boolean writeBundle; - BundleStatus(boolean hidden, boolean loadBundle) { + BundleStatus(boolean hidden, boolean loadBundle, boolean writeBundle) { this.hidden = hidden; this.loadBundle = loadBundle; + this.writeBundle = writeBundle; } boolean show() { @@ -95,6 +99,7 @@ boolean show() { Map pathSubstitutions = new HashMap<>(); private final List buildArgs; + private Collection updatedBuildArgs; private static final String bundleTempDirPrefix = "bundleRoot-"; private static final String bundleFileExtension = ".nib"; @@ -130,26 +135,9 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma bundleSupport = new BundleSupport(nativeImage, bundleStatus, bundleFilename); List buildArgs = bundleSupport.getBuildArgs(); for (int i = buildArgs.size() - 1; i >= 0; i--) { - String buildArg = buildArgs.get(i); - if (buildArg.startsWith(BUNDLE_OPTION)) { - assert !BundleStatus.valueOf(buildArg.substring(BUNDLE_OPTION.length() + 1)).loadBundle; - continue; - } - if (buildArg.startsWith(nativeImage.oHPath)) { - continue; - } - if (buildArg.equals(DefaultOptionHandler.verboseOption)) { - continue; - } - if (buildArg.startsWith("-Dllvm.bin.dir=")) { - Optional existing = nativeImage.config.getBuildArgs().stream().filter(arg -> arg.startsWith("-Dllvm.bin.dir=")).findFirst(); - if (existing.isPresent() && !existing.get().equals(buildArg)) { - throw NativeImage.showError("Bundle native-image argument '" + buildArg + "' conflicts with existing '" + existing.get() + "'."); - } - continue; - } - args.push(buildArg); + args.push(buildArgs.get(i)); } + bundleSupport.updatedBuildArgs = args.snapshot(); } else { bundleSupport = new BundleSupport(nativeImage, bundleStatus); } @@ -261,10 +249,6 @@ private BundleSupport(NativeImage nativeImage, BundleStatus status, String bundl } } - public boolean isBundleCreation() { - return !status.loadBundle; - } - public List getBuildArgs() { return buildArgs; } @@ -455,9 +439,7 @@ void complete() { } try { - if (isBundleCreation()) { - writeBundle(); - } + writeBundle(); } finally { nativeImage.deleteAllFiles(rootDir); } @@ -469,7 +451,19 @@ public void setBundleLocation(Path imagePath, String imageName) { } void writeBundle() { - assert isBundleCreation(); + if (!status.writeBundle) { + return; + } + + String originalOutputDirName = outputDir.getFileName().toString() + originalDirExtension; + Path originalOutputDir = rootDir.resolve(originalOutputDirName); + if (Files.exists(originalOutputDir)) { + nativeImage.deleteAllFiles(originalOutputDir); + } + Path metaInfDir = rootDir.resolve("META-INF"); + if (Files.exists(metaInfDir)) { + nativeImage.deleteAllFiles(metaInfDir); + } Path pathCanonicalizationsFile = stageDir.resolve("path_canonicalizations.json"); try (JsonWriter writer = new JsonWriter(pathCanonicalizationsFile)) { @@ -488,8 +482,29 @@ void writeBundle() { Path buildArgsFile = stageDir.resolve("build.json"); try (JsonWriter writer = new JsonWriter(buildArgsFile)) { + ArrayList cleanBuildArgs = new ArrayList<>(); + for (String buildArg : updatedBuildArgs != null ? updatedBuildArgs : buildArgs) { + if (buildArg.startsWith(BUNDLE_OPTION)) { + assert !BundleStatus.valueOf(buildArg.substring(BUNDLE_OPTION.length() + 1)).loadBundle; + continue; + } + if (buildArg.startsWith(nativeImage.oHPath)) { + continue; + } + if (buildArg.equals(DefaultOptionHandler.verboseOption)) { + continue; + } + if (buildArg.startsWith("-Dllvm.bin.dir=")) { + Optional existing = nativeImage.config.getBuildArgs().stream().filter(arg -> arg.startsWith("-Dllvm.bin.dir=")).findFirst(); + if (existing.isPresent() && !existing.get().equals(buildArg)) { + throw NativeImage.showError("Bundle native-image argument '" + buildArg + "' conflicts with existing '" + existing.get() + "'."); + } + continue; + } + cleanBuildArgs.add(buildArg); + } /* Printing as list with defined sort-order ensures useful diffs are possible */ - JsonPrinter.printCollection(writer, buildArgs, null, BundleSupport::printBuildArg); + JsonPrinter.printCollection(writer, cleanBuildArgs, null, BundleSupport::printBuildArg); } catch (IOException e) { throw NativeImage.showError("Failed to write bundle-file " + pathSubstitutionsFile, e); } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 7c284bfd3375..277507596bd0 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -176,6 +176,10 @@ public boolean isEmpty() { public int size() { return queue.size(); } + + public List snapshot() { + return new ArrayList<>(queue); + } } abstract static class OptionHandler { From a40b4c61c2e4d7c8026f1627d46d6966788cfb54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 16 Jan 2023 11:05:10 +0100 Subject: [PATCH 14/46] Adjust SubstrateOptions.CLibraryPath to new LocatableMultiOptionValue.Paths type --- .../src/com/oracle/svm/hosted/c/NativeLibraries.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/NativeLibraries.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/NativeLibraries.java index b47315d6921d..fd81b9c67349 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/NativeLibraries.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/NativeLibraries.java @@ -329,9 +329,8 @@ private static LinkedHashSet initCLibraryPath() { /* Probe for static JDK libraries in user-specified CLibraryPath directory */ if (staticLibsDir == null) { - for (String clibPathComponent : SubstrateOptions.CLibraryPath.getValue().values()) { - Path path = Paths.get(clibPathComponent); - Predicate hasStaticLibraryCLibraryPath = s -> Files.isRegularFile(path.resolve(getStaticLibraryName(s))); + for (Path clibPathComponent : SubstrateOptions.CLibraryPath.getValue().values()) { + Predicate hasStaticLibraryCLibraryPath = s -> Files.isRegularFile(clibPathComponent.resolve(getStaticLibraryName(s))); if (defaultBuiltInLibraries.stream().allMatch(hasStaticLibraryCLibraryPath)) { return libraryPaths; } From 99d9aca8541dde3ace769902af4be6a2a99adff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 16 Jan 2023 11:05:36 +0100 Subject: [PATCH 15/46] Streamline bundle option handling --- .../resources/HelpExtra.txt | 24 +++++----- .../com/oracle/svm/driver/BundleSupport.java | 44 +++++++------------ .../com/oracle/svm/driver/NativeImage.java | 2 +- 3 files changed, 29 insertions(+), 41 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt b/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt index 8ba4e0c71884..7c24208462c8 100644 --- a/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt +++ b/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt @@ -16,20 +16,18 @@ Non-standard options help: --diagnostics-mode Enables logging of image-build information to a diagnostics folder. --dry-run output the command line that would be used for building - --bundle-prepare create a bundle file (*.nib file) that allows building an image at a - later point from the bundle file alone. - --bundle-create in addition to building a native image a bundle file gets created that - allows rebuilding the image again at a later point from the bundle. - --bundle bundle-file - or + --bundle-create in addition to building a native image create a bundle file (*.nib file) + that allows rebuilding of that image again at a later point. --bundle-apply bundle-file - an image will be built from the given bundle with the exact same - arguments and files that have been passed to native-image originally to - create the bundle. - --bundle-update bundle-file - same as --bundle-apply but given extra arguments add to the arguments - provided by the bundle. After image build the additional arguments - are incorporated into an updated bundle file. + an image will be built from the given bundle file with the exact same + arguments and files that have been passed to native-image originally + to create the bundle. + --bundle-prepare create a bundle file that allows building an image at a later point from + the bundle file but do not also build the image right away. + --bundle-amend bundle-file + create a new bundle but use a given bundle as the starting point. Any + given extra arguments add to the arguments of the provided bundle and + are incorporated into the new updated bundle. -V= provide values for placeholders in native-image.properties files diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index b635d951e1dd..08ba3403a341 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -62,23 +62,19 @@ final class BundleSupport { static final String BUNDLE_OPTION = "--bundle"; enum BundleStatus { - prepare(false, false, true), - create(false, false, true), - update(false, true, true), - apply(true, true, false); + create(false, true, true), + apply(true, false, true), + prepare(false, true, false), + amend(true, true, false); - final boolean hidden; final boolean loadBundle; final boolean writeBundle; + final boolean buildImage; - BundleStatus(boolean hidden, boolean loadBundle, boolean writeBundle) { - this.hidden = hidden; + BundleStatus(boolean loadBundle, boolean writeBundle, boolean buildImage) { this.loadBundle = loadBundle; this.writeBundle = writeBundle; - } - - boolean show() { - return !hidden; + this.buildImage = buildImage; } } @@ -114,20 +110,13 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } BundleStatus bundleStatus; - if (bundleArg.equals(BUNDLE_OPTION)) { - /* Handle short form of --bundle-apply */ - bundleStatus = BundleStatus.apply; - } else { - String bundleVariant = bundleArg.substring(BUNDLE_OPTION.length() + 1); - try { - bundleStatus = BundleStatus.valueOf(bundleVariant); - } catch (IllegalArgumentException e) { - String suggestedVariants = Arrays.stream(BundleStatus.values()) - .filter(BundleStatus::show) - .map(v -> BUNDLE_OPTION + "-" + v) - .collect(Collectors.joining(", ")); - throw NativeImage.showError("Unknown option " + bundleArg + ". Valid variants are: " + suggestedVariants + "."); - } + try { + bundleStatus = BundleStatus.valueOf(bundleArg.substring(BUNDLE_OPTION.length() + 1)); + } catch (StringIndexOutOfBoundsException | IllegalArgumentException e) { + String suggestedVariants = Arrays.stream(BundleStatus.values()) + .map(v -> BUNDLE_OPTION + "-" + v) + .collect(Collectors.joining(", ")); + throw NativeImage.showError("Unknown option " + bundleArg + ". Valid variants are: " + suggestedVariants + "."); } BundleSupport bundleSupport; if (bundleStatus.loadBundle) { @@ -361,6 +350,7 @@ private Path substitutePath(Path origPath, Path destinationDir) { /* Refuse /, C:, D:, ... */ if (origPath.equals(rootDirectory)) { forbiddenPath = true; + break; } } if (forbiddenPath) { @@ -555,7 +545,7 @@ private PathMapParser(Map pathMap) { } @Override - public void parseAndRegister(Object json, URI origin) throws IOException { + public void parseAndRegister(Object json, URI origin) { for (var rawEntry : asList(json, "Expected a list of path substitution objects")) { var entry = asMap(rawEntry, "Expected a substitution object"); Object srcPathString = entry.get(substitutionMapSrcField); @@ -581,7 +571,7 @@ private BuildArgsParser(List args) { } @Override - public void parseAndRegister(Object json, URI origin) throws IOException { + public void parseAndRegister(Object json, URI origin) { for (var arg : asList(json, "Expected a list of arguments")) { args.add(arg.toString()); } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 277507596bd0..d7508a57abdf 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1412,7 +1412,7 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa showVerboseMessage(isVerbose() || dryRun, "]"); } - if (dryRun || useBundle() && bundleSupport.status == BundleSupport.BundleStatus.prepare) { + if (dryRun || useBundle() && !bundleSupport.status.buildImage) { return ExitStatus.OK.getValue(); } From cb643a9ec3b65db4f1c14e0cde5fd6f14b6c4108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 16 Jan 2023 12:37:32 +0100 Subject: [PATCH 16/46] Ensure bundleRoot directory layout is always set up correctly --- .../com/oracle/svm/driver/BundleSupport.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 08ba3403a341..cc2bf60faab5 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -146,7 +146,7 @@ private BundleSupport(NativeImage nativeImage, BundleStatus status) { Path classesDir = inputDir.resolve("classes"); classPathDir = Files.createDirectories(classesDir.resolve("cp")); modulePathDir = Files.createDirectories(classesDir.resolve("p")); - outputDir = Files.createDirectories(rootDir.resolve("output")); + outputDir = rootDir.resolve("output"); imagePathOutputDir = Files.createDirectories(outputDir.resolve("default")); auxiliaryOutputDir = Files.createDirectories(outputDir.resolve("other")); } catch (IOException e) { @@ -207,14 +207,18 @@ private BundleSupport(NativeImage nativeImage, BundleStatus status, String bundl bundlePath = bundleFile.getParent(); bundleName = bundleFileName; - Path inputDir = rootDir.resolve("input"); - stageDir = inputDir.resolve("stage"); - auxiliaryDir = inputDir.resolve("auxiliary"); - Path classesDir = inputDir.resolve("classes"); - classPathDir = classesDir.resolve("cp"); - modulePathDir = classesDir.resolve("p"); - imagePathOutputDir = outputDir.resolve("default"); - auxiliaryOutputDir = outputDir.resolve("other"); + try { + Path inputDir = rootDir.resolve("input"); + stageDir = Files.createDirectories(inputDir.resolve("stage")); + auxiliaryDir = Files.createDirectories(inputDir.resolve("auxiliary")); + Path classesDir = inputDir.resolve("classes"); + classPathDir = Files.createDirectories(classesDir.resolve("cp")); + modulePathDir = Files.createDirectories(classesDir.resolve("p")); + imagePathOutputDir = Files.createDirectories(outputDir.resolve("default")); + auxiliaryOutputDir = Files.createDirectories(outputDir.resolve("other")); + } catch (IOException e) { + throw NativeImage.showError("Unable to create bundle directory layout", e); + } Path pathCanonicalizationsFile = stageDir.resolve("path_canonicalizations.json"); try (Reader reader = Files.newBufferedReader(pathCanonicalizationsFile)) { From 350ba7787edc6f18c94b4ec2ef7fa56983d7f7f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 16 Jan 2023 15:10:36 +0100 Subject: [PATCH 17/46] Console output for bundle operations --- .../com/oracle/svm/driver/BundleSupport.java | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index cc2bf60faab5..dec651f104b2 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -428,27 +428,37 @@ private void copyFile(Path sourceFile, Path target, boolean overwrite) { } void complete() { - if (Files.exists(outputDir)) { - copyFiles(outputDir, bundlePath.resolve(nativeImage.imageName + "." + outputDir.getFileName()), true); + assert status.buildImage || status.writeBundle : "Superfluous bundle operations"; + + Path workDir = nativeImage.config.getWorkingDirectory(); + + if (status.buildImage) { + Path externalOutputDir = bundlePath.resolve(nativeImage.imageName + "." + outputDir.getFileName()); + copyFiles(outputDir, externalOutputDir, true); + nativeImage.showNewline(); + nativeImage.showMessage("Bundle output written to " + workDir.relativize(externalOutputDir)); } try { - writeBundle(); + if (status.writeBundle) { + Path bundleFilePath = writeBundle(); + if (!status.buildImage) { + nativeImage.showNewline(); + } + nativeImage.showMessage("Bundle written to " + workDir.relativize(bundleFilePath)); + } } finally { + nativeImage.showNewline(); nativeImage.deleteAllFiles(rootDir); } } - public void setBundleLocation(Path imagePath, String imageName) { + void setBundleLocation(Path imagePath, String imageName) { bundlePath = imagePath; bundleName = imageName + bundleFileExtension; } - void writeBundle() { - if (!status.writeBundle) { - return; - } - + private Path writeBundle() { String originalOutputDirName = outputDir.getFileName().toString() + originalDirExtension; Path originalOutputDir = rootDir.resolve(originalOutputDirName); if (Files.exists(originalOutputDir)) { @@ -503,7 +513,8 @@ void writeBundle() { throw NativeImage.showError("Failed to write bundle-file " + pathSubstitutionsFile, e); } - try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(bundlePath.resolve(bundleName)), new Manifest())) { + Path bundleFilePath = this.bundlePath.resolve(bundleName); + try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(bundleFilePath), new Manifest())) { try (Stream walk = Files.walk(rootDir)) { walk.forEach(bundleEntry -> { if (Files.isDirectory(bundleEntry)) { @@ -524,6 +535,8 @@ void writeBundle() { } catch (IOException e) { throw NativeImage.showError("Failed to create bundle file " + bundleName, e); } + + return bundleFilePath; } private static final String substitutionMapSrcField = "src"; From 43449e5d3f2f5307861eb60dddbb538db2a24a4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 16 Jan 2023 15:40:30 +0100 Subject: [PATCH 18/46] Style fixes --- .../src/com/oracle/svm/driver/APIOptionHandler.java | 1 + .../src/com/oracle/svm/driver/DefaultOptionHandler.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java index 189620e0070d..894147f684f1 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java @@ -451,6 +451,7 @@ static BuilderArgumentParts from(String builderArgument) { return new BuilderArgumentParts(optionName, optionOrigin, optionValue); } + @Override public String toString() { String nameAndOrigin = optionOrigin == null ? optionName : optionName + "@" + optionOrigin; return optionValue == null ? nameAndOrigin : nameAndOrigin + "=" + optionValue; diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java index b7338d8ab48a..bf58c02305e3 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java @@ -33,7 +33,6 @@ import java.util.ArrayList; import java.util.List; -import com.oracle.svm.common.option.MultiOptionValue; import com.oracle.svm.core.option.BundleMember; import com.oracle.svm.core.option.OptionOrigin; import com.oracle.svm.driver.NativeImage.ArgumentQueue; From 20dd552b329668d3f4aa3a2a3ab4da24840c9406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 16 Jan 2023 15:48:43 +0100 Subject: [PATCH 19/46] Do not write bundle output info as relative paths When native-image building is embedded in some other build system (maven or gradle) outputting bundle locations as relative paths is not useful. --- .../src/com/oracle/svm/driver/BundleSupport.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index dec651f104b2..b4daa5aa4d7e 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -430,13 +430,11 @@ private void copyFile(Path sourceFile, Path target, boolean overwrite) { void complete() { assert status.buildImage || status.writeBundle : "Superfluous bundle operations"; - Path workDir = nativeImage.config.getWorkingDirectory(); - if (status.buildImage) { Path externalOutputDir = bundlePath.resolve(nativeImage.imageName + "." + outputDir.getFileName()); copyFiles(outputDir, externalOutputDir, true); nativeImage.showNewline(); - nativeImage.showMessage("Bundle output written to " + workDir.relativize(externalOutputDir)); + nativeImage.showMessage("Bundle output written to " + externalOutputDir); } try { @@ -445,7 +443,7 @@ void complete() { if (!status.buildImage) { nativeImage.showNewline(); } - nativeImage.showMessage("Bundle written to " + workDir.relativize(bundleFilePath)); + nativeImage.showMessage("Bundle written to " + bundleFilePath); } } finally { nativeImage.showNewline(); From 547d21ef645fca6e82656ac6dd6dcbf6fce43dde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 16 Jan 2023 21:32:37 +0100 Subject: [PATCH 20/46] No readable-based bailout for output path substitutions --- .../svm/common/option/LocatableOption.java | 10 ++-- .../oracle/svm/driver/APIOptionHandler.java | 51 +++++++++++-------- .../com/oracle/svm/driver/BundleSupport.java | 6 ++- 3 files changed, 40 insertions(+), 27 deletions(-) diff --git a/substratevm/src/com.oracle.svm.common/src/com/oracle/svm/common/option/LocatableOption.java b/substratevm/src/com.oracle.svm.common/src/com/oracle/svm/common/option/LocatableOption.java index 62927c9a3a31..176bd31f98b9 100644 --- a/substratevm/src/com.oracle.svm.common/src/com/oracle/svm/common/option/LocatableOption.java +++ b/substratevm/src/com.oracle.svm.common/src/com/oracle/svm/common/option/LocatableOption.java @@ -27,10 +27,10 @@ public final class LocatableOption { - final String name; - final String origin; + public final String name; + public final String origin; - static LocatableOption from(String rawOptionName) { + public static LocatableOption from(String rawOptionName) { return new LocatableOption(rawOptionName); } @@ -54,6 +54,10 @@ public String toString() { return result + " from '" + origin + "'"; } + public String rawName() { + return origin == null ? name : name + '@' + origin; + } + private static final class LocatableOptionValue { private final Object value; private final String origin; diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java index 894147f684f1..fd8da3341a92 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/APIOptionHandler.java @@ -48,6 +48,7 @@ import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; +import com.oracle.svm.common.option.LocatableOption; import com.oracle.svm.common.option.MultiOptionValue; import com.oracle.svm.core.option.APIOption; import com.oracle.svm.core.option.APIOption.APIOptionKind; @@ -396,21 +397,29 @@ String translateOption(ArgumentQueue argQueue) { } String transformBuilderArgument(String builderArgument, BiFunction transformFunction) { - BuilderArgumentParts parts = BuilderArgumentParts.from(builderArgument); - if (parts.optionValue == null) { + BuilderArgumentParts argumentParts = BuilderArgumentParts.from(builderArgument); + if (argumentParts.optionValue == null) { + /* Option has no value that could need transforming -> early exit */ return builderArgument; } - PathsOptionInfo pathsOptionInfo = pathOptions.get(parts.optionName); + PathsOptionInfo pathsOptionInfo = pathOptions.get(argumentParts.option.name); if (pathsOptionInfo == null || pathsOptionInfo.role == BundleMember.Role.Ignore) { + /* Not an option that request value-transforming -> early exit */ return builderArgument; } + + /* + * Option requests value-transformations, first split value aggregate into individual values + */ List rawEntries; String delimiter = pathsOptionInfo.delimiter; if (delimiter.isEmpty()) { - rawEntries = List.of(parts.optionValue); + rawEntries = List.of(argumentParts.optionValue); } else { - rawEntries = List.of(StringUtil.split(parts.optionValue, delimiter)); + rawEntries = List.of(StringUtil.split(argumentParts.optionValue, delimiter)); } + + /* Perform value-transformation on individual values with given transformFunction */ try { String transformedOptionValue = rawEntries.stream() .filter(s -> !s.isEmpty()) @@ -418,43 +427,41 @@ String transformBuilderArgument(String builderArgument, BiFunction transformFunction.apply(src, pathsOptionInfo.role)) .map(Path::toString) .collect(Collectors.joining(delimiter)); - parts.optionValue = transformedOptionValue; - return parts.toString(); + /* Update argumentParts with transformed aggregate value and return as string */ + argumentParts.optionValue = transformedOptionValue; + return argumentParts.toString(); } catch (BundleSupport.BundlePathSubstitutionError error) { - Object optionOrigin = OptionOrigin.from(parts.optionOrigin, false); - if (optionOrigin == null && parts.optionOrigin != null) { - optionOrigin = parts.optionOrigin; + String originStr = argumentParts.option.origin; + Object optionOrigin = OptionOrigin.from(originStr, false); + if (optionOrigin == null && originStr != null) { + /* If we cannot get an OptionOrigin, fallback to the raw originStr */ + optionOrigin = originStr; } String fromPart = optionOrigin != null ? " from '" + optionOrigin + "'" : ""; - throw NativeImage.showError("Failed to prepare path entry '" + error.origPath + "' of option " + parts.optionName + fromPart + " for bundle inclusion.", error); + throw NativeImage.showError("Failed to prepare path entry '" + error.origPath + "' of option " + argumentParts.option.name + fromPart + " for bundle inclusion.", error); } } static final class BuilderArgumentParts { - final String optionName; - final String optionOrigin; + final LocatableOption option; String optionValue; - private BuilderArgumentParts(String optionName, String optionOrigin, String optionValue) { - this.optionName = optionName; - this.optionOrigin = optionOrigin; + private BuilderArgumentParts(LocatableOption option, String optionValue) { + this.option = option; this.optionValue = optionValue; } static BuilderArgumentParts from(String builderArgument) { String[] nameAndValue = StringUtil.split(builderArgument, "=", 2); String optionValue = nameAndValue.length != 2 ? null : nameAndValue[1]; - String[] nameAndOrigin = StringUtil.split(nameAndValue[0], "@", 2); - String optionName = nameAndOrigin[0]; - String optionOrigin = nameAndOrigin.length == 2 ? nameAndOrigin[1] : null; - return new BuilderArgumentParts(optionName, optionOrigin, optionValue); + return new BuilderArgumentParts(LocatableOption.from(nameAndValue[0]), optionValue); } @Override public String toString() { - String nameAndOrigin = optionOrigin == null ? optionName : optionName + "@" + optionOrigin; - return optionValue == null ? nameAndOrigin : nameAndOrigin + "=" + optionValue; + String optionName = option.rawName(); + return optionValue == null ? optionName : optionName + "=" + optionValue; } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index b4daa5aa4d7e..552fae381792 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -361,7 +361,9 @@ private Path substitutePath(Path origPath, Path destinationDir) { throw new BundlePathSubstitutionError("Bundles do not allow inclusion of directory " + origPath, origPath); } - if (!Files.isReadable(origPath)) { + boolean isOutputPath = destinationDir.startsWith(outputDir); + + if (!isOutputPath && !Files.isReadable(origPath)) { /* Prevent subsequent retries to substitute invalid paths */ pathSubstitutions.put(origPath, origPath); return origPath; @@ -388,7 +390,7 @@ private Path substitutePath(Path origPath, Path destinationDir) { substitutedPath = destinationDir.resolve(baseName + "_" + collisionIndex + extension); } - if (!destinationDir.startsWith(outputDir)) { + if (!isOutputPath) { copyFiles(origPath, substitutedPath, false); } From 0b4dbc2aaf71dadfa69ecbd3641bf64b554a7d5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 17 Jan 2023 09:41:52 +0100 Subject: [PATCH 21/46] Prevent accidental bundle-extension duplication --- .../src/com/oracle/svm/driver/BundleSupport.java | 12 ++++++++++-- .../src/com/oracle/svm/driver/NativeImage.java | 3 ++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 552fae381792..127865319ce5 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -453,9 +453,17 @@ void complete() { } } - void setBundleLocation(Path imagePath, String imageName) { + String setBundleLocation(Path imagePath, String imageName) { + String fixedImageName; + if (imageName.endsWith(bundleFileExtension)) { + int endIndex = imageName.length() - bundleFileExtension.length(); + fixedImageName = imageName.substring(0, endIndex); + } else { + fixedImageName = imageName; + } bundlePath = imagePath; - bundleName = imageName + bundleFileExtension; + bundleName = fixedImageName + bundleFileExtension; + return fixedImageName; } private Path writeBundle() { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index d7508a57abdf..fadd6818bb59 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1138,7 +1138,8 @@ private int completeImageBuild() { * that we can now use to derive the new bundle name from. For apply-mode setting * imagePath determines where to copy the bundle output to. */ - bundleSupport.setBundleLocation(imagePath, imageName); + imageName = bundleSupport.setBundleLocation(imagePath, imageName); + updateArgumentEntryValue(imageBuilderArgs, imageNameEntry, imageName); /* The imagePath has to be redirected to be within the bundle */ imagePath = bundleSupport.substituteImagePath(imagePath); /* and we need to adjust the argument that passes the imagePath to the builder */ From 3774f2d630976a0c98e67b0286198af14937e32b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 17 Jan 2023 14:34:18 +0100 Subject: [PATCH 22/46] Add missing copyright header --- .../oracle/svm/core/option/BundleMember.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/BundleMember.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/BundleMember.java index fd356952c8c5..565feb80b241 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/BundleMember.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/BundleMember.java @@ -1,3 +1,28 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + package com.oracle.svm.core.option; import java.lang.annotation.ElementType; From 7720a10b5dba7f4e69d23545017c05048bb03edb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 17 Jan 2023 14:35:15 +0100 Subject: [PATCH 23/46] Style fix --- .../src/com/oracle/svm/hosted/ServiceLoaderFeature.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ServiceLoaderFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ServiceLoaderFeature.java index 5ad5eff9bd61..d3cd01ac6816 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ServiceLoaderFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ServiceLoaderFeature.java @@ -107,7 +107,8 @@ public static class Options { public static final HostedOptionKey ServiceLoaderFeatureExcludeServices = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); @Option(help = "Comma-separated list of service providers that should be excluded", type = OptionType.Expert) // - public static final HostedOptionKey ServiceLoaderFeatureExcludeServiceProviders = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); + public static final HostedOptionKey ServiceLoaderFeatureExcludeServiceProviders = new HostedOptionKey<>( + LocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); } From e5f106ff03a7249ce92b0162f9b29b90b33829db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 17 Jan 2023 18:23:36 +0100 Subject: [PATCH 24/46] CheckStyle fix --- .../com/oracle/svm/core/option/LocatableMultiOptionValue.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/LocatableMultiOptionValue.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/LocatableMultiOptionValue.java index 6036f0a44249..4bc6f74b0adb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/LocatableMultiOptionValue.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/LocatableMultiOptionValue.java @@ -118,7 +118,7 @@ public String toString() { return "<" + ClassUtil.getUnqualifiedName(valueType).toLowerCase() + ">*"; } - public static class Strings extends LocatableMultiOptionValue { + public static final class Strings extends LocatableMultiOptionValue { private Strings(Strings other) { super(other); @@ -146,7 +146,7 @@ public static Strings buildWithDefaults(String... defaultStrings) { } } - public static class Paths extends LocatableMultiOptionValue { + public static final class Paths extends LocatableMultiOptionValue { private Paths(Paths other) { super(other); From 287f88836cbd98c530d99836dc3b255f96e6c628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 18 Jan 2023 16:07:22 +0100 Subject: [PATCH 25/46] Only write bundle output if not empty --- .../com/oracle/svm/driver/BundleSupport.java | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 127865319ce5..fac9b790d35b 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -431,20 +431,33 @@ private void copyFile(Path sourceFile, Path target, boolean overwrite) { void complete() { assert status.buildImage || status.writeBundle : "Superfluous bundle operations"; + final boolean[] firstMessage = {true}; + Runnable outputNewline = () -> { + if (firstMessage[0]) { + nativeImage.showNewline(); + firstMessage[0] = false; + } + }; if (status.buildImage) { - Path externalOutputDir = bundlePath.resolve(nativeImage.imageName + "." + outputDir.getFileName()); - copyFiles(outputDir, externalOutputDir, true); - nativeImage.showNewline(); - nativeImage.showMessage("Bundle output written to " + externalOutputDir); + boolean writeOutput; + try (Stream pathOutputFiles = Files.list(imagePathOutputDir); Stream auxiliaryOutputFiles = Files.list(auxiliaryOutputDir)) { + writeOutput = pathOutputFiles.findAny().isPresent() || auxiliaryOutputFiles.findAny().isPresent(); + } catch (IOException e) { + throw NativeImage.showError("Unable to determine if bundle output should be written."); + } + if (writeOutput) { + Path externalOutputDir = bundlePath.resolve(nativeImage.imageName + "." + outputDir.getFileName()); + copyFiles(outputDir, externalOutputDir, true); + outputNewline.run(); + nativeImage.showMessage("Bundle build output written to " + externalOutputDir); + } } try { if (status.writeBundle) { Path bundleFilePath = writeBundle(); - if (!status.buildImage) { - nativeImage.showNewline(); - } + outputNewline.run(); nativeImage.showMessage("Bundle written to " + bundleFilePath); } } finally { From 854bcabd4a40c7efec970d6493670b77eeff0737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 18 Jan 2023 16:08:17 +0100 Subject: [PATCH 26/46] Ensure imagePath target is relative in substitutions map --- .../src/com/oracle/svm/driver/BundleSupport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index fac9b790d35b..0a76084c37c6 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -287,7 +287,7 @@ Path substituteAuxiliaryPath(Path origPath, BundleMember.Role bundleMemberRole) } Path substituteImagePath(Path origPath) { - pathSubstitutions.put(origPath, imagePathOutputDir); + pathSubstitutions.put(origPath, rootDir.relativize(imagePathOutputDir)); return imagePathOutputDir; } From b9442030e001020a5deba7913a2a4bfe3e31d4f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 18 Jan 2023 16:09:59 +0100 Subject: [PATCH 27/46] Disallow bundle-use if CWD was implicitly added to classpath --- .../src/com/oracle/svm/driver/NativeImage.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index fadd6818bb59..c32e57e1d7ff 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1006,6 +1006,9 @@ private int completeImageBuild() { } if (shouldAddCWDToCP()) { + if (useBundle()) { + throw NativeImage.showError("Bundle support requires -cp or -p to be set (implicit current directory classpath unsupported)."); + } addImageClasspath(Paths.get(".")); } imageClasspath.addAll(customImageClasspath); From 904dbba8865b641d261e9a89b1e7a235c1abbbc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 18 Jan 2023 16:10:54 +0100 Subject: [PATCH 28/46] Fix manifest writing and add bundle properties writing and checking --- .../com/oracle/svm/driver/BundleSupport.java | 74 ++++++++++++++++++- .../com/oracle/svm/driver/NativeImage.java | 2 +- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 0a76084c37c6..faf0dc39da78 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -26,6 +26,7 @@ import java.io.File; import java.io.IOException; +import java.io.OutputStream; import java.io.Reader; import java.net.URI; import java.nio.file.CopyOption; @@ -41,6 +42,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Properties; +import java.util.jar.Attributes; import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.jar.JarOutputStream; @@ -83,6 +86,8 @@ enum BundleStatus { final BundleStatus status; final Path rootDir; + final Path bundlePropertiesFile; + final Path stageDir; final Path classPathDir; final Path modulePathDir; @@ -97,6 +102,10 @@ enum BundleStatus { private final List buildArgs; private Collection updatedBuildArgs; + private static final Path bundlePropertiesFileName = Path.of("META-INF/nibundle.properties"); + private static final int bundleFileFormatVersionMajor = 0; + private static final int bundleFileFormatVersionMinor = 9; + private static final String bundleTempDirPrefix = "bundleRoot-"; private static final String bundleFileExtension = ".nib"; private static final String originalDirExtension = ".orig"; @@ -140,6 +149,8 @@ private BundleSupport(NativeImage nativeImage, BundleStatus status) { this.status = status; try { rootDir = Files.createTempDirectory(bundleTempDirPrefix); + bundlePropertiesFile = rootDir.resolve(bundlePropertiesFileName); + Path inputDir = rootDir.resolve("input"); stageDir = Files.createDirectories(inputDir.resolve("stage")); auxiliaryDir = Files.createDirectories(inputDir.resolve("auxiliary")); @@ -179,6 +190,8 @@ private BundleSupport(NativeImage nativeImage, BundleStatus status, String bundl try { rootDir = Files.createTempDirectory(bundleTempDirPrefix); + bundlePropertiesFile = rootDir.resolve(bundlePropertiesFileName); + outputDir = rootDir.resolve("output"); String originalOutputDirName = outputDir.getFileName().toString() + originalDirExtension; @@ -207,6 +220,8 @@ private BundleSupport(NativeImage nativeImage, BundleStatus status, String bundl bundlePath = bundleFile.getParent(); bundleName = bundleFileName; + verifyBundleProperties(bundlePropertiesFile); + try { Path inputDir = rootDir.resolve("input"); stageDir = Files.createDirectories(inputDir.resolve("stage")); @@ -485,7 +500,8 @@ private Path writeBundle() { if (Files.exists(originalOutputDir)) { nativeImage.deleteAllFiles(originalOutputDir); } - Path metaInfDir = rootDir.resolve("META-INF"); + + Path metaInfDir = rootDir.resolve(JarFile.MANIFEST_NAME); if (Files.exists(metaInfDir)) { nativeImage.deleteAllFiles(metaInfDir); } @@ -534,8 +550,10 @@ private Path writeBundle() { throw NativeImage.showError("Failed to write bundle-file " + pathSubstitutionsFile, e); } + writeBundleProperties(bundlePropertiesFile); + Path bundleFilePath = this.bundlePath.resolve(bundleName); - try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(bundleFilePath), new Manifest())) { + try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(bundleFilePath), createManifest())) { try (Stream walk = Files.walk(rootDir)) { walk.forEach(bundleEntry -> { if (Files.isDirectory(bundleEntry)) { @@ -560,6 +578,58 @@ private Path writeBundle() { return bundleFilePath; } + private Manifest createManifest() { + Manifest mf = new Manifest(); + Attributes attributes = mf.getMainAttributes(); + attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); + /* If we add run-bundle-as-java-application a launcher mainclass would be added here */ + return mf; + } + + private void writeBundleProperties(Path bundlePropertiesFile) { + Properties nibp = new Properties(); + nibp.setProperty("BundleFileVersionMajor", String.valueOf(bundleFileFormatVersionMajor)); + nibp.setProperty("BundleFileVersionMinor", String.valueOf(bundleFileFormatVersionMinor)); + boolean imageBuilt = status.buildImage && !nativeImage.isDryRun(); + nibp.setProperty("ImageBuilt", String.valueOf(imageBuilt)); + if (imageBuilt) { + nibp.setProperty("BuildContainerized", String.valueOf(false)); + } + nibp.setProperty("NativeImagePlatform", NativeImage.platform); + nibp.setProperty("NativeImageVersion", NativeImage.graalvmVersion); + NativeImage.ensureDirectoryExists(bundlePropertiesFile.getParent()); + try (OutputStream outputStream = Files.newOutputStream(bundlePropertiesFile)) { + nibp.store(outputStream, "Native Image bundle file properties"); + } catch (IOException e) { + throw NativeImage.showError("Creating bundle properties file failed", e); + } + } + + private void verifyBundleProperties(Path propertiesFile) { + if (!Files.isReadable(propertiesFile)) { + throw NativeImage.showError("The given bundle file " + bundleName + " does not contain a bundle properties file"); + } + + Map bundleProperties = NativeImage.loadProperties(propertiesFile); + String fileVersionKey = "BundleFileVersionMajor"; + try { + int major = Integer.valueOf(bundleProperties.getOrDefault(fileVersionKey, "-1")); + fileVersionKey = "BundleFileVersionMinor"; + int minor = Integer.valueOf(bundleProperties.getOrDefault(fileVersionKey, "-1")); + String message = String.format("The given bundle file %s was created with newer bundle file version %d.%d." + + " Update to the latest version of native-image.", bundleName, major, minor); + if (major > bundleFileFormatVersionMajor) { + throw NativeImage.showError(message); + } else if (major == bundleFileFormatVersionMajor) { + if (minor > bundleFileFormatVersionMinor) { + NativeImage.showWarning(message); + } + } + } catch (NumberFormatException e) { + throw NativeImage.showError(fileVersionKey + " in " + bundlePropertiesFileName + " is missing or ill-defined", e); + } + } + private static final String substitutionMapSrcField = "src"; private static final String substitutionMapDstField = "dst"; diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index c32e57e1d7ff..f1b74059031d 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -772,7 +772,7 @@ protected Path getUserConfigDir() { return Paths.get(userHomeStr); } - protected static void ensureDirectoryExists(Path dir) { + static void ensureDirectoryExists(Path dir) { if (Files.exists(dir)) { if (!Files.isDirectory(dir)) { throw showError("File " + dir + " is not a directory"); From e4e92179842f8f79f131d44cc4af8503cc59bb37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 18 Jan 2023 16:11:52 +0100 Subject: [PATCH 29/46] Fix unnecessary boxing of ints --- .../src/com/oracle/svm/driver/BundleSupport.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index faf0dc39da78..5f232d94d14b 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -613,9 +613,9 @@ private void verifyBundleProperties(Path propertiesFile) { Map bundleProperties = NativeImage.loadProperties(propertiesFile); String fileVersionKey = "BundleFileVersionMajor"; try { - int major = Integer.valueOf(bundleProperties.getOrDefault(fileVersionKey, "-1")); + int major = Integer.parseInt(bundleProperties.getOrDefault(fileVersionKey, "-1")); fileVersionKey = "BundleFileVersionMinor"; - int minor = Integer.valueOf(bundleProperties.getOrDefault(fileVersionKey, "-1")); + int minor = Integer.parseInt(bundleProperties.getOrDefault(fileVersionKey, "-1")); String message = String.format("The given bundle file %s was created with newer bundle file version %d.%d." + " Update to the latest version of native-image.", bundleName, major, minor); if (major > bundleFileFormatVersionMajor) { From 1a5de837391c5b6b825073b99e0c0b6d9850801e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 18 Jan 2023 16:28:48 +0100 Subject: [PATCH 30/46] If bundle is loaded do not implicitly add CWD to classpath --- .../src/com/oracle/svm/driver/NativeImage.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index f1b74059031d..852d30a4fa3e 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1239,6 +1239,11 @@ private boolean shouldAddCWDToCP() { return false; } + if (useBundle() && bundleSupport.status.loadBundle) { + /* If bundle was loaded we have valid -cp and/or -p from within the bundle */ + return false; + } + /* If no customImageClasspath was specified put "." on classpath */ return customImageClasspath.isEmpty() && imageModulePath.isEmpty(); } From 5fb50c1829b912f8f151f1dece50bc972666d7e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 18 Jan 2023 16:57:05 +0100 Subject: [PATCH 31/46] Warn if bundle platform or graalvmVersion differ --- .../com/oracle/svm/driver/BundleSupport.java | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 5f232d94d14b..f234c9ecd6dc 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -587,19 +587,19 @@ private Manifest createManifest() { } private void writeBundleProperties(Path bundlePropertiesFile) { - Properties nibp = new Properties(); - nibp.setProperty("BundleFileVersionMajor", String.valueOf(bundleFileFormatVersionMajor)); - nibp.setProperty("BundleFileVersionMinor", String.valueOf(bundleFileFormatVersionMinor)); + Properties properties = new Properties(); + properties.setProperty("BundleFileVersionMajor", String.valueOf(bundleFileFormatVersionMajor)); + properties.setProperty("BundleFileVersionMinor", String.valueOf(bundleFileFormatVersionMinor)); boolean imageBuilt = status.buildImage && !nativeImage.isDryRun(); - nibp.setProperty("ImageBuilt", String.valueOf(imageBuilt)); + properties.setProperty("ImageBuilt", String.valueOf(imageBuilt)); if (imageBuilt) { - nibp.setProperty("BuildContainerized", String.valueOf(false)); + properties.setProperty("BuildContainerized", String.valueOf(false)); } - nibp.setProperty("NativeImagePlatform", NativeImage.platform); - nibp.setProperty("NativeImageVersion", NativeImage.graalvmVersion); + properties.setProperty("NativeImagePlatform", NativeImage.platform); + properties.setProperty("NativeImageVersion", NativeImage.graalvmVersion); NativeImage.ensureDirectoryExists(bundlePropertiesFile.getParent()); try (OutputStream outputStream = Files.newOutputStream(bundlePropertiesFile)) { - nibp.store(outputStream, "Native Image bundle file properties"); + properties.store(outputStream, "Native Image bundle file properties"); } catch (IOException e) { throw NativeImage.showError("Creating bundle properties file failed", e); } @@ -610,14 +610,14 @@ private void verifyBundleProperties(Path propertiesFile) { throw NativeImage.showError("The given bundle file " + bundleName + " does not contain a bundle properties file"); } - Map bundleProperties = NativeImage.loadProperties(propertiesFile); + Map properties = NativeImage.loadProperties(propertiesFile); String fileVersionKey = "BundleFileVersionMajor"; try { - int major = Integer.parseInt(bundleProperties.getOrDefault(fileVersionKey, "-1")); + int major = Integer.parseInt(properties.getOrDefault(fileVersionKey, "-1")); fileVersionKey = "BundleFileVersionMinor"; - int minor = Integer.parseInt(bundleProperties.getOrDefault(fileVersionKey, "-1")); - String message = String.format("The given bundle file %s was created with newer bundle file version %d.%d." + - " Update to the latest version of native-image.", bundleName, major, minor); + int minor = Integer.parseInt(properties.getOrDefault(fileVersionKey, "-1")); + String message = String.format("The given bundle file %s was created with newer bundle-file-format version %d.%d" + + " (current %d.%d). Update to the latest version of native-image.", bundleName, major, minor, bundleFileFormatVersionMajor, bundleFileFormatVersionMinor); if (major > bundleFileFormatVersionMajor) { throw NativeImage.showError(message); } else if (major == bundleFileFormatVersionMajor) { @@ -628,6 +628,14 @@ private void verifyBundleProperties(Path propertiesFile) { } catch (NumberFormatException e) { throw NativeImage.showError(fileVersionKey + " in " + bundlePropertiesFileName + " is missing or ill-defined", e); } + String bundlePlatform = properties.getOrDefault("NativeImagePlatform", "unknown"); + if (!bundlePlatform.equals(NativeImage.platform)) { + NativeImage.showWarning(String.format("The given bundle file %s was created on platform %s (current %s).", bundleName, bundlePlatform, NativeImage.platform)); + } + String bundleNativeImageVersion = properties.getOrDefault("NativeImageVersion", "unknown"); + if (!bundleNativeImageVersion.equals(NativeImage.graalvmVersion)) { + NativeImage.showWarning(String.format("The given bundle file %s was created with native-image %s (current %s).", bundleName, bundleNativeImageVersion, NativeImage.graalvmVersion)); + } } private static final String substitutionMapSrcField = "src"; From cb417541274d2314fa7cc4155628cc93b4b70265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 18 Jan 2023 17:20:15 +0100 Subject: [PATCH 32/46] Fix NativeImageVersion property value --- .../src/com/oracle/svm/driver/BundleSupport.java | 9 +++++---- .../com/oracle/svm/driver/CmdLineOptionHandler.java | 7 +------ .../src/com/oracle/svm/driver/NativeImage.java | 10 ++++++++++ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index f234c9ecd6dc..6092b608f3a8 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -596,7 +596,7 @@ private void writeBundleProperties(Path bundlePropertiesFile) { properties.setProperty("BuildContainerized", String.valueOf(false)); } properties.setProperty("NativeImagePlatform", NativeImage.platform); - properties.setProperty("NativeImageVersion", NativeImage.graalvmVersion); + properties.setProperty("NativeImageVersion", NativeImage.getNativeImageVersion()); NativeImage.ensureDirectoryExists(bundlePropertiesFile.getParent()); try (OutputStream outputStream = Files.newOutputStream(bundlePropertiesFile)) { properties.store(outputStream, "Native Image bundle file properties"); @@ -630,11 +630,12 @@ private void verifyBundleProperties(Path propertiesFile) { } String bundlePlatform = properties.getOrDefault("NativeImagePlatform", "unknown"); if (!bundlePlatform.equals(NativeImage.platform)) { - NativeImage.showWarning(String.format("The given bundle file %s was created on platform %s (current %s).", bundleName, bundlePlatform, NativeImage.platform)); + NativeImage.showWarning(String.format("The given bundle file %s was created on platform '%s' (current '%s').", bundleName, bundlePlatform, NativeImage.platform)); } String bundleNativeImageVersion = properties.getOrDefault("NativeImageVersion", "unknown"); - if (!bundleNativeImageVersion.equals(NativeImage.graalvmVersion)) { - NativeImage.showWarning(String.format("The given bundle file %s was created with native-image %s (current %s).", bundleName, bundleNativeImageVersion, NativeImage.graalvmVersion)); + if (!bundleNativeImageVersion.equals(NativeImage.getNativeImageVersion())) { + NativeImage.showWarning( + String.format("The given bundle file %s was created with native-image version '%s' (current '%s').", bundleName, bundleNativeImageVersion, NativeImage.getNativeImageVersion())); } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java index 3a2001dcb407..db3ca54ad33e 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java @@ -82,12 +82,7 @@ private boolean consume(ArgumentQueue args, String headArg) { case "--version": args.poll(); singleArgumentCheck(args, headArg); - String message; - if (NativeImage.IS_AOT) { - message = System.getProperty("java.vm.version"); - } else { - message = "native-image " + NativeImage.graalvmVersion + " " + NativeImage.graalvmConfig; - } + String message = NativeImage.getNativeImageVersion(); message += " (Java Version " + javaRuntimeVersion + ")"; nativeImage.showMessage(message); System.exit(ExitStatus.OK.getValue()); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 852d30a4fa3e..4ea738356fc1 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -109,6 +109,16 @@ private static String getPlatform() { return (OS.getCurrent().className + "-" + SubstrateUtil.getArchitectureName()).toLowerCase(); } + static String getNativeImageVersion() { + String message; + if (IS_AOT) { + message = System.getProperty("java.vm.version"); + } else { + message = "native-image " + graalvmVersion + " " + graalvmConfig; + } + return message; + } + static final String graalvmVersion = System.getProperty("org.graalvm.version", "dev"); static final String graalvmConfig = System.getProperty("org.graalvm.config", "CE"); static final String graalvmVendor = System.getProperty("org.graalvm.vendor", "Oracle Corporation"); From a0e96a3d90001ea1a4047087b0dffe9623e8d31a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 18 Jan 2023 17:37:34 +0100 Subject: [PATCH 33/46] Style fixes --- .../src/com/oracle/svm/driver/BundleSupport.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 6092b608f3a8..6318e1c4979a 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -578,7 +578,7 @@ private Path writeBundle() { return bundleFilePath; } - private Manifest createManifest() { + private static Manifest createManifest() { Manifest mf = new Manifest(); Attributes attributes = mf.getMainAttributes(); attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); @@ -586,7 +586,7 @@ private Manifest createManifest() { return mf; } - private void writeBundleProperties(Path bundlePropertiesFile) { + private void writeBundleProperties(Path propertiesFile) { Properties properties = new Properties(); properties.setProperty("BundleFileVersionMajor", String.valueOf(bundleFileFormatVersionMajor)); properties.setProperty("BundleFileVersionMinor", String.valueOf(bundleFileFormatVersionMinor)); @@ -597,8 +597,8 @@ private void writeBundleProperties(Path bundlePropertiesFile) { } properties.setProperty("NativeImagePlatform", NativeImage.platform); properties.setProperty("NativeImageVersion", NativeImage.getNativeImageVersion()); - NativeImage.ensureDirectoryExists(bundlePropertiesFile.getParent()); - try (OutputStream outputStream = Files.newOutputStream(bundlePropertiesFile)) { + NativeImage.ensureDirectoryExists(propertiesFile.getParent()); + try (OutputStream outputStream = Files.newOutputStream(propertiesFile)) { properties.store(outputStream, "Native Image bundle file properties"); } catch (IOException e) { throw NativeImage.showError("Creating bundle properties file failed", e); @@ -630,12 +630,14 @@ private void verifyBundleProperties(Path propertiesFile) { } String bundlePlatform = properties.getOrDefault("NativeImagePlatform", "unknown"); if (!bundlePlatform.equals(NativeImage.platform)) { - NativeImage.showWarning(String.format("The given bundle file %s was created on platform '%s' (current '%s').", bundleName, bundlePlatform, NativeImage.platform)); + NativeImage.showWarning(String.format("The given bundle file %s was created on platform '%s' (current '%s').", + bundleName, bundlePlatform, NativeImage.platform)); } String bundleNativeImageVersion = properties.getOrDefault("NativeImageVersion", "unknown"); if (!bundleNativeImageVersion.equals(NativeImage.getNativeImageVersion())) { NativeImage.showWarning( - String.format("The given bundle file %s was created with native-image version '%s' (current '%s').", bundleName, bundleNativeImageVersion, NativeImage.getNativeImageVersion())); + String.format("The given bundle file %s was created with native-image version '%s' (current '%s').", + bundleName, bundleNativeImageVersion, NativeImage.getNativeImageVersion())); } } From dc0a94c853e7d36ce5288c603cd5a29bf842a9f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 18 Jan 2023 18:03:14 +0100 Subject: [PATCH 34/46] Do not use undefined origin string --- .../com/oracle/svm/core/option/LocatableMultiOptionValue.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/LocatableMultiOptionValue.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/LocatableMultiOptionValue.java index 4bc6f74b0adb..9bb60bcf5289 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/LocatableMultiOptionValue.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/LocatableMultiOptionValue.java @@ -50,7 +50,7 @@ private LocatableMultiOptionValue(Class valueType, String delimiter, List this.valueType = valueType; this.delimiter = delimiter; values = new ArrayList<>(); - values.addAll(defaults.stream().map(val -> Pair.create(val, "default")).collect(Collectors.toList())); + values.addAll(defaults.stream().map(val -> Pair. createLeft(val)).collect(Collectors.toList())); } private LocatableMultiOptionValue(LocatableMultiOptionValue other) { From a28c9ee5c4a4dccd595fe8461961947ae8234a8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 23 Jan 2023 14:47:37 +0100 Subject: [PATCH 35/46] Reduce bundle options to --bundle-{create,apply} and allow combining --- substratevm/mx.substratevm/mx_substratevm.py | 2 +- .../resources/HelpExtra.txt | 25 +- .../com/oracle/svm/driver/BundleSupport.java | 418 ++++++++++-------- .../svm/driver/CmdLineOptionHandler.java | 25 +- .../svm/driver/DefaultOptionHandler.java | 8 - .../com/oracle/svm/driver/NativeImage.java | 40 +- 6 files changed, 297 insertions(+), 221 deletions(-) diff --git a/substratevm/mx.substratevm/mx_substratevm.py b/substratevm/mx.substratevm/mx_substratevm.py index 89b6228986b0..b18c1c5ef4a3 100644 --- a/substratevm/mx.substratevm/mx_substratevm.py +++ b/substratevm/mx.substratevm/mx_substratevm.py @@ -277,7 +277,7 @@ def query_native_image(all_args, option): stdoutdata = [] def stdout_collector(x): stdoutdata.append(x.rstrip()) - _native_image(['--dry-run'] + all_args, out=stdout_collector) + _native_image(['--dry-run', '--verbose'] + all_args, out=stdout_collector) def remove_quotes(val): if len(val) >= 2 and val.startswith("'") and val.endswith("'"): diff --git a/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt b/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt index 7c24208462c8..b460584fcac8 100644 --- a/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt +++ b/substratevm/src/com.oracle.svm.driver/resources/HelpExtra.txt @@ -16,18 +16,23 @@ Non-standard options help: --diagnostics-mode Enables logging of image-build information to a diagnostics folder. --dry-run output the command line that would be used for building - --bundle-create in addition to building a native image create a bundle file (*.nib file) - that allows rebuilding of that image again at a later point. - --bundle-apply bundle-file + --bundle-create[=new-bundle.nib] + in addition to image building, create a native image bundle file (*.nib + file) that allows rebuilding of that image again at a later point. If a + bundle-file gets passed the bundle will be created with the given name. + Otherwise, the bundle-file name is derived from the image name. Note + both bundle options can be combined with --dry-run to only perform the + bundle operations without any actual image building. + --bundle-apply=some-bundle.nib an image will be built from the given bundle file with the exact same arguments and files that have been passed to native-image originally - to create the bundle. - --bundle-prepare create a bundle file that allows building an image at a later point from - the bundle file but do not also build the image right away. - --bundle-amend bundle-file - create a new bundle but use a given bundle as the starting point. Any - given extra arguments add to the arguments of the provided bundle and - are incorporated into the new updated bundle. + to create the bundle. Note that if an extra --bundle-create gets passed + after --bundle-apply, a new bundle will be written based on the given + bundle args plus any additional arguments that haven been passed + afterwards. For example: + > native-image --bundle-apply=app.nib --bundle-create=app_dbg.nib -g + creates a new bundle app_dbg.nib based on the given app.nib bundle. + Both bundles are the same except the new one also uses the -g option. -V= provide values for placeholders in native-image.properties files diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 6318e1c4979a..50577349bdd7 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -41,6 +41,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Properties; import java.util.jar.Attributes; @@ -54,6 +55,7 @@ import org.graalvm.util.json.JSONParserException; import com.oracle.svm.core.OS; +import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.configure.ConfigurationParser; import com.oracle.svm.core.option.BundleMember; import com.oracle.svm.core.util.json.JsonPrinter; @@ -62,31 +64,9 @@ final class BundleSupport { - static final String BUNDLE_OPTION = "--bundle"; - - enum BundleStatus { - create(false, true, true), - apply(true, false, true), - prepare(false, true, false), - amend(true, true, false); - - final boolean loadBundle; - final boolean writeBundle; - final boolean buildImage; - - BundleStatus(boolean loadBundle, boolean writeBundle, boolean buildImage) { - this.loadBundle = loadBundle; - this.writeBundle = writeBundle; - this.buildImage = buildImage; - } - } - final NativeImage nativeImage; - final BundleStatus status; - final Path rootDir; - final Path bundlePropertiesFile; final Path stageDir; final Path classPathDir; @@ -102,54 +82,105 @@ enum BundleStatus { private final List buildArgs; private Collection updatedBuildArgs; - private static final Path bundlePropertiesFileName = Path.of("META-INF/nibundle.properties"); + boolean loadBundle; + boolean writeBundle; + private static final int bundleFileFormatVersionMajor = 0; private static final int bundleFileFormatVersionMinor = 9; + private static final String bundleInfoMessagePrefix = "GraalVM Native Image Bundle Support: "; private static final String bundleTempDirPrefix = "bundleRoot-"; - private static final String bundleFileExtension = ".nib"; private static final String originalDirExtension = ".orig"; private Path bundlePath; private String bundleName; + private final BundleProperties bundleProperties; + + static final String BUNDLE_OPTION = "--bundle"; + static final String BUNDLE_FILE_EXTENSION = ".nib"; + + private enum BundleOptionVariants { + create(), + apply() + } + static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeImage.ArgumentQueue args) { if (!nativeImage.userConfigProperties.isEmpty()) { throw NativeImage.showError("Bundle support cannot be combined with " + NativeImage.CONFIG_FILE_ENV_VAR_KEY + " environment variable use."); } - BundleStatus bundleStatus; try { - bundleStatus = BundleStatus.valueOf(bundleArg.substring(BUNDLE_OPTION.length() + 1)); + String variant = bundleArg.substring(BUNDLE_OPTION.length() + 1); + String bundleFilename = null; + String[] variantParts = SubstrateUtil.split(variant, "=", 2); + if (variantParts.length == 2) { + variant = variantParts[0]; + bundleFilename = variantParts[1]; + } + String applyOptionStr = BUNDLE_OPTION + "-" + BundleOptionVariants.apply; + String createOptionStr = BUNDLE_OPTION + "-" + BundleOptionVariants.create; + BundleSupport bundleSupport; + switch (BundleOptionVariants.valueOf(variant)) { + case apply: + if (nativeImage.useBundle()) { + if (nativeImage.bundleSupport.loadBundle) { + throw NativeImage.showError(String.format("Native-image allows option %s to be specified only once.", applyOptionStr)); + } + if (nativeImage.bundleSupport.writeBundle) { + throw NativeImage.showError(String.format("Native-image option %s is not allowed to be used after option %s.", applyOptionStr, createOptionStr)); + } + } + if (bundleFilename == null) { + throw NativeImage.showError(String.format("Native-image option %s requires a bundle file argument. E.g. %s=bundle-file.nib.", applyOptionStr, applyOptionStr)); + } + bundleSupport = new BundleSupport(nativeImage, bundleFilename); + /* Inject the command line args from the loaded bundle in-place */ + List buildArgs = bundleSupport.getBuildArgs(); + for (int i = buildArgs.size() - 1; i >= 0; i--) { + args.push(buildArgs.get(i)); + } + nativeImage.showVerboseMessage(nativeImage.isVerbose(), bundleInfoMessagePrefix + "Inject args: '" + String.join(" ", buildArgs) + "'"); + /* Snapshot args after in-place expansion (includes also args after this one) */ + bundleSupport.updatedBuildArgs = args.snapshot(); + break; + case create: + if (nativeImage.useBundle()) { + if (nativeImage.bundleSupport.writeBundle) { + throw NativeImage.showError(String.format("Native-image allows option %s to be specified only once.", bundleArg)); + } else { + bundleSupport = nativeImage.bundleSupport; + bundleSupport.writeBundle = true; + } + } else { + bundleSupport = new BundleSupport(nativeImage); + } + if (bundleFilename != null) { + bundleSupport.updateBundleLocation(Path.of(bundleFilename), true); + } + break; + default: + throw new IllegalArgumentException(); + } + return bundleSupport; + } catch (StringIndexOutOfBoundsException | IllegalArgumentException e) { - String suggestedVariants = Arrays.stream(BundleStatus.values()) + String suggestedVariants = Arrays.stream(BundleOptionVariants.values()) .map(v -> BUNDLE_OPTION + "-" + v) .collect(Collectors.joining(", ")); throw NativeImage.showError("Unknown option " + bundleArg + ". Valid variants are: " + suggestedVariants + "."); } - BundleSupport bundleSupport; - if (bundleStatus.loadBundle) { - String bundleFilename = args.poll(); - bundleSupport = new BundleSupport(nativeImage, bundleStatus, bundleFilename); - List buildArgs = bundleSupport.getBuildArgs(); - for (int i = buildArgs.size() - 1; i >= 0; i--) { - args.push(buildArgs.get(i)); - } - bundleSupport.updatedBuildArgs = args.snapshot(); - } else { - bundleSupport = new BundleSupport(nativeImage, bundleStatus); - } - return bundleSupport; } - private BundleSupport(NativeImage nativeImage, BundleStatus status) { - assert !status.loadBundle : "This constructor is only used when a new bundle gets created"; - + private BundleSupport(NativeImage nativeImage) { + Objects.requireNonNull(nativeImage); this.nativeImage = nativeImage; - this.status = status; + + loadBundle = false; + writeBundle = true; try { rootDir = Files.createTempDirectory(bundleTempDirPrefix); - bundlePropertiesFile = rootDir.resolve(bundlePropertiesFileName); + bundleProperties = new BundleProperties(); Path inputDir = rootDir.resolve("input"); stageDir = Files.createDirectories(inputDir.resolve("stage")); @@ -164,38 +195,27 @@ private BundleSupport(NativeImage nativeImage, BundleStatus status) { throw NativeImage.showError("Unable to create bundle directory layout", e); } this.buildArgs = Collections.unmodifiableList(nativeImage.config.getBuildArgs()); - - setBundleLocation(nativeImage.config.getWorkingDirectory(), "unknown"); } - private BundleSupport(NativeImage nativeImage, BundleStatus status, String bundleFilenameArg) { - assert status.loadBundle : "This constructor is used when a previously created bundle gets applied"; - + private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { + Objects.requireNonNull(nativeImage); this.nativeImage = nativeImage; - this.status = status; - Path bundleFile = Path.of(bundleFilenameArg).toAbsolutePath(); - String bundleFileName = bundleFile.getFileName().toString(); - if (!bundleFileName.endsWith(bundleFileExtension)) { - throw NativeImage.showError("The given bundle file " + bundleFileName + " does not end with '" + bundleFileExtension + "'"); - } + loadBundle = true; + writeBundle = false; - if (!Files.isReadable(bundleFile)) { - throw NativeImage.showError("The given bundle file " + bundleFileName + " cannot be read"); - } - - if (Files.isDirectory(bundleFile)) { - throw NativeImage.showError("The given bundle file " + bundleFileName + " is a directory and not a file"); - } + Objects.requireNonNull(bundleFilenameArg); + updateBundleLocation(Path.of(bundleFilenameArg), false); try { rootDir = Files.createTempDirectory(bundleTempDirPrefix); - bundlePropertiesFile = rootDir.resolve(bundlePropertiesFileName); + bundleProperties = new BundleProperties(); outputDir = rootDir.resolve("output"); String originalOutputDirName = outputDir.getFileName().toString() + originalDirExtension; - try (JarFile archive = new JarFile(bundleFile.toFile())) { + Path bundleFilePath = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION); + try (JarFile archive = new JarFile(bundleFilePath.toFile())) { archive.stream().forEach(jarEntry -> { Path bundleEntry = rootDir.resolve(jarEntry.getName()); if (bundleEntry.startsWith(outputDir)) { @@ -214,13 +234,10 @@ private BundleSupport(NativeImage nativeImage, BundleStatus status, String bundl }); } } catch (IOException e) { - throw NativeImage.showError("Unable to create bundle directory layout from file " + bundleFileName, e); + throw NativeImage.showError("Unable to expand bundle directory layout from bundle file " + bundleName + BUNDLE_FILE_EXTENSION, e); } - bundlePath = bundleFile.getParent(); - bundleName = bundleFileName; - - verifyBundleProperties(bundlePropertiesFile); + bundleProperties.loadAndVerify(); try { Path inputDir = rootDir.resolve("input"); @@ -263,26 +280,20 @@ public List getBuildArgs() { Path recordCanonicalization(Path before, Path after) { if (before.startsWith(rootDir)) { - if (nativeImage.isVerbose()) { - System.out.println(("RecordCanonicalization Skip: " + before)); - } + nativeImage.showVerboseMessage(nativeImage.isVVerbose(), "RecordCanonicalization Skip: " + before); return before; } if (after.startsWith(nativeImage.config.getJavaHome())) { return after; } - if (nativeImage.isVerbose()) { - System.out.println("RecordCanonicalization src: " + before + ", dst: " + after); - } + nativeImage.showVerboseMessage(nativeImage.isVVerbose(), "RecordCanonicalization src: " + before + ", dst: " + after); pathCanonicalizations.put(before, after); return after; } Path restoreCanonicalization(Path before) { Path after = pathCanonicalizations.get(before); - if (after != null && nativeImage.isVerbose()) { - System.out.println("RestoreCanonicalization src: " + before + ", dst: " + after); - } + nativeImage.showVerboseMessage(after != null && nativeImage.isVVerbose(), "RestoreCanonicalization src: " + before + ", dst: " + after); return after; } @@ -337,17 +348,13 @@ private Path substitutePath(Path origPath, Path destinationDir) { assert destinationDir.startsWith(rootDir); if (origPath.startsWith(rootDir)) { - if (nativeImage.isVerbose()) { - System.out.println(("RecordSubstitution/RestoreSubstitution Skip: " + origPath)); - } + nativeImage.showVerboseMessage(nativeImage.isVVerbose(), "RecordSubstitution/RestoreSubstitution Skip: " + origPath); return origPath; } Path previousRelativeSubstitutedPath = pathSubstitutions.get(origPath); if (previousRelativeSubstitutedPath != null) { - if (nativeImage.isVerbose()) { - System.out.println("RestoreSubstitution src: " + origPath + ", dst: " + previousRelativeSubstitutedPath); - } + nativeImage.showVerboseMessage(nativeImage.isVVerbose(), "RestoreSubstitution src: " + origPath + ", dst: " + previousRelativeSubstitutedPath); return rootDir.resolve(previousRelativeSubstitutedPath); } @@ -410,14 +417,13 @@ private Path substitutePath(Path origPath, Path destinationDir) { } Path relativeSubstitutedPath = rootDir.relativize(substitutedPath); - if (nativeImage.isVerbose()) { - System.out.println("RecordSubstitution src: " + origPath + ", dst: " + relativeSubstitutedPath); - } + nativeImage.showVerboseMessage(nativeImage.isVVerbose(), "RecordSubstitution src: " + origPath + ", dst: " + relativeSubstitutedPath); pathSubstitutions.put(origPath, relativeSubstitutedPath); return substitutedPath; } private void copyFiles(Path source, Path target, boolean overwrite) { + nativeImage.showVerboseMessage(nativeImage.isVVerbose(), "> Copy files from " + source + " to " + target); if (Files.isDirectory(source)) { try (Stream walk = Files.walk(source)) { walk.forEach(sourcePath -> copyFile(sourcePath, target.resolve(source.relativize(sourcePath)), overwrite)); @@ -431,9 +437,7 @@ private void copyFiles(Path source, Path target, boolean overwrite) { private void copyFile(Path sourceFile, Path target, boolean overwrite) { try { - if (nativeImage.isVerbose()) { - System.out.println("> Copy " + sourceFile + " to " + target); - } + nativeImage.showVerboseMessage(nativeImage.isVVVerbose(), "> Copy " + sourceFile + " to " + target); if (overwrite && Files.isDirectory(sourceFile) && Files.isDirectory(target)) { return; } @@ -445,8 +449,7 @@ private void copyFile(Path sourceFile, Path target, boolean overwrite) { } void complete() { - assert status.buildImage || status.writeBundle : "Superfluous bundle operations"; - final boolean[] firstMessage = {true}; + final boolean[] firstMessage = {!nativeImage.isDryRun()}; Runnable outputNewline = () -> { if (firstMessage[0]) { nativeImage.showNewline(); @@ -454,26 +457,34 @@ void complete() { } }; - if (status.buildImage) { - boolean writeOutput; - try (Stream pathOutputFiles = Files.list(imagePathOutputDir); Stream auxiliaryOutputFiles = Files.list(auxiliaryOutputDir)) { - writeOutput = pathOutputFiles.findAny().isPresent() || auxiliaryOutputFiles.findAny().isPresent(); - } catch (IOException e) { - throw NativeImage.showError("Unable to determine if bundle output should be written."); - } - if (writeOutput) { - Path externalOutputDir = bundlePath.resolve(nativeImage.imageName + "." + outputDir.getFileName()); - copyFiles(outputDir, externalOutputDir, true); - outputNewline.run(); - nativeImage.showMessage("Bundle build output written to " + externalOutputDir); - } + boolean writeOutput; + try (Stream pathOutputFiles = Files.list(imagePathOutputDir); Stream auxiliaryOutputFiles = Files.list(auxiliaryOutputDir)) { + writeOutput = pathOutputFiles.findAny().isPresent() || auxiliaryOutputFiles.findAny().isPresent(); + } catch (IOException e) { + throw NativeImage.showError("Unable to determine if bundle output should be written."); + } + + /* + * In the unlikely case of writing a bundle but no location got specified so far, provide a + * final fallback here. Can happen when something goes wrong in bundle processing itself. + */ + if (bundlePath == null) { + bundlePath = nativeImage.config.getWorkingDirectory(); + bundleName = "unknown"; + } + + if (writeOutput) { + Path externalOutputDir = bundlePath.resolve(bundleName + "." + outputDir.getFileName()); + copyFiles(outputDir, externalOutputDir, true); + outputNewline.run(); + nativeImage.showMessage(bundleInfoMessagePrefix + "Bundle build output written to " + externalOutputDir); } try { - if (status.writeBundle) { + if (writeBundle) { Path bundleFilePath = writeBundle(); outputNewline.run(); - nativeImage.showMessage("Bundle written to " + bundleFilePath); + nativeImage.showMessage(bundleInfoMessagePrefix + "Bundle written to " + bundleFilePath); } } finally { nativeImage.showNewline(); @@ -481,17 +492,41 @@ void complete() { } } - String setBundleLocation(Path imagePath, String imageName) { - String fixedImageName; - if (imageName.endsWith(bundleFileExtension)) { - int endIndex = imageName.length() - bundleFileExtension.length(); - fixedImageName = imageName.substring(0, endIndex); - } else { - fixedImageName = imageName; + void updateBundleLocation(Path bundleFile, boolean redefine) { + if (redefine) { + bundlePath = null; + bundleName = null; + } + + if (bundlePath != null) { + Objects.requireNonNull(bundleName); + /* Bundle location is already set */ + return; + } + Path bundleFilePath = bundleFile.toAbsolutePath(); + String bundleFileName = bundleFile.getFileName().toString(); + if (!bundleFileName.endsWith(BUNDLE_FILE_EXTENSION)) { + throw NativeImage.showError("The given bundle file " + bundleFileName + " does not end with '" + BUNDLE_FILE_EXTENSION + "'"); + } + if (Files.isDirectory(bundleFilePath)) { + throw NativeImage.showError("The given bundle file " + bundleFileName + " is a directory and not a file"); + } + if (loadBundle && !redefine) { + if (!Files.isReadable(bundleFilePath)) { + throw NativeImage.showError("The given bundle file " + bundleFileName + " cannot be read."); + } + } + Path newBundlePath = bundleFilePath.getParent(); + if (writeBundle) { + if (!Files.isWritable(newBundlePath)) { + throw NativeImage.showError("The bundle file directory " + newBundlePath + " is not writeable."); + } + if (Files.exists(bundleFilePath) && !Files.isWritable(bundleFilePath)) { + throw NativeImage.showError("The given bundle file " + bundleFileName + " is not writeable."); + } } - bundlePath = imagePath; - bundleName = fixedImageName + bundleFileExtension; - return fixedImageName; + bundlePath = newBundlePath; + bundleName = bundleFileName.substring(0, bundleFileName.length() - BUNDLE_FILE_EXTENSION.length()); } private Path writeBundle() { @@ -526,13 +561,15 @@ private Path writeBundle() { ArrayList cleanBuildArgs = new ArrayList<>(); for (String buildArg : updatedBuildArgs != null ? updatedBuildArgs : buildArgs) { if (buildArg.startsWith(BUNDLE_OPTION)) { - assert !BundleStatus.valueOf(buildArg.substring(BUNDLE_OPTION.length() + 1)).loadBundle; continue; } if (buildArg.startsWith(nativeImage.oHPath)) { continue; } - if (buildArg.equals(DefaultOptionHandler.verboseOption)) { + if (buildArg.equals(CmdLineOptionHandler.verboseOption)) { + continue; + } + if (buildArg.equals(CmdLineOptionHandler.dryRunOption)) { continue; } if (buildArg.startsWith("-Dllvm.bin.dir=")) { @@ -550,9 +587,9 @@ private Path writeBundle() { throw NativeImage.showError("Failed to write bundle-file " + pathSubstitutionsFile, e); } - writeBundleProperties(bundlePropertiesFile); + bundleProperties.write(); - Path bundleFilePath = this.bundlePath.resolve(bundleName); + Path bundleFilePath = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION); try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(bundleFilePath), createManifest())) { try (Stream walk = Files.walk(rootDir)) { walk.forEach(bundleEntry -> { @@ -567,12 +604,12 @@ private Path writeBundle() { Files.copy(bundleEntry, jarOutStream); jarOutStream.closeEntry(); } catch (IOException e) { - throw NativeImage.showError("Failed to copy " + bundleEntry + " into bundle file " + bundleName, e); + throw NativeImage.showError("Failed to copy " + bundleEntry + " into bundle file " + bundleFilePath.getFileName(), e); } }); } } catch (IOException e) { - throw NativeImage.showError("Failed to create bundle file " + bundleName, e); + throw NativeImage.showError("Failed to create bundle file " + bundleFilePath.getFileName(), e); } return bundleFilePath; @@ -586,61 +623,6 @@ private static Manifest createManifest() { return mf; } - private void writeBundleProperties(Path propertiesFile) { - Properties properties = new Properties(); - properties.setProperty("BundleFileVersionMajor", String.valueOf(bundleFileFormatVersionMajor)); - properties.setProperty("BundleFileVersionMinor", String.valueOf(bundleFileFormatVersionMinor)); - boolean imageBuilt = status.buildImage && !nativeImage.isDryRun(); - properties.setProperty("ImageBuilt", String.valueOf(imageBuilt)); - if (imageBuilt) { - properties.setProperty("BuildContainerized", String.valueOf(false)); - } - properties.setProperty("NativeImagePlatform", NativeImage.platform); - properties.setProperty("NativeImageVersion", NativeImage.getNativeImageVersion()); - NativeImage.ensureDirectoryExists(propertiesFile.getParent()); - try (OutputStream outputStream = Files.newOutputStream(propertiesFile)) { - properties.store(outputStream, "Native Image bundle file properties"); - } catch (IOException e) { - throw NativeImage.showError("Creating bundle properties file failed", e); - } - } - - private void verifyBundleProperties(Path propertiesFile) { - if (!Files.isReadable(propertiesFile)) { - throw NativeImage.showError("The given bundle file " + bundleName + " does not contain a bundle properties file"); - } - - Map properties = NativeImage.loadProperties(propertiesFile); - String fileVersionKey = "BundleFileVersionMajor"; - try { - int major = Integer.parseInt(properties.getOrDefault(fileVersionKey, "-1")); - fileVersionKey = "BundleFileVersionMinor"; - int minor = Integer.parseInt(properties.getOrDefault(fileVersionKey, "-1")); - String message = String.format("The given bundle file %s was created with newer bundle-file-format version %d.%d" + - " (current %d.%d). Update to the latest version of native-image.", bundleName, major, minor, bundleFileFormatVersionMajor, bundleFileFormatVersionMinor); - if (major > bundleFileFormatVersionMajor) { - throw NativeImage.showError(message); - } else if (major == bundleFileFormatVersionMajor) { - if (minor > bundleFileFormatVersionMinor) { - NativeImage.showWarning(message); - } - } - } catch (NumberFormatException e) { - throw NativeImage.showError(fileVersionKey + " in " + bundlePropertiesFileName + " is missing or ill-defined", e); - } - String bundlePlatform = properties.getOrDefault("NativeImagePlatform", "unknown"); - if (!bundlePlatform.equals(NativeImage.platform)) { - NativeImage.showWarning(String.format("The given bundle file %s was created on platform '%s' (current '%s').", - bundleName, bundlePlatform, NativeImage.platform)); - } - String bundleNativeImageVersion = properties.getOrDefault("NativeImageVersion", "unknown"); - if (!bundleNativeImageVersion.equals(NativeImage.getNativeImageVersion())) { - NativeImage.showWarning( - String.format("The given bundle file %s was created with native-image version '%s' (current '%s').", - bundleName, bundleNativeImageVersion, NativeImage.getNativeImageVersion())); - } - } - private static final String substitutionMapSrcField = "src"; private static final String substitutionMapDstField = "dst"; @@ -696,4 +678,82 @@ public void parseAndRegister(Object json, URI origin) { } } } + + private static final Path bundlePropertiesFileName = Path.of("META-INF/nibundle.properties"); + + private final class BundleProperties { + + private static final String PROPERTY_KEY_BUNDLE_FILE_VERSION_MAJOR = "BundleFileVersionMajor"; + private static final String PROPERTY_KEY_BUNDLE_FILE_VERSION_MINOR = "BundleFileVersionMinor"; + private static final String PROPERTY_KEY_IMAGE_BUILT = "ImageBuilt"; + private static final String PROPERTY_KEY_BUILT_WITH_CONTAINER = "BuiltWithContainer"; + private static final String PROPERTY_KEY_NATIVE_IMAGE_PLATFORM = "NativeImagePlatform"; + private static final String PROPERTY_KEY_NATIVE_IMAGE_VERSION = "NativeImageVersion"; + + private final Path bundlePropertiesFile; + private final Map properties; + + private BundleProperties() { + Objects.requireNonNull(rootDir); + Objects.requireNonNull(nativeImage); + + bundlePropertiesFile = rootDir.resolve(bundlePropertiesFileName); + properties = new HashMap<>(); + } + + private void loadAndVerify() { + Objects.requireNonNull(bundleName); + + String bundleFileName = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION).toString(); + if (!Files.isReadable(bundlePropertiesFile)) { + throw NativeImage.showError("The given bundle file " + bundleFileName + " does not contain a bundle properties file"); + } + + properties.putAll(NativeImage.loadProperties(bundlePropertiesFile)); + String fileVersionKey = PROPERTY_KEY_BUNDLE_FILE_VERSION_MAJOR; + try { + int major = Integer.parseInt(properties.getOrDefault(fileVersionKey, "-1")); + fileVersionKey = PROPERTY_KEY_BUNDLE_FILE_VERSION_MINOR; + int minor = Integer.parseInt(properties.getOrDefault(fileVersionKey, "-1")); + String message = String.format("The given bundle file %s was created with newer bundle-file-format version %d.%d" + + " (current %d.%d). Update to the latest version of native-image.", bundleFileName, major, minor, bundleFileFormatVersionMajor, bundleFileFormatVersionMinor); + if (major > bundleFileFormatVersionMajor) { + throw NativeImage.showError(message); + } else if (major == bundleFileFormatVersionMajor) { + if (minor > bundleFileFormatVersionMinor) { + NativeImage.showWarning(message); + } + } + } catch (NumberFormatException e) { + throw NativeImage.showError(fileVersionKey + " in " + bundlePropertiesFileName + " is missing or ill-defined", e); + } + String bundleVersion = properties.getOrDefault(PROPERTY_KEY_NATIVE_IMAGE_VERSION, "unknown"); + String currentVersion = bundleVersion.equals(NativeImage.getNativeImageVersion()) ? "" : " != '" + NativeImage.getNativeImageVersion() + "'"; + String bundlePlatform = properties.getOrDefault(PROPERTY_KEY_NATIVE_IMAGE_PLATFORM, "unknown"); + String currentPlatform = bundlePlatform.equals(NativeImage.platform) ? "" : " != '" + NativeImage.platform + "'"; + nativeImage.showNewline(); + nativeImage.showMessage(String.format("%sLoaded Bundle from %s", bundleInfoMessagePrefix, bundleFileName)); + nativeImage.showMessage(String.format("%sVersion: '%s'%s, Platform: '%s'%s", bundleInfoMessagePrefix, bundleVersion, currentVersion, bundlePlatform, currentPlatform)); + } + + private void write() { + properties.put(PROPERTY_KEY_BUNDLE_FILE_VERSION_MAJOR, String.valueOf(bundleFileFormatVersionMajor)); + properties.put(PROPERTY_KEY_BUNDLE_FILE_VERSION_MINOR, String.valueOf(bundleFileFormatVersionMinor)); + boolean imageBuilt = !nativeImage.isDryRun(); + properties.put(PROPERTY_KEY_IMAGE_BUILT, String.valueOf(imageBuilt)); + if (imageBuilt) { + properties.put(PROPERTY_KEY_BUILT_WITH_CONTAINER, String.valueOf(false)); + } + properties.put(PROPERTY_KEY_NATIVE_IMAGE_PLATFORM, NativeImage.platform); + properties.put(PROPERTY_KEY_NATIVE_IMAGE_VERSION, NativeImage.getNativeImageVersion()); + NativeImage.ensureDirectoryExists(bundlePropertiesFile.getParent()); + try (OutputStream outputStream = Files.newOutputStream(bundlePropertiesFile)) { + Properties p = new Properties(); + p.putAll(properties); + p.store(outputStream, "Native Image bundle file properties"); + } catch (IOException e) { + throw NativeImage.showError("Creating bundle properties file " + bundlePropertiesFileName + " failed", e); + } + } + } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java index db3ca54ad33e..08f89f1f8d64 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java @@ -26,6 +26,7 @@ import java.io.File; import java.nio.file.Paths; +import java.util.List; import java.util.regex.Pattern; import org.graalvm.compiler.options.OptionType; @@ -40,12 +41,13 @@ class CmdLineOptionHandler extends NativeImage.OptionHandler { private static final String helpText = NativeImage.getResource("/Help.txt"); private static final String helpExtraText = NativeImage.getResource("/HelpExtra.txt"); + static final String verboseOption = "--verbose"; + static final String dryRunOption = "--dry-run"; + static final String debugAttachOption = "--debug-attach"; /* Defunct legacy options that we have to accept to maintain backward compatibility */ private static final String verboseServerOption = "--verbose-server"; private static final String serverOptionPrefix = "--server-"; - public static final String DEBUG_ATTACH_OPTION = "--debug-attach"; - private static final String javaRuntimeVersion = System.getProperty("java.runtime.version"); boolean useDebugAttach = false; @@ -119,11 +121,11 @@ private boolean consume(ArgumentQueue args, String headArg) { } nativeImage.addExcludeConfig(Pattern.compile(excludeJar), Pattern.compile(excludeConfig)); return true; - case DefaultOptionHandler.verboseOption: + case verboseOption: args.poll(); - nativeImage.setVerbose(true); + nativeImage.addVerbose(); return true; - case "--dry-run": + case dryRunOption: args.poll(); nativeImage.setDryRun(true); return true; @@ -151,13 +153,13 @@ private boolean consume(ArgumentQueue args, String headArg) { return true; } - if (headArg.startsWith(DEBUG_ATTACH_OPTION)) { + if (headArg.startsWith(debugAttachOption)) { if (useDebugAttach) { - throw NativeImage.showError("The " + DEBUG_ATTACH_OPTION + " option can only be used once."); + throw NativeImage.showError("The " + debugAttachOption + " option can only be used once."); } useDebugAttach = true; String debugAttachArg = args.poll(); - String addressSuffix = debugAttachArg.substring(DEBUG_ATTACH_OPTION.length()); + String addressSuffix = debugAttachArg.substring(debugAttachOption.length()); String address = addressSuffix.isEmpty() ? "8000" : addressSuffix.substring(1); /* Using agentlib to allow interoperability with other agents */ nativeImage.addImageBuilderJavaArgs("-agentlib:jdwp=transport=dt_socket,server=y,address=" + address + ",suspend=y"); @@ -188,4 +190,11 @@ private static void singleArgumentCheck(ArgumentQueue args, String arg) { NativeImage.showError("Option " + arg + " cannot be combined with other options."); } } + + @Override + void addFallbackBuildArgs(List buildArgs) { + if (nativeImage.isVerbose()) { + buildArgs.add(verboseOption); + } + } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java index bf58c02305e3..1cef750e4dcd 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java @@ -39,7 +39,6 @@ class DefaultOptionHandler extends NativeImage.OptionHandler { - static final String verboseOption = "--verbose"; private static final String requireValidJarFileMessage = "-jar requires a valid jarfile"; private static final String newStyleClasspathOptionName = "--class-path"; @@ -437,11 +436,4 @@ private void handleJarFileArg(Path jarFilePath) { } nativeImage.addCustomImageClasspath(finalFilePath); } - - @Override - void addFallbackBuildArgs(List buildArgs) { - if (nativeImage.isVerbose()) { - buildArgs.add(verboseOption); - } - } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 4ea738356fc1..2cbb09427189 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -265,7 +265,7 @@ private static String oR(OptionKey option) { final Map userConfigProperties = new HashMap<>(); private final Map propertyFileSubstitutionValues = new HashMap<>(); - private boolean verbose = Boolean.valueOf(System.getenv("VERBOSE_GRAALVM_LAUNCHERS")); + private int verbose = Boolean.valueOf(System.getenv("VERBOSE_GRAALVM_LAUNCHERS")) ? 1 : 0; private boolean diagnostics = false; String diagnosticsDir; private boolean jarOptionMode = false; @@ -693,6 +693,7 @@ private ArrayList createFallbackBuildArgs() { buildArgs.add(FallbackExecutor.class.getName()); buildArgs.add(imageName); + defaultOptionHandler.addFallbackBuildArgs(buildArgs); for (OptionHandler handler : optionHandlers) { handler.addFallbackBuildArgs(buildArgs); } @@ -1148,11 +1149,11 @@ private int completeImageBuild() { if (useBundle()) { /* * In creation-mode, we are at the point where we know the final imagePath and imageName - * that we can now use to derive the new bundle name from. For apply-mode setting - * imagePath determines where to copy the bundle output to. + * that we can now use to derive a bundle name in case non was set so far. */ - imageName = bundleSupport.setBundleLocation(imagePath, imageName); - updateArgumentEntryValue(imageBuilderArgs, imageNameEntry, imageName); + String bundleName = imageName.endsWith(BundleSupport.BUNDLE_FILE_EXTENSION) ? imageName : imageName + BundleSupport.BUNDLE_FILE_EXTENSION; + bundleSupport.updateBundleLocation(imagePath.resolve(bundleName), false); + /* The imagePath has to be redirected to be within the bundle */ imagePath = bundleSupport.substituteImagePath(imagePath); /* and we need to adjust the argument that passes the imagePath to the builder */ @@ -1249,7 +1250,7 @@ private boolean shouldAddCWDToCP() { return false; } - if (useBundle() && bundleSupport.status.loadBundle) { + if (useBundle() && bundleSupport.loadBundle) { /* If bundle was loaded we have valid -cp and/or -p from within the bundle */ return false; } @@ -1275,7 +1276,7 @@ private List getAgentArguments() { if (!agentOptions.isEmpty()) { if (useDebugAttach()) { - throw NativeImage.showError(CmdLineOptionHandler.DEBUG_ATTACH_OPTION + " cannot be used with class initialization/object instantiation tracing (" + oHTraceClassInitialization + + throw NativeImage.showError(CmdLineOptionHandler.debugAttachOption + " cannot be used with class initialization/object instantiation tracing (" + oHTraceClassInitialization + "/ + " + oHTraceObjectInstantiation + ")."); } args.add("-agentlib:native-image-diagnostics-agent=" + agentOptions); @@ -1426,12 +1427,12 @@ protected int buildImage(List javaArgs, LinkedHashSet cp, LinkedHa // write to the diagnostics dir ReportUtils.report("command line arguments", diagnosticsDir, "command-line", "txt", printWriter -> printWriter.write(commandLine)); } else { - showVerboseMessage(isVerbose() || dryRun, "Executing ["); - showVerboseMessage(isVerbose() || dryRun, commandLine); - showVerboseMessage(isVerbose() || dryRun, "]"); + showVerboseMessage(isVerbose(), "Executing ["); + showVerboseMessage(isVerbose(), commandLine); + showVerboseMessage(isVerbose(), "]"); } - if (dryRun || useBundle() && !bundleSupport.status.buildImage) { + if (dryRun) { return ExitStatus.OK.getValue(); } @@ -1751,15 +1752,16 @@ void addCustomJavaArgs(String javaArg) { customJavaArgs.add(javaArg); } - void setVerbose(boolean val) { - verbose = val; + void addVerbose() { + verbose += 1; } void setDiagnostics(boolean val) { diagnostics = val; diagnosticsDir = Paths.get("reports", ReportUtils.timeStampedFileName("diagnostics", "")).toString(); if (val) { - verbose = true; + addVerbose(); + addVerbose(); } } @@ -1780,7 +1782,15 @@ private void enableModulePathBuild() { } boolean isVerbose() { - return verbose; + return verbose > 0; + } + + boolean isVVerbose() { + return verbose > 1; + } + + boolean isVVVerbose() { + return verbose > 2; } boolean isDiagnostics() { From 727c94b6df50804d7e49481419a112956c07c564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 24 Jan 2023 13:35:48 +0100 Subject: [PATCH 36/46] Allow subdirectories within /tmp to be captured in bundles --- .../src/com/oracle/svm/driver/BundleSupport.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 50577349bdd7..3b22aa56ffe8 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -39,11 +39,13 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Properties; +import java.util.Set; import java.util.jar.Attributes; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -365,10 +367,16 @@ private Path substitutePath(Path origPath, Path destinationDir) { boolean forbiddenPath = false; if (!OS.WINDOWS.isCurrent()) { - for (Path path : ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES) { - if (origPath.startsWith(path)) { - forbiddenPath = true; - break; + Path tmpPath = ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES_ROOT.resolve("tmp"); + boolean subdirInTmp = origPath.startsWith(tmpPath) && !origPath.equals(tmpPath); + if (!subdirInTmp) { + Set forbiddenPaths = new HashSet<>(ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES); + forbiddenPaths.add(rootDir); + for (Path path : forbiddenPaths) { + if (origPath.startsWith(path)) { + forbiddenPath = true; + break; + } } } } From cfd883acb42419166d75469b9d0f610a7e12fcfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 24 Jan 2023 13:36:30 +0100 Subject: [PATCH 37/46] Add BundleFileCreationTimestamp to BundleProperties --- .../com/oracle/svm/driver/BundleSupport.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 3b22aa56ffe8..9ed3bd9e0484 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -34,6 +34,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.format.FormatStyle; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -693,6 +697,7 @@ private final class BundleProperties { private static final String PROPERTY_KEY_BUNDLE_FILE_VERSION_MAJOR = "BundleFileVersionMajor"; private static final String PROPERTY_KEY_BUNDLE_FILE_VERSION_MINOR = "BundleFileVersionMinor"; + private static final String PROPERTY_KEY_BUNDLE_FILE_CREATION_TIMESTAMP = "BundleFileCreationTimestamp"; private static final String PROPERTY_KEY_IMAGE_BUILT = "ImageBuilt"; private static final String PROPERTY_KEY_BUILT_WITH_CONTAINER = "BuiltWithContainer"; private static final String PROPERTY_KEY_NATIVE_IMAGE_PLATFORM = "NativeImagePlatform"; @@ -739,14 +744,24 @@ private void loadAndVerify() { String currentVersion = bundleVersion.equals(NativeImage.getNativeImageVersion()) ? "" : " != '" + NativeImage.getNativeImageVersion() + "'"; String bundlePlatform = properties.getOrDefault(PROPERTY_KEY_NATIVE_IMAGE_PLATFORM, "unknown"); String currentPlatform = bundlePlatform.equals(NativeImage.platform) ? "" : " != '" + NativeImage.platform + "'"; + String bundleCreationTimestamp = properties.getOrDefault(PROPERTY_KEY_BUNDLE_FILE_CREATION_TIMESTAMP, ""); + String localDateStr; + try { + ZonedDateTime dateTime = ZonedDateTime.parse(bundleCreationTimestamp, DateTimeFormatter.ISO_DATE_TIME); + localDateStr = dateTime.format(DateTimeFormatter.ofLocalizedDateTime(FormatStyle.FULL)); + } catch (DateTimeParseException e) { + localDateStr = "unknown time"; + } nativeImage.showNewline(); nativeImage.showMessage(String.format("%sLoaded Bundle from %s", bundleInfoMessagePrefix, bundleFileName)); - nativeImage.showMessage(String.format("%sVersion: '%s'%s, Platform: '%s'%s", bundleInfoMessagePrefix, bundleVersion, currentVersion, bundlePlatform, currentPlatform)); + nativeImage.showMessage(String.format("%sBundle created at '%s'", bundleInfoMessagePrefix, localDateStr)); + nativeImage.showMessage(String.format("%sUsing version: '%s'%s on platform: '%s'%s", bundleInfoMessagePrefix, bundleVersion, currentVersion, bundlePlatform, currentPlatform)); } private void write() { properties.put(PROPERTY_KEY_BUNDLE_FILE_VERSION_MAJOR, String.valueOf(bundleFileFormatVersionMajor)); properties.put(PROPERTY_KEY_BUNDLE_FILE_VERSION_MINOR, String.valueOf(bundleFileFormatVersionMinor)); + properties.put(PROPERTY_KEY_BUNDLE_FILE_CREATION_TIMESTAMP, ZonedDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME)); boolean imageBuilt = !nativeImage.isDryRun(); properties.put(PROPERTY_KEY_IMAGE_BUILT, String.valueOf(imageBuilt)); if (imageBuilt) { From 95d7fdab240448e6ef6ff32768a53c19aebafc69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 25 Jan 2023 20:03:20 +0100 Subject: [PATCH 38/46] Fold isDirectory-filtering into stream --- .../src/com/oracle/svm/driver/BundleSupport.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 9ed3bd9e0484..f7ed76270eed 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -50,6 +50,7 @@ import java.util.Optional; import java.util.Properties; import java.util.Set; +import java.util.function.Predicate; import java.util.jar.Attributes; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -604,10 +605,7 @@ private Path writeBundle() { Path bundleFilePath = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION); try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(bundleFilePath), createManifest())) { try (Stream walk = Files.walk(rootDir)) { - walk.forEach(bundleEntry -> { - if (Files.isDirectory(bundleEntry)) { - return; - } + walk.filter(Predicate.not(Files::isDirectory)).forEach(bundleEntry -> { String jarEntryName = rootDir.relativize(bundleEntry).toString(); JarEntry entry = new JarEntry(jarEntryName.replace(File.separator, "/")); try { From 5e2f223e645d01718f7b11a910e4c99d3b27d7fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 25 Jan 2023 20:07:31 +0100 Subject: [PATCH 39/46] Fix typos --- .../src/com/oracle/svm/driver/BundleSupport.java | 8 ++++---- .../src/com/oracle/svm/driver/NativeImage.java | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index f7ed76270eed..56060e3a2376 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -132,14 +132,14 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma case apply: if (nativeImage.useBundle()) { if (nativeImage.bundleSupport.loadBundle) { - throw NativeImage.showError(String.format("Native-image allows option %s to be specified only once.", applyOptionStr)); + throw NativeImage.showError(String.format("native-image allows option %s to be specified only once.", applyOptionStr)); } if (nativeImage.bundleSupport.writeBundle) { - throw NativeImage.showError(String.format("Native-image option %s is not allowed to be used after option %s.", applyOptionStr, createOptionStr)); + throw NativeImage.showError(String.format("native-image option %s is not allowed to be used after option %s.", applyOptionStr, createOptionStr)); } } if (bundleFilename == null) { - throw NativeImage.showError(String.format("Native-image option %s requires a bundle file argument. E.g. %s=bundle-file.nib.", applyOptionStr, applyOptionStr)); + throw NativeImage.showError(String.format("native-image option %s requires a bundle file argument. E.g. %s=bundle-file.nib.", applyOptionStr, applyOptionStr)); } bundleSupport = new BundleSupport(nativeImage, bundleFilename); /* Inject the command line args from the loaded bundle in-place */ @@ -154,7 +154,7 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma case create: if (nativeImage.useBundle()) { if (nativeImage.bundleSupport.writeBundle) { - throw NativeImage.showError(String.format("Native-image allows option %s to be specified only once.", bundleArg)); + throw NativeImage.showError(String.format("native-image allows option %s to be specified only once.", bundleArg)); } else { bundleSupport = nativeImage.bundleSupport; bundleSupport.writeBundle = true; diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 2cbb09427189..bac4fc03c6d4 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1149,7 +1149,7 @@ private int completeImageBuild() { if (useBundle()) { /* * In creation-mode, we are at the point where we know the final imagePath and imageName - * that we can now use to derive a bundle name in case non was set so far. + * that we can now use to derive a bundle name in case none was set so far. */ String bundleName = imageName.endsWith(BundleSupport.BUNDLE_FILE_EXTENSION) ? imageName : imageName + BundleSupport.BUNDLE_FILE_EXTENSION; bundleSupport.updateBundleLocation(imagePath.resolve(bundleName), false); From 95ff820286c1126302a5270671e5f1a6d826d730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 26 Jan 2023 12:47:28 +0100 Subject: [PATCH 40/46] Add --enable-experimental-bundle-support unlocking option --- .../src/com/oracle/svm/driver/BundleSupport.java | 10 ++++++++++ .../com/oracle/svm/driver/CmdLineOptionHandler.java | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 56060e3a2376..3387c3a62313 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -104,6 +104,9 @@ final class BundleSupport { private final BundleProperties bundleProperties; + static boolean allowBundleSupport; + static final String UNLOCK_BUNDLE_SUPPORT_OPTION = "--enable-experimental-bundle-support"; + static final String BUNDLE_OPTION = "--bundle"; static final String BUNDLE_FILE_EXTENSION = ".nib"; @@ -113,6 +116,10 @@ private enum BundleOptionVariants { } static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeImage.ArgumentQueue args) { + if (!allowBundleSupport) { + throw NativeImage.showError("Bundle support is still experimental and needs to be unlocked with " + UNLOCK_BUNDLE_SUPPORT_OPTION); + } + if (!nativeImage.userConfigProperties.isEmpty()) { throw NativeImage.showError("Bundle support cannot be combined with " + NativeImage.CONFIG_FILE_ENV_VAR_KEY + " environment variable use."); } @@ -573,6 +580,9 @@ private Path writeBundle() { try (JsonWriter writer = new JsonWriter(buildArgsFile)) { ArrayList cleanBuildArgs = new ArrayList<>(); for (String buildArg : updatedBuildArgs != null ? updatedBuildArgs : buildArgs) { + if (buildArg.equals(UNLOCK_BUNDLE_SUPPORT_OPTION)) { + continue; + } if (buildArg.startsWith(BUNDLE_OPTION)) { continue; } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java index 08f89f1f8d64..9760961da362 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java @@ -142,6 +142,10 @@ private boolean consume(ArgumentQueue args, String headArg) { String optionNames = args.poll(); nativeImage.setPrintFlagsWithExtraHelpOptionQuery(optionNames); return true; + case BundleSupport.UNLOCK_BUNDLE_SUPPORT_OPTION: + args.poll(); + BundleSupport.allowBundleSupport = true; + return true; case verboseServerOption: args.poll(); NativeImage.showWarning("Ignoring server-mode native-image argument " + headArg + "."); From 1a39de07f48b3e53e03fa1bc566b8751c6358e85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 26 Jan 2023 16:00:20 +0100 Subject: [PATCH 41/46] Resolve class path with base name + asterisk early --- .../oracle/svm/core/util/ClasspathUtils.java | 31 --------- .../oracle/svm/driver/MacroOptionHandler.java | 13 ++-- .../com/oracle/svm/driver/NativeImage.java | 68 +++++++++---------- .../metainf/NativeImageMetaInfWalker.java | 34 +++------- .../hosted/NativeImageClassLoaderSupport.java | 21 +++--- 5 files changed, 58 insertions(+), 109 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/ClasspathUtils.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/ClasspathUtils.java index 53f335f716dc..e660b98a650c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/ClasspathUtils.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/ClasspathUtils.java @@ -24,44 +24,13 @@ */ package com.oracle.svm.core.util; -import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.regex.Pattern; - -import com.oracle.svm.core.OS; public final class ClasspathUtils { - public static final String cpWildcardSubstitute = "$JavaCla$$pathWildcard$ubstitute$"; - - public static Path stringToClasspath(String cp) { - String separators = Pattern.quote(File.separator); - if (OS.getCurrent().equals(OS.WINDOWS)) { - separators += "/"; /* on Windows also / is accepted as valid separator */ - } - String[] components = cp.split("[" + separators + "]", Integer.MAX_VALUE); - for (int i = 0; i < components.length; i++) { - if (components[i].equals("*")) { - components[i] = cpWildcardSubstitute; - } - } - return Paths.get(String.join(File.separator, components)); - } - - public static String classpathToString(Path cp) { - String[] components = cp.toString().split(Pattern.quote(File.separator), Integer.MAX_VALUE); - for (int i = 0; i < components.length; i++) { - if (components[i].equals(cpWildcardSubstitute)) { - components[i] = "*"; - } - } - return String.join(File.separator, components); - } - public static boolean isJar(Path p) { Path fn = p.getFileName(); assert fn != null; diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java index 5965ddcd1b01..a07601398aa6 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java @@ -30,7 +30,6 @@ import com.oracle.svm.core.OS; import com.oracle.svm.core.option.OptionUtils.InvalidMacroException; -import com.oracle.svm.core.util.ClasspathUtils; import com.oracle.svm.driver.MacroOption.AddedTwiceException; import com.oracle.svm.driver.MacroOption.VerboseInvalidMacroException; import com.oracle.svm.driver.NativeImage.ArgumentQueue; @@ -94,12 +93,12 @@ private void applyEnabled(MacroOption.EnabledOption enabledOption, String argume config.modulePathBuild = modulePathBuild; } - enabledOption.forEachPropertyValue(config, "ImageBuilderClasspath", entry -> nativeImage.addImageBuilderClasspath(ClasspathUtils.stringToClasspath(entry)), PATH_SEPARATOR_REGEX); - - boolean explicitImageModulePath = enabledOption.forEachPropertyValue( - config, "ImageModulePath", entry -> nativeImage.addImageModulePath(ClasspathUtils.stringToClasspath(entry)), PATH_SEPARATOR_REGEX); - boolean explicitImageClasspath = enabledOption.forEachPropertyValue( - config, "ImageClasspath", entry -> nativeImage.addImageClasspath(ClasspathUtils.stringToClasspath(entry)), PATH_SEPARATOR_REGEX); + enabledOption.forEachPropertyValue(config, + "ImageBuilderClasspath", entry -> nativeImage.addImageBuilderClasspath(Path.of(entry)), PATH_SEPARATOR_REGEX); + boolean explicitImageModulePath = enabledOption.forEachPropertyValue(config, + "ImageModulePath", entry -> nativeImage.addImageModulePath(Path.of((entry))), PATH_SEPARATOR_REGEX); + boolean explicitImageClasspath = enabledOption.forEachPropertyValue(config, + "ImageClasspath", entry -> nativeImage.addImageClasspath(Path.of((entry))), PATH_SEPARATOR_REGEX); if (!explicitImageModulePath && !explicitImageClasspath) { NativeImage.getJars(imageJarsDirectory).forEach(nativeImage::addImageClasspath); } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index bac4fc03c6d4..28cea8cb52c1 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -674,7 +674,7 @@ private ArrayList createFallbackBuildArgs() { return path; } }) - .map(ClasspathUtils::classpathToString) + .map(Path::toString) .collect(Collectors.joining(File.pathSeparator)); if (!isPortable[0]) { showWarning("The produced fallback image will not be portable, because not all classpath entries" + @@ -927,23 +927,6 @@ static String injectHostedOptionOrigin(String option, String origin) { return option; } - static void processManifestMainAttributes(Path path, BiConsumer manifestConsumer) { - if (path.endsWith(ClasspathUtils.cpWildcardSubstitute)) { - if (!Files.isDirectory(path.getParent())) { - throw NativeImage.showError("Cannot expand wildcard: '" + path + "' is not a directory"); - } - try { - Files.list(path.getParent()) - .filter(ClasspathUtils::isJar) - .forEach(p -> processJarManifestMainAttributes(p, manifestConsumer)); - } catch (IOException e) { - throw NativeImage.showError("Error while expanding wildcard for '" + path + "'", e); - } - } else if (ClasspathUtils.isJar(path)) { - processJarManifestMainAttributes(path, manifestConsumer); - } - } - static boolean processJarManifestMainAttributes(Path jarFilePath, BiConsumer manifestConsumer) { try (JarFile jarFile = new JarFile(jarFilePath.toFile())) { Manifest manifest = jarFile.getManifest(); @@ -971,7 +954,7 @@ void handleClassPathAttribute(LinkedHashSet destination, Path jarFilePath, /* Missing Class-Path Attribute is tolerable */ if (classPathValue != null) { for (String cp : classPathValue.split(" +")) { - Path manifestClassPath = ClasspathUtils.stringToClasspath(cp); + Path manifestClassPath = Path.of(cp); if (!manifestClassPath.isAbsolute()) { /* Resolve relative manifestClassPath against directory containing jar */ manifestClassPath = jarFilePath.getParent().resolve(manifestClassPath); @@ -1333,7 +1316,7 @@ protected static List createImageBuilderArgs(List imageArgs, Lis List result = new ArrayList<>(); if (!imagecp.isEmpty()) { result.add(SubstrateOptions.IMAGE_CLASSPATH_PREFIX); - result.add(imagecp.stream().map(ClasspathUtils::classpathToString).collect(Collectors.joining(File.pathSeparator))); + result.add(imagecp.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator))); } if (!imagemp.isEmpty()) { result.add(SubstrateOptions.IMAGE_MODULEPATH_PREFIX); @@ -1570,24 +1553,14 @@ Path canonicalize(Path path, boolean strict) { if (!strict) { return bundleSupport != null ? bundleSupport.recordCanonicalization(path, absolutePath) : absolutePath; } - boolean hasWildcard = absolutePath.endsWith(ClasspathUtils.cpWildcardSubstitute); - if (hasWildcard) { - absolutePath = absolutePath.getParent(); - } try { Path realPath = absolutePath.toRealPath(); if (!Files.isReadable(realPath)) { - showError("Path entry " + ClasspathUtils.classpathToString(path) + " is not readable"); - } - if (hasWildcard) { - if (!Files.isDirectory(realPath)) { - showError("Path entry with wildcard " + ClasspathUtils.classpathToString(path) + " is not a directory"); - } - realPath = realPath.resolve(ClasspathUtils.cpWildcardSubstitute); + showError("Path entry " + path + " is not readable"); } return bundleSupport != null ? bundleSupport.recordCanonicalization(path, realPath) : realPath; } catch (IOException e) { - throw showError("Invalid Path entry " + ClasspathUtils.classpathToString(path), e); + throw showError("Invalid Path entry " + path, e); } } @@ -1712,7 +1685,32 @@ void addImageModulePath(Path modulePathEntry, boolean strict) { * strings same as java -cp (is tolerant against invalid classpath entries). */ void addCustomImageClasspath(String classpath) { - addImageClasspathEntry(customImageClasspath, ClasspathUtils.stringToClasspath(classpath), false); + for (Path path : expandAsteriskClassPathElement(classpath)) { + addImageClasspathEntry(customImageClasspath, path, false); + } + } + + public List expandAsteriskClassPathElement(String cp) { + String separators = Pattern.quote(File.separator); + if (OS.getCurrent().equals(OS.WINDOWS)) { + separators += "/"; /* on Windows also / is accepted as valid separator */ + } + List components = new ArrayList<>(List.of(cp.split("[" + separators + "]"))); + int lastElementIndex = components.size() - 1; + if (lastElementIndex >= 0 && "*".equals(components.get(lastElementIndex))) { + components.remove(lastElementIndex); + Path searchDir = Path.of(String.join(File.separator, components)); + try (Stream filesInSearchDir = Files.list(searchDir)) { + return filesInSearchDir.filter(NativeImage::hasJarFileSuffix).collect(Collectors.toList()); + } catch (IOException e) { + throw NativeImage.showError("Class path element asterisk (*) expansion failed for directory " + searchDir); + } + } + return List.of(Path.of(cp)); + } + + private static boolean hasJarFileSuffix(Path p) { + return p.getFileName().toString().toLowerCase().endsWith(".jar"); } /** @@ -1743,7 +1741,9 @@ private void addImageClasspathEntry(LinkedHashSet destination, Path classp Path classpathEntryFinal = bundleSupport != null ? bundleSupport.substituteModulePath(classpathEntry) : classpathEntry; if (!imageClasspath.contains(classpathEntryFinal) && !customImageClasspath.contains(classpathEntryFinal)) { destination.add(classpathEntryFinal); - processManifestMainAttributes(classpathEntryFinal, (jarFilePath, attributes) -> handleClassPathAttribute(destination, jarFilePath, attributes)); + if (ClasspathUtils.isJar(classpathEntryFinal)) { + processJarManifestMainAttributes(classpathEntryFinal, (jarFilePath, attributes) -> handleClassPathAttribute(destination, jarFilePath, attributes)); + } processClasspathNativeImageMetaInf(classpathEntryFinal); } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/metainf/NativeImageMetaInfWalker.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/metainf/NativeImageMetaInfWalker.java index df5d2335411d..389331073061 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/metainf/NativeImageMetaInfWalker.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/metainf/NativeImageMetaInfWalker.java @@ -24,19 +24,20 @@ */ package com.oracle.svm.driver.metainf; -import com.oracle.svm.core.util.ClasspathUtils; import java.io.IOException; import java.net.URI; import java.nio.file.FileSystem; import java.nio.file.FileSystemNotFoundException; import java.nio.file.FileSystems; import java.nio.file.Files; -import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.oracle.svm.core.util.ClasspathUtils; public class NativeImageMetaInfWalker { @@ -60,38 +61,25 @@ public static void walkMetaInfForCPEntry(Path classpathEntry, NativeImageMetaInf Path nativeImageMetaInfBase = classpathEntry.resolve(Paths.get(nativeImageMetaInf)); processNativeImageMetaInf(classpathEntry, nativeImageMetaInfBase, metaInfProcessor); } else { - List jarFileMatches = Collections.emptyList(); - if (classpathEntry.endsWith(ClasspathUtils.cpWildcardSubstitute)) { - try { - jarFileMatches = Files.list(classpathEntry.getParent()) - .filter(ClasspathUtils::isJar) - .collect(Collectors.toList()); - } catch (NoSuchFileException e) { - /* Fallthrough */ - } - } else if (ClasspathUtils.isJar(classpathEntry)) { - jarFileMatches = Collections.singletonList(classpathEntry); - } - - for (Path jarFile : jarFileMatches) { - URI jarFileURI = URI.create("jar:" + jarFile.toUri()); + if (ClasspathUtils.isJar(classpathEntry)) { + URI jarFileURI = URI.create("jar:" + classpathEntry.toUri()); FileSystem probeJarFS; try { probeJarFS = FileSystems.newFileSystem(jarFileURI, Collections.emptyMap()); } catch (UnsupportedOperationException e) { probeJarFS = null; - metaInfProcessor.showWarning(ClasspathUtils.classpathToString(classpathEntry) + " does not describe valid jarfile" + (jarFileMatches.size() > 1 ? "s" : "")); + metaInfProcessor.showWarning(classpathEntry + " does not describe valid jar-file"); } if (probeJarFS != null) { try (FileSystem jarFS = probeJarFS) { Path nativeImageMetaInfBase = jarFS.getPath("/" + nativeImageMetaInf); - processNativeImageMetaInf(jarFile, nativeImageMetaInfBase, metaInfProcessor); + processNativeImageMetaInf(classpathEntry, nativeImageMetaInfBase, metaInfProcessor); } } } } } catch (IOException | FileSystemNotFoundException e) { - throw new MetaInfWalkException("Invalid classpath entry " + ClasspathUtils.classpathToString(classpathEntry), e); + throw new MetaInfWalkException("Invalid classpath entry " + classpathEntry, e); } } @@ -99,10 +87,8 @@ private static void processNativeImageMetaInf(Path classpathEntry, Path nativeIm if (Files.isDirectory(nativeImageMetaInfBase)) { for (MetaInfFileType fileType : MetaInfFileType.values()) { List nativeImageMetaInfFiles; - try { - nativeImageMetaInfFiles = Files.walk(nativeImageMetaInfBase) - .filter(p -> p.endsWith(fileType.fileName)) - .collect(Collectors.toList()); + try (Stream pathStream = Files.walk(nativeImageMetaInfBase)) { + nativeImageMetaInfFiles = pathStream.filter(p -> p.endsWith(fileType.fileName)).collect(Collectors.toList()); } catch (IOException e) { throw new MetaInfWalkException("Processing " + nativeImageMetaInfBase.toUri() + " failed.", e); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java index 05287c453e77..686ba5943b43 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java @@ -71,7 +71,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import com.oracle.svm.util.ClassUtil; import org.graalvm.collections.EconomicMap; import org.graalvm.collections.EconomicSet; import org.graalvm.collections.MapCursor; @@ -90,6 +89,7 @@ import com.oracle.svm.core.util.VMError; import com.oracle.svm.hosted.annotation.SubstrateAnnotationExtractor; import com.oracle.svm.hosted.option.HostedOptionParser; +import com.oracle.svm.util.ClassUtil; import com.oracle.svm.util.ModuleSupport; import com.oracle.svm.util.ReflectionUtil; @@ -218,7 +218,7 @@ public boolean noEntryForURI(EconomicSet set) { protected static class Util { static URL[] verifyClassPathAndConvertToURLs(String[] classpath) { - Stream pathStream = new LinkedHashSet<>(Arrays.asList(classpath)).stream().flatMap(Util::toClassPathEntries); + Stream pathStream = new LinkedHashSet<>(Arrays.asList(classpath)).stream().map(Path::of).filter(Util::verifyClassPathEntry); return pathStream.map(v -> { try { return toRealPath(v).toUri().toURL(); @@ -236,19 +236,14 @@ static Path toRealPath(Path p) { } } - static Stream toClassPathEntries(String classPathEntry) { - Path entry = ClasspathUtils.stringToClasspath(classPathEntry); - if (entry.endsWith(ClasspathUtils.cpWildcardSubstitute)) { - try { - return Files.list(entry.getParent()).filter(ClasspathUtils::isJar); - } catch (IOException e) { - return Stream.empty(); - } + private static boolean verifyClassPathEntry(Path cpEntry) { + if (ClasspathUtils.isJar(cpEntry)) { + return true; } - if (Files.isReadable(entry)) { - return Stream.of(entry); + if (Files.isDirectory(cpEntry) && Files.isReadable(cpEntry)) { + return true; } - return Stream.empty(); + return false; } static Path urlToPath(URL url) { From c5db297dc563c0694d1b5a29806cf3317427e006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 26 Jan 2023 17:25:44 +0100 Subject: [PATCH 42/46] Restore asterisk expansion in ImageClasspath property of macro options --- .../src/com/oracle/svm/driver/MacroOptionHandler.java | 2 +- .../src/com/oracle/svm/driver/NativeImage.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java index a07601398aa6..102ff101f804 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOptionHandler.java @@ -98,7 +98,7 @@ private void applyEnabled(MacroOption.EnabledOption enabledOption, String argume boolean explicitImageModulePath = enabledOption.forEachPropertyValue(config, "ImageModulePath", entry -> nativeImage.addImageModulePath(Path.of((entry))), PATH_SEPARATOR_REGEX); boolean explicitImageClasspath = enabledOption.forEachPropertyValue(config, - "ImageClasspath", entry -> nativeImage.addImageClasspath(Path.of((entry))), PATH_SEPARATOR_REGEX); + "ImageClasspath", entry -> NativeImage.expandAsteriskClassPathElement(entry).forEach(nativeImage::addImageClasspath), PATH_SEPARATOR_REGEX); if (!explicitImageModulePath && !explicitImageClasspath) { NativeImage.getJars(imageJarsDirectory).forEach(nativeImage::addImageClasspath); } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 28cea8cb52c1..4c51360e0c45 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1690,7 +1690,7 @@ void addCustomImageClasspath(String classpath) { } } - public List expandAsteriskClassPathElement(String cp) { + public static List expandAsteriskClassPathElement(String cp) { String separators = Pattern.quote(File.separator); if (OS.getCurrent().equals(OS.WINDOWS)) { separators += "/"; /* on Windows also / is accepted as valid separator */ From 353c12ef8c6b544756c20ea38953f7059a40eda4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 30 Jan 2023 10:33:09 +0000 Subject: [PATCH 43/46] Avoid spaces in JSON output --- .../src/com/oracle/svm/driver/BundleSupport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 3387c3a62313..dec47e25b914 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -648,7 +648,7 @@ private static Manifest createManifest() { private static void printPathMapping(Map.Entry entry, JsonWriter w) throws IOException { w.append('{').quote(substitutionMapSrcField).append(" : ").quote(entry.getKey()); - w.append(", ").quote(substitutionMapDstField).append(" : ").quote(entry.getValue()); + w.append(',').quote(substitutionMapDstField).append(':').quote(entry.getValue()); w.append('}'); } From 4fbec8c06f8b496f2acebb52f78c3cd48d30ef24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 30 Jan 2023 12:29:06 +0100 Subject: [PATCH 44/46] Follow naming conventions of static final fields --- .../com/oracle/svm/driver/BundleSupport.java | 46 +++++++++---------- .../svm/driver/CmdLineOptionHandler.java | 40 ++++++++-------- .../com/oracle/svm/driver/NativeImage.java | 2 +- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index dec47e25b914..4bc0937f4bde 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -92,12 +92,12 @@ final class BundleSupport { boolean loadBundle; boolean writeBundle; - private static final int bundleFileFormatVersionMajor = 0; - private static final int bundleFileFormatVersionMinor = 9; + private static final int BUNDLE_FILE_FORMAT_VERSION_MAJOR = 0; + private static final int BUNDLE_FILE_FORMAT_VERSION_MINOR = 9; - private static final String bundleInfoMessagePrefix = "GraalVM Native Image Bundle Support: "; - private static final String bundleTempDirPrefix = "bundleRoot-"; - private static final String originalDirExtension = ".orig"; + private static final String BUNDLE_INFO_MESSAGE_PREFIX = "GraalVM Native Image Bundle Support: "; + private static final String BUNDLE_TEMP_DIR_PREFIX = "bundleRoot-"; + private static final String ORIGINAL_DIR_EXTENSION = ".orig"; private Path bundlePath; private String bundleName; @@ -154,7 +154,7 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma for (int i = buildArgs.size() - 1; i >= 0; i--) { args.push(buildArgs.get(i)); } - nativeImage.showVerboseMessage(nativeImage.isVerbose(), bundleInfoMessagePrefix + "Inject args: '" + String.join(" ", buildArgs) + "'"); + nativeImage.showVerboseMessage(nativeImage.isVerbose(), BUNDLE_INFO_MESSAGE_PREFIX + "Inject args: '" + String.join(" ", buildArgs) + "'"); /* Snapshot args after in-place expansion (includes also args after this one) */ bundleSupport.updatedBuildArgs = args.snapshot(); break; @@ -193,7 +193,7 @@ private BundleSupport(NativeImage nativeImage) { loadBundle = false; writeBundle = true; try { - rootDir = Files.createTempDirectory(bundleTempDirPrefix); + rootDir = Files.createTempDirectory(BUNDLE_TEMP_DIR_PREFIX); bundleProperties = new BundleProperties(); Path inputDir = rootDir.resolve("input"); @@ -222,11 +222,11 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { updateBundleLocation(Path.of(bundleFilenameArg), false); try { - rootDir = Files.createTempDirectory(bundleTempDirPrefix); + rootDir = Files.createTempDirectory(BUNDLE_TEMP_DIR_PREFIX); bundleProperties = new BundleProperties(); outputDir = rootDir.resolve("output"); - String originalOutputDirName = outputDir.getFileName().toString() + originalDirExtension; + String originalOutputDirName = outputDir.getFileName().toString() + ORIGINAL_DIR_EXTENSION; Path bundleFilePath = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION); try (JarFile archive = new JarFile(bundleFilePath.toFile())) { @@ -497,14 +497,14 @@ void complete() { Path externalOutputDir = bundlePath.resolve(bundleName + "." + outputDir.getFileName()); copyFiles(outputDir, externalOutputDir, true); outputNewline.run(); - nativeImage.showMessage(bundleInfoMessagePrefix + "Bundle build output written to " + externalOutputDir); + nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Bundle build output written to " + externalOutputDir); } try { if (writeBundle) { Path bundleFilePath = writeBundle(); outputNewline.run(); - nativeImage.showMessage(bundleInfoMessagePrefix + "Bundle written to " + bundleFilePath); + nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Bundle written to " + bundleFilePath); } } finally { nativeImage.showNewline(); @@ -550,7 +550,7 @@ void updateBundleLocation(Path bundleFile, boolean redefine) { } private Path writeBundle() { - String originalOutputDirName = outputDir.getFileName().toString() + originalDirExtension; + String originalOutputDirName = outputDir.getFileName().toString() + ORIGINAL_DIR_EXTENSION; Path originalOutputDir = rootDir.resolve(originalOutputDirName); if (Files.exists(originalOutputDir)) { nativeImage.deleteAllFiles(originalOutputDir); @@ -589,10 +589,10 @@ private Path writeBundle() { if (buildArg.startsWith(nativeImage.oHPath)) { continue; } - if (buildArg.equals(CmdLineOptionHandler.verboseOption)) { + if (buildArg.equals(CmdLineOptionHandler.VERBOSE_OPTION)) { continue; } - if (buildArg.equals(CmdLineOptionHandler.dryRunOption)) { + if (buildArg.equals(CmdLineOptionHandler.DRY_RUN_OPTION)) { continue; } if (buildArg.startsWith("-Dllvm.bin.dir=")) { @@ -737,11 +737,11 @@ private void loadAndVerify() { fileVersionKey = PROPERTY_KEY_BUNDLE_FILE_VERSION_MINOR; int minor = Integer.parseInt(properties.getOrDefault(fileVersionKey, "-1")); String message = String.format("The given bundle file %s was created with newer bundle-file-format version %d.%d" + - " (current %d.%d). Update to the latest version of native-image.", bundleFileName, major, minor, bundleFileFormatVersionMajor, bundleFileFormatVersionMinor); - if (major > bundleFileFormatVersionMajor) { + " (current %d.%d). Update to the latest version of native-image.", bundleFileName, major, minor, BUNDLE_FILE_FORMAT_VERSION_MAJOR, BUNDLE_FILE_FORMAT_VERSION_MINOR); + if (major > BUNDLE_FILE_FORMAT_VERSION_MAJOR) { throw NativeImage.showError(message); - } else if (major == bundleFileFormatVersionMajor) { - if (minor > bundleFileFormatVersionMinor) { + } else if (major == BUNDLE_FILE_FORMAT_VERSION_MAJOR) { + if (minor > BUNDLE_FILE_FORMAT_VERSION_MINOR) { NativeImage.showWarning(message); } } @@ -761,14 +761,14 @@ private void loadAndVerify() { localDateStr = "unknown time"; } nativeImage.showNewline(); - nativeImage.showMessage(String.format("%sLoaded Bundle from %s", bundleInfoMessagePrefix, bundleFileName)); - nativeImage.showMessage(String.format("%sBundle created at '%s'", bundleInfoMessagePrefix, localDateStr)); - nativeImage.showMessage(String.format("%sUsing version: '%s'%s on platform: '%s'%s", bundleInfoMessagePrefix, bundleVersion, currentVersion, bundlePlatform, currentPlatform)); + nativeImage.showMessage(String.format("%sLoaded Bundle from %s", BUNDLE_INFO_MESSAGE_PREFIX, bundleFileName)); + nativeImage.showMessage(String.format("%sBundle created at '%s'", BUNDLE_INFO_MESSAGE_PREFIX, localDateStr)); + nativeImage.showMessage(String.format("%sUsing version: '%s'%s on platform: '%s'%s", BUNDLE_INFO_MESSAGE_PREFIX, bundleVersion, currentVersion, bundlePlatform, currentPlatform)); } private void write() { - properties.put(PROPERTY_KEY_BUNDLE_FILE_VERSION_MAJOR, String.valueOf(bundleFileFormatVersionMajor)); - properties.put(PROPERTY_KEY_BUNDLE_FILE_VERSION_MINOR, String.valueOf(bundleFileFormatVersionMinor)); + properties.put(PROPERTY_KEY_BUNDLE_FILE_VERSION_MAJOR, String.valueOf(BUNDLE_FILE_FORMAT_VERSION_MAJOR)); + properties.put(PROPERTY_KEY_BUNDLE_FILE_VERSION_MINOR, String.valueOf(BUNDLE_FILE_FORMAT_VERSION_MINOR)); properties.put(PROPERTY_KEY_BUNDLE_FILE_CREATION_TIMESTAMP, ZonedDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME)); boolean imageBuilt = !nativeImage.isDryRun(); properties.put(PROPERTY_KEY_IMAGE_BUILT, String.valueOf(imageBuilt)); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java index 9760961da362..2a7aaaeae4b1 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/CmdLineOptionHandler.java @@ -38,17 +38,17 @@ class CmdLineOptionHandler extends NativeImage.OptionHandler { - private static final String helpText = NativeImage.getResource("/Help.txt"); - private static final String helpExtraText = NativeImage.getResource("/HelpExtra.txt"); + private static final String HELP_TEXT = NativeImage.getResource("/Help.txt"); + private static final String HELP_EXTRA_TEXT = NativeImage.getResource("/HelpExtra.txt"); - static final String verboseOption = "--verbose"; - static final String dryRunOption = "--dry-run"; - static final String debugAttachOption = "--debug-attach"; + static final String VERBOSE_OPTION = "--verbose"; + static final String DRY_RUN_OPTION = "--dry-run"; + static final String DEBUG_ATTACH_OPTION = "--debug-attach"; /* Defunct legacy options that we have to accept to maintain backward compatibility */ - private static final String verboseServerOption = "--verbose-server"; - private static final String serverOptionPrefix = "--server-"; + private static final String VERBOSE_SERVER_OPTION = "--verbose-server"; + private static final String SERVER_OPTION_PREFIX = "--server-"; - private static final String javaRuntimeVersion = System.getProperty("java.runtime.version"); + private static final String JAVA_RUNTIME_VERSION = System.getProperty("java.runtime.version"); boolean useDebugAttach = false; @@ -73,7 +73,7 @@ private boolean consume(ArgumentQueue args, String headArg) { case "--help": args.poll(); singleArgumentCheck(args, headArg); - nativeImage.showMessage(helpText); + nativeImage.showMessage(HELP_TEXT); nativeImage.showNewline(); nativeImage.apiOptionHandler.printOptions(nativeImage::showMessage, false); nativeImage.showNewline(); @@ -85,14 +85,14 @@ private boolean consume(ArgumentQueue args, String headArg) { args.poll(); singleArgumentCheck(args, headArg); String message = NativeImage.getNativeImageVersion(); - message += " (Java Version " + javaRuntimeVersion + ")"; + message += " (Java Version " + JAVA_RUNTIME_VERSION + ")"; nativeImage.showMessage(message); System.exit(ExitStatus.OK.getValue()); return true; case "--help-extra": args.poll(); singleArgumentCheck(args, headArg); - nativeImage.showMessage(helpExtraText); + nativeImage.showMessage(HELP_EXTRA_TEXT); nativeImage.apiOptionHandler.printOptions(nativeImage::showMessage, true); nativeImage.showNewline(); nativeImage.optionRegistry.showOptions(OptionUtils.MacroOptionKind.Macro, true, nativeImage::showMessage); @@ -121,11 +121,11 @@ private boolean consume(ArgumentQueue args, String headArg) { } nativeImage.addExcludeConfig(Pattern.compile(excludeJar), Pattern.compile(excludeConfig)); return true; - case verboseOption: + case VERBOSE_OPTION: args.poll(); nativeImage.addVerbose(); return true; - case dryRunOption: + case DRY_RUN_OPTION: args.poll(); nativeImage.setDryRun(true); return true; @@ -146,7 +146,7 @@ private boolean consume(ArgumentQueue args, String headArg) { args.poll(); BundleSupport.allowBundleSupport = true; return true; - case verboseServerOption: + case VERBOSE_SERVER_OPTION: args.poll(); NativeImage.showWarning("Ignoring server-mode native-image argument " + headArg + "."); return true; @@ -157,13 +157,13 @@ private boolean consume(ArgumentQueue args, String headArg) { return true; } - if (headArg.startsWith(debugAttachOption)) { + if (headArg.startsWith(DEBUG_ATTACH_OPTION)) { if (useDebugAttach) { - throw NativeImage.showError("The " + debugAttachOption + " option can only be used once."); + throw NativeImage.showError("The " + DEBUG_ATTACH_OPTION + " option can only be used once."); } useDebugAttach = true; String debugAttachArg = args.poll(); - String addressSuffix = debugAttachArg.substring(debugAttachOption.length()); + String addressSuffix = debugAttachArg.substring(DEBUG_ATTACH_OPTION.length()); String address = addressSuffix.isEmpty() ? "8000" : addressSuffix.substring(1); /* Using agentlib to allow interoperability with other agents */ nativeImage.addImageBuilderJavaArgs("-agentlib:jdwp=transport=dt_socket,server=y,address=" + address + ",suspend=y"); @@ -172,10 +172,10 @@ private boolean consume(ArgumentQueue args, String headArg) { return true; } - if (headArg.startsWith(serverOptionPrefix)) { + if (headArg.startsWith(SERVER_OPTION_PREFIX)) { args.poll(); NativeImage.showWarning("Ignoring server-mode native-image argument " + headArg + "."); - String serverOptionCommand = headArg.substring(serverOptionPrefix.length()); + String serverOptionCommand = headArg.substring(SERVER_OPTION_PREFIX.length()); if (!serverOptionCommand.startsWith("session=")) { /* * All but the --server-session=... option used to exit(0). We want to simulate that @@ -198,7 +198,7 @@ private static void singleArgumentCheck(ArgumentQueue args, String arg) { @Override void addFallbackBuildArgs(List buildArgs) { if (nativeImage.isVerbose()) { - buildArgs.add(verboseOption); + buildArgs.add(VERBOSE_OPTION); } } } diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 4c51360e0c45..1a00fa86eaed 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1259,7 +1259,7 @@ private List getAgentArguments() { if (!agentOptions.isEmpty()) { if (useDebugAttach()) { - throw NativeImage.showError(CmdLineOptionHandler.debugAttachOption + " cannot be used with class initialization/object instantiation tracing (" + oHTraceClassInitialization + + throw NativeImage.showError(CmdLineOptionHandler.DEBUG_ATTACH_OPTION + " cannot be used with class initialization/object instantiation tracing (" + oHTraceClassInitialization + "/ + " + oHTraceObjectInstantiation + ")."); } args.add("-agentlib:native-image-diagnostics-agent=" + agentOptions); From 154966b9e9b512d879c00c1e931f666484d7d73b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 30 Jan 2023 12:46:36 +0100 Subject: [PATCH 45/46] Simplify newline before bundle output info --- .../src/com/oracle/svm/driver/BundleSupport.java | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 4bc0937f4bde..bca4d5e7dace 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -469,14 +469,6 @@ private void copyFile(Path sourceFile, Path target, boolean overwrite) { } void complete() { - final boolean[] firstMessage = {!nativeImage.isDryRun()}; - Runnable outputNewline = () -> { - if (firstMessage[0]) { - nativeImage.showNewline(); - firstMessage[0] = false; - } - }; - boolean writeOutput; try (Stream pathOutputFiles = Files.list(imagePathOutputDir); Stream auxiliaryOutputFiles = Files.list(auxiliaryOutputDir)) { writeOutput = pathOutputFiles.findAny().isPresent() || auxiliaryOutputFiles.findAny().isPresent(); @@ -493,17 +485,19 @@ void complete() { bundleName = "unknown"; } + if (!nativeImage.isDryRun() && (writeOutput || writeBundle)) { + nativeImage.showNewline(); + } + if (writeOutput) { Path externalOutputDir = bundlePath.resolve(bundleName + "." + outputDir.getFileName()); copyFiles(outputDir, externalOutputDir, true); - outputNewline.run(); nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Bundle build output written to " + externalOutputDir); } try { if (writeBundle) { Path bundleFilePath = writeBundle(); - outputNewline.run(); nativeImage.showMessage(BUNDLE_INFO_MESSAGE_PREFIX + "Bundle written to " + bundleFilePath); } } finally { From de68a51902cd89c1f64b6acadaa37c8ea6aa2fe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 30 Jan 2023 12:54:43 +0100 Subject: [PATCH 46/46] Use enhanced switch in substituteAuxiliaryPath --- .../com/oracle/svm/driver/BundleSupport.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index bca4d5e7dace..881a0ddd80b7 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -312,16 +312,13 @@ Path restoreCanonicalization(Path before) { } Path substituteAuxiliaryPath(Path origPath, BundleMember.Role bundleMemberRole) { - Path destinationDir; - switch (bundleMemberRole) { - case Input: - destinationDir = auxiliaryDir; - break; - case Output: - destinationDir = auxiliaryOutputDir; - break; - default: - return origPath; + Path destinationDir = switch (bundleMemberRole) { + case Input -> auxiliaryDir; + case Output -> auxiliaryOutputDir; + case Ignore -> null; + }; + if (destinationDir == null) { + return origPath; } return substitutePath(origPath, destinationDir); }