Skip to content

Use @argFiles to prevent issues with long command line invocations #3734

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 7, 2021
Merged

Use @argFiles to prevent issues with long command line invocations #3734

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
27c55a1
Add @argFile option to the Native Image Driver
lazar-mitrovic Aug 27, 2021
7c4a040
Use @argFiles for NativeImageGeneratorRunner's vm invocation on JDK9+
lazar-mitrovic Aug 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions substratevm/src/com.oracle.svm.driver/resources/Help.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@ Usage: native-image [options] class [imagename] [options]
(to build an image for a class)
or native-image [options] -jar jarfile [imagename] [options]
(to build an image for a jar file)

where options include:

@argument files one or more argument files containing options
-cp <class search path of directories and zip/jar files>
-classpath <class search path of directories and zip/jar files>
--class-path <class search path of directories and zip/jar files>
Expand Down
220 changes: 220 additions & 0 deletions substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/DefaultOptionHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,12 @@
package com.oracle.svm.driver;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;
Expand Down Expand Up @@ -58,6 +61,7 @@ class DefaultOptionHandler extends NativeImage.OptionHandler<NativeImage> {
}

boolean useDebugAttach = false;
boolean disableAtFiles = false;

private static void singleArgumentCheck(ArgumentQueue args, String arg) {
if (!args.isEmpty()) {
Expand Down Expand Up @@ -210,6 +214,10 @@ public boolean consume(ArgumentQueue args) {
nativeImage.showNewline();
System.exit(0);
return true;
case "--disable-@files":
args.poll();
disableAtFiles = true;
return true;
}

String debugAttach = "--debug-attach";
Expand Down Expand Up @@ -296,9 +304,221 @@ public boolean consume(ArgumentQueue args) {
}
return true;
}
if (headArg.startsWith("@") && !disableAtFiles) {
args.poll();
headArg = headArg.substring(1);
Path argFile = Paths.get(headArg);
NativeImage.NativeImageArgsProcessor processor = nativeImage.new NativeImageArgsProcessor(argFile.toString());
readArgFile(argFile).forEach(processor::accept);
List<String> leftoverArgs = processor.apply(false);
if (leftoverArgs.size() > 0) {
NativeImage.showError("Found unrecognized options while parsing argument file '" + argFile + "':\n" + String.join("\n", leftoverArgs));
}
return true;
}
return false;
}

// Ported from JDK11's java.base/share/native/libjli/args.c
enum PARSER_STATE {
FIND_NEXT,
IN_COMMENT,
IN_QUOTE,
IN_ESCAPE,
SKIP_LEAD_WS,
IN_TOKEN
}

class CTX_ARGS {
PARSER_STATE state;
int cptr;
int eob;
char quoteChar;
List<String> parts;
String options;
}

// Ported from JDK11's java.base/share/native/libjli/args.c
private List<String> readArgFile(Path file) {
List<String> arguments = new ArrayList<>();
// Use of the at sign (@) to recursively interpret files isn't supported.
arguments.add("--disable-@files");

String options = null;
try {
options = new String(Files.readAllBytes(file), StandardCharsets.UTF_8);
} catch (IOException e) {
NativeImage.showError("Error reading argument file", e);
}

CTX_ARGS ctx = new CTX_ARGS();
ctx.state = PARSER_STATE.FIND_NEXT;
ctx.parts = new ArrayList<>(4);
ctx.quoteChar = '"';
ctx.cptr = 0;
ctx.eob = options.length();
ctx.options = options;

String token = nextToken(ctx);
while (token != null) {
arguments.add(token);
token = nextToken(ctx);
}

// remaining partial token
if (ctx.state == PARSER_STATE.IN_TOKEN || ctx.state == PARSER_STATE.IN_QUOTE) {
if (ctx.parts.size() != 0) {
token = String.join("", ctx.parts);
arguments.add(token);
}
}
return arguments;
}

// Ported from JDK11's java.base/share/native/libjli/args.c
@SuppressWarnings("fallthrough")
private static String nextToken(CTX_ARGS ctx) {
int nextc = ctx.cptr;
int eob = ctx.eob;
int anchor = nextc;
String token;

for (; nextc < eob; nextc++) {
char ch = ctx.options.charAt(nextc);

// Skip white space characters
if (ctx.state == PARSER_STATE.FIND_NEXT || ctx.state == PARSER_STATE.SKIP_LEAD_WS) {
while (ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t' || ch == '\f') {
nextc++;
if (nextc >= eob) {
return null;
}
ch = ctx.options.charAt(nextc);
}
ctx.state = (ctx.state == PARSER_STATE.FIND_NEXT) ? PARSER_STATE.IN_TOKEN : PARSER_STATE.IN_QUOTE;
anchor = nextc;
// Deal with escape sequences
} else if (ctx.state == PARSER_STATE.IN_ESCAPE) {
// concatenation directive
if (ch == '\n' || ch == '\r') {
ctx.state = PARSER_STATE.SKIP_LEAD_WS;
} else {
// escaped character
char[] escaped = new char[2];
escaped[1] = '\0';
switch (ch) {
case 'n':
escaped[0] = '\n';
break;
case 'r':
escaped[0] = '\r';
break;
case 't':
escaped[0] = '\t';
break;
case 'f':
escaped[0] = '\f';
break;
default:
escaped[0] = ch;
break;
}
ctx.parts.add(String.valueOf(escaped));
ctx.state = PARSER_STATE.IN_QUOTE;
}
// anchor to next character
anchor = nextc + 1;
continue;
// ignore comment to EOL
} else if (ctx.state == PARSER_STATE.IN_COMMENT) {
while (ch != '\n' && ch != '\r') {
nextc++;
if (nextc >= eob) {
return null;
}
ch = ctx.options.charAt(nextc);
}
anchor = nextc + 1;
ctx.state = PARSER_STATE.FIND_NEXT;
continue;
}

assert (ctx.state != PARSER_STATE.IN_ESCAPE);
assert (ctx.state != PARSER_STATE.FIND_NEXT);
assert (ctx.state != PARSER_STATE.SKIP_LEAD_WS);
assert (ctx.state != PARSER_STATE.IN_COMMENT);

switch (ch) {
case ' ':
case '\t':
case '\f':
if (ctx.state == PARSER_STATE.IN_QUOTE) {
continue;
}
// fall through
case '\n':
case '\r':
if (ctx.parts.size() == 0) {
token = ctx.options.substring(anchor, nextc);
} else {
ctx.parts.add(ctx.options.substring(anchor, nextc));
token = String.join("", ctx.parts);
ctx.parts = new ArrayList<>();
}
ctx.cptr = nextc + 1;
ctx.state = PARSER_STATE.FIND_NEXT;
return token;
case '#':
if (ctx.state == PARSER_STATE.IN_QUOTE) {
continue;
}
ctx.state = PARSER_STATE.IN_COMMENT;
anchor = nextc + 1;
break;
case '\\':
if (ctx.state != PARSER_STATE.IN_QUOTE) {
continue;
}
ctx.parts.add(ctx.options.substring(anchor, nextc));
ctx.state = PARSER_STATE.IN_ESCAPE;
// anchor after backslash character
anchor = nextc + 1;
break;
case '\'':
case '"':
if (ctx.state == PARSER_STATE.IN_QUOTE && ctx.quoteChar != ch) {
// not matching quote
continue;
}
// partial before quote
if (anchor != nextc) {
ctx.parts.add(ctx.options.substring(anchor, nextc));
}
// anchor after quote character
anchor = nextc + 1;
if (ctx.state == PARSER_STATE.IN_TOKEN) {
ctx.quoteChar = ch;
ctx.state = PARSER_STATE.IN_QUOTE;
} else {
ctx.state = PARSER_STATE.IN_TOKEN;
}
break;
default:
break;
}
}

assert (nextc == eob);
// Only need partial token, not comment or whitespaces
if (ctx.state == PARSER_STATE.IN_TOKEN || ctx.state == PARSER_STATE.IN_QUOTE) {
if (anchor < nextc) {
// not yet return until end of stream, we have part of a token.
ctx.parts.add(ctx.options.substring(anchor, nextc));
}
}
return null;
}

private void processClasspathArgs(String cpArgs) {
for (String cp : cpArgs.split(File.pathSeparator, Integer.MAX_VALUE)) {
/* Conform to `java` command empty cp entry handling. */
Expand Down
58 changes: 35 additions & 23 deletions substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -815,7 +815,7 @@ protected NativeImage(BuildConfiguration config) {
/* Discover supported MacroOptions */
optionRegistry = new MacroOption.Registry();

/* Default handler needs to be fist */
/* Default handler needs to be first */
defaultOptionHandler = new DefaultOptionHandler(this);
registerOptionHandler(defaultOptionHandler);
apiOptionHandler = new APIOptionHandler(this);
Expand Down Expand Up @@ -1372,21 +1372,24 @@ protected static List<String> createImageBuilderArgs(ArrayList<String> imageArgs
return result;
}

protected static String createVMInvocationArgumentFile(List<String> arguments) {
try {
Path argsFile = Files.createTempFile("vminvocation", ".args");
String joinedOptions = String.join("\n", arguments);
Files.write(argsFile, joinedOptions.getBytes());
argsFile.toFile().deleteOnExit();
return "@" + argsFile;
} catch (IOException e) {
throw showError(e.getMessage());
}
}

protected static String createImageBuilderArgumentFile(List<String> imageBuilderArguments) {
try {
Path argsFile = Files.createTempFile("native-image", "args");
Path argsFile = Files.createTempFile("native-image", ".args");
String joinedOptions = String.join("\0", imageBuilderArguments);
Files.write(argsFile, joinedOptions.getBytes());
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
try {
Files.delete(argsFile);
} catch (IOException e) {
System.err.println("Failed to delete temporary image builder arguments file: " + argsFile.toString());
}
}
});
argsFile.toFile().deleteOnExit();
return NativeImageGeneratorRunner.IMAGE_BUILDER_ARG_FILE_OPTION + argsFile.toString();
} catch (IOException e) {
throw showError(e.getMessage());
Expand All @@ -1395,37 +1398,46 @@ public void run() {

protected int buildImage(List<String> javaArgs, LinkedHashSet<Path> bcp, LinkedHashSet<Path> cp, LinkedHashSet<Path> mp, ArrayList<String> imageArgs, LinkedHashSet<Path> imagecp,
LinkedHashSet<Path> imagemp) {
/* Construct ProcessBuilder command from final arguments */
List<String> command = new ArrayList<>();
command.add(canonicalize(config.getJavaExecutable()).toString());
command.addAll(javaArgs);
List<String> arguments = new ArrayList<>();
arguments.addAll(javaArgs);
if (!bcp.isEmpty()) {
command.add(bcp.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator, "-Xbootclasspath/a:", "")));
arguments.add(bcp.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator, "-Xbootclasspath/a:", "")));
}

if (!cp.isEmpty()) {
command.addAll(Arrays.asList("-cp", cp.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator))));
arguments.addAll(Arrays.asList("-cp", cp.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator))));
}
if (!mp.isEmpty()) {
List<String> strings = Arrays.asList("--module-path", mp.stream().map(Path::toString).collect(Collectors.joining(File.pathSeparator)));
command.addAll(strings);
arguments.addAll(strings);
}

if (USE_NI_JPMS) {
command.addAll(Arrays.asList("--module", DEFAULT_GENERATOR_MODULE_NAME + "/" + DEFAULT_GENERATOR_CLASS_NAME));
arguments.addAll(Arrays.asList("--module", DEFAULT_GENERATOR_MODULE_NAME + "/" + DEFAULT_GENERATOR_CLASS_NAME));
} else {
command.add(config.getGeneratorMainClass());
arguments.add(config.getGeneratorMainClass());
}
if (IS_AOT && OS.getCurrent().hasProcFS) {
/*
* GR-8254: Ensure image-building VM shuts down even if native-image dies unexpected
* (e.g. using CTRL-C in Gradle daemon mode)
*/
command.addAll(Arrays.asList(SubstrateOptions.WATCHPID_PREFIX, "" + ProcessProperties.getProcessID()));
arguments.addAll(Arrays.asList(SubstrateOptions.WATCHPID_PREFIX, "" + ProcessProperties.getProcessID()));
}
List<String> finalImageBuilderArgs = createImageBuilderArgs(imageArgs, imagecp, imagemp);
List<String> completeCommandList = Stream.concat(command.stream(), finalImageBuilderArgs.stream()).collect(Collectors.toList());

/* Construct ProcessBuilder command from final arguments */
List<String> command = new ArrayList<>();
command.add(canonicalize(config.getJavaExecutable()).toString());
List<String> completeCommandList = new ArrayList<>(command);
if (config.useJavaModules()) { // Only in JDK9+ 'java' executable supports @argFiles.
command.add(createVMInvocationArgumentFile(arguments));
} else {
command.addAll(arguments);
}
command.add(createImageBuilderArgumentFile(finalImageBuilderArgs));

completeCommandList.addAll(Stream.concat(arguments.stream(), finalImageBuilderArgs.stream()).collect(Collectors.toList()));
final String commandLine = SubstrateUtil.getShellCommandString(completeCommandList, true);
if (isDiagnostics()) {
// write to the diagnostics dir
Expand Down