diff --git a/framework/lib/dlju/.gitignore b/framework/lib/dlju/.gitignore new file mode 100644 index 000000000..58ba77a6c --- /dev/null +++ b/framework/lib/dlju/.gitignore @@ -0,0 +1,4 @@ +classes* +options* +build*.sh +all-argfiles diff --git a/framework/lib/dlju/README.md b/framework/lib/dlju/README.md new file mode 100644 index 000000000..89768e30b --- /dev/null +++ b/framework/lib/dlju/README.md @@ -0,0 +1,216 @@ +# do-like-junit + +`do-like-junit` (or `dlju`) is a tool that extracts test information from build processes +of a given Java project, and extracts command(s) to run the project's tests directly. + +`do-like-junit` supports projects built with + + - Apache Ant + - Apache Maven + +The `dlju` executable (in `bin/dlju`) specifically works with `defects4j` projects. + +## Dependencies + +Python 3 is required, as well as [do-like-javac](https://github.com/kelloggm/do-like-javac) +(or `dljc`) and `defects4j` on your path. + +You should also have + + - the build tool of choice, and + - any project dependencies + +installed as well. + +## Installation + +After cloning this repo and making sure you have the above dependencies, you can add a symlink to the location of the `bin/dlju` +executable somewhere on your path, for example: + +``` +ln -s /path/to/dlju $HOME/bin/dlju +``` + +## Running + +Suppose we want to analyze the build system of Lang, bug 1 (fixed version). Run the following in the terminal (does not have to be in any particular directory): + +``` +dlju Lang 1f ant compile test +``` + +This checks out Lang, version 1f, to a working directory `/tmp/d4j/Lang-1f`, and runs +`ant clean`, the runs `ant compile` and `ant test` using `dljc`. An executable file called `run_junit` +is produced in the working directory. Then you can test if test run extraction was successful: + +``` +$ cd /tmp/d4j/Lang-1f +$ ./run_junit +JUnit version 4.10 +.......... +Time: 0.034 + +OK (10 tests) + +JUnit version 4.10 +............. +Time: 0.01 + +OK (13 tests) + +JUnit version 4.10 +....................................................... +Time: 0.024 + +OK (55 tests) +# etc +``` + +You should see output similar to the above. + +In general, to run on any `defects4j` project, use + +``` +dlju +``` + +### Troubleshooting + +If a run fails, you can inspect the `dljc` build logs, which will be in `/tmp/d4j/-/.compile.logs` and `/tmp/d4j/-/.test.logs`. + +#### Common issue: package does not exist + +For example, this build output (from `/tmp/d4j/Lang-1f`) + +``` +[javac] /private/tmp/d4j/Lang-1f/src/test/java/org/apache/commons/lang3/CharSequenceUtilsTest.java:21: error: package org.junit does not exist +[javac] import static org.junit.Assert.assertNotNull; +``` + +This most likely indicates that the `build.properties` file in the working +directory either + + - doesn't exist, + - does exist, but doesn't point to the actual locations of build dependencies such as `junit`. + +#### Common issue: Apache Rat causes build to fail + +Sometimes, the compile and test log directories and files can trip up the maven rat plugin, +which assumes that these log directories and files are unlicensed. As a workaround, +you can manually change the `dlju` executable so that it outputs logs inside the +build directory for the project, or you can add `.test.logs` and +`.compile.logs` to the project's `.gitignore`. + + +# dljc_to_argfile + +Takes the `javac.json` output from [do-like-javac](https://github.com/kelloggm/do-like-javac) +and turns it into an argfile that you can give to `javac`. + +## Requirements + +Requires + + - Python 3 + - either Mac OS X or Linux + - `dljc` (the `do-like-javac` executable) on your path + +## How to Use + +``` +usage: javac_to_argfile.py [-h] [-o OPTIONS] [-c CLASSES] [-d ARGFILES_DIR] + [-n] [-b BUILD_SCRIPT] [-N] + file + +Turns the json javac output from do-like-javac into a javac argfile + +positional arguments: + file The location of the json do-like-javac output to turn + into an argfile + +optional arguments: + -h, --help show this help message and exit + -o OPTIONS, --options OPTIONS + Name/location of the options file. Defaults to + "options" + -c CLASSES, --classes CLASSES + Name/location of the classes file to be output. + Defaults to "classes" + -d ARGFILES_DIR, --argfiles-dir ARGFILES_DIR + The directory to put all generated argfiles. By + default, if there is only one javac call in the json + file, it puts the argfiles into the current directory. + However, if there are multiple javac calls, it will + default to putting them into a directory called all- + argfiles, which will be in the current directory, + unless you turn on the flag --no-dirs. + -n, --no-dir If this option is provided and no --argfiles-dir is + specified, then it will always output all argfiles + into the current directory, even if there are multiple + javac calls. Note that if --argfiles-dir/-d is given, + then this option is ignored. + -b BUILD_SCRIPT, --build-script BUILD_SCRIPT + If provided, this will automagically generate a shell + script to run that will execute all of the javac + commands found in the provided javac.json file. Even + if the option isn't provided, if there is more than + one javac call in the javac json file, it will by + default call it build_all.sh, and it will be placed + either in the argfiles directory (with no -n/--no-dir + specified) or in the current directory (with -n/--no- + dir specified). To turn off this default behavior, use + the option -N/--no-build-script. + -N, --no-build-script + If -N/--no-build-script is given and -b/--build-script + is not specified, then javac_to_argfile.py will not + attempt to automagically generated a shell script that + runs all of the javac commands. If -b/--build-script + is specified, then this option is ignored. +``` + +### Overview of features +Note that paths containing spaces are always wrapped in double-quotes. + +#### Generation of argfiles + +For the kth `javac` call that is encoded in the `javac.json` file, `javac_to_argfile.py` +generates two files: one named by default `optionsk` and the other named `classesk`, +which contain the `javac` switches and sourcefiles respectively. If there is only +one `javac` call in the `javac.json` file given, these files are named `options` +and `classes` instead. + +#### Generation of a script for running the javac calls +This script, which is by default named `build_all.sh` (which can be changed +using the option `-b/--build-script`), will create any directories necessary +and call `javac` on the generated argfiles. + +The script is created automatically if there is more than one `javac` call. To +turn off this behavior, use `-N/--no-build-script`. + +The build script assumes that it is being called in the same directory as the +argfiles, and so it uses relative paths to access the argfiles. + +#### Output directory for argfiles and build script + +To specify a directory for the argfiles and the build script, use the option +`-d/--argfiles-dir`. + +If there is more than one `javac` call in the file given, then unless the option +`-n/--no-dir` is given, `javac_to_argfile.py` will put the argfiles and the +generated build scirpt in a folder named `all-argfiles`. + +### Other notes + +All of the paths given by `do-like-javac/dljc` are absolute, so these argfiles +could technically be given to javac anywhere. The same however is not true of +the buildfile, as noted earlier. + +## Tips for do-like-javac +The `dljc` executable works fairly well with Maven and Ant, as long as you +have run `ant clean` or `mvn clean` beforehand. However, `dljc` may fail to +find `javac` calls with Gradle for any number of reasons: + +- Gradle was being run too often, and so the daemon disappeared prematurely +- The option `--rerun-tasks` wasn't given to Gradle, and so cached build results + were used instead of actually re-building the targets (this can happen even if + you run `gradle clean`) diff --git a/framework/lib/dlju/bin/dlju b/framework/lib/dlju/bin/dlju new file mode 100755 index 000000000..4a8580e5a --- /dev/null +++ b/framework/lib/dlju/bin/dlju @@ -0,0 +1,142 @@ +#!/usr/bin/env bash + +USAGE="Usage: dlju project_name project_version build_command compile_target test_target" + +if [ "$#" -ne 5 ]; then + echo "Error, not enough arguments. 5 are required." + echo $USAGE + exit 1 +fi + +PROJECT_NAME=$1 +PROJECT_VERSION=$2 +BUILD_COMMAND=$3 +COMPILE_TARGET=$4 +TEST_TARGET=$5 + +DLJU_DIR="$(dirname "$(dirname "$0")")" +echo $DLJU_DIR + +get_javac="javac_to_argfile.py" +ant_junit="capture_junit.py" +maven_junit="maven_junit.py" + +SHOULD_EXIT="exit 1" + +PWD=`pwd` + +function die { + cd $PWD + exit 1 +} + +function cantFindPythonScript { + local script=$1 + if ! [ -f "$DLJU_DIR/$script" ]; then + echo "Error: Could not find python script \"$script\" in directory DLJU_DIR=\"$DLJU_DIR\". Are you sure you ran \"$DLJU_DIR/init\"?" + SHOULD_EXIT="exit 0" + fi +} + +# Only accept either ant or mvn (maven) build commands, since we only have +# scripts for extra test info from ant and maven. +if ! [[ "$BUILD_COMMAND" = ant* || "$BUILD_COMMAND" = mvn* ]]; then + echo "Error: build command \"$BUILD_COMMAND\" matches neither \"ant\" nor \"mvn\". Exiting." + exit 1 +fi + + +# Check to make sure that these scripts all exist in the place +# we think they should exist at, which is $DLJU_DIR +cantFindPythonScript $get_javac +cantFindPythonScript $ant_junit +cantFindPythonScript $maven_junit + +if ($SHOULD_EXIT); then + echo "Exiting." + die +fi + +# Working directory for defects4j +TMP_WORK_DIR="/tmp/d4j/$PROJECT_NAME-$PROJECT_VERSION" + +# make sure that this directory inside of the temp directory exists +mkdir /tmp/d4j + +# Check out defects4j project +if [ -e "$TMP_WORK_DIR" ]; then + echo "Working directory already exists." +else + if defects4j checkout -p $PROJECT_NAME -v $PROJECT_VERSION -w $TMP_WORK_DIR; then + : + else + echo "Error, defects4j checkout of project \"$PROJECT_NAME\" at version \"$PROJECT_VERSION\" failed." + echo "Please check that defects4j is on the \$PATH, and \"$PROJECT_NAME\" with version \"$PROJECT_VERSION\" exists." + exit 1 + fi +fi + + + +cd $TMP_WORK_DIR + +if ! [ -e "build.properties" ] && [ -e "build.properties.sample" ]; then + cp build.properties.sample build.properties +fi + +# Clean the checked out dir just in case +$BUILD_COMMAND clean + +COMPILE_LOGS_DIR="$BUILD_COMMAND.compile.logs" +TEST_LOGS_DIR="$BUILD_COMMAND.test.logs" + + +# The name of the script to output to run testing +JUNIT_RUNNER="run_junit" + +# Compile and run on dljc +if dljc -o $COMPILE_LOGS_DIR -- $BUILD_COMMAND $COMPILE_TARGET; then + # Succeeded, we don't really want to run the rest unless this has succeeded + + # Now get argfiles for compilation options and the classes + python3 "$DLJU_DIR/$get_javac" "$COMPILE_LOGS_DIR/javac.json" -o "compileOptions" -c "compileClasses" + + # run dljc on test target + if dljc -o $TEST_LOGS_DIR -- $BUILD_COMMAND $TEST_TARGET; then + # Succeeded + + # Get argfiles for test compilation options and classes + python3 "$DLJU_DIR/$get_javac" "$TEST_LOGS_DIR/javac.json" -o "testOptions" -c "testClasses" + if [[ $BUILD_COMMAND = mvn* ]]; then + # Extract testing commands from maven surefire output + python3 "$DLJU_DIR/$maven_junit" -l $TEST_LOGS_DIR -o $JUNIT_RUNNER + if (exit $!); then + : + else + echo "Error: Could not extract junit tests from output. Exiting." + die + fi + else + # Extract testing commands from ant junit output + python3 "$DLJU_DIR/$ant_junit" -l $TEST_LOGS_DIR -o $JUNIT_RUNNER + if (exit $!); then + : + else + echo "Error: Could not extract junit tests from output. Exiting." + die + fi + fi + + else + # if running dljc on test target failed + echo "Warning: the command \"dljc -o $TEST_LOGS_DIR -- $BUILD_COMMAND $TEST_TARGET\" failed" + die + fi +else + # if running dljc on compile target failed + echo "Warning: the command \"dljc -o $COMPILE_LOGS_DIR -- $BUILD_COMMAND $COMPILE_TARGET\" failed" + die +fi + +# make the file $JUNIT_RUNNER executable +chmod +x $JUNIT_RUNNER diff --git a/framework/lib/dlju/capture_junit.py b/framework/lib/dlju/capture_junit.py new file mode 100755 index 000000000..ac3d103fa --- /dev/null +++ b/framework/lib/dlju/capture_junit.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python +""" Parse junit output from the do-like-javac build output, and create a file + containing all of the commands to run tests + (Outputted commands did not work last time I checked, but this script can be + used to extract classpaths, test classes, etc.) +""" + + +import os +import re +import argparse +import path_utils as putils +import json + +def writeCommands(commandsObject, fileName): + """ Outputs the junit commands to a file + + Parameters + ---------- + commandsObject : list of str + Description of parameter `commandsObject`. + fileName : str + The file to write the commands to + + """ + with open(fileName, "w") as f: + f.write("\n".join(commandsObject)) + f.flush() +def parse_args(outputSuffix=""): + """ Set up and parse command line arguments + + Parameters + ---------- + outputSuffix : str + Optional suffix to append to the output. Defaults to "". + + Returns + ------- + object containing args + The result of calling argparse.ArgumentParser.parse_args + + """ + junit_output_default="junit_commands{}".format(outputSuffix) + parser = argparse.ArgumentParser() + parser.add_argument("-l", "--logs", + type=str, + help="Location of the logs from dljc") + parser.add_argument("-o", "--output", + type=str, + default=junit_output_default, + help="File to store output commands in. Defaults to \"{}\"".format(junit_output_default)) + parser.add_argument("-i", "--info-only", + action="store_true", + help="Only produce information, and don't create commands. Will produce the classpath and the test classes instead.") + return parser.parse_args() + +def parse_junit_tasks(output): + """ Parse junit tasks from the build output + + Parameters + ---------- + output : list of str + A list of the strings in the build output file from dljc + + Returns + ------- + list of junit tasks + A list containing the junit tasks + + """ + i = 0 + output = [o.strip() for o in output] + def getQuotedString(mystr): + if re.search(r"\'[^\']+\'", mystr): + return re.sub(r"[^\']+\'([^\']+)\'.*", r"\1", mystr) + return None + def isOption(mystr): + return mystr.startswith("-") + def add_junit_task(beginIndex): + print("Beginning") + index = beginIndex + obj = [] + lastWasOption = False + lastOption = "" + while output[index].startswith("[junit]") and output[index].find("The \' characters around the executable and arguments") == -1: + unquoted = getQuotedString(output[index]) + if unquoted is not None: + if isOption(unquoted): + lastOption = unquoted + lastWasOption = True + pass + elif lastWasOption: + obj.append((lastOption, unquoted)) + lastWasOption = False + pass + else: + if unquoted.find("=") > 0 and unquoted.find("=") < len(unquoted) - 1: + # Add system property + obj.append(("", "-D{}".format(unquoted))) + pass + else: + obj.append(("", unquoted)) + pass + pass + pass + index += 1 + pass + return (obj, index) + tasks = [] + print("len: {}".format(len(output))) + while i < len(output): + line = output[i].strip() + stupid_bool = True if line.startswith("[junit]") and re.match(r"\[junit\]\s*Executing\s*\'[-a-zA-Z0-9./@_ ]+\' with arguments:", line) else False + #if line.startswith("[junit]") and re.match(r"\[junit\]\s*Executing\s*\'[a-zA-Z0-9./ ]+\' with arguments:", line): + if stupid_bool: + print("Hello???") + oldi = i + task, i = add_junit_task(i) + print("i, oldi: {} {}".format(i, oldi)) + tasks.append(task) + else: + i += 1 + print(tasks) + return tasks + +def editClasspath(cp): + """ Remove ant jars from the classpath + + Parameters + ---------- + cp : str + The classpath for junit + + Returns + ------- + str + A classpath string for running tests + + """ + paths = cp.split(":") + paths = [p for p in paths if not re.match(r"ant.*\.jar", os.path.basename(p))] + return ":".join(paths) +def getJUnitVersion(commandsList): + """ Find the junit version + + Parameters + ---------- + commandsList : list of str + A list of the parts of a command for running junit + + Returns + ------- + None, int, or str + Returns None if it cannot find the string junit in the classpath string + Returns -1 if it cannot find a string of the format `junit-\d+\.\d+` + Returns a str of the major junit verison + + """ + if any(c.startswith("-classpath") or c.startswith("-cp") for c in commandsList): + classpaths = [c for c in commandsList if c.startswith("-classpath") or c.startswith("-cp")] + if len(classpaths) == 1: + splits = classpaths[0].split(" ") + if len(splits) == 2: + cp = splits[1].split(":") + if any(p.find("junit") >= 0 for p in cp): + junitPaths = [p for p in cp if p.find("junit") >= 0] + match = [re.search(r"junit-(\d+\.\d+)", p) for p in junitPaths] + match = [m for m in match if m] + match = [int(re.sub(r"(\d+).\d+", r"\1", m.group(1))) for m in match] + matchset = set(match) + if len(matchset) == 1: + return list(match)[0] + return -1 + +def getJunitTestRunnerClass(version): + """ Get the correct junit test running class for the given junit version + + Parameters + ---------- + version : int + The major version for junit + + Returns + ------- + str or None + Returns str if `version` is either 3 or 4 + Returns None otherwise + + """ + if version == 4: + return "org.junit.runner.JUnitCore" + elif version == 3: + # Use the JUnit 3 test batch runner + # info here: http://www.geog.leeds.ac.uk/people/a.turner/src/andyt/java/grids/lib/junit-3.8.1/doc/cookbook/cookbook.htm + # Does anyone actually use this version of junit????? + return "junit.textui.TestRunner" + return None + +def reorderCommands(commandList): + """ Reorder the parts of the junit command so that it is in the order of + + 1. `java` call + 2. `java` runtime options + 3. `java` classes to run + + Parameters + ---------- + commandList : list of str + A list of the parts of the junit command + + Returns + ------- + list of str + A list of the parts of the junit command, in the given order + + """ + version = getJUnitVersion(commandList) + # print(version) + java_call = commandList.pop(0) + options = [c for c in commandList if c.startswith("-")] + classes = [c for c in commandList if not c.startswith("-")] + # Remove the ant junit test runner and replace with a normal junit test runner + testRunner = "org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner" + if testRunner in classes: + index = classes.index(testRunner) + if index >= 0: + classes.pop(index) + junitClass = getJunitTestRunnerClass(version) + if junitClass: + # Only add it if it is not None + classes = [junitClass] + classes + return [java_call] + options + classes + +def commandify(tasks, partsOnly=False): + """ Turn the junit tasks, which are stored as parts, into single command + strings. + + Parameters + ---------- + tasks : list of list of tuples of strings + tasks level: [ + task level: [ + option level: (_ , _) # The former part of the tuple is empty if + # this isn't a switch + ] + ] + + Returns + ------- + list of str + A list of junit command strings + + """ + def combinator(a, b): + if len(a) == 0: + return b + elif a.endswith("classpath") or a.endswith("cp"): + return "{} {}".format(a, editClasspath(b)) + return "{} {}".format(a, b) + if len(tasks) > 0: + all_combined = [] + if partsOnly: + for task in tasks: + simplified = [combinator(partA, partB) for partA, partB in task] + if len(simplified > 1): + simplified.pop(0) + classpaths = [c for c in simplified if c.startswith("-classpath") or c.startswith("-cp")] + classes = [c for c in simplified if not c.startrswith("-")] + all_combined.append((classpaths, classes)) + pass + pass + pass + else: + for task in tasks: + combined = reorderCommands([combinator(partA, partB) for partA, partB in task]) + all_combined.append(" ".join(combined)) + pass + pass + return all_combined + return "" + +def getOutput(logsDir): + """ Find the build output in the dljc logs directory + + Parameters + ---------- + logsDir : str + Path to the dljc logs directory + + Returns + ------- + list of str + The lines of the dljc build output file + + """ + if not os.path.exists(os.path.join(logsDir, "build_output.txt")): + print("Warning: the directory {} does not exist!".format(args.logs)) + exit(1) + pass + + output = [] + with open(os.path.join(logsDir, "build_output.txt"), "r") as f: + output = f.readlines() + pass + return output + +def main(): + args = parse_args() + + output = getOutput(args.logs) + # print(output) + junit_tasks = parse_junit_tasks(output) + commands = commandify(junit_tasks, partsOnly=args.info_only) + if args.info_only: + print(commands) + pass + else: + writeCommands(commandify(junit_tasks, partsOnly=args.info_only), args.output) + + +if __name__ == '__main__': + main() diff --git a/framework/lib/dlju/javac_to_argfile.py b/framework/lib/dlju/javac_to_argfile.py new file mode 100755 index 000000000..3531b8f86 --- /dev/null +++ b/framework/lib/dlju/javac_to_argfile.py @@ -0,0 +1,440 @@ +""" Take the output from do-like-javac (dljc) and turn this into an argfile for + javac. +""" + +import json +import argparse +import os +import path_utils as putils +import re +import readline +import os_utils as otils + +def setup_args(): + """ Setup commandline arguments + + Returns + ------- + object containing args + The result of calling `argparse.ArgumentParser.parse_args` + + """ + parser = argparse.ArgumentParser(description="Turns the json javac output from do-like-javac into a javac argfile") + + parser.add_argument("file", type=str, + help="The location of the json do-like-javac output to turn into an argfile") + parser.add_argument("-o", "--options", + type=str, + default="options", + help="Name/location of the options file. Defaults to \"options\"") + parser.add_argument("-c", "--classes", + type=str, + default="classes", + help="Name/location of the classes file to be output. Defaults to \"classes\"") + parser.add_argument("-d", "--argfiles-dir", + type=str, + default=None, + help="The directory to put all generated argfiles. By default, if there is only one javac call in the json file, it puts the argfiles into the current directory. However, if there are multiple javac calls, it will default to putting them into a directory called all-argfiles, which will be in the current directory, unless you turn on the flag --no-dirs.") + parser.add_argument("-n", "--no-dir", + action='store_true', + help="If this option is provided and no --argfiles-dir is specified, then it will always output all argfiles into the current directory, even if there are multiple javac calls. Note that if --argfiles-dir/-d is given, then this option is ignored.") + parser.add_argument("-b", "--build-script", + type=str, + default=None, + help="If provided, this will automagically generate a shell script to run that will execute all of the javac commands found in the provided javac.json file. Even if the option isn't provided, if there is more than one javac call in the javac json file, it will by default call it build_all.sh, and it will be placed either in the argfiles directory (with no -n/--no-dir specified) or in the current directory (with -n/--no-dir specified). To turn off this default behavior, use the option -N/--no-build-script.") + parser.add_argument("-N", "--no-build-script", + action='store_true', + help="If -N/--no-build-script is given and -b/--build-script is not specified, then javac_to_argfile.py will not attempt to automagically generated a shell script that runs all of the javac commands. If -b/--build-script is specified, then this option is ignored.") + parser.add_argument("--bootclasspath", + type=str, + nargs=2, + default=None, + help="Give the java version (1.7, 1.8, etc.) and bootclasspath to use for that version.") + parser.add_argument("--always-ask", + action='store_true', + help="Turns on always asking which bootclasspath to use, whenever -Werrror and -source or -target is specified, since if you compile using a javac for a higher java version than either -source or -target, then you will get a warning that makes compilation fail. By default, it just uses the previously used bootclasspath for the given java version.") + args = parser.parse_args() + return args + + +def wrap_paths_with_spaces_with_quote_marks(possiblePath): + """ Helper function to find spaces in paths, and if so, wraps the path in + quotation marks. + + Parameters + ---------- + possiblePath : str + A string that may be a path + + Returns + ------- + str + A path that has been properly wrapped in quotation marks, if needed + + """ + # TODO: This seems to actually do the _opposite_ of what the name suggests + if putils.path_exists_or_is_creatable(possiblePath): + if possiblePath.find(" ") == -1: + return "\"{}\"".format(possiblePath) + return possiblePath + +def configure_target_or_source(switch, key): + """ Obtains the java version to be used for the target or source switch, if + the given `switch` is either `target` or `source`. + + Parameters + ---------- + switch : str + A javac switch, without the preceding `-` + key : str or bool + The value given to the switch, if any + + Returns + ------- + str or bool + If the switch is either target or source, and the key is not a boolean, + then a string containing the java version without the preceding `1.` + Else, returns the key, unaltered. + + """ + if switch in ["source", "target"]: + if not isinstance(key, bool) and key.startswith("1."): + return key[2:] + return key + +def mayNeedBootClassPath(switches): + """ Indicates that to make the build work, we will need to suppress + potential `bootclasspath not found` warnings that have been promoted to + errors. + + Parameters + ---------- + switches : list of str + A list of all of the javac switches found in the dljc output + + Returns + ------- + bool + True if the bootclasspath should be specified. + + """ + return "Werror" in switches and ("source" in switches or "target" in switches) + +def getPath(version): + """ Ask the user to provide a path for the bootclasspath + + Parameters + ---------- + version : str + A string describing the Java version + + Returns + ------- + str + The path, if specified, otherwise an empty string + + """ + jarname = "rt" + if otils.isMacOsX() and version in ["1.1", "1.2", "1.3", "1.5", "5", "1.6", "6"]: + print("Please specify the location of the Java {} classes.jar file (usually rt.jar on Mac OS X for Java versions >= 7)".format(version)) + jarname = "classes" + else: + print("Please specify the location of the Java {} rt.jar file".format(version)) + pass + path = input("{}.jar location: ".format(jarname)) + while path != "" and not os.path.exists(path): + print("Sorry, the path {} does not exist.".format(path)) + path = input("{}.jar location (or hit enter to cancel setting the bootclasspath): ".format(jarname)) + pass + return path + +def addBootClassPath(switches, + switches_key, + call, + java_bootclasspath_version_locations, + always_ask): + """ Add the boot class path, if needed + + Parameters + ---------- + switches : list of str + The list of the javac switches to examine + switches_key : str + The key for the switches in the call + call : dict + A dictionary (obtained from JSON) describing the javac call + java_bootclasspath_version_locations : dict + A dictionary for containing the java rt.jar or classes.jar locations, + where the keys are the java version. This is altered by this function + (by reference) + always_ask : bool + Whether to always ask for the boot classpath from the user. + + """ + src = call[switches_key]["source"] if "source" in call[switches_key] else "" + trgt = call[switches_key]["target"] if "target" in call[switches_key] else "" + these_options_names = [switch for switch in call[switches_key] if switch == "source" or switch == "target"] + these_options_names = sorted(these_options_names) + these_options = [src, trgt] if len(src) > 0 and len(trgt) > 0 else ([src] if len(trgt) == 0 else [trgt]) + print("-Werror specified, in addition to {}".format( + "{}: {}".format( + ",".join(list(map(lambda x: "-{}".format(x), these_options_names))), + ",".join(these_options)))) + print("Warning: if the bootclasspath is not set correctly for the source/target Java version, you will get a warning that will make your compilation fail, due to -Werror being set.") + boot_paths = [] + def add_path_to_bootclasspath(path): + if path != "" and len(boot_paths) > 0: + print("Setting -bootclasspath to {}".format(":".join(boot_paths))) + switches.append("-bootclasspath {}".format(":".join(boot_paths))) + elif len(boot_paths) == 0: + print("No boot paths to add.") + pass + unique_java_versions = list(set([version for version in these_options if version not in java_bootclasspath_version_locations])) + if any(version not in java_bootclasspath_version_locations for version in these_options): + want_to_specify_bootclasspath = input("Would you like to to specify the bootclasspath? (y/n)") + if want_to_specify_bootclasspath.lower() == "y": + if otils.isMacOsX(): + print("Detected operating system: Mac OS X") + pass + if otils.isLinux(): + print("Detected operating system: Linux") + pass + + for v in unique_java_versions: + path = getPath(v) + if path == "": + print("Cancelling setting the classpath....") + break + else: + java_bootclasspath_version_locations[v] = path + boot_paths.append(path) + add_path_to_bootclasspath(path) + pass + else: + print("Skipping setting the bootclasspath") + pass + pass + else: + # In this instance, you've already previously specified what the boot + # classpath should be for the particular versions of java asked for + unique_java_versions = list(set(these_options)) + if not always_ask: + path = "" + for version in unique_java_versions: + print("Using old bootclasspath {} for Java {} by default...".format(java_bootclasspath_version_locations[version], version)) + path = java_bootclasspath_version_locations[version] + boot_paths.append(path) + pass + add_path_to_bootclasspath(path) + pass + else: + want_to_use_bootclasspath = input("Would you like to set the -bootclasspath option for this javac call? (y/n): ") + if want_to_use_bootclasspath.lower() == "y": + want_to_use_previous_bootclasspaths = input("Would you like to just use the previously given paths for Java version{} {}? (y/n): ".format("" if len(unique_java_versions) == 1 else "s", ", ".join(unique_java_versions))) + if want_to_use_previous_bootclasspaths.lower() == "y": + print("Okay, using previous settings:") + for version in unique_java_versions: + print("Using path {} for Java {}".format(java_bootclasspath_version_locations[version], version)) + pass + pass + else: + path = "" + for version in unique_java_versions: + use_new = input("Would you like to specify a new rt.jar for Java {}? (y/n): ".format(version)) + if use_new.lower() == "y": + path = getPath(version) + if path == "": + print("Cancelling setting the bootclasspath") + break + else: + boot_paths.append(path) + pass + pass + if use_new.lower() == "n": + print("Using old bootclasspath {} for Java {} by default...".format(java_bootclasspath_version_locations[version], version)) + boot_paths.append(java_bootclasspath_version_locations[version]) + pass + pass + add_path_to_bootclasspath(path) + pass + pass + else: + print("Skipping setting the -bootclasspath option...") + pass + pass + pass + pass + + +def checkFileName(fileName): + """ Check that the file name given is JSON and exists -- this file should + be the javac file created by dljc + + Parameters + ---------- + fileName : str + The name of the file containing the javac call + + """ + if not os.path.exists(fileName): + print("Warning: file named {} does not exist. Please check to make sure you spelled it correctly and have the right file.".format(fileName)) + print("Exiting now.") + exit(1) + pass + elif not fileName.endswith(".json"): + print("Warning: file {} does not have a json file extension.".format(fileName)) + pass + pass + +def main(): + # Setup commandline parsing with tab completion + readline.set_completer_delims(' \t\n=') + readline.parse_and_bind("tab:complete") + args = setup_args() + fileName = args.file + print(args.file) + + argsfileDir = "" + + if args.argfiles_dir is not None: + argsfileDir = args.argfiles_dir + + if not os.path.exists(argsfileDir): + os.makedirs(argsfileDir) + pass + pass + + + + checkFileName(fileName) + + + javac_calls = [] + try: + with open(fileName, "r") as f: + javac_calls = json.loads(f.read()) + pass + pass + except Exception as e: + print("Please make sure that the json file that you provided has content in it.") + print(e) + pass + + buildScript = "" + + if args.build_script is not None: + buildScript = args.build_script + if len(buildScript) == 0: + print("Expected a build script name, but got an empty string. Please provide the name/location of a build script, such as using -b some/path/my_script.sh or --build-script other/path/my_build_script.sh") + exit(1) + pass + else: + dirs, name = os.path.split(buildScript) + if len(dirs) > 0 and not os.path.exists(dirs): + os.makedirs(dirs) + pass + pass + pass + + + if len(javac_calls) > 1: + if args.argfiles_dir is None and not args.no_dir: + argsfileDir = "all-argfiles" + if not os.path.exists(argsfileDir): + os.mkdir(argsfileDir) + pass + pass + if args.build_script is None and not args.no_build_script: + buildScript = "build_all.sh" + pass + pass + + # print(len(javac_calls)) + buildfile_lines = [] + java_bootclasspath_version_locations = {} + i = 0 + for call in javac_calls: + # We create an argfile per call + i += 1 + # This provides a suffix for the created argfiles + s = str(i) + if len(javac_calls) == 1: + s = "" + pass + + classes_file = "{}{}".format(args.classes, s) + classes_file = os.path.join(argsfileDir, classes_file) if len(argsfileDir) > 0 else classes_file + options_file = "{}{}".format(args.options, s) + options_file = os.path.join(argsfileDir, options_file) if len(argsfileDir) > 0 else options_file + files_key = "java_files" + switches_key = "javac_switches" + if files_key in call and switches_key in call: + files = [wrap_paths_with_spaces_with_quote_marks(f) for f in call[files_key]] + switches = [(switch, arg) + for switch, arg in call[switches_key].items() + if not (switch == "sourcepath" and isinstance(arg, str) and len(arg) == 0)] + switches = ["{} {}".format("-{}".format(switch) + if not isinstance(arg, bool) or arg + else "", + wrap_paths_with_spaces_with_quote_marks(arg) + if not isinstance(arg, bool) and not re.match(r"source|target", switch) + else (configure_target_or_source(switch, arg) + if re.match(r"source|target", switch) + else "")) + for switch, arg in switches] + if mayNeedBootClassPath(call[switches_key]) and "bootclasspath" not in call[switches_key]: + addBootClassPath(switches, switches_key, call, java_bootclasspath_version_locations, args.always_ask) + print("Outputting classes into {} and options in {}".format(classes_file, options_file)) + with open(classes_file, "w") as f: + if len(call[files_key]) >= 1: + f.write("\n".join(files)) + pass + else: + f.write("") + pass + pass + with open(options_file, "w") as f: + if len(call[switches_key].keys()) >= 1: + f.write("\n".join(switches)) + pass + else: + f.write("") + pass + if len(buildScript) > 0: # We're actually going to output a build script + # Directories given in switches -d, -s MUST exist -- javac will not create them + if "d" in call[switches_key] and len(call[switches_key]["d"]) > 0: + d_val = wrap_paths_with_spaces_with_quote_marks(call[switches_key]["d"]) + # d_val = d_val if d_val.find(" ") == -1 else "\"{}\"".format(d_val) + buildfile_lines.append("# Since -d {} given and the -d DIR option requires DIR to already exist,".format(d_val)) + buildfile_lines.append("# have to try to create this directory.") + buildfile_lines.append("mkdir -p {}".format(d_val)) + pass + if "s" in call[switches_key] and len(call[switches_key]["s"]) > 0: + s_val = wrap_paths_with_spaces_with_quote_marks(call[switches_key]["s"]) + # s_val = s_val if s_val.find(" ") == -1 else "\"{}\"".format(s_val) + buildfile_lines.append("mkdir -p {}".format(s_val)) + pass + classes_loc = os.path.basename(classes_file) if not args.no_dir else classes_file + options_loc = os.path.basename(options_file) if not args.no_dir else options_file + buildfile_lines.append("javac @{} @{}".format(classes_loc, options_loc)) + buildfile_lines.append("echo \"Done with javac @{} @{}\"".format(classes_loc, options_loc)) + pass + pass + pass + if len(buildfile_lines) > 0: + with open(os.path.join(argsfileDir, buildScript), "w") as f: + #if not buildScript.endswith(".sh"): + f.write("#!/usr/bin/env bash\n\n") + f.write("# Must call this script from the same directory as the classes and options argfiles\n") + f.write("\n".join(buildfile_lines)) + f.flush() + pass + pass + pass + + + + +if __name__ == '__main__': + main() + pass diff --git a/framework/lib/dlju/maven_junit.py b/framework/lib/dlju/maven_junit.py new file mode 100644 index 000000000..1b3bac5a9 --- /dev/null +++ b/framework/lib/dlju/maven_junit.py @@ -0,0 +1,221 @@ +import capture_junit as cj +import re +import os + +tests_begin = """------------------------------------------------------- + T E S T S +-------------------------------------------------------""" + +def findBeginningOfTests(output): + """ Find where the test running begins + + Parameters + ---------- + output : list of str + A list of the build output strings + + Returns + ------- + int + The index in the `output` list where the actual tests begin + If it can't find that index, then it returns -1 + """ + testLines = tests_begin.split("\n") + # print(testLines) + for i, o in enumerate(output): + if o == testLines[0].strip() and (i + 2) < len(output): + if output[i+1] == testLines[1].strip() and output[i + 2] == testLines[2].strip(): + return i + 3 + return -1 + +def findSurefireCommand(output): + """ Find where the maven surefire plugin (which is the test plugin) + begins its command + + Parameters + ---------- + output : list of str + The lines of the build output + + Returns + ------- + str + The line that contains the java command to run the maven surefire plugin + + """ + for o in output: + print(o) + if o.startswith("Forking command line"): + return o + return None + +def createCommands(javaCommand, options, lines, ranTests): + """ Return a list of the junit commands to run + + Parameters + ---------- + javaCommand : str + The `java` invocation + options : list of str + The command line options for `java` + lines : list of str + Lines containing the test classes and classPathUrl + ranTests : list of str + A list of the tests that were ran + + Returns + ------- + list of str + A list containing commands to run each test class + + """ + print(lines) + testClasses = [l for l in lines if l.startswith("tc")] + classPaths = [l for l in lines if l.startswith("classPathUrl")] + print(len(classPaths)) + testClasses = [l.split("=") for l in testClasses] + classPaths = [l.split("=") for l in classPaths] + print(classPaths) + testClasses = [l[1] for l in testClasses if l[1] in ranTests] + classPaths = [l[1] for l in classPaths] + + classPath = "-classpath {}".format(":".join(classPaths)) + junitVersion = cj.getJUnitVersion([classPath]) + junitClass = cj.getJunitTestRunnerClass(junitVersion) + if junitClass is None: + print("Error: could not find JUnit version for classpath {}".format(classPath)) + exit(1) + return ["{} {} {} {} {}".format(javaCommand, " ".join(options), classPath, junitClass, tc) for tc in testClasses] +def findRunTests(lines): + """ From the lines of the build output, figures out which tests were run + + Parameters + ---------- + lines : list of str + The lines of the build output file + + Returns + ------- + list of str + A list of the names of the tests that were run + + """ + ran = [] + for l in lines: + if l.startswith("Running"): + splits = l.split(" ") + if len(splits) == 2: + ran.append(splits[1]) + if l.startswith("Results"): + break + return ran + +def isArgumentlessJavaOption(line): + """ Determine whether a given line contains a command line option that does + not take arguments. + + Parameters + ---------- + line : str + A line of the build output + + Returns + ------- + bool + True if the line contains an option that doesn't take arguments + + """ + argumentlessOptions = ["agentlib", + "agentpath", + "disableassertions", + "D", + "da", + "enableassertions", + "ea", + "enablesystemassertions", + "esa", + "disablesystemassertions", + "dsa", + "javaagent", + "jre-restrict-search", + "no-jre-restrict-search", + "showversion", + "splash", + "verbose", + "version", + "X"] + for a in argumentlessOptions: + if line.startswith("-{}".format(a)): + return True + return False + +def getSurefireDir(tmps): + for t in tmps: + if os.path.isdir(t): + return t + return None + +if __name__ == '__main__': + args = cj.parse_args(outputSuffix="_maven") + output = cj.getOutput(args.logs) + print(len(output)) + output = [o.rstrip() for o in output] + output = [re.sub(r"\[[^\]]+\]", "", o) for o in output] + output = [o.strip() for o in output] + beginIndex = findBeginningOfTests(output) + print(beginIndex) + surefire = findSurefireCommand(output[beginIndex:]) + ran = findRunTests(output[beginIndex:]) + if surefire is not None: + if surefire.find("&&") >= 0: + splits = surefire.split("&&") + if len(splits) >= 2: + secondPart = splits[1] + commandParts = secondPart.split(" ") + i = 0 + javaCommand = "" + options = [] + tmpFiles = [] + print(commandParts) + while i < len(commandParts): + command = commandParts[i] + print(f"Command: {command}") + if command.find("bin/java") >= 0: + javaCommand = command + elif command.find("-jar") >= 0: + i += 1 + elif command.startswith("-"): + print(f"An option: {command}") + # print("Skipping {}".format(command)) + if isArgumentlessJavaOption(command) or command.find("=") > -1: + options.append(command) + else: + options.append("{} {}".format(command, commandParts[i + 1])) + i += 1 + elif len(command) > 0: + tmpFiles.append(command) + i += 1 + pass + tmpFiles = [tmp for tmp in tmpFiles if tmp.find("surefire") > -1] + + print("tmpFiles: {}".format(tmpFiles)) + surefireDir = getSurefireDir(tmpFiles) + tmpFiles = [t for t in tmpFiles if t != surefireDir] + for tmp in tmpFiles: + lines = [] + with open(os.path.join(surefireDir, tmp), "r") as f: + lines = f.readlines() + lines = [l.strip() for l in lines] + if any(l.startswith("tc") for l in lines): + # This is one that we want + newCommands = createCommands(javaCommand, options, lines, ran) + with open(args.output, "w") as f: + f.write("\n".join(newCommands)) + f.flush() + break + pass + pass + pass + pass + else: + print("Surefire is none") diff --git a/framework/lib/dlju/os_utils.py b/framework/lib/dlju/os_utils.py new file mode 100644 index 000000000..c284ee68c --- /dev/null +++ b/framework/lib/dlju/os_utils.py @@ -0,0 +1,12 @@ +""" Figure out which operating system is being used +""" +import platform + +def isMacOsX(): + return platform.system() == "Darwin" + +def isLinux(): + return platform.system() == "Linux" + +def isWindows(): + return platform.system() == "Windows" or platform.system().lower().startswith("cygwin") diff --git a/framework/lib/dlju/path_utils.py b/framework/lib/dlju/path_utils.py new file mode 100644 index 000000000..bf62c037e --- /dev/null +++ b/framework/lib/dlju/path_utils.py @@ -0,0 +1,118 @@ +import errno, os, sys +''' +Taken from https://stackoverflow.com/questions/9532499/check-whether-a-path-is-valid-in-python-without-creating-a-file-at-the-paths-ta/9532586#9532586 +because apparently python neglects to provide a decent or easy way of actually +validating whether a pathname makes any sense. I don't want to know if a +path actually exists, I just want to know if the OS will throw a hissy fit +when I try to create a path. + +This is windows-compatible, technically, but note that I have not in any way +designed the rest of this code to be windows-friendly. +''' + +# Sadly, Python fails to provide the following magic number for us. +ERROR_INVALID_NAME = 123 +''' +Windows-specific error code indicating an invalid pathname. + +See Also +---------- +https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499- + Official listing of all such codes. +''' + +def is_pathname_valid(pathname: str) -> bool: + ''' + `True` if the passed pathname is a valid pathname for the current OS; + `False` otherwise. + ''' + # If this pathname is either not a string or is but is empty, this pathname + # is invalid. + try: + if not isinstance(pathname, str) or not pathname: + return False + + # Strip this pathname's Windows-specific drive specifier (e.g., `C:\`) + # if any. Since Windows prohibits path components from containing `:` + # characters, failing to strip this `:`-suffixed prefix would + # erroneously invalidate all valid absolute Windows pathnames. + _, pathname = os.path.splitdrive(pathname) + + # Directory guaranteed to exist. If the current OS is Windows, this is + # the drive to which Windows was installed (e.g., the "%HOMEDRIVE%" + # environment variable); else, the typical root directory. + root_dirname = os.environ.get('HOMEDRIVE', 'C:') \ + if sys.platform == 'win32' else os.path.sep + assert os.path.isdir(root_dirname) # ...Murphy and her ironclad Law + + # Append a path separator to this directory if needed. + root_dirname = root_dirname.rstrip(os.path.sep) + os.path.sep + + # Test whether each path component split from this pathname is valid or + # not, ignoring non-existent and non-readable path components. + for pathname_part in pathname.split(os.path.sep): + try: + os.lstat(root_dirname + pathname_part) + # If an OS-specific exception is raised, its error code + # indicates whether this pathname is valid or not. Unless this + # is the case, this exception implies an ignorable kernel or + # filesystem complaint (e.g., path not found or inaccessible). + # + # Only the following exceptions indicate invalid pathnames: + # + # * Instances of the Windows-specific "WindowsError" class + # defining the "winerror" attribute whose value is + # "ERROR_INVALID_NAME". Under Windows, "winerror" is more + # fine-grained and hence useful than the generic "errno" + # attribute. When a too-long pathname is passed, for example, + # "errno" is "ENOENT" (i.e., no such file or directory) rather + # than "ENAMETOOLONG" (i.e., file name too long). + # * Instances of the cross-platform "OSError" class defining the + # generic "errno" attribute whose value is either: + # * Under most POSIX-compatible OSes, "ENAMETOOLONG". + # * Under some edge-case OSes (e.g., SunOS, *BSD), "ERANGE". + except OSError as exc: + if hasattr(exc, 'winerror'): + if exc.winerror == ERROR_INVALID_NAME: + return False + elif exc.errno in {errno.ENAMETOOLONG, errno.ERANGE}: + return False + # If a "TypeError" exception was raised, it almost certainly has the + # error message "embedded NUL character" indicating an invalid pathname. + except TypeError as exc: + return False + # If no exception was raised, all path components and hence this + # pathname itself are valid. (Praise be to the curmudgeonly python.) + else: + return True + # If any other exception was raised, this is an unrelated fatal issue + # (e.g., a bug). Permit this exception to unwind the call stack. + # + # Did we mention this should be shipped with Python already? + +def is_path_creatable(pathname: str) -> bool: + ''' + `True` if the current user has sufficient permissions to create the passed + pathname; `False` otherwise. + ''' + # Parent directory of the passed path. If empty, we substitute the current + # working directory (CWD) instead. + dirname = os.path.dirname(pathname) or os.getcwd() + return os.access(dirname, os.W_OK) + +def path_exists_or_is_creatable(pathname: str) -> bool: + ''' + `True` if the passed pathname is a valid pathname for the current OS _and_ + either currently exists or is hypothetically creatable; `False` otherwise. + This function is guaranteed to _never_ raise exceptions. + ''' + try: + # To prevent "os" module calls from raising undesirable exceptions on + # invalid pathnames, is_pathname_valid() is explicitly called first. + return is_pathname_valid(pathname) and ( + os.path.exists(pathname) or is_path_creatable(pathname)) + # Report failure on non-fatal filesystem complaints (e.g., connection + # timeouts, permissions issues) implying this path to be inaccessible. All + # other exceptions are unrelated fatal issues and should not be caught here. + except OSError: + return False