diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt index fcfb6abc0c..4e72c32be1 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt @@ -1260,13 +1260,6 @@ private fun StringBuilder.appendOptional(name: String, value: Map<*, *>) { } } -/** - * Enum that represents different type of engines that produce tests. - */ -enum class UtExecutionCreator { - FUZZER, SYMBOLIC_ENGINE -} - /** * Entity that represents cluster information that should appear in the comment. */ @@ -1280,7 +1273,6 @@ data class UtClusterInfo( */ data class UtExecutionCluster(val clusterInfo: UtClusterInfo, val executions: List) - /** * Entity that represents various types of statements in comments. */ diff --git a/utbot-summary-tests/src/test/kotlin/math/SummaryIntMathTest.kt b/utbot-summary-tests/src/test/kotlin/math/SummaryIntMathTest.kt index e0dd005b12..a237fc6340 100644 --- a/utbot-summary-tests/src/test/kotlin/math/SummaryIntMathTest.kt +++ b/utbot-summary-tests/src/test/kotlin/math/SummaryIntMathTest.kt @@ -137,7 +137,7 @@ class SummaryIntMathTest : SummaryTestCaseGeneratorTest( ) val clusterInfo = listOf( - Pair(UtClusterInfo("SUCCESSFUL EXECUTIONS for method pow(int, int)", null), 14) + Pair(UtClusterInfo("SYMBOLIC EXECUTION ENGINE: SUCCESSFUL EXECUTIONS for method pow(int, int)", null), 14) ) val method = IntMath::pow diff --git a/utbot-summary-tests/src/test/kotlin/math/SummaryOfMathTest.kt b/utbot-summary-tests/src/test/kotlin/math/SummaryOfMathTest.kt index 48abef0d4f..7348fd1179 100644 --- a/utbot-summary-tests/src/test/kotlin/math/SummaryOfMathTest.kt +++ b/utbot-summary-tests/src/test/kotlin/math/SummaryOfMathTest.kt @@ -219,10 +219,10 @@ class SummaryOfMathTest : SummaryTestCaseGeneratorTest( ) val clusterInfo = listOf( - Pair(UtClusterInfo("SUCCESSFUL EXECUTIONS #0 for method ofDoubles(double[])", null), 3), + Pair(UtClusterInfo("SYMBOLIC EXECUTION ENGINE: SUCCESSFUL EXECUTIONS #0 for method ofDoubles(double[])", null), 3), Pair( UtClusterInfo( - "SUCCESSFUL EXECUTIONS #1 for method ofDoubles(double[])", "\n" + + "SYMBOLIC EXECUTION ENGINE: SUCCESSFUL EXECUTIONS #1 for method ofDoubles(double[])", "\n" + "Common steps:\n" + "
\n" +
                             "Tests execute conditions:\n" +
@@ -246,7 +246,7 @@ class SummaryOfMathTest : SummaryTestCaseGeneratorTest(
                             "
" ), 3 ), - Pair(UtClusterInfo("ERROR SUITE for method ofDoubles(double[])", null), 1) + Pair(UtClusterInfo("SYMBOLIC EXECUTION ENGINE: ERROR SUITE for method ofDoubles(double[])", null), 1) ) summaryCheck(method, mockStrategy, coverage, summaryKeys, methodNames, displayNames, clusterInfo) diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt index 133fa77ec6..cdc94ee3bd 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt @@ -16,13 +16,20 @@ import org.utbot.summary.UtSummarySettings.GENERATE_NAMES import org.utbot.summary.analysis.ExecutionStructureAnalysis import org.utbot.summary.ast.JimpleToASTMap import org.utbot.summary.ast.SourceCodeParser -import org.utbot.summary.comment.SimpleClusterCommentBuilder +import org.utbot.summary.comment.SymbolicExecutionClusterCommentBuilder import org.utbot.summary.comment.SimpleCommentBuilder import org.utbot.summary.name.SimpleNameBuilder import java.io.File import java.nio.file.Path import java.nio.file.Paths import mu.KotlinLogging +import org.utbot.framework.plugin.api.UtConcreteExecutionFailure +import org.utbot.framework.plugin.api.UtExecutionSuccess +import org.utbot.framework.plugin.api.UtExplicitlyThrownException +import org.utbot.framework.plugin.api.UtImplicitlyThrownException +import org.utbot.framework.plugin.api.UtOverflowFailure +import org.utbot.framework.plugin.api.UtSandboxFailure +import org.utbot.framework.plugin.api.UtTimeoutException import org.utbot.fuzzer.FuzzedMethodDescription import org.utbot.fuzzer.FuzzedValue import org.utbot.fuzzer.UtFuzzedExecution @@ -40,7 +47,7 @@ fun UtMethodTestSet.summarize(sourceFile: File?, searchDirectory: Path = Paths.g makeDiverseExecutions(this) val invokeDescriptions = invokeDescriptions(this, searchDirectory) // every cluster has summary and list of executions - val executionClusters = Summarization(sourceFile, invokeDescriptions).summary(this) + val executionClusters = Summarization(sourceFile, invokeDescriptions).fillSummaries(this) val updatedExecutions = executionClusters.flatMap { it.executions } var pos = 0 val clustersInfo = executionClusters.map { @@ -49,7 +56,10 @@ fun UtMethodTestSet.summarize(sourceFile: File?, searchDirectory: Path = Paths.g pos += clusterSize it.clusterInfo to indices } - this.copy(executions = updatedExecutions, clustersInfo = clustersInfo) // TODO: looks weird and don't create the real copy + this.copy( + executions = updatedExecutions, + clustersInfo = clustersInfo + ) // TODO: looks weird and don't create the real copy } catch (e: Throwable) { logger.info(e) { "Summary generation error" } this @@ -64,7 +74,7 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List { + fun fillSummaries(testSet: UtMethodTestSet): List { val namesCounter = mutableMapOf() if (testSet.executions.isEmpty()) { @@ -83,28 +93,61 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List() + val successfulFuzzerExecutions = mutableListOf() + val unsuccessfulFuzzerExecutions = mutableListOf() if (executionsProducedByFuzzer.isNotEmpty()) { executionsProducedByFuzzer.forEach { utExecution -> val nameSuggester = sequenceOf(ModelBasedNameSuggester(), MethodBasedNameSuggester()) val testMethodName = try { - nameSuggester.flatMap { it.suggest(utExecution.fuzzedMethodDescription as FuzzedMethodDescription, utExecution.fuzzingValues as List, utExecution.result) }.firstOrNull() + nameSuggester.flatMap { + it.suggest( + utExecution.fuzzedMethodDescription as FuzzedMethodDescription, + utExecution.fuzzingValues as List, + utExecution.result + ) + }.firstOrNull() } catch (t: Throwable) { logger.error(t) { "Cannot create suggested test name for $utExecution" } // TODO: add better explanation or default behavoiur null } utExecution.testMethodName = testMethodName?.testName - utExecution.displayName = testMethodName?.displayName + utExecution.displayName = testMethodName?.displayName + + when (utExecution.result) { + is UtConcreteExecutionFailure -> unsuccessfulFuzzerExecutions.add(utExecution) + is UtExplicitlyThrownException -> unsuccessfulFuzzerExecutions.add(utExecution) + is UtImplicitlyThrownException -> unsuccessfulFuzzerExecutions.add(utExecution) + is UtOverflowFailure -> unsuccessfulFuzzerExecutions.add(utExecution) + is UtSandboxFailure -> unsuccessfulFuzzerExecutions.add(utExecution) + is UtTimeoutException -> unsuccessfulFuzzerExecutions.add(utExecution) + is UtExecutionSuccess -> successfulFuzzerExecutions.add(utExecution) + } } - clustersToReturn.add( - UtExecutionCluster( - UtClusterInfo(), // TODO: add something https://github.com/UnitTestBot/UTBotJava/issues/430 - executionsProducedByFuzzer + if (successfulFuzzerExecutions.isNotEmpty()) { + val clusterHeader = buildFuzzerClusterHeaderForSuccessfulExecutions(testSet) + + clustersToReturn.add( + UtExecutionCluster( + UtClusterInfo(clusterHeader, null), + successfulFuzzerExecutions + ) ) - ) + } + + if (unsuccessfulFuzzerExecutions.isNotEmpty()) { + val clusterHeader = buildFuzzerClusterHeaderForUnsuccessfulExecutions(testSet) + + clustersToReturn.add( + UtExecutionCluster( + UtClusterInfo(clusterHeader, null), + unsuccessfulFuzzerExecutions + ) + ) + } } // handles tests produced by symbolic engine, but with empty paths @@ -113,14 +156,14 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List 1 // there is more than one successful execution && clusterTraceTags.traceTags.size > 1 // add if there is more than 1 execution ) { - SimpleClusterCommentBuilder(clusterTraceTags.commonStepsTraceTag, sootToAST) + SymbolicExecutionClusterCommentBuilder(clusterTraceTags.commonStepsTraceTag, sootToAST) .buildString(methodUnderTest) .takeIf { it.isNotBlank() } ?.let { @@ -204,6 +247,20 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List().filter { it.path.isNotEmpty() } @@ -278,7 +335,8 @@ private fun makeDiverseExecutions(testSet: UtMethodTestSet) { } private fun invokeDescriptions(testSet: UtMethodTestSet, searchDirectory: Path): List { - val sootInvokes = testSet.executions.filterIsInstance().flatMap { it.path.invokeJimpleMethods() }.toSet() + val sootInvokes = + testSet.executions.filterIsInstance().flatMap { it.path.invokeJimpleMethods() }.toSet() return sootInvokes //TODO(SAT-1170) .filterNot { "\$lambda" in it.declaringClass.name } diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt index 9f7f4d9c92..bf91ba5e48 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt @@ -37,7 +37,7 @@ class TagGenerator { // we only want to find intersections if there is more than one successful execution if (numberOfSuccessfulClusters > 1 && REMOVE_INTERSECTIONS) { val commonStepsInSuccessfulEx = listOfSplitSteps - .filterIndexed { i, _ -> clusteredExecutions[i] is SuccessfulExecutionCluster } //search only in successful + .filterIndexed { i, _ -> clusteredExecutions[i] is SuccessfulExecutionCluster } // search only in successful .map { it.commonSteps } .filter { it.isNotEmpty() } if (commonStepsInSuccessfulEx.size > 1) { @@ -98,11 +98,12 @@ private fun generateExecutionTags(executions: List, splitSt private fun toClusterExecutions(testSet: UtMethodTestSet): List { val methodExecutions = testSet.executions.filterIsInstance() val clusters = mutableListOf() - val commentPostfix = "for method ${testSet.method.displayName}" + val commentPrefix = "SYMBOLIC EXECUTION ENGINE:" + val commentPostfix = "for method ${testSet.method.humanReadableName}" val grouped = methodExecutions.groupBy { it.result.clusterKind() } - val successfulExecutions = grouped[ClusterKind.SUCCESSFUL_EXECUTIONS] ?: emptyList() + val successfulExecutions = grouped[ExecutionGroup.SUCCESSFUL_EXECUTIONS] ?: emptyList() if (successfulExecutions.isNotEmpty()) { val clustered = if (successfulExecutions.size >= MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING) { @@ -113,28 +114,29 @@ private fun toClusterExecutions(testSet: UtMethodTestSet): List kind == ClusterKind.SUCCESSFUL_EXECUTIONS } + .filterNot { (kind, _) -> kind == ExecutionGroup.SUCCESSFUL_EXECUTIONS } .map { (suffixId, group) -> - FailedExecutionCluster("${suffixId.displayName} $commentPostfix", group) + FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group) } return clusters } -enum class ClusterKind { +/** The group of execution to be presented in the generated source file with tests. */ +enum class ExecutionGroup { SUCCESSFUL_EXECUTIONS, ERROR_SUITE, CHECKED_EXCEPTIONS, @@ -148,13 +150,13 @@ enum class ClusterKind { } private fun UtExecutionResult.clusterKind() = when (this) { - is UtExecutionSuccess -> ClusterKind.SUCCESSFUL_EXECUTIONS - is UtImplicitlyThrownException -> if (this.exception.isCheckedException) ClusterKind.CHECKED_EXCEPTIONS else ClusterKind.ERROR_SUITE - is UtExplicitlyThrownException -> if (this.exception.isCheckedException) ClusterKind.CHECKED_EXCEPTIONS else ClusterKind.EXPLICITLY_THROWN_UNCHECKED_EXCEPTIONS - is UtOverflowFailure -> ClusterKind.OVERFLOWS - is UtTimeoutException -> ClusterKind.TIMEOUTS - is UtConcreteExecutionFailure -> ClusterKind.CRASH_SUITE - is UtSandboxFailure -> ClusterKind.SECURITY + is UtExecutionSuccess -> ExecutionGroup.SUCCESSFUL_EXECUTIONS + is UtImplicitlyThrownException -> if (this.exception.isCheckedException) ExecutionGroup.CHECKED_EXCEPTIONS else ExecutionGroup.ERROR_SUITE + is UtExplicitlyThrownException -> if (this.exception.isCheckedException) ExecutionGroup.CHECKED_EXCEPTIONS else ExecutionGroup.EXPLICITLY_THROWN_UNCHECKED_EXCEPTIONS + is UtOverflowFailure -> ExecutionGroup.OVERFLOWS + is UtTimeoutException -> ExecutionGroup.TIMEOUTS + is UtConcreteExecutionFailure -> ExecutionGroup.CRASH_SUITE + is UtSandboxFailure -> ExecutionGroup.SECURITY } /** @@ -185,7 +187,7 @@ private const val REMOVE_INTERSECTIONS: Boolean = true * Contains the entities required for summarization */ data class TraceTagCluster( - var summary: String, + var clusterHeader: String, val traceTags: List, val commonStepsTraceTag: TraceTagWithoutExecution, val isSuccessful: Boolean diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/Util.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/Util.kt index c7c3b7e334..28cd355b55 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/Util.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/Util.kt @@ -78,7 +78,7 @@ val UtMethod.javaConstructor: Constructor<*>? val UtMethod.javaMethod: Method? get() = (callable as? KFunction<*>)?.javaMethod ?: (callable as? KProperty<*>)?.getter?.javaMethod -val UtMethod.displayName: String +val UtMethod.humanReadableName: String get() { val methodName = this.callable.name val javaMethod = this.javaMethod ?: this.javaConstructor diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/comment/SimpleClusterCommentBuilder.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/comment/SymbolicExecutionClusterCommentBuilder.kt similarity index 99% rename from utbot-summary/src/main/kotlin/org/utbot/summary/comment/SimpleClusterCommentBuilder.kt rename to utbot-summary/src/main/kotlin/org/utbot/summary/comment/SymbolicExecutionClusterCommentBuilder.kt index 3fff8b1fba..fa95017174 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/comment/SimpleClusterCommentBuilder.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/comment/SymbolicExecutionClusterCommentBuilder.kt @@ -20,7 +20,7 @@ import soot.jimple.internal.JVirtualInvokeExpr /** * Inherits from SimpleCommentBuilder */ -class SimpleClusterCommentBuilder( +class SymbolicExecutionClusterCommentBuilder( traceTag: TraceTagWithoutExecution, sootToAST: MutableMap ) : SimpleCommentBuilder(traceTag, sootToAST, stringTemplates = StringsTemplatesPlural()) { diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/fuzzer/names/ModelBasedNameSuggester.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/fuzzer/names/ModelBasedNameSuggester.kt index 3cb7db4982..38160e4ddd 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/fuzzer/names/ModelBasedNameSuggester.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/fuzzer/names/ModelBasedNameSuggester.kt @@ -18,23 +18,27 @@ class ModelBasedNameSuggester( PrimitiveModelNameSuggester, ArrayModelNameSuggester, ) -): NameSuggester { +) : NameSuggester { var maxNumberOfParametersWhenNameIsSuggested = 3 set(value) { field = maxOf(0, value) } - override fun suggest(description: FuzzedMethodDescription, values: List, result: UtExecutionResult?): Sequence { + override fun suggest( + description: FuzzedMethodDescription, + values: List, + result: UtExecutionResult? + ): Sequence { if (description.parameters.size > maxNumberOfParametersWhenNameIsSuggested) { return emptySequence() } return sequenceOf( TestSuggestedInfo( - testName = createTestName(description, values, result), - displayName = createDisplayName(description, values, result) - ) + testName = createTestName(description, values, result), + displayName = createDisplayName(description, values, result) + ) ) } @@ -48,20 +52,27 @@ class ModelBasedNameSuggester( * 3. *With return value*: `testMethodReturnZeroWithNonEmptyString` * 4. *When throws an exception*: `testMethodThrowsNPEWithEmptyString` */ - private fun createTestName(description: FuzzedMethodDescription, values: List, result: UtExecutionResult?): String { + private fun createTestName( + description: FuzzedMethodDescription, + values: List, + result: UtExecutionResult? + ): String { val returnString = when (result) { is UtExecutionSuccess -> (result.model as? UtPrimitiveModel)?.value?.let { v -> when (v) { is Number -> prettifyNumber(v) is Boolean -> v.toString() .replaceFirstChar { if (it.isLowerCase()) it.titlecase(Locale.getDefault()) else it.toString() } + else -> null }?.let { "Returns$it" } } + is UtExplicitlyThrownException, is UtImplicitlyThrownException -> result.exceptionOrNull()?.let { t -> prettifyException(t).let { "Throws$it" } } - else -> null + + else -> null // TODO: handle other types of the UtExecutionResult } ?: "" val parameters = values.asSequence() @@ -96,7 +107,11 @@ class ModelBasedNameSuggester( * 1. **Full name**: `firstArg = 12, secondArg < 100.0, thirdArg = empty string -> throw IllegalArgumentException` * 2. **Name without appropriate information**: `arg_0 = 0 and others -> return 0` */ - private fun createDisplayName(description: FuzzedMethodDescription, values: List, result: UtExecutionResult?): String { + private fun createDisplayName( + description: FuzzedMethodDescription, + values: List, + result: UtExecutionResult? + ): String { val summaries = values.asSequence() .mapIndexed { index, value -> value.summary?.replace("%var%", description.parameterNameMap(index) ?: "arg_$index") @@ -111,7 +126,7 @@ class ModelBasedNameSuggester( } val parameters = summaries.joinToString(postfix = postfix) - val returnValue = when(result) { + val returnValue = when (result) { is UtExecutionSuccess -> result.model.let { m -> when { m is UtPrimitiveModel && m.classId != voidClassId -> "-> return " + m.value @@ -119,6 +134,7 @@ class ModelBasedNameSuggester( else -> null } } + is UtExplicitlyThrownException, is UtImplicitlyThrownException -> "-> throw ${(result as UtExecutionFailure).exception::class.java.simpleName}" else -> null } @@ -136,6 +152,7 @@ class ModelBasedNameSuggester( value.isInfinite() -> "Infinity" else -> null } + (value is Byte || value is Short || value is Int || value is Long) && value.toLong() in 0..99999 -> value.toString() else -> null } diff --git a/utbot-summary/src/test/kotlin/org/utbot/summary/comment/SimpleClusterCommentBuilderTest.kt b/utbot-summary/src/test/kotlin/org/utbot/summary/comment/SymbolicExecutionClusterCommentBuilderTest.kt similarity index 90% rename from utbot-summary/src/test/kotlin/org/utbot/summary/comment/SimpleClusterCommentBuilderTest.kt rename to utbot-summary/src/test/kotlin/org/utbot/summary/comment/SymbolicExecutionClusterCommentBuilderTest.kt index 42291edb58..ab4c93c06e 100644 --- a/utbot-summary/src/test/kotlin/org/utbot/summary/comment/SimpleClusterCommentBuilderTest.kt +++ b/utbot-summary/src/test/kotlin/org/utbot/summary/comment/SymbolicExecutionClusterCommentBuilderTest.kt @@ -18,7 +18,7 @@ import soot.jimple.internal.JReturnStmt private const val EMPTY_STRING = "" @TestInstance(TestInstance.Lifecycle.PER_CLASS) -class SimpleClusterCommentBuilderTest { +class SymbolicExecutionClusterCommentBuilderTest { private lateinit var traceTag: TraceTag private lateinit var jimpleToASTMap: JimpleToASTMap private lateinit var sootToAst: MutableMap @@ -48,14 +48,14 @@ class SimpleClusterCommentBuilderTest { @Test fun `builds empty comment if execution result is null`() { - val commentBuilder = SimpleClusterCommentBuilder(traceTag, sootToAst) + val commentBuilder = SymbolicExecutionClusterCommentBuilder(traceTag, sootToAst) val comment = commentBuilder.buildString(sootMethod) assertEquals(EMPTY_STRING, comment) } @Test fun `does not build any statements for javadoc if execution result is null`() { - val commentBuilder = SimpleClusterCommentBuilder(traceTag, sootToAst) + val commentBuilder = SymbolicExecutionClusterCommentBuilder(traceTag, sootToAst) val statements = commentBuilder.buildDocStmts(sootMethod) assertEquals(statements.size, 0) }