Adds a cluster comment for tests produced by Smart Fuzzer (#858)

* Proposed an initial solution

* Fix tests

* Fix formatting

* Fix formatting

Author: amandelpie

utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt

@@ -1260,13 +1260,6 @@ private fun StringBuilder.appendOptional(name: String, value: Map<*, *>) {
     }
 }
 
-/**
- * Enum that represents different type of engines that produce tests.
- */
-enum class UtExecutionCreator {
-    FUZZER, SYMBOLIC_ENGINE
-}
-
 /**
  * Entity that represents cluster information that should appear in the comment.
  */
@@ -1280,7 +1273,6 @@ data class UtClusterInfo(
  */
 data class UtExecutionCluster(val clusterInfo: UtClusterInfo, val executions: List<UtExecution>)
 
-
 /**
  * Entity that represents various types of statements in comments.
  */

utbot-summary-tests/src/test/kotlin/math/SummaryIntMathTest.kt

@@ -137,7 +137,7 @@ class SummaryIntMathTest : SummaryTestCaseGeneratorTest(
         )
 
         val clusterInfo = listOf(
-            Pair(UtClusterInfo("SUCCESSFUL EXECUTIONS for method pow(int, int)", null), 14)
+            Pair(UtClusterInfo("SYMBOLIC EXECUTION ENGINE: SUCCESSFUL EXECUTIONS for method pow(int, int)", null), 14)
         )
 
         val method = IntMath::pow

utbot-summary-tests/src/test/kotlin/math/SummaryOfMathTest.kt

@@ -219,10 +219,10 @@ class SummaryOfMathTest : SummaryTestCaseGeneratorTest(
         )
 
         val clusterInfo = listOf(
-            Pair(UtClusterInfo("SUCCESSFUL EXECUTIONS #0 for method ofDoubles(double[])", null), 3),
+            Pair(UtClusterInfo("SYMBOLIC EXECUTION ENGINE: SUCCESSFUL EXECUTIONS #0 for method ofDoubles(double[])", null), 3),
             Pair(
                 UtClusterInfo(
-                    "SUCCESSFUL EXECUTIONS #1 for method ofDoubles(double[])", "\n" +
+                    "SYMBOLIC EXECUTION ENGINE: SUCCESSFUL EXECUTIONS #1 for method ofDoubles(double[])", "\n" +
                             "Common steps:\n" +
                             "<pre>\n" +
                             "Tests execute conditions:\n" +
@@ -246,7 +246,7 @@ class SummaryOfMathTest : SummaryTestCaseGeneratorTest(
                             "</pre>"
                 ), 3
             ),
-            Pair(UtClusterInfo("ERROR SUITE for method ofDoubles(double[])", null), 1)
+            Pair(UtClusterInfo("SYMBOLIC EXECUTION ENGINE: ERROR SUITE for method ofDoubles(double[])", null), 1)
         )
 
         summaryCheck(method, mockStrategy, coverage, summaryKeys, methodNames, displayNames, clusterInfo)

utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt

@@ -16,13 +16,20 @@ import org.utbot.summary.UtSummarySettings.GENERATE_NAMES
 import org.utbot.summary.analysis.ExecutionStructureAnalysis
 import org.utbot.summary.ast.JimpleToASTMap
 import org.utbot.summary.ast.SourceCodeParser
-import org.utbot.summary.comment.SimpleClusterCommentBuilder
+import org.utbot.summary.comment.SymbolicExecutionClusterCommentBuilder
 import org.utbot.summary.comment.SimpleCommentBuilder
 import org.utbot.summary.name.SimpleNameBuilder
 import java.io.File
 import java.nio.file.Path
 import java.nio.file.Paths
 import mu.KotlinLogging
+import org.utbot.framework.plugin.api.UtConcreteExecutionFailure
+import org.utbot.framework.plugin.api.UtExecutionSuccess
+import org.utbot.framework.plugin.api.UtExplicitlyThrownException
+import org.utbot.framework.plugin.api.UtImplicitlyThrownException
+import org.utbot.framework.plugin.api.UtOverflowFailure
+import org.utbot.framework.plugin.api.UtSandboxFailure
+import org.utbot.framework.plugin.api.UtTimeoutException
 import org.utbot.fuzzer.FuzzedMethodDescription
 import org.utbot.fuzzer.FuzzedValue
 import org.utbot.fuzzer.UtFuzzedExecution
@@ -40,7 +47,7 @@ fun UtMethodTestSet.summarize(sourceFile: File?, searchDirectory: Path = Paths.g
         makeDiverseExecutions(this)
         val invokeDescriptions = invokeDescriptions(this, searchDirectory)
         // every cluster has summary and list of executions
-        val executionClusters = Summarization(sourceFile, invokeDescriptions).summary(this)
+        val executionClusters = Summarization(sourceFile, invokeDescriptions).fillSummaries(this)
         val updatedExecutions = executionClusters.flatMap { it.executions }
         var pos = 0
         val clustersInfo = executionClusters.map {
@@ -49,7 +56,10 @@ fun UtMethodTestSet.summarize(sourceFile: File?, searchDirectory: Path = Paths.g
             pos += clusterSize
             it.clusterInfo to indices
         }
-        this.copy(executions = updatedExecutions, clustersInfo = clustersInfo) // TODO: looks weird and don't create the real copy
+        this.copy(
+            executions = updatedExecutions,
+            clustersInfo = clustersInfo
+        ) // TODO: looks weird and don't create the real copy
     } catch (e: Throwable) {
         logger.info(e) { "Summary generation error" }
         this
@@ -64,7 +74,7 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
     private val tagGenerator = TagGenerator()
     private val jimpleBodyAnalysis = ExecutionStructureAnalysis()
 
-    fun summary(testSet: UtMethodTestSet): List<UtExecutionCluster> {
+    fun fillSummaries(testSet: UtMethodTestSet): List<UtExecutionCluster> {
         val namesCounter = mutableMapOf<String, Int>()
 
         if (testSet.executions.isEmpty()) {
@@ -83,28 +93,61 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
 
         // handles tests produced by fuzzing
         val executionsProducedByFuzzer = testSet.executions.filterIsInstance<UtFuzzedExecution>()
+        val successfulFuzzerExecutions = mutableListOf<UtFuzzedExecution>()
+        val unsuccessfulFuzzerExecutions = mutableListOf<UtFuzzedExecution>()
 
         if (executionsProducedByFuzzer.isNotEmpty()) {
             executionsProducedByFuzzer.forEach { utExecution ->
 
                 val nameSuggester = sequenceOf(ModelBasedNameSuggester(), MethodBasedNameSuggester())
                 val testMethodName = try {
-                    nameSuggester.flatMap { it.suggest(utExecution.fuzzedMethodDescription as FuzzedMethodDescription, utExecution.fuzzingValues as List<FuzzedValue>, utExecution.result) }.firstOrNull()
+                    nameSuggester.flatMap {
+                        it.suggest(
+                            utExecution.fuzzedMethodDescription as FuzzedMethodDescription,
+                            utExecution.fuzzingValues as List<FuzzedValue>,
+                            utExecution.result
+                        )
+                    }.firstOrNull()
                 } catch (t: Throwable) {
                     logger.error(t) { "Cannot create suggested test name for $utExecution" } // TODO: add better explanation or default behavoiur
                     null
                 }
 
                 utExecution.testMethodName = testMethodName?.testName
-                utExecution.displayName =  testMethodName?.displayName
+                utExecution.displayName = testMethodName?.displayName
+
+                when (utExecution.result) {
+                    is UtConcreteExecutionFailure -> unsuccessfulFuzzerExecutions.add(utExecution)
+                    is UtExplicitlyThrownException -> unsuccessfulFuzzerExecutions.add(utExecution)
+                    is UtImplicitlyThrownException -> unsuccessfulFuzzerExecutions.add(utExecution)
+                    is UtOverflowFailure -> unsuccessfulFuzzerExecutions.add(utExecution)
+                    is UtSandboxFailure -> unsuccessfulFuzzerExecutions.add(utExecution)
+                    is UtTimeoutException -> unsuccessfulFuzzerExecutions.add(utExecution)
+                    is UtExecutionSuccess -> successfulFuzzerExecutions.add(utExecution)
+                }
             }
 
-            clustersToReturn.add(
-                UtExecutionCluster(
-                    UtClusterInfo(), // TODO: add something https://github.com/UnitTestBot/UTBotJava/issues/430
-                    executionsProducedByFuzzer
+            if (successfulFuzzerExecutions.isNotEmpty()) {
+                val clusterHeader = buildFuzzerClusterHeaderForSuccessfulExecutions(testSet)
+
+                clustersToReturn.add(
+                    UtExecutionCluster(
+                        UtClusterInfo(clusterHeader, null),
+                        successfulFuzzerExecutions
+                    )
                 )
-            )
+            }
+
+            if (unsuccessfulFuzzerExecutions.isNotEmpty()) {
+                val clusterHeader = buildFuzzerClusterHeaderForUnsuccessfulExecutions(testSet)
+
+                clustersToReturn.add(
+                    UtExecutionCluster(
+                        UtClusterInfo(clusterHeader, null),
+                        unsuccessfulFuzzerExecutions
+                    )
+                )
+            }
         }
 
         // handles tests produced by symbolic engine, but with empty paths
@@ -113,14 +156,14 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
         if (executionsWithEmptyPaths.isNotEmpty()) {
             executionsWithEmptyPaths.forEach {
                 logger.info {
-                    "Test is created by Symbolic Engine. The path for test ${it.testMethodName} " +
+                    "Test is created by Symbolic Execution Engine. The path for test ${it.testMethodName} " +
                             "for method ${testSet.method.clazz.qualifiedName} is empty and summaries could not be generated."
                 }
             }
 
             clustersToReturn.add(
                 UtExecutionCluster(
-                    UtClusterInfo(), // TODO: https://github.com/UnitTestBot/UTBotJava/issues/430
+                    UtClusterInfo(),
                     executionsWithEmptyPaths
                 )
             )
@@ -135,13 +178,13 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
             jimpleBodyAnalysis.traceStructuralAnalysis(jimpleBody, clusteredTags, methodUnderTest, invokeDescriptions)
             val numberOfSuccessfulClusters = clusteredTags.filter { it.isSuccessful }.size
             for (clusterTraceTags in clusteredTags) {
-                val clusterHeader = clusterTraceTags.summary.takeIf { GENERATE_CLUSTER_COMMENTS }
+                val clusterHeader = clusterTraceTags.clusterHeader.takeIf { GENERATE_CLUSTER_COMMENTS }
                 val clusterContent = if (
                     GENERATE_CLUSTER_COMMENTS && clusterTraceTags.isSuccessful // add only for successful executions
                     && numberOfSuccessfulClusters > 1 // there is more than one successful execution
                     && clusterTraceTags.traceTags.size > 1 // add if there is more than 1 execution
                 ) {
-                    SimpleClusterCommentBuilder(clusterTraceTags.commonStepsTraceTag, sootToAST)
+                    SymbolicExecutionClusterCommentBuilder(clusterTraceTags.commonStepsTraceTag, sootToAST)
                         .buildString(methodUnderTest)
                         .takeIf { it.isNotBlank() }
                         ?.let {
@@ -204,6 +247,20 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
         return listOf(UtExecutionCluster(UtClusterInfo(), testSet.executions))
     }
 
+    private fun buildFuzzerClusterHeaderForSuccessfulExecutions(testSet: UtMethodTestSet): String {
+        val commentPrefix = "FUZZER:"
+        val commentPostfix = "for method ${testSet.method.humanReadableName}"
+
+        return "$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix"
+    }
+
+    private fun buildFuzzerClusterHeaderForUnsuccessfulExecutions(testSet: UtMethodTestSet): String {
+        val commentPrefix = "FUZZER:"
+        val commentPostfix = "for method ${testSet.method.humanReadableName}"
+
+        return "$commentPrefix ${ExecutionGroup.EXPLICITLY_THROWN_UNCHECKED_EXCEPTIONS} $commentPostfix"
+    }
+
     private fun prepareTestSetForByteCodeAnalysis(testSet: UtMethodTestSet): UtMethodTestSet {
         val executions =
             testSet.executions.filterIsInstance<UtSymbolicExecution>().filter { it.path.isNotEmpty() }
@@ -278,7 +335,8 @@ private fun makeDiverseExecutions(testSet: UtMethodTestSet) {
 }
 
 private fun invokeDescriptions(testSet: UtMethodTestSet, searchDirectory: Path): List<InvokeDescription> {
-    val sootInvokes = testSet.executions.filterIsInstance<UtSymbolicExecution>().flatMap { it.path.invokeJimpleMethods() }.toSet()
+    val sootInvokes =
+        testSet.executions.filterIsInstance<UtSymbolicExecution>().flatMap { it.path.invokeJimpleMethods() }.toSet()
     return sootInvokes
         //TODO(SAT-1170)
         .filterNot { "\$lambda" in it.declaringClass.name }

utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt

@@ -37,7 +37,7 @@ class TagGenerator {
             // we only want to find intersections if there is more than one successful execution
             if (numberOfSuccessfulClusters > 1 && REMOVE_INTERSECTIONS) {
                 val commonStepsInSuccessfulEx = listOfSplitSteps
-                    .filterIndexed { i, _ -> clusteredExecutions[i] is SuccessfulExecutionCluster } //search only in successful
+                    .filterIndexed { i, _ -> clusteredExecutions[i] is SuccessfulExecutionCluster } // search only in successful
                     .map { it.commonSteps }
                     .filter { it.isNotEmpty() }
                 if (commonStepsInSuccessfulEx.size > 1) {
@@ -98,11 +98,12 @@ private fun generateExecutionTags(executions: List<UtSymbolicExecution>, splitSt
 private fun toClusterExecutions(testSet: UtMethodTestSet): List<ExecutionCluster> {
     val methodExecutions = testSet.executions.filterIsInstance<UtSymbolicExecution>()
     val clusters = mutableListOf<ExecutionCluster>()
-    val commentPostfix = "for method ${testSet.method.displayName}"
+    val commentPrefix =  "SYMBOLIC EXECUTION ENGINE:"
+    val commentPostfix = "for method ${testSet.method.humanReadableName}"
 
     val grouped = methodExecutions.groupBy { it.result.clusterKind() }
 
-    val successfulExecutions = grouped[ClusterKind.SUCCESSFUL_EXECUTIONS] ?: emptyList()
+    val successfulExecutions = grouped[ExecutionGroup.SUCCESSFUL_EXECUTIONS] ?: emptyList()
     if (successfulExecutions.isNotEmpty()) {
         val clustered =
             if (successfulExecutions.size >= MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING) {
@@ -113,28 +114,29 @@ private fun toClusterExecutions(testSet: UtMethodTestSet): List<ExecutionCluster
             for (c in clustered) {
                 clusters +=
                     SuccessfulExecutionCluster(
-                        "${ClusterKind.SUCCESSFUL_EXECUTIONS.displayName} #${clustered.keys.indexOf(c.key)} $commentPostfix",
+                        "$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} #${clustered.keys.indexOf(c.key)} $commentPostfix",
                         c.value.toList()
                     )
             }
         } else {
             clusters +=
                 SuccessfulExecutionCluster(
-                    "${ClusterKind.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix",
+                    "$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix",
                     successfulExecutions.toList()
                 )
         }
     }
 
     clusters += grouped
-        .filterNot { (kind, _) -> kind == ClusterKind.SUCCESSFUL_EXECUTIONS }
+        .filterNot { (kind, _) -> kind == ExecutionGroup.SUCCESSFUL_EXECUTIONS }
         .map { (suffixId, group) ->
-        FailedExecutionCluster("${suffixId.displayName} $commentPostfix", group)
+        FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group)
     }
     return clusters
 }
 
-enum class ClusterKind {
+/** The group of execution to be presented in the generated source file with tests. */
+enum class ExecutionGroup {
     SUCCESSFUL_EXECUTIONS,
     ERROR_SUITE,
     CHECKED_EXCEPTIONS,
@@ -148,13 +150,13 @@ enum class ClusterKind {
 }
 
 private fun UtExecutionResult.clusterKind() = when (this) {
-    is UtExecutionSuccess -> ClusterKind.SUCCESSFUL_EXECUTIONS
-    is UtImplicitlyThrownException -> if (this.exception.isCheckedException) ClusterKind.CHECKED_EXCEPTIONS else ClusterKind.ERROR_SUITE
-    is UtExplicitlyThrownException -> if (this.exception.isCheckedException) ClusterKind.CHECKED_EXCEPTIONS else ClusterKind.EXPLICITLY_THROWN_UNCHECKED_EXCEPTIONS
-    is UtOverflowFailure -> ClusterKind.OVERFLOWS
-    is UtTimeoutException -> ClusterKind.TIMEOUTS
-    is UtConcreteExecutionFailure -> ClusterKind.CRASH_SUITE
-    is UtSandboxFailure -> ClusterKind.SECURITY
+    is UtExecutionSuccess -> ExecutionGroup.SUCCESSFUL_EXECUTIONS
+    is UtImplicitlyThrownException -> if (this.exception.isCheckedException) ExecutionGroup.CHECKED_EXCEPTIONS else ExecutionGroup.ERROR_SUITE
+    is UtExplicitlyThrownException -> if (this.exception.isCheckedException) ExecutionGroup.CHECKED_EXCEPTIONS else ExecutionGroup.EXPLICITLY_THROWN_UNCHECKED_EXCEPTIONS
+    is UtOverflowFailure -> ExecutionGroup.OVERFLOWS
+    is UtTimeoutException -> ExecutionGroup.TIMEOUTS
+    is UtConcreteExecutionFailure -> ExecutionGroup.CRASH_SUITE
+    is UtSandboxFailure -> ExecutionGroup.SECURITY
 }
 
 /**
@@ -185,7 +187,7 @@ private const val REMOVE_INTERSECTIONS: Boolean = true
  * Contains the entities required for summarization
  */
 data class TraceTagCluster(
-    var summary: String,
+    var clusterHeader: String,
     val traceTags: List<TraceTag>,
     val commonStepsTraceTag: TraceTagWithoutExecution,
     val isSuccessful: Boolean

utbot-summary/src/main/kotlin/org/utbot/summary/Util.kt

@@ -78,7 +78,7 @@ val <R> UtMethod<R>.javaConstructor: Constructor<*>?
 val <R> UtMethod<R>.javaMethod: Method?
     get() = (callable as? KFunction<*>)?.javaMethod ?: (callable as? KProperty<*>)?.getter?.javaMethod
 
-val <R> UtMethod<R>.displayName: String
+val <R> UtMethod<R>.humanReadableName: String
     get() {
         val methodName = this.callable.name
         val javaMethod = this.javaMethod ?: this.javaConstructor

utbot-summary/src/main/kotlin/org/utbot/summary/comment/SimpleClusterCommentBuilder.kt renamed to utbot-summary/src/main/kotlin/org/utbot/summary/comment/SymbolicExecutionClusterCommentBuilder.kt

@@ -20,7 +20,7 @@ import soot.jimple.internal.JVirtualInvokeExpr
 /**
  * Inherits from SimpleCommentBuilder
  */
-class SimpleClusterCommentBuilder(
+class SymbolicExecutionClusterCommentBuilder(
     traceTag: TraceTagWithoutExecution,
     sootToAST: MutableMap<SootMethod, JimpleToASTMap>
 ) : SimpleCommentBuilder(traceTag, sootToAST, stringTemplates = StringsTemplatesPlural()) {