diff --git a/buildSrc/src/main/kotlin/CIJobsExtensions.kt b/buildSrc/src/main/kotlin/CIJobsExtensions.kt index 1e284329e4c..3f837f27f6b 100644 --- a/buildSrc/src/main/kotlin/CIJobsExtensions.kt +++ b/buildSrc/src/main/kotlin/CIJobsExtensions.kt @@ -5,104 +5,105 @@ import org.gradle.api.Task import org.gradle.kotlin.dsl.extra /** - * Checks if a task is affected by git changes + * Returns the task's path, given affected projects, if this task or its dependencies are affected by git changes. */ -internal fun isAffectedBy(baseTask: Task, affectedProjects: Map>): String? { - val visited = mutableSetOf() - val queue = mutableListOf(baseTask) +internal fun findAffectedTaskPath(baseTask: Task, affectedProjects: Map>): String? { + val visited = mutableSetOf() + val queue = mutableListOf(baseTask) + + while (queue.isNotEmpty()) { + val t = queue.removeAt(0) + if (visited.contains(t)) { + continue + } + visited.add(t) - while (queue.isNotEmpty()) { - val t = queue.removeAt(0) - if (visited.contains(t)) { - continue - } - visited.add(t) - - val affectedTasks = affectedProjects[t.project] - if (affectedTasks != null) { - if (affectedTasks.contains("all")) { - return "${t.project.path}:${t.name}" - } - if (affectedTasks.contains(t.name)) { - return "${t.project.path}:${t.name}" - } - } - - t.taskDependencies.getDependencies(t).forEach { queue.add(it) } + val affectedTasks = affectedProjects[t.project] + if (affectedTasks != null) { + if (affectedTasks.contains("all")) { + return "${t.project.path}:${t.name}" + } + if (affectedTasks.contains(t.name)) { + return "${t.project.path}:${t.name}" + } } - return null + + t.taskDependencies.getDependencies(t).forEach { queue.add(it) } + } + return null } /** * Creates a single aggregate root task that depends on matching subproject tasks */ private fun Project.createRootTask( - rootTaskName: String, - subProjTaskName: String, - includePrefixes: List, - excludePrefixes: List, - forceCoverage: Boolean + rootTaskName: String, + subProjTaskName: String, + includePrefixes: List, + excludePrefixes: List, + forceCoverage: Boolean ) { - val coverage = forceCoverage || rootProject.hasProperty("checkCoverage") - tasks.register(rootTaskName) { - subprojects.forEach { subproject -> - val activePartition = subproject.extra.get("activePartition") as Boolean - if (activePartition && - includePrefixes.any { subproject.path.startsWith(it) } && - !excludePrefixes.any { subproject.path.startsWith(it) }) { - - val testTask = subproject.tasks.findByName(subProjTaskName) - var isAffected = true - - if (testTask != null) { - val useGitChanges = rootProject.extra.get("useGitChanges") as Boolean - if (useGitChanges) { - @Suppress("UNCHECKED_CAST") - val affectedProjects = rootProject.extra.get("affectedProjects") as Map> - val fileTrigger = isAffectedBy(testTask, affectedProjects) - if (fileTrigger != null) { - logger.warn("Selecting ${subproject.path}:$subProjTaskName (triggered by $fileTrigger)") - } else { - logger.warn("Skipping ${subproject.path}:$subProjTaskName (not affected by changed files)") - isAffected = false - } - } - if (isAffected) { - dependsOn(testTask) - } - } - - if (isAffected && coverage) { - val coverageTask = subproject.tasks.findByName("jacocoTestReport") - if (coverageTask != null) { - dependsOn(coverageTask) - } - val verificationTask = subproject.tasks.findByName("jacocoTestCoverageVerification") - if (verificationTask != null) { - dependsOn(verificationTask) - } - } + val coverage = forceCoverage || rootProject.providers.gradleProperty("checkCoverage").isPresent + tasks.register(rootTaskName) { + subprojects.forEach { subproject -> + val activePartition = subproject.extra.get("activePartition") as Boolean + if ( + activePartition && + includePrefixes.any { subproject.path.startsWith(it) } && + !excludePrefixes.any { subproject.path.startsWith(it) } + ) { + val testTask = subproject.tasks.findByName(subProjTaskName) + var isAffected = true + + if (testTask != null) { + val useGitChanges = rootProject.extra.get("useGitChanges") as Boolean + if (useGitChanges) { + @Suppress("UNCHECKED_CAST") + val affectedProjects = rootProject.extra.get("affectedProjects") as Map> + val affectedTaskPath = findAffectedTaskPath(testTask, affectedProjects) + if (affectedTaskPath != null) { + logger.warn("Selecting ${subproject.path}:$subProjTaskName (affected by $affectedTaskPath)") + } else { + logger.warn("Skipping ${subproject.path}:$subProjTaskName (not affected by changed files)") + isAffected = false } + } + if (isAffected) { + dependsOn(testTask) + } + } + + if (isAffected && coverage) { + val coverageTask = subproject.tasks.findByName("jacocoTestReport") + if (coverageTask != null) { + dependsOn(coverageTask) + } + val verificationTask = subproject.tasks.findByName("jacocoTestCoverageVerification") + if (verificationTask != null) { + dependsOn(verificationTask) + } } + } } + } } /** * Creates aggregate test tasks for CI using createRootTask() above - * + * * Creates three subtasks for the given base task name: * - ${baseTaskName}Test - runs allTests * - ${baseTaskName}LatestDepTest - runs allLatestDepTests * - ${baseTaskName}Check - runs check */ fun Project.testAggregate( - baseTaskName: String, - includePrefixes: List, - excludePrefixes: List = emptyList(), - forceCoverage: Boolean = false + baseTaskName: String, + includePrefixes: List, + excludePrefixes: List = emptyList(), + forceCoverage: Boolean = false ) { - createRootTask("${baseTaskName}Test", "allTests", includePrefixes, excludePrefixes, forceCoverage) - createRootTask("${baseTaskName}LatestDepTest", "allLatestDepTests", includePrefixes, excludePrefixes, forceCoverage) - createRootTask("${baseTaskName}Check", "check", includePrefixes, excludePrefixes, forceCoverage) + createRootTask("${baseTaskName}Test", "allTests", includePrefixes, excludePrefixes, forceCoverage) + createRootTask("${baseTaskName}LatestDepTest", "allLatestDepTests", includePrefixes, excludePrefixes, forceCoverage) + createRootTask("${baseTaskName}Check", "check", includePrefixes, excludePrefixes, forceCoverage) } diff --git a/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts b/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts index a7b495c9096..8d49d688cdb 100644 --- a/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts +++ b/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts @@ -4,129 +4,127 @@ * with -PgitBaseRef. */ -import datadog.gradle.plugin.ci.isAffectedBy +import datadog.gradle.plugin.ci.findAffectedTaskPath import java.io.File import kotlin.math.abs // Set up activePartition property on all projects allprojects { - extra.set("activePartition", true) - - val shouldUseTaskPartitions = rootProject.hasProperty("taskPartitionCount") && rootProject.hasProperty("taskPartition") - if (shouldUseTaskPartitions) { - val taskPartitionCount = rootProject.property("taskPartitionCount") as String - val taskPartition = rootProject.property("taskPartition") as String - val currentTaskPartition = abs(project.path.hashCode() % taskPartitionCount.toInt()) - extra.set("activePartition", currentTaskPartition == taskPartition.toInt()) - } + extra.set("activePartition", true) + + val taskPartitionCountProvider = rootProject.providers.gradleProperty("taskPartitionCount") + val taskPartitionProvider = rootProject.providers.gradleProperty("taskPartition") + if (taskPartitionCountProvider.isPresent && taskPartitionProvider.isPresent) { + val taskPartitionCount = taskPartitionCountProvider.get() + val taskPartition = taskPartitionProvider.get() + val currentTaskPartition = abs(project.path.hashCode() % taskPartitionCount.toInt()) + extra.set("activePartition", currentTaskPartition == taskPartition.toInt()) + } } fun relativeToGitRoot(f: File): File { - return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile() + return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile() } fun getChangedFiles(baseRef: String, newRef: String): List { - val stdout = StringBuilder() - val stderr = StringBuilder() + val stdout = StringBuilder() + val stderr = StringBuilder() - val proc = Runtime.getRuntime().exec(arrayOf("git", "diff", "--name-only", "$baseRef..$newRef")) - proc.inputStream.bufferedReader().use { stdout.append(it.readText()) } - proc.errorStream.bufferedReader().use { stderr.append(it.readText()) } - proc.waitFor() - require(proc.exitValue() == 0) { "git diff command failed, stderr: $stderr" } + val proc = Runtime.getRuntime().exec(arrayOf("git", "diff", "--name-only", "$baseRef..$newRef")) + proc.inputStream.bufferedReader().use { stdout.append(it.readText()) } + proc.errorStream.bufferedReader().use { stderr.append(it.readText()) } + proc.waitFor() + require(proc.exitValue() == 0) { "git diff command failed, stderr: $stderr" } - val out = stdout.toString().trim() - if (out.isEmpty()) { - return emptyList() - } + val out = stdout.toString().trim() + if (out.isEmpty()) { + return emptyList() + } - logger.debug("git diff output: $out") - return out.split("\n").map { File(rootProject.projectDir, it.trim()) } + logger.debug("git diff output: $out") + return out.split("\n").map { File(rootProject.projectDir, it.trim()) } } // Initialize git change tracking rootProject.extra.set("useGitChanges", false) -if (rootProject.hasProperty("gitBaseRef")) { - val baseRef = rootProject.property("gitBaseRef") as String - val newRef = if (rootProject.hasProperty("gitNewRef")) { - rootProject.property("gitNewRef") as String - } else { - "HEAD" - } - - val changedFiles = getChangedFiles(baseRef, newRef) - rootProject.extra.set("changedFiles", changedFiles) - rootProject.extra.set("useGitChanges", true) - - val ignoredFiles = fileTree(rootProject.projectDir) { - include(".gitignore", ".editorconfig") - include("*.md", "**/*.md") - include("gradlew", "gradlew.bat", "mvnw", "mvnw.cmd") - include("NOTICE") - include("static-analysis.datadog.yml") - } +val gitBaseRefProvider = rootProject.providers.gradleProperty("gitBaseRef") +if (gitBaseRefProvider.isPresent) { + val baseRef = gitBaseRefProvider.get() + val newRef = rootProject.providers.gradleProperty("gitNewRef").orElse("HEAD").get() + + val changedFiles = getChangedFiles(baseRef, newRef) + rootProject.extra.set("changedFiles", changedFiles) + rootProject.extra.set("useGitChanges", true) + + val ignoredFiles = fileTree(rootProject.projectDir) { + include(".gitignore", ".editorconfig") + include("*.md", "**/*.md") + include("gradlew", "gradlew.bat", "mvnw", "mvnw.cmd") + include("NOTICE") + include("static-analysis.datadog.yml") + } - changedFiles.forEach { f -> - if (ignoredFiles.contains(f)) { - logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}") - } + changedFiles.forEach { f -> + if (ignoredFiles.contains(f)) { + logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}") } + } - val filteredChangedFiles = changedFiles.filter { !ignoredFiles.contains(it) } - rootProject.extra.set("changedFiles", filteredChangedFiles) - - val globalEffectFiles = fileTree(rootProject.projectDir) { - include(".gitlab/**") - include("build.gradle") - include("gradle/**") - } - - for (f in filteredChangedFiles) { - if (globalEffectFiles.contains(f)) { - logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)") - rootProject.extra.set("useGitChanges", false) - break - } + val filteredChangedFiles = changedFiles.filter { !ignoredFiles.contains(it) } + rootProject.extra.set("changedFiles", filteredChangedFiles) + + val globalEffectFiles = fileTree(rootProject.projectDir) { + include(".gitlab/**") + include("build.gradle") + include("gradle/**") + } + + for (f in filteredChangedFiles) { + if (globalEffectFiles.contains(f)) { + logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)") + rootProject.extra.set("useGitChanges", false) + break } + } + + if (rootProject.extra.get("useGitChanges") as Boolean) { + logger.warn("Git change tracking is enabled: $baseRef..$newRef") - if (rootProject.extra.get("useGitChanges") as Boolean) { - logger.warn("Git change tracking is enabled: $baseRef..$newRef") - - val projects = subprojects.sortedByDescending { it.projectDir.path.length } - val affectedProjects = mutableMapOf>() + val projects = subprojects.sortedByDescending { it.projectDir.path.length } + val affectedProjects = mutableMapOf>() - // Path prefixes mapped to affected task names. A file not matching any of these prefixes will affect all tasks in - // the project ("all" can be used a task name to explicitly state the same). Only the first matching prefix is used. - val matchers = listOf( - mapOf("prefix" to "src/testFixtures/", "task" to "testFixturesClasses"), - mapOf("prefix" to "src/test/", "task" to "testClasses"), - mapOf("prefix" to "src/jmh/", "task" to "jmhCompileGeneratedClasses") - ) + // Path prefixes mapped to affected task names. A file not matching any of these prefixes will affect all tasks in + // the project ("all" can be used a task name to explicitly state the same). Only the first matching prefix is used. + val matchers = listOf( + mapOf("prefix" to "src/testFixtures/", "task" to "testFixturesClasses"), + mapOf("prefix" to "src/test/", "task" to "testClasses"), + mapOf("prefix" to "src/jmh/", "task" to "jmhCompileGeneratedClasses") + ) - for (f in filteredChangedFiles) { - val p = projects.find { f.toString().startsWith(it.projectDir.path + "/") } - if (p == null) { - logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no task will be skipped)") - rootProject.extra.set("useGitChanges", false) - break - } + for (f in filteredChangedFiles) { + val p = projects.find { f.toString().startsWith(it.projectDir.path + "/") } + if (p == null) { + logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no task will be skipped)") + rootProject.extra.set("useGitChanges", false) + break + } - // Make sure path separator is / - val relPath = p.projectDir.toPath().relativize(f.toPath()).joinToString("/") - val task = matchers.find { relPath.startsWith(it["prefix"]!!) }?.get("task") ?: "all" - logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} ($task)") - affectedProjects.computeIfAbsent(p) { mutableSetOf() }.add(task) - } - - rootProject.extra.set("affectedProjects", affectedProjects) + // Make sure path separator is / + val relPath = p.projectDir.toPath().relativize(f.toPath()).joinToString("/") + val task = matchers.find { relPath.startsWith(it["prefix"]!!) }?.get("task") ?: "all" + logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} ($task)") + affectedProjects.computeIfAbsent(p) { mutableSetOf() }.add(task) } + + rootProject.extra.set("affectedProjects", affectedProjects) + } } tasks.register("runMuzzle") { - val muzzleSubprojects = subprojects.filter { p -> - val activePartition = p.extra.get("activePartition") as Boolean - activePartition && p.plugins.hasPlugin("java") && p.plugins.hasPlugin("muzzle") - } - dependsOn(muzzleSubprojects.map { p -> "${p.path}:muzzle" }) + val muzzleSubprojects = subprojects.filter { p -> + val activePartition = p.extra.get("activePartition") as Boolean + activePartition && p.plugins.hasPlugin("java") && p.plugins.hasPlugin("muzzle") + } + dependsOn(muzzleSubprojects.map { p -> "${p.path}:muzzle" }) }