From 9ee6788b46730b95a38a97303aa5fd8698c25074 Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Tue, 21 Oct 2025 19:08:47 -0400 Subject: [PATCH 1/4] Turn ci_jobs into a convention plugin and extension --- build.gradle.kts | 15 +- buildSrc/src/main/kotlin/CIJobsExtensions.kt | 98 +++++ .../main/kotlin/datadog.ci-jobs.gradle.kts | 131 ++++++ gradle/ci_jobs.gradle | 394 +++++++++--------- 4 files changed, 440 insertions(+), 198 deletions(-) create mode 100644 buildSrc/src/main/kotlin/CIJobsExtensions.kt create mode 100644 buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts diff --git a/build.gradle.kts b/build.gradle.kts index eb7d87b4c43..41ecb4b63e1 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,6 +5,7 @@ plugins { id("datadog.dependency-locking") id("datadog.tracer-version") id("datadog.dump-hanged-test") + id("datadog.ci-jobs") id("com.diffplug.spotless") version "6.13.0" id("com.github.spotbugs") version "5.0.14" @@ -137,4 +138,16 @@ allprojects { } } -apply(from = "$rootDir/gradle/ci_jobs.gradle") +testAggregate("smoke", listOf(":dd-smoke-tests"), emptyList()) +testAggregate("instrumentation", listOf(":dd-java-agent:instrumentation"), emptyList()) +testAggregate("profiling", listOf(":dd-java-agent:agent-profiling"), emptyList()) +testAggregate("debugger", listOf(":dd-java-agent:agent-debugger"), emptyList(), true) +testAggregate( + "base", listOf(":"), + listOf( + ":dd-java-agent:instrumentation", + ":dd-smoke-tests", + ":dd-java-agent:agent-profiling", + ":dd-java-agent:agent-debugger" + ) +) diff --git a/buildSrc/src/main/kotlin/CIJobsExtensions.kt b/buildSrc/src/main/kotlin/CIJobsExtensions.kt new file mode 100644 index 00000000000..0ac106a53a6 --- /dev/null +++ b/buildSrc/src/main/kotlin/CIJobsExtensions.kt @@ -0,0 +1,98 @@ +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.kotlin.dsl.extra + +/** + * Checks if a task is affected by git changes + */ +internal fun isAffectedBy(baseTask: Task, affectedProjects: Map>): String? { + val visited = mutableSetOf() + val queue = mutableListOf(baseTask) + + while (queue.isNotEmpty()) { + val t = queue.removeAt(0) + if (visited.contains(t)) { + continue + } + visited.add(t) + + val affectedTasks = affectedProjects[t.project] + if (affectedTasks != null) { + if (affectedTasks.contains("all")) { + return "${t.project.path}:${t.name}" + } + if (affectedTasks.contains(t.name)) { + return "${t.project.path}:${t.name}" + } + } + + t.taskDependencies.getDependencies(t).forEach { queue.add(it) } + } + return null +} + +/** + * Creates aggregate test tasks for CI + * + * Creates three tasks for the given base name: + * - ${baseTaskName}Test - runs allTests + * - ${baseTaskName}LatestDepTest - runs allLatestDepTests + * - ${baseTaskName}Check - runs check + */ +fun Project.testAggregate( + baseTaskName: String, + includePrefixes: List, + excludePrefixes: List, + forceCoverage: Boolean = false +) { + fun createRootTask(rootTaskName: String, subProjTaskName: String) { + val coverage = forceCoverage || rootProject.hasProperty("checkCoverage") + val proj = this@testAggregate + tasks.register(rootTaskName) { + proj.subprojects.forEach { subproject -> + val activePartition = subproject.extra.get("activePartition") as Boolean + if (activePartition && + includePrefixes.any { subproject.path.startsWith(it) } && + !excludePrefixes.any { subproject.path.startsWith(it) }) { + + val testTask = subproject.tasks.findByName(subProjTaskName) + var isAffected = true + + if (testTask != null) { + val useGitChanges = proj.rootProject.extra.get("useGitChanges") as Boolean + if (useGitChanges) { + @Suppress("UNCHECKED_CAST") + val affectedProjects = proj.rootProject.extra.get("affectedProjects") as Map> + val fileTrigger = isAffectedBy(testTask, affectedProjects) + if (fileTrigger != null) { + proj.logger.warn("Selecting ${subproject.path}:$subProjTaskName (triggered by $fileTrigger)") + } else { + proj.logger.warn("Skipping ${subproject.path}:$subProjTaskName (not affected by changed files)") + isAffected = false + } + } + if (isAffected) { + dependsOn(testTask) + } + } + + if (isAffected && coverage) { + val coverageTask = subproject.tasks.findByName("jacocoTestReport") + if (coverageTask != null) { + dependsOn(coverageTask) + } + val verificationTask = subproject.tasks.findByName("jacocoTestCoverageVerification") + if (verificationTask != null) { + dependsOn(verificationTask) + } + } + } + } + } + } + + createRootTask("${baseTaskName}Test", "allTests") + createRootTask("${baseTaskName}LatestDepTest", "allLatestDepTests") + createRootTask("${baseTaskName}Check", "check") +} + diff --git a/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts b/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts new file mode 100644 index 00000000000..4705059c4e1 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts @@ -0,0 +1,131 @@ +/* + * This plugin defines a set of tasks to be used in CI. These aggregate tasks support partitioning (to parallelize + * jobs) with -PtaskPartitionCount and -PtaskPartition, and limiting tasks to those affected by git changes + * with -PgitBaseRef. + */ + +import java.io.File +import kotlin.math.abs + +// Set up activePartition property on all projects +allprojects { + extra.set("activePartition", true) + + val shouldUseTaskPartitions = rootProject.hasProperty("taskPartitionCount") && rootProject.hasProperty("taskPartition") + if (shouldUseTaskPartitions) { + val taskPartitionCount = rootProject.property("taskPartitionCount") as String + val taskPartition = rootProject.property("taskPartition") as String + val currentTaskPartition = abs(project.path.hashCode() % taskPartitionCount.toInt()) + extra.set("activePartition", currentTaskPartition == taskPartition.toInt()) + } +} + +fun relativeToGitRoot(f: File): File { + return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile() +} + +fun getChangedFiles(baseRef: String, newRef: String): List { + val stdout = StringBuilder() + val stderr = StringBuilder() + + val proc = Runtime.getRuntime().exec(arrayOf("git", "diff", "--name-only", "$baseRef..$newRef")) + proc.inputStream.bufferedReader().use { stdout.append(it.readText()) } + proc.errorStream.bufferedReader().use { stderr.append(it.readText()) } + proc.waitFor() + require(proc.exitValue() == 0) { "git diff command failed, stderr: $stderr" } + + val out = stdout.toString().trim() + if (out.isEmpty()) { + return emptyList() + } + + logger.debug("git diff output: $out") + return out.split("\n").map { File(rootProject.projectDir, it.trim()) } +} + +// Initialize git change tracking +rootProject.extra.set("useGitChanges", false) + +if (rootProject.hasProperty("gitBaseRef")) { + val baseRef = rootProject.property("gitBaseRef") as String + val newRef = if (rootProject.hasProperty("gitNewRef")) { + rootProject.property("gitNewRef") as String + } else { + "HEAD" + } + + val changedFiles = getChangedFiles(baseRef, newRef) + rootProject.extra.set("changedFiles", changedFiles) + rootProject.extra.set("useGitChanges", true) + + val ignoredFiles = fileTree(rootProject.projectDir) { + include(".gitignore", ".editorconfig") + include("*.md", "**/*.md") + include("gradlew", "gradlew.bat", "mvnw", "mvnw.cmd") + include("NOTICE") + include("static-analysis.datadog.yml") + } + + changedFiles.forEach { f -> + if (ignoredFiles.contains(f)) { + logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}") + } + } + + val filteredChangedFiles = changedFiles.filter { !ignoredFiles.contains(it) } + rootProject.extra.set("changedFiles", filteredChangedFiles) + + val globalEffectFiles = fileTree(rootProject.projectDir) { + include(".gitlab/**") + include("build.gradle") + include("gradle/**") + } + + for (f in filteredChangedFiles) { + if (globalEffectFiles.contains(f)) { + logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)") + rootProject.extra.set("useGitChanges", false) + break + } + } + + if (rootProject.extra.get("useGitChanges") as Boolean) { + logger.warn("Git change tracking is enabled: $baseRef..$newRef") + + val projects = subprojects.sortedByDescending { it.projectDir.path.length } + val affectedProjects = mutableMapOf>() + + // Path prefixes mapped to affected task names. A file not matching any of these prefixes will affect all tasks in + // the project ("all" can be used a task name to explicitly state the same). Only the first matching prefix is used. + val matchers = listOf( + mapOf("prefix" to "src/testFixtures/", "task" to "testFixturesClasses"), + mapOf("prefix" to "src/test/", "task" to "testClasses"), + mapOf("prefix" to "src/jmh/", "task" to "jmhCompileGeneratedClasses") + ) + + for (f in filteredChangedFiles) { + val p = projects.find { f.toString().startsWith(it.projectDir.path + "/") } + if (p == null) { + logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no task will be skipped)") + rootProject.extra.set("useGitChanges", false) + break + } + + // Make sure path separator is / + val relPath = p.projectDir.toPath().relativize(f.toPath()).joinToString("/") + val task = matchers.find { relPath.startsWith(it["prefix"]!!) }?.get("task") ?: "all" + logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} ($task)") + affectedProjects.computeIfAbsent(p) { mutableSetOf() }.add(task) + } + + rootProject.extra.set("affectedProjects", affectedProjects) + } +} + +tasks.register("runMuzzle") { + val muzzleSubprojects = subprojects.filter { p -> + val activePartition = p.extra.get("activePartition") as Boolean + activePartition && p.plugins.hasPlugin("java") && p.plugins.hasPlugin("muzzle") + } + dependsOn(muzzleSubprojects.map { p -> "${p.path}:muzzle" }) +} diff --git a/gradle/ci_jobs.gradle b/gradle/ci_jobs.gradle index 4fd8a2350db..c4ccc094e7d 100644 --- a/gradle/ci_jobs.gradle +++ b/gradle/ci_jobs.gradle @@ -1,197 +1,197 @@ -/** - * This script defines a set of tasks to be used in CI. These aggregate tasks support partitioning (to parallelize - * jobs) with -PtaskPartitionCount and -PtaskPartition, and limiting tasks to those affected by git changes - * with -PgitBaseRef. - */ -import java.nio.file.Paths - -allprojects { project -> - project.ext { - activePartition = true - } - final boolean shouldUseTaskPartitions = project.rootProject.hasProperty("taskPartitionCount") && project.rootProject.hasProperty("taskPartition") - if (shouldUseTaskPartitions) { - final int taskPartitionCount = project.rootProject.property("taskPartitionCount") as int - final int taskPartition = project.rootProject.property("taskPartition") as int - final currentTaskPartition = Math.abs(project.path.hashCode() % taskPartitionCount) - project.setProperty("activePartition", currentTaskPartition == taskPartition) - } -} - -File relativeToGitRoot(File f) { - return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile() -} - -String isAffectedBy(Task baseTask, Map> affectedProjects) { - HashSet visited = [] - LinkedList queue = [baseTask] - while (!queue.isEmpty()) { - Task t = queue.poll() - if (visited.contains(t)) { - continue - } - visited.add(t) - - final Set affectedTasks = affectedProjects.get(t.project) - if (affectedTasks != null) { - if (affectedTasks.contains("all")) { - return "${t.project.path}:${t.name}" - } - if (affectedTasks.contains(t.name)) { - return "${t.project.path}:${t.name}" - } - } - - t.taskDependencies.each { queue.addAll(it.getDependencies(t)) } - } - return null -} - -List getChangedFiles(String baseRef, String newRef) { - final stdout = new StringBuilder() - final stderr = new StringBuilder() - final proc = "git diff --name-only ${baseRef}..${newRef}".execute() - proc.consumeProcessOutput(stdout, stderr) - proc.waitForOrKill(1000) - assert proc.exitValue() == 0, "git diff command failed, stderr: ${stderr}" - def out = stdout.toString().trim() - if (out.isEmpty()) { - return [] - } - logger.debug("git diff output: ${out}") - return out.split("\n").collect { - new File(rootProject.projectDir, it.trim()) - } -} - -rootProject.ext { - useGitChanges = false -} - -if (rootProject.hasProperty("gitBaseRef")) { - final String baseRef = rootProject.property("gitBaseRef") - final String newRef = rootProject.hasProperty("gitNewRef") ? rootProject.property("gitNewRef") : "HEAD" - - rootProject.ext { - it.changedFiles = getChangedFiles(baseRef, newRef) - useGitChanges = true - } - - final ignoredFiles = fileTree(rootProject.projectDir) { - include '.gitignore', '.editorconfig' - include '*.md', '**/*.md' - include 'gradlew', 'gradlew.bat', 'mvnw', 'mvnw.cmd' - include 'NOTICE' - include 'static-analysis.datadog.yml' - } - rootProject.changedFiles.each { File f -> - if (ignoredFiles.contains(f)) { - logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}") - } - } - rootProject.changedFiles = rootProject.changedFiles.findAll { !ignoredFiles.contains(it) } - - final globalEffectFiles = fileTree(rootProject.projectDir) { - include '.gitlab/**' - include 'build.gradle' - include 'gradle/**' - } - - for (File f in rootProject.changedFiles) { - if (globalEffectFiles.contains(f)) { - logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)") - rootProject.useGitChanges = false - break - } - } - - if (rootProject.useGitChanges) { - logger.warn("Git change tracking is enabled: ${baseRef}..${newRef}") - - final projects = subprojects.sort { a, b -> b.projectDir.path.length() <=> a.projectDir.path.length() } - Map> _affectedProjects = [:] - // Path prefixes mapped to affected task names. A file not matching any of these prefixes will affect all tasks in - // the project ("all" can be used a task name to explicitly state the same). Only the first matching prefix is used. - final List> matchers = [ - [prefix: 'src/testFixtures/', task: 'testFixturesClasses'], - [prefix: 'src/test/', task: 'testClasses'], - [prefix: 'src/jmh/', task: 'jmhCompileGeneratedClasses'] - ] - for (File f in rootProject.changedFiles) { - Project p = projects.find { f.toString().startsWith(it.projectDir.path + "/") } - if (p == null) { - logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no task will be skipped)") - rootProject.useGitChanges = false - break - } - // Make sure path separator is / - final relPath = Paths.get(p.projectDir.path).relativize(f.toPath()).collect { it.toString() }.join('/') - final String task = matchers.find { relPath.startsWith(it.prefix) }?.task ?: "all" - logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (${task})") - _affectedProjects.computeIfAbsent(p, { new HashSet() }).add(task) - } - rootProject.ext { - it.affectedProjects = _affectedProjects - } - } -} - -def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) { - def createRootTask = { String rootTaskName, String subProjTaskName -> - def coverage = forceCoverage || rootProject.hasProperty("checkCoverage") - tasks.register(rootTaskName) { aggTest -> - subprojects { subproject -> - if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) { - Task testTask = subproject.tasks.findByName(subProjTaskName) - boolean isAffected = true - if (testTask != null) { - if (rootProject.useGitChanges) { - final fileTrigger = isAffectedBy(testTask, rootProject.property("affectedProjects")) - if (fileTrigger != null) { - logger.warn("Selecting ${subproject.path}:${subProjTaskName} (triggered by ${fileTrigger})") - } else { - logger.warn("Skipping ${subproject.path}:${subProjTaskName} (not affected by changed files)") - isAffected = false - } - } - if (isAffected) { - aggTest.dependsOn(testTask) - } - } - if (isAffected && coverage) { - def coverageTask = subproject.tasks.findByName("jacocoTestReport") - if (coverageTask != null) { - aggTest.dependsOn(coverageTask) - } - coverageTask = subproject.tasks.findByName("jacocoTestCoverageVerification") - if (coverageTask != null) { - aggTest.dependsOn(coverageTask) - } - } - } - } - } - } - - createRootTask "${baseTaskName}Test", 'allTests' - createRootTask "${baseTaskName}LatestDepTest", 'allLatestDepTests' - createRootTask "${baseTaskName}Check", 'check' -} - -testAggregate("smoke", [":dd-smoke-tests"], []) -testAggregate("instrumentation", [":dd-java-agent:instrumentation"], []) -testAggregate("profiling", [":dd-java-agent:agent-profiling"], []) -testAggregate("debugger", [":dd-java-agent:agent-debugger"], [], true) -testAggregate("base", [":"], [ - ":dd-java-agent:instrumentation", - ":dd-smoke-tests", - ":dd-java-agent:agent-profiling", - ":dd-java-agent:agent-debugger" -]) - - -tasks.register('runMuzzle') { - dependsOn subprojects.findAll { p -> - p.activePartition && p.plugins.hasPlugin('java') && p.plugins.hasPlugin('muzzle') - }.collect { p -> p.path + ":muzzle" } -} +// /** +// * This script defines a set of tasks to be used in CI. These aggregate tasks support partitioning (to parallelize +// * jobs) with -PtaskPartitionCount and -PtaskPartition, and limiting tasks to those affected by git changes +// * with -PgitBaseRef. +// */ +// import java.nio.file.Paths + +// allprojects { project -> +// project.ext { +// activePartition = true +// } +// final boolean shouldUseTaskPartitions = project.rootProject.hasProperty("taskPartitionCount") && project.rootProject.hasProperty("taskPartition") +// if (shouldUseTaskPartitions) { +// final int taskPartitionCount = project.rootProject.property("taskPartitionCount") as int +// final int taskPartition = project.rootProject.property("taskPartition") as int +// final currentTaskPartition = Math.abs(project.path.hashCode() % taskPartitionCount) +// project.setProperty("activePartition", currentTaskPartition == taskPartition) +// } +// } + +// File relativeToGitRoot(File f) { +// return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile() +// } + +// String isAffectedBy(Task baseTask, Map> affectedProjects) { +// HashSet visited = [] +// LinkedList queue = [baseTask] +// while (!queue.isEmpty()) { +// Task t = queue.poll() +// if (visited.contains(t)) { +// continue +// } +// visited.add(t) + +// final Set affectedTasks = affectedProjects.get(t.project) +// if (affectedTasks != null) { +// if (affectedTasks.contains("all")) { +// return "${t.project.path}:${t.name}" +// } +// if (affectedTasks.contains(t.name)) { +// return "${t.project.path}:${t.name}" +// } +// } + +// t.taskDependencies.each { queue.addAll(it.getDependencies(t)) } +// } +// return null +// } + +// List getChangedFiles(String baseRef, String newRef) { +// final stdout = new StringBuilder() +// final stderr = new StringBuilder() +// final proc = "git diff --name-only ${baseRef}..${newRef}".execute() +// proc.consumeProcessOutput(stdout, stderr) +// proc.waitForOrKill(1000) +// assert proc.exitValue() == 0, "git diff command failed, stderr: ${stderr}" +// def out = stdout.toString().trim() +// if (out.isEmpty()) { +// return [] +// } +// logger.debug("git diff output: ${out}") +// return out.split("\n").collect { +// new File(rootProject.projectDir, it.trim()) +// } +// } + +// rootProject.ext { +// useGitChanges = false +// } + +// if (rootProject.hasProperty("gitBaseRef")) { +// final String baseRef = rootProject.property("gitBaseRef") +// final String newRef = rootProject.hasProperty("gitNewRef") ? rootProject.property("gitNewRef") : "HEAD" + +// rootProject.ext { +// it.changedFiles = getChangedFiles(baseRef, newRef) +// useGitChanges = true +// } + +// final ignoredFiles = fileTree(rootProject.projectDir) { +// include '.gitignore', '.editorconfig' +// include '*.md', '**/*.md' +// include 'gradlew', 'gradlew.bat', 'mvnw', 'mvnw.cmd' +// include 'NOTICE' +// include 'static-analysis.datadog.yml' +// } +// rootProject.changedFiles.each { File f -> +// if (ignoredFiles.contains(f)) { +// logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}") +// } +// } +// rootProject.changedFiles = rootProject.changedFiles.findAll { !ignoredFiles.contains(it) } + +// final globalEffectFiles = fileTree(rootProject.projectDir) { +// include '.gitlab/**' +// include 'build.gradle' +// include 'gradle/**' +// } + +// for (File f in rootProject.changedFiles) { +// if (globalEffectFiles.contains(f)) { +// logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)") +// rootProject.useGitChanges = false +// break +// } +// } + +// if (rootProject.useGitChanges) { +// logger.warn("Git change tracking is enabled: ${baseRef}..${newRef}") + +// final projects = subprojects.sort { a, b -> b.projectDir.path.length() <=> a.projectDir.path.length() } +// Map> _affectedProjects = [:] +// // Path prefixes mapped to affected task names. A file not matching any of these prefixes will affect all tasks in +// // the project ("all" can be used a task name to explicitly state the same). Only the first matching prefix is used. +// final List> matchers = [ +// [prefix: 'src/testFixtures/', task: 'testFixturesClasses'], +// [prefix: 'src/test/', task: 'testClasses'], +// [prefix: 'src/jmh/', task: 'jmhCompileGeneratedClasses'] +// ] +// for (File f in rootProject.changedFiles) { +// Project p = projects.find { f.toString().startsWith(it.projectDir.path + "/") } +// if (p == null) { +// logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no task will be skipped)") +// rootProject.useGitChanges = false +// break +// } +// // Make sure path separator is / +// final relPath = Paths.get(p.projectDir.path).relativize(f.toPath()).collect { it.toString() }.join('/') +// final String task = matchers.find { relPath.startsWith(it.prefix) }?.task ?: "all" +// logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (${task})") +// _affectedProjects.computeIfAbsent(p, { new HashSet() }).add(task) +// } +// rootProject.ext { +// it.affectedProjects = _affectedProjects +// } +// } +// } + +// def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) { +// def createRootTask = { String rootTaskName, String subProjTaskName -> +// def coverage = forceCoverage || rootProject.hasProperty("checkCoverage") +// tasks.register(rootTaskName) { aggTest -> +// subprojects { subproject -> +// if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) { +// Task testTask = subproject.tasks.findByName(subProjTaskName) +// boolean isAffected = true +// if (testTask != null) { +// if (rootProject.useGitChanges) { +// final fileTrigger = isAffectedBy(testTask, rootProject.property("affectedProjects")) +// if (fileTrigger != null) { +// logger.warn("Selecting ${subproject.path}:${subProjTaskName} (triggered by ${fileTrigger})") +// } else { +// logger.warn("Skipping ${subproject.path}:${subProjTaskName} (not affected by changed files)") +// isAffected = false +// } +// } +// if (isAffected) { +// aggTest.dependsOn(testTask) +// } +// } +// if (isAffected && coverage) { +// def coverageTask = subproject.tasks.findByName("jacocoTestReport") +// if (coverageTask != null) { +// aggTest.dependsOn(coverageTask) +// } +// coverageTask = subproject.tasks.findByName("jacocoTestCoverageVerification") +// if (coverageTask != null) { +// aggTest.dependsOn(coverageTask) +// } +// } +// } +// } +// } +// } + +// createRootTask "${baseTaskName}Test", 'allTests' +// createRootTask "${baseTaskName}LatestDepTest", 'allLatestDepTests' +// createRootTask "${baseTaskName}Check", 'check' +// } + +// testAggregate("smoke", [":dd-smoke-tests"], []) +// testAggregate("instrumentation", [":dd-java-agent:instrumentation"], []) +// testAggregate("profiling", [":dd-java-agent:agent-profiling"], []) +// testAggregate("debugger", [":dd-java-agent:agent-debugger"], [], true) +// testAggregate("base", [":"], [ +// ":dd-java-agent:instrumentation", +// ":dd-smoke-tests", +// ":dd-java-agent:agent-profiling", +// ":dd-java-agent:agent-debugger" +// ]) + + +// tasks.register('runMuzzle') { +// dependsOn subprojects.findAll { p -> +// p.activePartition && p.plugins.hasPlugin('java') && p.plugins.hasPlugin('muzzle') +// }.collect { p -> p.path + ":muzzle" } +// } From 0bbe3d71f25297fbc02ce76d54aa8a416970db60 Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Wed, 22 Oct 2025 15:45:35 -0400 Subject: [PATCH 2/4] Remove ci_jobs file --- gradle/ci_jobs.gradle | 197 ------------------------------------------ 1 file changed, 197 deletions(-) delete mode 100644 gradle/ci_jobs.gradle diff --git a/gradle/ci_jobs.gradle b/gradle/ci_jobs.gradle deleted file mode 100644 index c4ccc094e7d..00000000000 --- a/gradle/ci_jobs.gradle +++ /dev/null @@ -1,197 +0,0 @@ -// /** -// * This script defines a set of tasks to be used in CI. These aggregate tasks support partitioning (to parallelize -// * jobs) with -PtaskPartitionCount and -PtaskPartition, and limiting tasks to those affected by git changes -// * with -PgitBaseRef. -// */ -// import java.nio.file.Paths - -// allprojects { project -> -// project.ext { -// activePartition = true -// } -// final boolean shouldUseTaskPartitions = project.rootProject.hasProperty("taskPartitionCount") && project.rootProject.hasProperty("taskPartition") -// if (shouldUseTaskPartitions) { -// final int taskPartitionCount = project.rootProject.property("taskPartitionCount") as int -// final int taskPartition = project.rootProject.property("taskPartition") as int -// final currentTaskPartition = Math.abs(project.path.hashCode() % taskPartitionCount) -// project.setProperty("activePartition", currentTaskPartition == taskPartition) -// } -// } - -// File relativeToGitRoot(File f) { -// return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile() -// } - -// String isAffectedBy(Task baseTask, Map> affectedProjects) { -// HashSet visited = [] -// LinkedList queue = [baseTask] -// while (!queue.isEmpty()) { -// Task t = queue.poll() -// if (visited.contains(t)) { -// continue -// } -// visited.add(t) - -// final Set affectedTasks = affectedProjects.get(t.project) -// if (affectedTasks != null) { -// if (affectedTasks.contains("all")) { -// return "${t.project.path}:${t.name}" -// } -// if (affectedTasks.contains(t.name)) { -// return "${t.project.path}:${t.name}" -// } -// } - -// t.taskDependencies.each { queue.addAll(it.getDependencies(t)) } -// } -// return null -// } - -// List getChangedFiles(String baseRef, String newRef) { -// final stdout = new StringBuilder() -// final stderr = new StringBuilder() -// final proc = "git diff --name-only ${baseRef}..${newRef}".execute() -// proc.consumeProcessOutput(stdout, stderr) -// proc.waitForOrKill(1000) -// assert proc.exitValue() == 0, "git diff command failed, stderr: ${stderr}" -// def out = stdout.toString().trim() -// if (out.isEmpty()) { -// return [] -// } -// logger.debug("git diff output: ${out}") -// return out.split("\n").collect { -// new File(rootProject.projectDir, it.trim()) -// } -// } - -// rootProject.ext { -// useGitChanges = false -// } - -// if (rootProject.hasProperty("gitBaseRef")) { -// final String baseRef = rootProject.property("gitBaseRef") -// final String newRef = rootProject.hasProperty("gitNewRef") ? rootProject.property("gitNewRef") : "HEAD" - -// rootProject.ext { -// it.changedFiles = getChangedFiles(baseRef, newRef) -// useGitChanges = true -// } - -// final ignoredFiles = fileTree(rootProject.projectDir) { -// include '.gitignore', '.editorconfig' -// include '*.md', '**/*.md' -// include 'gradlew', 'gradlew.bat', 'mvnw', 'mvnw.cmd' -// include 'NOTICE' -// include 'static-analysis.datadog.yml' -// } -// rootProject.changedFiles.each { File f -> -// if (ignoredFiles.contains(f)) { -// logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}") -// } -// } -// rootProject.changedFiles = rootProject.changedFiles.findAll { !ignoredFiles.contains(it) } - -// final globalEffectFiles = fileTree(rootProject.projectDir) { -// include '.gitlab/**' -// include 'build.gradle' -// include 'gradle/**' -// } - -// for (File f in rootProject.changedFiles) { -// if (globalEffectFiles.contains(f)) { -// logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)") -// rootProject.useGitChanges = false -// break -// } -// } - -// if (rootProject.useGitChanges) { -// logger.warn("Git change tracking is enabled: ${baseRef}..${newRef}") - -// final projects = subprojects.sort { a, b -> b.projectDir.path.length() <=> a.projectDir.path.length() } -// Map> _affectedProjects = [:] -// // Path prefixes mapped to affected task names. A file not matching any of these prefixes will affect all tasks in -// // the project ("all" can be used a task name to explicitly state the same). Only the first matching prefix is used. -// final List> matchers = [ -// [prefix: 'src/testFixtures/', task: 'testFixturesClasses'], -// [prefix: 'src/test/', task: 'testClasses'], -// [prefix: 'src/jmh/', task: 'jmhCompileGeneratedClasses'] -// ] -// for (File f in rootProject.changedFiles) { -// Project p = projects.find { f.toString().startsWith(it.projectDir.path + "/") } -// if (p == null) { -// logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no task will be skipped)") -// rootProject.useGitChanges = false -// break -// } -// // Make sure path separator is / -// final relPath = Paths.get(p.projectDir.path).relativize(f.toPath()).collect { it.toString() }.join('/') -// final String task = matchers.find { relPath.startsWith(it.prefix) }?.task ?: "all" -// logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (${task})") -// _affectedProjects.computeIfAbsent(p, { new HashSet() }).add(task) -// } -// rootProject.ext { -// it.affectedProjects = _affectedProjects -// } -// } -// } - -// def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) { -// def createRootTask = { String rootTaskName, String subProjTaskName -> -// def coverage = forceCoverage || rootProject.hasProperty("checkCoverage") -// tasks.register(rootTaskName) { aggTest -> -// subprojects { subproject -> -// if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) { -// Task testTask = subproject.tasks.findByName(subProjTaskName) -// boolean isAffected = true -// if (testTask != null) { -// if (rootProject.useGitChanges) { -// final fileTrigger = isAffectedBy(testTask, rootProject.property("affectedProjects")) -// if (fileTrigger != null) { -// logger.warn("Selecting ${subproject.path}:${subProjTaskName} (triggered by ${fileTrigger})") -// } else { -// logger.warn("Skipping ${subproject.path}:${subProjTaskName} (not affected by changed files)") -// isAffected = false -// } -// } -// if (isAffected) { -// aggTest.dependsOn(testTask) -// } -// } -// if (isAffected && coverage) { -// def coverageTask = subproject.tasks.findByName("jacocoTestReport") -// if (coverageTask != null) { -// aggTest.dependsOn(coverageTask) -// } -// coverageTask = subproject.tasks.findByName("jacocoTestCoverageVerification") -// if (coverageTask != null) { -// aggTest.dependsOn(coverageTask) -// } -// } -// } -// } -// } -// } - -// createRootTask "${baseTaskName}Test", 'allTests' -// createRootTask "${baseTaskName}LatestDepTest", 'allLatestDepTests' -// createRootTask "${baseTaskName}Check", 'check' -// } - -// testAggregate("smoke", [":dd-smoke-tests"], []) -// testAggregate("instrumentation", [":dd-java-agent:instrumentation"], []) -// testAggregate("profiling", [":dd-java-agent:agent-profiling"], []) -// testAggregate("debugger", [":dd-java-agent:agent-debugger"], [], true) -// testAggregate("base", [":"], [ -// ":dd-java-agent:instrumentation", -// ":dd-smoke-tests", -// ":dd-java-agent:agent-profiling", -// ":dd-java-agent:agent-debugger" -// ]) - - -// tasks.register('runMuzzle') { -// dependsOn subprojects.findAll { p -> -// p.activePartition && p.plugins.hasPlugin('java') && p.plugins.hasPlugin('muzzle') -// }.collect { p -> p.path + ":muzzle" } -// } From 6c609527f5f5523131e5cb4f624a52a30ea79bcb Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Thu, 23 Oct 2025 09:46:43 -0400 Subject: [PATCH 3/4] Address review comments --- build.gradle.kts | 1 + buildSrc/src/main/kotlin/CIJobsExtensions.kt | 112 ++++++++++-------- .../main/kotlin/datadog.ci-jobs.gradle.kts | 1 + 3 files changed, 63 insertions(+), 51 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 41ecb4b63e1..96d51be26a0 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,4 +1,5 @@ import com.diffplug.gradle.spotless.SpotlessExtension +import datadog.ci.testAggregate plugins { id("datadog.gradle-debug") diff --git a/buildSrc/src/main/kotlin/CIJobsExtensions.kt b/buildSrc/src/main/kotlin/CIJobsExtensions.kt index 0ac106a53a6..20f3a03c4f0 100644 --- a/buildSrc/src/main/kotlin/CIJobsExtensions.kt +++ b/buildSrc/src/main/kotlin/CIJobsExtensions.kt @@ -1,3 +1,5 @@ +package datadog.ci + import org.gradle.api.Project import org.gradle.api.Task import org.gradle.kotlin.dsl.extra @@ -32,67 +34,75 @@ internal fun isAffectedBy(baseTask: Task, affectedProjects: Map, excludePrefixes: List, - forceCoverage: Boolean = false + forceCoverage: Boolean ) { - fun createRootTask(rootTaskName: String, subProjTaskName: String) { - val coverage = forceCoverage || rootProject.hasProperty("checkCoverage") - val proj = this@testAggregate - tasks.register(rootTaskName) { - proj.subprojects.forEach { subproject -> - val activePartition = subproject.extra.get("activePartition") as Boolean - if (activePartition && - includePrefixes.any { subproject.path.startsWith(it) } && - !excludePrefixes.any { subproject.path.startsWith(it) }) { - - val testTask = subproject.tasks.findByName(subProjTaskName) - var isAffected = true - - if (testTask != null) { - val useGitChanges = proj.rootProject.extra.get("useGitChanges") as Boolean - if (useGitChanges) { - @Suppress("UNCHECKED_CAST") - val affectedProjects = proj.rootProject.extra.get("affectedProjects") as Map> - val fileTrigger = isAffectedBy(testTask, affectedProjects) - if (fileTrigger != null) { - proj.logger.warn("Selecting ${subproject.path}:$subProjTaskName (triggered by $fileTrigger)") - } else { - proj.logger.warn("Skipping ${subproject.path}:$subProjTaskName (not affected by changed files)") - isAffected = false - } - } - if (isAffected) { - dependsOn(testTask) + val coverage = forceCoverage || rootProject.hasProperty("checkCoverage") + tasks.register(rootTaskName) { + subprojects.forEach { subproject -> + val activePartition = subproject.extra.get("activePartition") as Boolean + if (activePartition && + includePrefixes.any { subproject.path.startsWith(it) } && + !excludePrefixes.any { subproject.path.startsWith(it) }) { + + val testTask = subproject.tasks.findByName(subProjTaskName) + var isAffected = true + + if (testTask != null) { + val useGitChanges = rootProject.extra.get("useGitChanges") as Boolean + if (useGitChanges) { + @Suppress("UNCHECKED_CAST") + val affectedProjects = rootProject.extra.get("affectedProjects") as Map> + val fileTrigger = isAffectedBy(testTask, affectedProjects) + if (fileTrigger != null) { + logger.warn("Selecting ${subproject.path}:$subProjTaskName (triggered by $fileTrigger)") + } else { + logger.warn("Skipping ${subproject.path}:$subProjTaskName (not affected by changed files)") + isAffected = false } } - - if (isAffected && coverage) { - val coverageTask = subproject.tasks.findByName("jacocoTestReport") - if (coverageTask != null) { - dependsOn(coverageTask) - } - val verificationTask = subproject.tasks.findByName("jacocoTestCoverageVerification") - if (verificationTask != null) { - dependsOn(verificationTask) - } + if (isAffected) { + dependsOn(testTask) + } + } + + if (isAffected && coverage) { + val coverageTask = subproject.tasks.findByName("jacocoTestReport") + if (coverageTask != null) { + dependsOn(coverageTask) + } + val verificationTask = subproject.tasks.findByName("jacocoTestCoverageVerification") + if (verificationTask != null) { + dependsOn(verificationTask) } } } } } - - createRootTask("${baseTaskName}Test", "allTests") - createRootTask("${baseTaskName}LatestDepTest", "allLatestDepTests") - createRootTask("${baseTaskName}Check", "check") +} + +/** + * Creates aggregate test tasks for CI using createRootTask() above + * + * Creates three subtasks for the given base task name: + * - ${baseTaskName}Test - runs allTests + * - ${baseTaskName}LatestDepTest - runs allLatestDepTests + * - ${baseTaskName}Check - runs check + */ +fun Project.testAggregate( + baseTaskName: String, + includePrefixes: List, + excludePrefixes: List, + forceCoverage: Boolean = false +) { + createRootTask("${baseTaskName}Test", "allTests", includePrefixes, excludePrefixes, forceCoverage) + createRootTask("${baseTaskName}LatestDepTest", "allLatestDepTests", includePrefixes, excludePrefixes, forceCoverage) + createRootTask("${baseTaskName}Check", "check", includePrefixes, excludePrefixes, forceCoverage) } diff --git a/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts b/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts index 4705059c4e1..eb45f9a58ab 100644 --- a/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts +++ b/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts @@ -4,6 +4,7 @@ * with -PgitBaseRef. */ +import datadog.ci.isAffectedBy import java.io.File import kotlin.math.abs From 1ea522f2be856408277d314d54b1727a055ec46b Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Fri, 24 Oct 2025 11:02:21 -0400 Subject: [PATCH 4/4] Address review comments pt 2 --- build.gradle.kts | 4 ++-- buildSrc/src/main/kotlin/CIJobsExtensions.kt | 4 ++-- buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 96d51be26a0..abcd900cd3f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,5 +1,5 @@ import com.diffplug.gradle.spotless.SpotlessExtension -import datadog.ci.testAggregate +import datadog.gradle.plugin.ci.testAggregate plugins { id("datadog.gradle-debug") @@ -142,7 +142,7 @@ allprojects { testAggregate("smoke", listOf(":dd-smoke-tests"), emptyList()) testAggregate("instrumentation", listOf(":dd-java-agent:instrumentation"), emptyList()) testAggregate("profiling", listOf(":dd-java-agent:agent-profiling"), emptyList()) -testAggregate("debugger", listOf(":dd-java-agent:agent-debugger"), emptyList(), true) +testAggregate("debugger", listOf(":dd-java-agent:agent-debugger"), forceCoverage = true) testAggregate( "base", listOf(":"), listOf( diff --git a/buildSrc/src/main/kotlin/CIJobsExtensions.kt b/buildSrc/src/main/kotlin/CIJobsExtensions.kt index 20f3a03c4f0..1e284329e4c 100644 --- a/buildSrc/src/main/kotlin/CIJobsExtensions.kt +++ b/buildSrc/src/main/kotlin/CIJobsExtensions.kt @@ -1,4 +1,4 @@ -package datadog.ci +package datadog.gradle.plugin.ci import org.gradle.api.Project import org.gradle.api.Task @@ -98,7 +98,7 @@ private fun Project.createRootTask( fun Project.testAggregate( baseTaskName: String, includePrefixes: List, - excludePrefixes: List, + excludePrefixes: List = emptyList(), forceCoverage: Boolean = false ) { createRootTask("${baseTaskName}Test", "allTests", includePrefixes, excludePrefixes, forceCoverage) diff --git a/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts b/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts index eb45f9a58ab..a7b495c9096 100644 --- a/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts +++ b/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts @@ -4,7 +4,7 @@ * with -PgitBaseRef. */ -import datadog.ci.isAffectedBy +import datadog.gradle.plugin.ci.isAffectedBy import java.io.File import kotlin.math.abs