Skip to content

ci_jobs plugin clean up #9858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
+168 −169
Open

ci_jobs plugin clean up #9858

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d329dd4
Make CIJobsExtensions lazy
sarahchen6 Oct 24, 2025
f3aff4d
More CIJobsExtensions lazy API
sarahchen6 Oct 24, 2025
fa99f07
Fix spacing
sarahchen6 Oct 24, 2025
6d4f182
Make ci-jobs plugin lazy
sarahchen6 Oct 24, 2025
bb1e9ff
Remove some provider usage
sarahchen6 Oct 29, 2025
8b4821c
Rename some CIJobsExtensions variables
sarahchen6 Oct 29, 2025
adc18f0
Improve comment
sarahchen6 Oct 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 76 additions & 75 deletions buildSrc/src/main/kotlin/CIJobsExtensions.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,104 +5,105 @@ import org.gradle.api.Task
import org.gradle.kotlin.dsl.extra

/**
* Checks if a task is affected by git changes
* Returns the task's path, given affected projects, if this task or its dependencies are affected by git changes.
*/
internal fun isAffectedBy(baseTask: Task, affectedProjects: Map<Project, Set<String>>): String? {
val visited = mutableSetOf<Task>()
val queue = mutableListOf(baseTask)
internal fun findAffectedTaskPath(baseTask: Task, affectedProjects: Map<Project, Set<String>>): String? {
val visited = mutableSetOf<Task>()
val queue = mutableListOf(baseTask)

while (queue.isNotEmpty()) {
val t = queue.removeAt(0)
if (visited.contains(t)) {
continue
}
visited.add(t)

while (queue.isNotEmpty()) {
val t = queue.removeAt(0)
if (visited.contains(t)) {
continue
}
visited.add(t)

val affectedTasks = affectedProjects[t.project]
if (affectedTasks != null) {
if (affectedTasks.contains("all")) {
return "${t.project.path}:${t.name}"
}
if (affectedTasks.contains(t.name)) {
return "${t.project.path}:${t.name}"
}
}

t.taskDependencies.getDependencies(t).forEach { queue.add(it) }
val affectedTasks = affectedProjects[t.project]
if (affectedTasks != null) {
if (affectedTasks.contains("all")) {
return "${t.project.path}:${t.name}"
}
if (affectedTasks.contains(t.name)) {
return "${t.project.path}:${t.name}"
}
}
return null

t.taskDependencies.getDependencies(t).forEach { queue.add(it) }
}
return null
}

/**
* Creates a single aggregate root task that depends on matching subproject tasks
*/
private fun Project.createRootTask(
rootTaskName: String,
subProjTaskName: String,
includePrefixes: List<String>,
excludePrefixes: List<String>,
forceCoverage: Boolean
rootTaskName: String,
subProjTaskName: String,
includePrefixes: List<String>,
excludePrefixes: List<String>,
forceCoverage: Boolean
) {
val coverage = forceCoverage || rootProject.hasProperty("checkCoverage")
tasks.register(rootTaskName) {
subprojects.forEach { subproject ->
Copy link
Contributor

@bric3 bric3 Oct 28, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

note: Actually, I was wrong! The that code here is not executed unless the actual root task is requested. So providers won't provide speed-up there.

Copy link
Contributor Author

@sarahchen6 sarahchen6 Oct 29, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah I see! I'll revert this use of providers and for the code below in buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts that I believe is a similar situation because it registers the task, so should already be lazy.

tasks.register("runMuzzle") {
    val muzzleSubprojects = subprojects.filter { p ->
        val activePartition = p.extra.get("activePartition") as Boolean
        activePartition && p.plugins.hasPlugin("java") && p.plugins.hasPlugin("muzzle")
    }
    dependsOn(muzzleSubprojects.map { p -> "${p.path}:muzzle" })
}

val activePartition = subproject.extra.get("activePartition") as Boolean
if (activePartition &&
includePrefixes.any { subproject.path.startsWith(it) } &&
!excludePrefixes.any { subproject.path.startsWith(it) }) {

val testTask = subproject.tasks.findByName(subProjTaskName)
var isAffected = true

if (testTask != null) {
val useGitChanges = rootProject.extra.get("useGitChanges") as Boolean
if (useGitChanges) {
@Suppress("UNCHECKED_CAST")
val affectedProjects = rootProject.extra.get("affectedProjects") as Map<Project, Set<String>>
val fileTrigger = isAffectedBy(testTask, affectedProjects)
if (fileTrigger != null) {
logger.warn("Selecting ${subproject.path}:$subProjTaskName (triggered by $fileTrigger)")
} else {
logger.warn("Skipping ${subproject.path}:$subProjTaskName (not affected by changed files)")
isAffected = false
}
}
if (isAffected) {
dependsOn(testTask)
}
}

if (isAffected && coverage) {
val coverageTask = subproject.tasks.findByName("jacocoTestReport")
if (coverageTask != null) {
dependsOn(coverageTask)
}
val verificationTask = subproject.tasks.findByName("jacocoTestCoverageVerification")
if (verificationTask != null) {
dependsOn(verificationTask)
}
}
val coverage = forceCoverage || rootProject.providers.gradleProperty("checkCoverage").isPresent
tasks.register(rootTaskName) {
subprojects.forEach { subproject ->
val activePartition = subproject.extra.get("activePartition") as Boolean
if (
activePartition &&
includePrefixes.any { subproject.path.startsWith(it) } &&
!excludePrefixes.any { subproject.path.startsWith(it) }
) {
val testTask = subproject.tasks.findByName(subProjTaskName)
var isAffected = true

if (testTask != null) {
val useGitChanges = rootProject.extra.get("useGitChanges") as Boolean
if (useGitChanges) {
@Suppress("UNCHECKED_CAST")
val affectedProjects = rootProject.extra.get("affectedProjects") as Map<Project, Set<String>>
val affectedTaskPath = findAffectedTaskPath(testTask, affectedProjects)
if (affectedTaskPath != null) {
logger.warn("Selecting ${subproject.path}:$subProjTaskName (affected by $affectedTaskPath)")
} else {
logger.warn("Skipping ${subproject.path}:$subProjTaskName (not affected by changed files)")
isAffected = false
}
}
if (isAffected) {
dependsOn(testTask)
}
}

if (isAffected && coverage) {
val coverageTask = subproject.tasks.findByName("jacocoTestReport")
if (coverageTask != null) {
dependsOn(coverageTask)
}
val verificationTask = subproject.tasks.findByName("jacocoTestCoverageVerification")
if (verificationTask != null) {
dependsOn(verificationTask)
}
}
}
}
}
}

/**
* Creates aggregate test tasks for CI using createRootTask() above
*
*
* Creates three subtasks for the given base task name:
* - ${baseTaskName}Test - runs allTests
* - ${baseTaskName}LatestDepTest - runs allLatestDepTests
* - ${baseTaskName}Check - runs check
*/
fun Project.testAggregate(
baseTaskName: String,
includePrefixes: List<String>,
excludePrefixes: List<String> = emptyList(),
forceCoverage: Boolean = false
baseTaskName: String,
includePrefixes: List<String>,
excludePrefixes: List<String> = emptyList(),
forceCoverage: Boolean = false
) {
createRootTask("${baseTaskName}Test", "allTests", includePrefixes, excludePrefixes, forceCoverage)
createRootTask("${baseTaskName}LatestDepTest", "allLatestDepTests", includePrefixes, excludePrefixes, forceCoverage)
createRootTask("${baseTaskName}Check", "check", includePrefixes, excludePrefixes, forceCoverage)
createRootTask("${baseTaskName}Test", "allTests", includePrefixes, excludePrefixes, forceCoverage)
createRootTask("${baseTaskName}LatestDepTest", "allLatestDepTests", includePrefixes, excludePrefixes, forceCoverage)
createRootTask("${baseTaskName}Check", "check", includePrefixes, excludePrefixes, forceCoverage)
}

Loading