Develocity and Gradle Enterprise remote build cache connectors are supported (#807)

* Support Develocity and Gradle Enterprise remote build cache connectors

Adds a new command line argument for Gradle experiment 5, `-y` or
`--remote-build-cache-type`, allowing the user to specify a remote build
cache type to use. When specified, valid values are: `develocity`,
`gradle-enterprise`, or `http`. When the argument is specified, it will
always take precedence over what is configured in the build.

If the argument is not specified, the remote build cache type of the
first build will be fetched from the Develocity API and used for the
second build.

If the argument is not specified and the user does not have access to
the Develocity API, then the existing remote build cache configuration
will be used.

* Update user-facing text

* Update release notes

* Extract init script logic to helper functions

* Add 'connector' to help, interactive, and error text

* Bump com.gradle:build-scan-summary from 1.0.3-2024.1 to 1.0.4-2024.1

Bumps com.gradle:build-scan-summary from 1.0.3-2024.1 to 1.0.4-2024.1.

---
updated-dependencies:
- dependency-name: com.gradle:build-scan-summary
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <[email protected]>

---------

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

Author: erichaagdev

build.gradle.kts

@@ -61,7 +61,7 @@ val isDevelopmentRelease = !hasProperty("finalRelease")
 val releaseVersion = releaseVersion()
 val releaseNotes = releaseNotes()
 val distributionVersion = distributionVersion()
-val buildScanSummaryVersion = "1.0.3-2024.1"
+val buildScanSummaryVersion = "1.0.4-2024.1"
 
 allprojects {
     version = releaseVersion.get()

components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh

@@ -40,6 +40,7 @@ develocity_server=''
 interactive_mode=''
 
 ci_build_scan_url=''
+remote_build_cache_type=''
 remote_build_cache_url=''
 mapping_file=''
 
@@ -110,9 +111,9 @@ wizard_execute() {
   collect_gradle_details
 
   print_bl
-  explain_remote_build_cache_url
+  explain_collect_remote_build_cache
   print_bl
-  collect_remote_build_cache_url
+  collect_remote_build_cache
   explain_command_to_repeat_experiment_after_collecting_parameters
 
   print_bl
@@ -136,6 +137,7 @@ wizard_execute() {
 
 map_additional_script_args() {
   ci_build_scan_url="${_arg_first_build_ci}"
+  remote_build_cache_type="${_arg_remote_build_cache_type}"
   remote_build_cache_url="${_arg_remote_build_cache_url}"
   mapping_file="${_arg_mapping_file}"
 }
@@ -151,6 +153,10 @@ validate_required_args() {
   if [[ "${enable_develocity}" == "on" && -z "${develocity_server}" ]]; then
     _PRINT_HELP=yes die "ERROR: Missing required argument when enabling Develocity on a project not already connected: --develocity-server" "${INVALID_INPUT}"
   fi
+
+  if [[ -n "${remote_build_cache_type}" && "${remote_build_cache_type}" != 'http' && "${remote_build_cache_type}" != 'gradle-enterprise' && "${remote_build_cache_type}" != 'develocity' ]]; then
+    _PRINT_HELP=yes die "ERROR: Invalid value for argument --remote-build-cache-type. Values are 'develocity', 'gradle-enterprise', or 'http'." "${INVALID_INPUT}"
+  fi
 }
 
 fetch_build_params_from_build_scan() {
@@ -160,6 +166,9 @@ fetch_build_params_from_build_scan() {
 }
 
 read_build_params_from_build_scan_data() {
+  if [[ "${remote_build_cache_types[0]}" == "disabled" ]]; then
+    die "ERROR: Remote build cache was disabled for the first build. Enable the remote build cache in the build and restart the experiment."
+  fi
   if [ -z "${git_repo}" ]; then
     git_repo="${git_repos[0]}"
     project_name="$(basename -s .git "${git_repo}")"
@@ -170,6 +179,9 @@ read_build_params_from_build_scan_data() {
   if [ -z "${git_commit_id}" ]; then
     git_commit_id="${git_commit_ids[0]}"
   fi
+  if [[ -z "${remote_build_cache_type}" && "${remote_build_cache_types[0]}" != "unknown" ]]; then
+    remote_build_cache_type="${remote_build_cache_types[0]}"
+  fi
   if [ -z "${remote_build_cache_url}" ]; then
     remote_build_cache_url="${remote_build_cache_urls[0]}"
   fi
@@ -196,6 +208,9 @@ validate_build_config() {
 execute_build() {
   local args
   args=(--build-cache --init-script "${INIT_SCRIPTS_DIR}/configure-remote-build-caching.gradle")
+  if [ -n "${remote_build_cache_type}" ]; then
+    args+=("-Ddevelocity.build-validation.remoteBuildCacheType=${remote_build_cache_type}")
+  fi
   if [ -n "${remote_build_cache_url}" ]; then
     args+=("-Ddevelocity.build-validation.remoteBuildCacheUrl=${remote_build_cache_url}")
   fi
@@ -360,7 +375,7 @@ EOF
   print_interactive_text "${text}"
 }
 
-explain_remote_build_cache_url() {
+explain_collect_remote_build_cache() {
   local text
   IFS='' read -r -d '' text <<EOF
 The local build will connect to the given remote build cache. The remote build
@@ -369,11 +384,30 @@ EOF
   print_interactive_text "${text}"
 }
 
+collect_remote_build_cache() {
+  collect_remote_build_cache_type
+  collect_remote_build_cache_url
+}
+
+collect_remote_build_cache_type() {
+  local default_remote_cache_type="<project default>"
+  prompt_for_setting "What is the remote build cache connector type to use? [develocity, gradle-enterprise, http, or <BLANK>]" "${remote_build_cache_type}" "${default_remote_cache_type}" remote_build_cache_type
+
+  if [[ -n "${remote_build_cache_type}" && "${remote_build_cache_type}" != 'http' && "${remote_build_cache_type}" != 'gradle-enterprise' && "${remote_build_cache_type}" != 'develocity' ]]; then
+    print_bl
+    die "ERROR: Invalid value for remote build cache connector type. Values are 'develocity', 'gradle-enterprise', 'http', or <BLANK> for project default." "${INVALID_INPUT}"
+  fi
+
+  if [[ "${remote_build_cache_type}" == "${default_remote_cache_type}" ]]; then
+    remote_build_cache_type=''
+  fi
+}
+
 collect_remote_build_cache_url() {
-  local default_remote_cache="<project default>"
-  prompt_for_setting "What is the remote build cache url to use?" "${remote_build_cache_url}" "${default_remote_cache}" remote_build_cache_url
+  local default_remote_cache_url="<project default>"
+  prompt_for_setting "What is the remote build cache url to use?" "${remote_build_cache_url}" "${default_remote_cache_url}" remote_build_cache_url
 
-  if [[ "${remote_build_cache_url}" == "${default_remote_cache}" ]]; then
+  if [[ "${remote_build_cache_url}" == "${default_remote_cache_url}" ]]; then
     remote_build_cache_url=''
   fi
 }
@@ -465,6 +499,10 @@ generate_command_to_repeat_experiment() {
     cmd+=("-m" "${mapping_file}")
   fi
 
+  if [ -n "${remote_build_cache_type}" ]; then
+    cmd+=("-y" "${remote_build_cache_type}")
+  fi
+
   if [ -n "${remote_build_cache_url}" ]; then
     cmd+=("-u" "${remote_build_cache_url}")
   fi

components/scripts/gradle/gradle-init-scripts/configure-remote-build-caching.gradle

@@ -4,23 +4,134 @@ static getInputParam(Gradle gradle, String name) {
     return gradle.startParameter.systemPropertiesArgs[name] ?: System.getProperty(name) ?: System.getenv(envVarName)
 }
 
+def isTopLevelBuild = !gradle.parent
+
+def expDir = getInputParam(gradle, 'develocity.build-validation.expDir')
+def remoteBuildCacheType = getInputParam(gradle, 'develocity.build-validation.remoteBuildCacheType')
 def remoteBuildCacheUrl = getInputParam(gradle, 'develocity.build-validation.remoteBuildCacheUrl')
 
-settingsEvaluated { settings ->
+def docsRoot = 'https://docs.gradle.com/develocity/gradle-plugin'
+
+settingsEvaluated { Settings settings ->
     settings.buildCache {
         local {
             enabled = false
         }
-        remote(HttpBuildCache) {
-            enabled = true
-            push = false
-            if (remoteBuildCacheUrl) {
-                url = withPathTrailingSlash(new URI(remoteBuildCacheUrl))
+
+        if (remoteBuildCacheType) {
+            if (isInvalidRemoteBuildCacheType(remoteBuildCacheType)) {
+                failInvalidRemoteBuildCacheType(remoteBuildCacheType, expDir)
+            }
+
+            if (isTopLevelBuild && missingRequiredPlugin(settings, remoteBuildCacheType)) {
+                failMissingRequiredPlugin(remoteBuildCacheType, expDir, docsRoot)
+            }
+
+            def remoteBuildCacheImplementation = getRemoteBuildCacheImplementation(settings, remoteBuildCacheType)
+            if (remoteBuildCacheImplementation) {
+                logger.debug("Configuring remote build cache implementation for '${settings.rootProject.name}' as: ${remoteBuildCacheImplementation}")
+                remote(remoteBuildCacheImplementation)
+            }
+        }
+
+        logger.debug("Remote build cache implementation for '${settings.rootProject.name}' is: ${remote?.class?.name}")
+        if (remote) {
+            remote {
+                enabled = true
+            }
+
+            def remoteBuildCacheUri = remoteBuildCacheUrl ? withPathTrailingSlash(new URI(remoteBuildCacheUrl)) : null
+            if (remote instanceof HttpBuildCache) {
+                if (remoteBuildCacheUrl) {
+                    remote.url = remoteBuildCacheUri
+                } else if (!remote.url) {
+                    failMissingUrlForHttpBuildCache(expDir, docsRoot)
+                }
+            } else if (isBuildCacheImplementationFor(remote, 'com.gradle.develocity') || isBuildCacheImplementationFor(remote, 'com.gradle.enterprise')) {
+                if (remoteBuildCacheUri) {
+                    remote.server = toServerPart(remoteBuildCacheUri)
+                    remote.path = remoteBuildCacheUri.path
+                }
             }
+        } else if (isTopLevelBuild) {
+            failMissingRemoteBuildCacheConfiguration(expDir, docsRoot)
         }
     }
 }
 
+static boolean isInvalidRemoteBuildCacheType(String remoteBuildCacheType) {
+    return !['develocity', 'gradle-enterprise', 'http'].contains(remoteBuildCacheType)
+}
+
+static boolean missingRequiredPlugin(Settings settings, String type) {
+    return type == "develocity" && !settings.pluginManager.hasPlugin('com.gradle.develocity')
+        || type == "gradle-enterprise" && !settings.pluginManager.hasPlugin('com.gradle.enterprise')
+}
+
+static Class<? extends BuildCache> getRemoteBuildCacheImplementation(Settings settings, String type) {
+    if (type == "develocity") {
+        return settings.develocity.buildCache
+    } else if (type == "gradle-enterprise") {
+        return settings.gradleEnterprise.buildCache
+    } else if (type == "http") {
+        return HttpBuildCache
+    }
+    return null
+}
+
+static boolean isBuildCacheImplementationFor(BuildCache buildCache, String implementation) {
+    return buildCache.class.name.startsWith(implementation)
+}
+
 static URI withPathTrailingSlash(URI uri) {
-    uri.path.endsWith("/") ? uri : new URI(uri.scheme, uri.userInfo, uri.host, uri.port, uri.path + "/", uri.query, uri.fragment)
+    return uri.path.endsWith("/") ? uri : new URI(uri.scheme, uri.userInfo, uri.host, uri.port, uri.path + "/", uri.query, uri.fragment)
+}
+
+static String toServerPart(URI uri) {
+    return new URI(uri.scheme, uri.userInfo, uri.host, uri.port, null, uri.query, uri.fragment)
+}
+
+// The scripts already fail if the value of --remote-build-cache-type isn't valid.
+// This is for the sake of completeness since this init script assumes it's valid.
+static void failInvalidRemoteBuildCacheType(String remoteBuildCacheType, String expDir) {
+    def errorFile = new File(expDir, 'errors.txt')
+    def message = "Invalid value '${remoteBuildCacheType}' for remote build cache connector type. Values are 'develocity', 'gradle-enterprise', or 'http'."
+    errorFile.text = message
+    throw new IllegalStateException(message)
+}
+
+// Included builds may not have the necessary plugin applied.
+// Only fail if the top-level build is missing the required extension.
+static void failMissingRequiredPlugin(String remoteBuildCacheType, String expDir, String docsRoot) {
+    def errorFile = new File(expDir, 'errors.txt')
+    errorFile.text = "Remote build cache connector type '${remoteBuildCacheType}' requested, but the required plugin is not applied."
+    if (remoteBuildCacheType == 'develocity') {
+        throw new IllegalStateException("Remote build cache connector type 'develocity' requested,\n" +
+            "but the Develocity Gradle plugin is not applied.\n" +
+            "Either apply it directly (see $docsRoot/current/#applying_the_plugin),\n" +
+            "use --enable-develocity to enable the plugin,\n" +
+            "or use --remote-build-cache-type to choose a different remote build cache connector type\n" +
+            "when running the build validation script.")
+    } else {
+        throw new IllegalStateException("Remote build cache connector type 'gradle-enterprise' requested,\n" +
+            "but the Gradle Enterprise Gradle plugin is not applied (see $docsRoot/legacy/#applying_the_plugin).")
+    }
+}
+
+// Gradle already fails in this case, but handling it here means we can fail the experiment more
+// gracefully and provide guidance the user.
+static void failMissingUrlForHttpBuildCache(String expDir, String docsRoot) {
+    def errorFile = new File(expDir, 'errors.txt')
+    errorFile.text = 'A remote build cache URL has not been configured in the project or on the command line.'
+    throw new IllegalStateException("A remote build cache URL is not configured.\n"
+        + "Either configure it directly (see $docsRoot/current/#using_gradles_built_in_http_connector) in the project,\n"
+        + "or use --remote-build-cache-url when running the build validation script.")
+}
+
+static void failMissingRemoteBuildCacheConfiguration(String expDir, String docsRoot) {
+    def errorFile = new File(expDir, 'errors.txt')
+    errorFile.text = "Remote build cache is not configured for the project."
+    throw new IllegalStateException("Remote build cache is not configured for the project.\n" +
+        "Either configure it directly (see $docsRoot/current/#using_the_develocity_connector),\n" +
+        "or use --remote-build-cache-type when running the build validation script.")
 }

components/scripts/lib/build-scan-parse.sh

@@ -10,6 +10,8 @@ git_branches=()
 git_commit_ids=()
 requested_tasks=()
 build_outcomes=()
+remote_build_cache_types=()
+remote_build_cache_class_names=()
 remote_build_cache_urls=()
 remote_build_cache_shards=()
 
@@ -77,7 +79,7 @@ parse_build_scan_row() {
 
   local run_num
 
-  while IFS=, read -r run_num field_1 field_2 field_3 field_4 field_5 field_6 field_7 field_8 field_9 field_10 field_11 field_12 field_13 field_14 field_15 field_16 field_17 field_18 field_19 field_20; do
+  while IFS=, read -r run_num field_1 field_2 field_3 field_4 field_5 field_6 field_7 field_8 field_9 field_10 field_11 field_12 field_13 field_14 field_15 field_16 field_17 field_18 field_19 field_20 field_21 field_22; do
     debug "Build Scan $field_4 is for build $run_num"
 
     # The project_name should be overridden by Build Scan data if it is
@@ -127,21 +129,23 @@ parse_build_scan_row() {
     # The below fields are always set by Build Scan data regardless of their
     # previous value and are always safe to override.
 
-    remote_build_cache_urls[run_num]="${field_10}"
-    remote_build_cache_shards[run_num]="${field_11}"
+    remote_build_cache_types[run_num]="${field_10}"
+    remote_build_cache_class_names[run_num]="${field_11}"
+    remote_build_cache_urls[run_num]="${field_12}"
+    remote_build_cache_shards[run_num]="${field_13}"
 
     # Build caching performance metrics
-    avoided_up_to_date_num_tasks[run_num]="${field_12}"
-    avoided_up_to_date_avoidance_savings[run_num]="${field_13}"
-    avoided_from_cache_num_tasks[run_num]="${field_14}"
-    avoided_from_cache_avoidance_savings[run_num]="${field_15}"
-    executed_cacheable_num_tasks[run_num]="${field_16}"
-    executed_cacheable_duration[run_num]="${field_17}"
-    executed_not_cacheable_num_tasks[run_num]="${field_18}"
-    executed_not_cacheable_duration[run_num]="${field_19}"
+    avoided_up_to_date_num_tasks[run_num]="${field_14}"
+    avoided_up_to_date_avoidance_savings[run_num]="${field_15}"
+    avoided_from_cache_num_tasks[run_num]="${field_16}"
+    avoided_from_cache_avoidance_savings[run_num]="${field_17}"
+    executed_cacheable_num_tasks[run_num]="${field_18}"
+    executed_cacheable_duration[run_num]="${field_19}"
+    executed_not_cacheable_num_tasks[run_num]="${field_20}"
+    executed_not_cacheable_duration[run_num]="${field_21}"
 
     # Other build metrics
-    serialization_factors[run_num]="${field_20}"
+    serialization_factors[run_num]="${field_22}"
   done <<< "${build_scan_row}"
 }
 

components/scripts/lib/cli-parsers/gradle/05-cli-parser.m4

@@ -3,6 +3,7 @@
 # Created by argbash-init v2.10.0
 # ARG_OPTIONAL_SINGLE([first-build-ci],[1],[])
 # ARG_OPTIONAL_SINGLE([mapping-file],[m],[])
+# ARG_OPTIONAL_SINGLE([remote-build-cache-type],[y],[])
 # ARG_OPTIONAL_SINGLE([remote-build-cache-url],[u],[])
 # ARG_OPTIONAL_BOOLEAN([fail-if-not-fully-cacheable],[f],[])
 # ARG_HELP([This function is overridden later on.])
@@ -30,6 +31,7 @@ function print_help() {
   print_option_usage -p
   print_option_usage -t
   print_option_usage -a
+  print_option_usage "-y, --remote-build-cache-type" "Specifies the remote build cache connector type to use in the second build run locally. Values are 'develocity', 'gradle-enterprise', or 'http'."
   print_option_usage "-u, --remote-build-cache-url" "Specifies the URL for the remote build cache to access in the second build run locally."
   print_option_usage -s
   print_option_usage -e

components/scripts/lib/summary.sh

@@ -119,20 +119,34 @@ detect_warnings_from_build_scans() {
     if [ -z "${build_outcomes[i]}" ]; then
       warnings+=("Failed to fetch build scan data for the ${ORDINALS[i]} build.")
     fi
+    if [ "${remote_build_cache_types[i]}" == "unknown" ]; then
+      # "Develocity Build Validation Scripts" is specifically mentioned as to
+      # not imply Develocity itself does not work with other remote build cache
+      # implementations.
+      warnings+=("The ${ORDINALS[i]} build ran using a remote build cache implementation not officially supported by the experiment.")
+    fi
   done
 
   local value_mismatch=false
   if [[ "${project_names[0]}" != "${project_names[1]}" ]] ||
      [[ "${git_repos[0]}" != "${git_repos[1]}" ]] ||
      [[ "${git_branches[0]}" != "${git_branches[1]}" ]] ||
      [[ "${git_commit_ids[0]}" != "${git_commit_ids[1]}" ]] ||
-     [[ "${requested_tasks[0]}" != "${requested_tasks[1]}" ]]; then
+     [[ "${requested_tasks[0]}" != "${requested_tasks[1]}" ]] ||
+     [[ "${remote_build_cache_urls[0]}" != "${remote_build_cache_urls[1]}" ]] ||
+     [[ "${remote_build_cache_class_names[0]}" != "${remote_build_cache_class_names[1]}" ]]; then
     value_mismatch=true
   fi
 
   if [[ "${value_mismatch}" == "true" ]]; then
     warnings+=("Differences were detected between the two builds. This may skew the outcome of the experiment.")
   fi
+  if [ "${remote_build_cache_urls[0]}" != "${remote_build_cache_urls[1]}" ]; then
+    warnings+=("The two builds ran with different remote build cache URLs configured.")
+  fi
+  if [ "${remote_build_cache_class_names[0]}" != "${remote_build_cache_class_names[1]}" ]; then
+    warnings+=("The two builds ran using different remote build cache implementations.")
+  fi
   if [[ "${unknown_values}" == "true" ]]; then
     warnings+=("Some of the build properties could not be determined. This makes it uncertain if the experiment has run correctly.")
   fi

components/scripts/maven/04-validate-remote-build-caching-ci-local.sh

@@ -158,6 +158,9 @@ fetch_build_params_from_build_scan() {
 }
 
 read_build_params_from_build_scan_data() {
+  if [[ "${remote_build_cache_types[0]}" == "disabled" ]]; then
+    die "ERROR: Remote build cache was disabled for the first build. Enable the remote build cache in the build and restart the experiment."
+  fi
   if [ -z "${git_repo}" ]; then
     git_repo="${git_repos[0]}"
     project_name="$(basename -s .git "${git_repo}")"

release/changes.md

@@ -1,6 +1,8 @@
 > [!IMPORTANT]
 > The distributions of the Develocity Build Validation Scripts prefixed with `gradle-enterprise` are deprecated and will be removed in a future release. Migrate to the distributions prefixed with `develocity` instead.
 
+- [NEW] Support Develocity and Gradle Enterprise remote build cache connectors in the Gradle CI/Local experiment  
+- [NEW] Better handling of remote build cache misconfigurations
 - [FIX] Scripts do not wait long enough for build scans to become available when `--fail-if-not-fully-cacheable` is used
 - [FIX] Successful exit code returned when performance characteristics are unknown and `--fail-if-not-fully-cacheable` is used
 - [FIX] Gradle experiments do not disable background Build Scan publication