ci: build agent from snapshot DRA (#9048)

pkoutsovasilis · web-flow · commit a155660c4394 · 2025-08-07T11:43:24.000+03:00
* feat: rework .package-version and mage integration:UpdatePackageVersion to make CI build always from snapshot DRA * feat: incorporate USE_PACKAGE_VERSION in mage * experiment: bump version.go * Revert "experiment: bump version.go" This reverts commit a57ee10. * chore: bump .package-version * feat: allow AGENT_VERSION to be overridden by env var * fix: use named args for all args in integration_tests_tf.ps1 * feat: panic on err of initPackageVersion * fix: don't panic when .package-version file doesn't exist, log it instead * feat: rework fabrication of CI_ELASTIC_AGENT_DOCKER_IMAGE * feat: use os.WriteFile in writePackageVersion * chore: bump to latest snapshot DRA * fix: always DownloadManifest if PackagingFromManifest is set in mage package * fix: check err of filepath.Abs(dropPath)
diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml
@@ -111,13 +111,9 @@ steps:
         env:
           PACKAGES: "docker"
           PLATFORMS: "linux/amd64"
-          CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
-          CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud"
         command: |
-          source .buildkite/scripts/common.sh
           .buildkite/scripts/steps/integration-package.sh
-          echo "~~~ Pushing cloud image"
-          mage cloud:push
+          .buildkite/scripts/steps/integration-cloud-image-push.sh
         artifact_paths:
           - build/distributions/**
         agents:
@@ -149,13 +145,9 @@ steps:
           PACKAGES: "docker"
           PLATFORMS: "linux/amd64"
           FIPS: "true"
-          CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
-          CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips"
         command: |
-          source .buildkite/scripts/common.sh
           .buildkite/scripts/steps/integration-package.sh
-          echo "~~~ Pushing cloud image"
-          mage cloud:push
+          .buildkite/scripts/steps/integration-cloud-image-push.sh
         artifact_paths:
           - build/distributions/**
         agents:
diff --git a/.buildkite/scripts/buildkite-integration-tests.ps1 b/.buildkite/scripts/buildkite-integration-tests.ps1
@@ -25,13 +25,28 @@ go install gotest.tools/gotestsum
 gotestsum --version
 
 $env:TEST_BINARY_NAME = "elastic-agent"
-if (-not $env:AGENT_VERSION) {
-    # Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925
-    $AGENT_VERSION = (Get-Content version/version.go | Select-String -Pattern 'const defaultBeatVersion =' | ForEach-Object { $_ -replace '.*?"(.*?)".*', '$1' })
-    $env:AGENT_VERSION = $AGENT_VERSION + "-SNAPSHOT"
+
+if (-not $env:AGENT_VERSION)
+{
+    if (Test-Path .package-version)
+    {
+        $packageContent = Get-Content .package-version -Raw | ConvertFrom-Json
+        $env:AGENT_VERSION = $packageContent.version
+        Write-Output "~~~ Agent version: $env:AGENT_VERSION (from .package-version)"
+    }
+    else
+    {
+        # Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925
+        $AGENT_VERSION = (Get-Content version/version.go | Select-String -Pattern 'const defaultBeatVersion =' | ForEach-Object { $_ -replace '.*?"(.*?)".*', '$1' })
+        $env:AGENT_VERSION = $AGENT_VERSION + "-SNAPSHOT"
+        Write-Output "~~~ Agent version: $env:AGENT_VERSION (from version/version.go)"
+    }
+}
+else
+{
+    Write-Output "~~~ Agent version: $env:AGENT_VERSION (specified by env var)"
 }
 
-Write-Output "~~~ Agent version: $env:AGENT_VERSION"
 $env:SNAPSHOT = $true
 
 Write-Host "~~~ Running integration tests as $env:USERNAME"
diff --git a/.buildkite/scripts/buildkite-integration-tests.sh b/.buildkite/scripts/buildkite-integration-tests.sh
@@ -38,14 +38,20 @@ echo "~~~ Running integration tests as $USER"
 make install-gotestsum
 
 if [[ -z "${AGENT_VERSION:-}" ]]; then
-  # Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925
-  AGENT_VERSION=$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2)
-  AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT"
+  if [[ -f "${WORKSPACE}/.package-version" ]]; then
+    AGENT_VERSION="$(jq -r '.version' .package-version)"
+    echo "~~~ Agent version: ${AGENT_VERSION} (from .package-version)"
+  else
+    AGENT_VERSION=$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2)
+    AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT"
+    echo "~~~ Agent version: ${AGENT_VERSION} (from version/version.go)"
+  fi
+
+  export AGENT_VERSION
+else
+  echo "~~~ Agent version: ${AGENT_VERSION} (specified by env var)"
 fi
 
-export AGENT_VERSION
-echo "~~~ Agent version: ${AGENT_VERSION}"
-
 os_data=$(uname -spr | tr ' ' '_')
 root_suffix=""
 if [ "$TEST_SUDO" == "true" ]; then
diff --git a/.buildkite/scripts/buildkite-k8s-integration-tests.sh b/.buildkite/scripts/buildkite-k8s-integration-tests.sh
@@ -9,9 +9,18 @@ DOCKER_VARIANTS="${DOCKER_VARIANTS:-basic,wolfi,complete,complete-wolfi,service,
 CLUSTER_NAME="${K8S_VERSION}-kubernetes"
 
 if [[ -z "${AGENT_VERSION:-}" ]]; then
-  # If not specified, use the version in version/version.go
-  AGENT_VERSION="$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2)"
-  AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT"
+  if [[ -f "${WORKSPACE}/.package-version" ]]; then
+    AGENT_VERSION="$(jq -r '.version' .package-version)"
+    echo "~~~ Agent version: ${AGENT_VERSION} (from .package-version)"
+  else
+    AGENT_VERSION="$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2)"
+    AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT"
+    echo "~~~ Agent version: ${AGENT_VERSION} (from version/version.go)"
+  fi
+
+  export AGENT_VERSION
+else
+  echo "~~~ Agent version: ${AGENT_VERSION} (specified by env var)"
 fi
 
 echo "~~~ Create kind cluster '${CLUSTER_NAME}'"
diff --git a/.buildkite/scripts/steps/ess.ps1 b/.buildkite/scripts/steps/ess.ps1
@@ -1,6 +1,7 @@
 function ess_up {
   param (
       [string]$StackVersion,
+      [string]$StackBuildId = "",
       [string]$EssRegion = "gcp-us-west2"
   )
 
@@ -22,6 +23,7 @@ function ess_up {
   & terraform init
   & terraform apply -auto-approve `
       -var="stack_version=$StackVersion" `
+      -var="stack_build_id=$StackBuildId" `
       -var="ess_region=$EssRegion" `
       -var="creator=$BuildkiteBuildCreator" `
       -var="buildkite_id=$BuildkiteBuildNumber" `
@@ -99,12 +101,13 @@ function Retry-Command {
 
 function Get-Ess-Stack {
   param (
-      [string]$StackVersion
+      [string]$StackVersion,
+      [string]$StackBuildId = ""
   )
 
   if ($Env:BUILDKITE_RETRY_COUNT -gt 0) {
       Write-Output "The step is retried, starting the ESS stack again"
-      ess_up $StackVersion
+      ess_up $StackVersion $StackBuildId
       Write-Output "ESS stack is up. ES_HOST: $Env:ELASTICSEARCH_HOST"
   } else {
       # TODO: Use a metadata prefix for "fips." if we ever need to test Windows artifacts for FIPS.
diff --git a/.buildkite/scripts/steps/ess.sh b/.buildkite/scripts/steps/ess.sh
@@ -6,7 +6,8 @@ function ess_up() {
   local WORKSPACE=$(git rev-parse --show-toplevel)
   local TF_DIR="${WORKSPACE}/test_infra/ess/"
   local STACK_VERSION=$1
-  local ESS_REGION=${2:-"gcp-us-west2"}
+  local STACK_BUILD_ID=${2:-""}
+  local ESS_REGION=${3:-"gcp-us-west2"}
     
   if [ -z "$STACK_VERSION" ]; then
     echo "Error: Specify stack version: ess_up [stack_version]" >&2
@@ -22,6 +23,7 @@ function ess_up() {
   terraform apply \
     -auto-approve \
     -var="stack_version=${STACK_VERSION}" \
+    -var="stack_build_id=${STACK_BUILD_ID}" \
     -var="ess_region=${ESS_REGION}" \
     -var="creator=${BUILDKITE_BUILD_CREATOR}" \
     -var="buildkite_id=${BUILDKITE_BUILD_NUMBER}" \
diff --git a/.buildkite/scripts/steps/ess_start.sh b/.buildkite/scripts/steps/ess_start.sh
@@ -4,10 +4,10 @@ set -euo pipefail
 source .buildkite/scripts/steps/ess.sh
 source .buildkite/scripts/steps/fleet.sh
 
-OVERRIDE_STACK_VERSION="$(cat .package-version)"
-OVERRIDE_STACK_VERSION=${OVERRIDE_STACK_VERSION}"-SNAPSHOT"
+STACK_VERSION="$(jq -r '.version' .package-version)"
+STACK_BUILD_ID="$(jq -r '.stack_build_id' .package-version)"
 
-ess_up $OVERRIDE_STACK_VERSION
+ess_up "$STACK_VERSION" "$STACK_BUILD_ID"
 
 preinstall_fleet_packages
 
diff --git a/.buildkite/scripts/steps/integration-cloud-image-push.sh b/.buildkite/scripts/steps/integration-cloud-image-push.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+source .buildkite/scripts/common.sh
+
+echo "~~~ Pushing cloud image"
+
+suffix=""
+if [ "${FIPS:-false}" == "true" ]; then
+  suffix="-fips"
+fi
+
+CI_ELASTIC_AGENT_DOCKER_IMAGE="docker.elastic.co/beats-ci/elastic-agent-cloud${suffix}"
+export CI_ELASTIC_AGENT_DOCKER_IMAGE
+echo "CI_ELASTIC_AGENT_DOCKER_IMAGE: ${CI_ELASTIC_AGENT_DOCKER_IMAGE}"
+
+
+export CUSTOM_IMAGE_TAG="git-${BUILDKITE_COMMIT:0:12}"
+export USE_PACKAGE_VERSION="true"
+
+mage cloud:push
diff --git a/.buildkite/scripts/steps/integration-package.sh b/.buildkite/scripts/steps/integration-package.sh
@@ -5,5 +5,6 @@ source .buildkite/scripts/common.sh
 
 export SNAPSHOT="true"
 export EXTERNAL="true"
+export USE_PACKAGE_VERSION="true"
 
 mage package
diff --git a/.buildkite/scripts/steps/integration_tests.sh b/.buildkite/scripts/steps/integration_tests.sh
@@ -7,19 +7,9 @@ STACK_PROVISIONER="${1:-"stateful"}"
 MAGE_TARGET="${2:-"integration:test"}"
 MAGE_SUBTARGET="${3:-""}"
 
-
-# Override the stack version from `.package-version` contents
-# There is a time when the current snapshot is not available on cloud yet, so we cannot use the latest version automatically
-# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready.
-
-STACK_VERSION="$(cat .package-version)"
-if [[ -n "$STACK_VERSION" ]]; then
-    STACK_VERSION=${STACK_VERSION}"-SNAPSHOT"
-fi
-
 # Run integration tests
 set +e
-AGENT_STACK_VERSION="${STACK_VERSION}" TEST_INTEG_CLEAN_ON_EXIT=true  STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage $MAGE_TARGET $MAGE_SUBTARGET
+USE_PACKAGE_VERSION=true TEST_INTEG_CLEAN_ON_EXIT=true  STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage $MAGE_TARGET $MAGE_SUBTARGET
 TESTS_EXIT_STATUS=$?
 set -e
 
diff --git a/.buildkite/scripts/steps/integration_tests_tf.ps1 b/.buildkite/scripts/steps/integration_tests_tf.ps1
@@ -9,10 +9,15 @@ $PSVersionTable.PSVersion
 
 . "$PWD\.buildkite\scripts\steps\ess.ps1"
 
-# Read package version from .package-version file
-$PACKAGE_VERSION = Get-Content .package-version -ErrorAction SilentlyContinue
-if ($PACKAGE_VERSION) {
-    $PACKAGE_VERSION = "${PACKAGE_VERSION}-SNAPSHOT"
+# Override the stack version from `.package-version` contents
+# There is a time when the current snapshot is not available on cloud yet, so we cannot use the latest version automatically
+# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready
+$packageVersionContent = Get-Content .package-version -Raw -ErrorAction SilentlyContinue | ConvertFrom-Json
+if ($packageVersionContent -and $packageVersionContent.version ) {
+    $STACK_VERSION = $packageVersionContent.version
+}
+if ($packageVersionContent -and $packageVersionContent.stack_build_id ) {
+    $STACK_BUILD_ID = $packageVersionContent.stack_build_id
 }
 
 Write-Output "~~~ Building test binaries"
@@ -27,7 +32,7 @@ $TestsExitCode = 0
 try {
     Write-Output "~~~ Running integration tests"
     # Get-Ess-Stack will start the ESS stack if it is a BK retry, otherwise it will retrieve ESS stack metadata
-    Get-Ess-Stack -StackVersion $PACKAGE_VERSION
+    Get-Ess-Stack -StackVersion $STACK_VERSION -StackBuildId $STACK_BUILD_ID
     & "$PWD\.buildkite\scripts\buildkite-integration-tests.ps1" $GROUP_NAME $TEST_SUDO
     $TestsExitCode = $LASTEXITCODE
     if ($TestsExitCode -ne 0)
diff --git a/.buildkite/scripts/steps/integration_tests_tf.sh b/.buildkite/scripts/steps/integration_tests_tf.sh
@@ -20,11 +20,11 @@ if [ -z "$TEST_SUDO" ]; then
   exit 1
 fi
 
-# Override the agent package version using a string with format <major>.<minor>.<patch>
-# There is a time when the snapshot is not built yet, so we cannot use the latest version automatically
+# Override the stack version from `.package-version` contents
+# There is a time when the current snapshot is not available on cloud yet, so we cannot use the latest version automatically
 # This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready.
-OVERRIDE_STACK_VERSION="$(cat .package-version)"
-OVERRIDE_STACK_VERSION=${OVERRIDE_STACK_VERSION}"-SNAPSHOT"
+STACK_VERSION="$(jq -r '.version' .package-version)"
+STACK_BUILD_ID="$(jq -r '.stack_build_id' .package-version)"
 
 echo "~~~ Building test binaries"
 mage build:testBinaries
@@ -35,7 +35,7 @@ mage build:testBinaries
 if [[ "${BUILDKITE_RETRY_COUNT}" -gt 0 || "${FORCE_ESS_CREATE:-false}" == "true" ]]; then
   echo "~~~ The steps is retried, starting the ESS stack again"
   trap 'ess_down' EXIT
-  ess_up $OVERRIDE_STACK_VERSION || (echo -e "^^^ +++\nFailed to start ESS stack")
+  ess_up "$STACK_VERSION" "$STACK_BUILD_ID" || (echo -e "^^^ +++\nFailed to start ESS stack")
 else
   # For the first run, we start the stack in the start_ess.sh step and it sets the meta-data
   echo "~~~ Receiving ESS stack metadata"
diff --git a/.github/workflows/bump-agent-versions.sh b/.github/workflows/bump-agent-versions.sh
@@ -25,7 +25,7 @@ else
     git add testing/integration/testdata/.upgrade-test-agent-versions.yml .package-version
 
     nl=$'\n' # otherwise the new line character is not recognized properly
-    commit_desc="These files are used for picking the starting (pre-upgrade) or ending (post-upgrade) agent versions in upgrade integration tests.${nl}${nl}The content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}Package version: ${package_version}${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`"
+    commit_desc="These files are used for picking the starting (pre-upgrade) or ending (post-upgrade) agent versions in upgrade integration tests.${nl}${nl}The content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}Package version: \`\`\`json${nl}${package_version}${nl}\`\`\`${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`"
 
     git commit -m "[$current_ref][Automation] Update versions" -m "$commit_desc"
     git push --set-upstream origin "$pr_branch"
diff --git a/.package-version b/.package-version
@@ -1 +1,8 @@
-9.2.0
+{
+  "version": "9.2.0-SNAPSHOT",
+  "build_id": "9.2.0-ffb19b67",
+  "manifest_url": "https://snapshots.elastic.co/9.2.0-ffb19b67/manifest-9.2.0-SNAPSHOT.json",
+  "summary_url": "https://snapshots.elastic.co/9.2.0-ffb19b67/summary-9.2.0-SNAPSHOT.html",
+  "core_version": "9.2.0",
+  "stack_build_id": "9.2.0-ffb19b67-SNAPSHOT"
+}
diff --git a/dev-tools/mage/manifest/manifest.go b/dev-tools/mage/manifest/manifest.go
@@ -152,10 +152,10 @@ func DownloadComponents(ctx context.Context, expectedBinaries []packaging.Binary
 			if err != nil {
 				return fmt.Errorf("failed to create directory %s", targetPath)
 			}
-			log.Printf("+++ Prepare to download [%s] project [%s] for [%s]", spec.BinaryName, spec.ProjectName, platform)
+			fmt.Printf("Prepare to download [%s] project [%s] for [%s]\n", spec.BinaryName, spec.ProjectName, platform)
 
 			if !spec.SupportsPlatform(platform) {
-				log.Printf(">>>>>>>>> Binary [%s] does not support platform [%s] ", spec.BinaryName, platform)
+				fmt.Printf("Binary [%s] does not support platform [%s]\n", spec.BinaryName, platform)
 				continue
 			}
 
@@ -165,7 +165,7 @@ func DownloadComponents(ctx context.Context, expectedBinaries []packaging.Binary
 			}
 
 			for _, p := range resolvedPackage.URLs {
-				log.Printf(">>>>>>>>> Downloading [%s] [%s] ", spec.BinaryName, p)
+				fmt.Printf("Downloading [%s] [%s]\n", spec.BinaryName, p)
 				pkgFilename := path.Base(p)
 				downloadTarget := filepath.Join(targetPath, pkgFilename)
 				if _, err := os.Stat(downloadTarget); err != nil {
@@ -182,7 +182,7 @@ func DownloadComponents(ctx context.Context, expectedBinaries []packaging.Binary
 		return fmt.Errorf("error downloading files: %w", err)
 	}
 
-	log.Printf("Downloads for manifest %q complete.", manifest)
+	fmt.Printf("Downloads for manifest %q complete.\n", manifest)
 	return nil
 }
 
diff --git a/dev-tools/mage/packageversion.go b/dev-tools/mage/packageversion.go
diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go
diff --git a/magefile.go b/magefile.go
diff --git a/test_infra/ess/deployment.tf b/test_infra/ess/deployment.tf
diff --git a/testing/integration/ess/upgrade_broken_package_test.go b/testing/integration/ess/upgrade_broken_package_test.go
