Publish elastic-agent-service container directly to serverless from main (#9583)

* Update job to build and push.

* Fix artifacts.

* Add to core pipeline for main branch.

---------

Co-authored-by: Francisco Ramon <[email protected]>

Author: blakerouse

.buildkite/pipeline.agentless-app-release.yaml

@@ -1,14 +1,46 @@
 env:
   VERSION: "${BUILDKITE_COMMIT:0:12}"
 steps:
-  - label: "Mirror Elastic-Agent Snapshot DRA to internal registry"
+  - label: "Packaging: Service Container linux/amd64"
+    key: packaging-service-container-amd64
+    env:
+      PACKAGES: "docker"
+      PLATFORMS: "linux/amd64"
+      DOCKER_VARIANTS: "service"
+    command: |
+      .buildkite/scripts/steps/integration-package.sh
+    artifact_paths:
+      - "build/distributions/elastic-agent-service-git-*.docker.tar.gz"
+    agents:
+      provider: "gcp"
+      machineType: "c2-standard-16"
+      diskSizeGb: 400
+
+  - label: "Packaging: Service Container linux/arm64"
+    key: packaging-service-container-arm64
+    env:
+      PACKAGES: "docker"
+      PLATFORMS: "linux/arm64"
+      DOCKER_VARIANTS: "service"
+    command: |
+       .buildkite/scripts/steps/integration-package.sh
+    artifact_paths:
+      - "build/distributions/elastic-agent-service-git-*.docker.tar.gz"
+    agents:
+      provider: "aws"
+      instanceType: "t4g.2xlarge"
+      imagePrefix: "core-ubuntu-2204-aarch64"
+      diskSizeGb: 400
+
+  # wait for packaging to be done
+  - wait: ~
+
+  - label: "Publish to internal registry"
     key: "mirror-elastic-agent"
     command: ".buildkite/scripts/steps/ecp-internal-release.sh"
     agents:
       image: docker.elastic.co/ci-agent-images/serverless-helm-builder:0.0.2@sha256:d00e8a7a0ab3618cfaacb0a7b1e1b06ee29728eb2b44de602374bd8f6b9b92ac
 
-
-
   # wait for metadata to be set
   - wait: ~
 

.buildkite/pipeline.yml

@@ -371,3 +371,10 @@ steps:
     build:
       commit: "${BUILDKITE_COMMIT}"
       branch: "${BUILDKITE_BRANCH}"
+
+  # wait for packaging to be done
+  - wait: ~
+
+  - label: "Publish to serverless"
+    branches: main
+    trigger: "agentless-serverless-release"

.buildkite/scripts/steps/ecp-internal-release.sh

@@ -20,7 +20,6 @@ set -eu
 _SELF=$(dirname $0)
 source "${_SELF}/../common.sh"
 
-
 # annotate create temp markdown file if not exists
 # this file will be later used to annotate the build
 # it appends to the file the message passed as argument
@@ -33,19 +32,9 @@ write_annotation() {
     cat $BUILDKITE_ANNOTATE_FILE | buildkite-agent annotate --style info
 }
 
+BUILD_VERSION="$(jq -r '.version' .package-version)"
+DOCKER_TAG="git-${VERSION}"
 PRIVATE_REPO="docker.elastic.co/observability-ci/ecp-elastic-agent-service"
-SNAPSHOT_DRA_URL=https://snapshots.elastic.co/latest/master.json
-
-DRA_RESULT=$(curl -s -X GET "$SNAPSHOT_DRA_URL")
-echo "$DRA_RESULT"
-BUILD_ID=$(echo "$DRA_RESULT" | jq '.build_id' | tr -d '"')
-BUILD_VERSION=$(echo "$DRA_RESULT" | jq '.version' | tr -d '"')
-
-MANIFEST_URL="https://snapshots.elastic.co/$BUILD_ID/agent-package/agent-artifacts-$BUILD_VERSION.json"
-GIT_COMMIT=$(curl -s -X GET "$MANIFEST_URL" | jq '.projects["elastic-agent-core"]["commit_hash"]' | tr -d '"')
-GIT_SHORT_COMMIT=$(echo "$GIT_COMMIT" | cut -c1-12)
-
-DOCKER_TAG="git-${GIT_SHORT_COMMIT}"
 PRIVATE_IMAGE="${PRIVATE_REPO}:${DOCKER_TAG}"
 
 # TODO: let's avoid accessing vault directly but use the vault plugin itself
@@ -57,13 +46,19 @@ DOCKER_REGISTRY="docker.elastic.co"
 DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
 DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
 skopeo login --username "${DOCKER_USERNAME_SECRET}" --password "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}"
-skopeo copy --all "docker://docker.elastic.co/cloud-release/elastic-agent-service:$BUILD_ID-SNAPSHOT" "docker://$PRIVATE_IMAGE"
+
+# download the amd64 and arm64 builds of the image from the previous steps
+buildkite-agent artifact download "build/distributions/**" . --step "packaging-service-container-amd64"
+buildkite-agent artifact download "build/distributions/**" . --step "packaging-service-container-arm64"
+
+# copy the images into the private image location
+skopeo copy --all "docker-archive:./build/distributions/elastic-agent-service-$DOCKER_TAG-$BUILD_VERSION-linux-amd64.docker.tar.gz" "docker://$PRIVATE_IMAGE"
+skopeo copy --all "docker-archive:./build/distributions/elastic-agent-service-$DOCKER_TAG-$BUILD_VERSION-linux-arm64.docker.tar.gz" "docker://$PRIVATE_IMAGE"
 
 annotate "* Image: $PRIVATE_IMAGE"
-annotate "* Short commit: $GIT_SHORT_COMMIT"
-annotate "* Commit: https://github.com/elastic/elastic-agent/commit/$GIT_COMMIT"
-annotate "* Manifest: $MANIFEST_URL"
+annotate "* Short commit: $VERSION"
+annotate "* Commit: https://github.com/elastic/elastic-agent/commit/$VERSION"
 
-buildkite-agent meta-data set "git-short-commit" "$GIT_SHORT_COMMIT"
+buildkite-agent meta-data set "git-short-commit" "$VERSION"
 
 write_annotation