[8.x](backport #7690) Fips packaging (#7790)

* Make components in packages configurable (#7602)

* Redefine ExpectedBinaries as YAML config

* Move ExpectedBinaries closer to package spec file

* Fix error formatting in downloadDRAArtifacts

* add packageName template to ExpectedPackages

* use a relaxed dependencies version for IAR releases

* Remove FIPS hack introduced in PR #7486

* Allow for a looser match on relaxing dependencies versions

* Add debug logging when packaging with EXTERNAL=true

* move package tests to dedicated package

* Fips packaging (#7690)

* Add component list to specs

* extract component dependencies from the packages to be built

* Refactor component extraction from package specs

* Fix package tests error handling

* Inject dependencies and remove references to ExpectedBinaries

* Remove ExpectedBinaries global

* Add rootdir to components

* Extract actual version matched on the package file and use it to render RootDir

* Package elastic-agent FIPS specs when FIPS=true is specified

* refactor ResolveManifestPackage

* Move FIPS compile settings in packages.yml

* Add more FIPS components

* Properly handle dependenciesVersion when calling mage package

* Refactor ChecksumsWithoutManifest to use list of dependencies instead of globbing files

* Rework useDRAAgentBinaryForPackage for repackaging agent

Define elastic-agent-core components (both FIPS and non-FIPS variants)
and define package name and root dir templates.
Implement some filtering on component list to extract the correct
component definition according to the FIPSBuild flag.
Refactor code that downloads pre-compiled elastic-agent binaries and
places them in the golangCrossBuild directory to make use of the new
component definition.

* Write spec FIPS flag into manifest.yaml when packaging

* Add FIPS elastic agent basic and cloud docker images

* Build FIPS docker images in CI packaging

* Fix FIPS .tar.gz package tests

* Restructure package tests

* Extend FIPS check to all binaries in components directory

* Create FIPS elastic-agent-core artifacts in elastic-agent-binary-dra pipeline

* Cleanup ChecksumsWithManifest and improve godoc

* Improve godoc for BinarySpec

* Correctly inject dependency list when packaging using DROP_PATH (#7795)

* Restore qualifier=core for elastic-agent-core packaging specs (#7805)

Restore qualifier for elastic-agent-core packaging specs to avoid
changing the rootDir name of the archives.
The qualifier had been removed in PR #7690 trying to use the spec name:
this worked to get the desired file name but changed the root Dir name
which uses '{{.BeatName}}{{if .Qualifier}}-{{.Qualifier}}{{end}}' in the
template definition instead of '{{.Name}}' which would render the spec
name.

* Modify fips core spec qualifier and name (#7818)

* Reintroduce cloud-defend component

* Filter components by package-type

---------

Co-authored-by: Paolo Chilà <[email protected]>

Author: 2 people

.buildkite/integration.pipeline.yml

@@ -34,23 +34,6 @@ steps:
           machineType: "n2-standard-8"
           image: "${IMAGE_UBUNTU_2204_X86_64}"
 
-      - label: "Packaging: Ubuntu x86_64 FIPS"
-        key: "packaging-ubuntu-x86-64-fips"
-        env:
-          PACKAGES: "tar.gz"
-          PLATFORMS: "linux/amd64"
-          FIPS: "true"
-        command: ".buildkite/scripts/steps/integration-package.sh"
-        artifact_paths:
-          - build/distributions/**
-        retry:
-          automatic:
-            limit: 1
-        agents:
-          provider: "gcp"
-          machineType: "n2-standard-4"
-          image: "${IMAGE_UBUNTU_2204_X86_64}"
-
       # Packaging linux/arm64
       - label: "Packaging: linux/arm64 tar.gz"
         key: packaging-ubuntu-arm64
@@ -68,23 +51,6 @@ steps:
           instanceType: "c6g.4xlarge"
           image: "${IMAGE_UBUNTU_2204_ARM_64}"
 
-      - label: "Packaging: Ubuntu arm64 FIPS"
-        key: "packaging-ubuntu-arm64-fips"
-        env:
-          PACKAGES: "tar.gz"
-          PLATFORMS: "linux/arm64"
-          FIPS: "true"
-        command: ".buildkite/scripts/steps/integration-package.sh"
-        artifact_paths:
-          - build/distributions/**
-        retry:
-          automatic:
-            limit: 1
-        agents:
-          provider: "aws"
-          instanceType: "c6g.4xlarge"
-          image: "${IMAGE_UBUNTU_2204_ARM_64}"
-
       - label: "Packaging: windows/amd64 zip"
         key: packaging-windows
         env:
@@ -106,13 +72,8 @@ steps:
         env:
           PACKAGES: "docker"
           PLATFORMS: "linux/amd64"
-<<<<<<< HEAD
-        command: ".buildkite/scripts/steps/integration-package.sh"
-=======
         command: |
           .buildkite/scripts/steps/integration-package.sh
-          .buildkite/scripts/steps/integration-cloud-image-push.sh
->>>>>>> a155660c4 (ci: build agent from snapshot DRA (#9048))
         artifact_paths:
           - build/distributions/**
         agents:
@@ -136,44 +97,6 @@ steps:
           diskSizeGb: 200
           image: "${IMAGE_UBUNTU_2204_ARM_64}"
 
-<<<<<<< HEAD
-=======
-      - label: "Packaging: Containers linux/amd64 FIPS"
-        key: packaging-containers-x86-64-fips
-        env:
-          PACKAGES: "docker"
-          PLATFORMS: "linux/amd64"
-          FIPS: "true"
-        command: |
-          .buildkite/scripts/steps/integration-package.sh
-          .buildkite/scripts/steps/integration-cloud-image-push.sh
-        artifact_paths:
-          - build/distributions/**
-        agents:
-          provider: "gcp"
-          machineType: "n2-standard-8"
-          diskSizeGb: 200
-          image: "${IMAGE_UBUNTU_2204_X86_64}"
-        plugins:
-          - *vault_docker_login
-
-      - label: "Packaging: Containers linux/arm64 FIPS"
-        key: packaging-containers-arm64-fips
-        env:
-          PACKAGES: "docker"
-          PLATFORMS: "linux/arm64"
-          FIPS: "true"
-        command: |
-          .buildkite/scripts/steps/integration-package.sh
-        artifact_paths:
-          - build/distributions/**
-        agents:
-          provider: "aws"
-          instanceType: "c6g.4xlarge"
-          diskSizeGb: 200
-          image: "${IMAGE_UBUNTU_2204_ARM_64}"
-
->>>>>>> a155660c4 (ci: build agent from snapshot DRA (#9048))
   - label: "Triggering Integration tests"
     depends_on:
       - int-packaging

dev-tools/mage/build.go

@@ -73,6 +73,7 @@ func DefaultBuildArgs() BuildArgs {
 	args := BuildArgs{
 		Name: BeatName,
 		CGO:  build.Default.CgoEnabled,
+		Env:  map[string]string{},
 		Vars: map[string]string{
 			elasticAgentModulePath + "/version.buildTime": "{{ date }}",
 			elasticAgentModulePath + "/version.commit":    "{{ commit }}",
@@ -87,11 +88,6 @@ func DefaultBuildArgs() BuildArgs {
 		args.ExtraFlags = append(args.ExtraFlags, "-buildmode", "pie")
 	}
 
-	if FIPSBuild {
-		args.ExtraFlags = append(args.ExtraFlags, "-tags=requirefips")
-		args.CGO = true
-	}
-
 	if DevBuild {
 		// Disable optimizations (-N) and inlining (-l) for debugging.
 		args.ExtraFlags = append(args.ExtraFlags, `-gcflags=all=-N -l`)
@@ -191,11 +187,6 @@ func Build(params BuildArgs) error {
 		cgoEnabled = "1"
 	}
 
-	if FIPSBuild {
-		cgoEnabled = "1"
-		env["GOEXPERIMENT"] = "systemcrypto"
-	}
-
 	env["CGO_ENABLED"] = cgoEnabled
 
 	// Spec