(backport #7690) Fips packaging (#7790)

* Make components in packages configurable (#7602)

* Redefine ExpectedBinaries as YAML config

* Move ExpectedBinaries closer to package spec file

* Fix error formatting in downloadDRAArtifacts

* add packageName template to ExpectedPackages

* use a relaxed dependencies version for IAR releases

* Remove FIPS hack introduced in PR #7486

* Allow for a looser match on relaxing dependencies versions

* Add debug logging when packaging with EXTERNAL=true

* move package tests to dedicated package

* Fips packaging (#7690)

* Add component list to specs

* extract component dependencies from the packages to be built

* Refactor component extraction from package specs

* Fix package tests error handling

* Inject dependencies and remove references to ExpectedBinaries

* Remove ExpectedBinaries global

* Add rootdir to components

* Extract actual version matched on the package file and use it to render RootDir

* Package elastic-agent FIPS specs when FIPS=true is specified

* refactor ResolveManifestPackage

* Move FIPS compile settings in packages.yml

* Add more FIPS components

* Properly handle dependenciesVersion when calling mage package

* Refactor ChecksumsWithoutManifest to use list of dependencies instead of globbing files

* Rework useDRAAgentBinaryForPackage for repackaging agent

Define elastic-agent-core components (both FIPS and non-FIPS variants)
and define package name and root dir templates.
Implement some filtering on component list to extract the correct
component definition according to the FIPSBuild flag.
Refactor code that downloads pre-compiled elastic-agent binaries and
places them in the golangCrossBuild directory to make use of the new
component definition.

* Write spec FIPS flag into manifest.yaml when packaging

* Add FIPS elastic agent basic and cloud docker images

* Build FIPS docker images in CI packaging

* Fix FIPS .tar.gz package tests

* Restructure package tests

* Extend FIPS check to all binaries in components directory

* Create FIPS elastic-agent-core artifacts in elastic-agent-binary-dra pipeline

* Cleanup ChecksumsWithManifest and improve godoc

* Improve godoc for BinarySpec

* Correctly inject dependency list when packaging using DROP_PATH (#7795)

* Restore qualifier=core for elastic-agent-core packaging specs (#7805)

Restore qualifier for elastic-agent-core packaging specs to avoid
changing the rootDir name of the archives.
The qualifier had been removed in PR #7690 trying to use the spec name:
this worked to get the desired file name but changed the root Dir name
which uses '{{.BeatName}}{{if .Qualifier}}-{{.Qualifier}}{{end}}' in the
template definition instead of '{{.Name}}' which would render the spec
name.

* Modify fips core spec qualifier and name (#7818)

* Reintroduce cloud-defend component

* Filter components by package-type

---------

Co-authored-by: Paolo Chilà <[email protected]>
# Conflicts:
#	.buildkite/integration.pipeline.yml
#	dev-tools/mage/checksums.go
#	dev-tools/mage/dockerbuilder.go
#	dev-tools/mage/manifest/manifest.go
#	dev-tools/mage/pkgtypes.go
#	dev-tools/mage/settings.go
#	dev-tools/packaging/packages.yml
#	dev-tools/packaging/testing/package_test.go
#	testing/integration/ess/upgrade_standalone_same_commit_test.go

Author: 2 people

dev-tools/mage/build.go

@@ -18,6 +18,8 @@ import (
 	"github.com/magefile/mage/sh"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
+
+	"github.com/elastic/elastic-agent/dev-tools/packaging"
 )
 
 // BuildArgs are the arguments used for the "build" target and they define how
@@ -73,6 +75,7 @@ func DefaultBuildArgs() BuildArgs {
 	args := BuildArgs{
 		Name: BeatName,
 		CGO:  build.Default.CgoEnabled,
+		Env:  map[string]string{},
 		Vars: map[string]string{
 			elasticAgentModulePath + "/version.buildTime": "{{ date }}",
 			elasticAgentModulePath + "/version.commit":    "{{ commit }}",
@@ -88,8 +91,16 @@ func DefaultBuildArgs() BuildArgs {
 	}
 
 	if FIPSBuild {
-		args.ExtraFlags = append(args.ExtraFlags, "-tags=requirefips")
-		args.CGO = true
+
+		fipsConfig := packaging.Settings().FIPS
+
+		for _, tag := range fipsConfig.Compile.Tags {
+			args.ExtraFlags = append(args.ExtraFlags, "-tags="+tag)
+		}
+		args.CGO = args.CGO || fipsConfig.Compile.CGO
+		for varName, value := range fipsConfig.Compile.Env {
+			args.Env[varName] = value
+		}
 	}
 
 	if DevBuild {
@@ -191,11 +202,6 @@ func Build(params BuildArgs) error {
 		cgoEnabled = "1"
 	}
 
-	if FIPSBuild {
-		cgoEnabled = "1"
-		env["GOEXPERIMENT"] = "systemcrypto"
-	}
-
 	env["CGO_ENABLED"] = cgoEnabled
 
 	// Spec

dev-tools/mage/checksums.go

@@ -9,12 +9,12 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/magefile/mage/mg"
 	"github.com/otiai10/copy"
 
 	"github.com/elastic/elastic-agent/dev-tools/mage/manifest"
+	"github.com/elastic/elastic-agent/dev-tools/packaging"
 )
 
 const ComponentSpecFileSuffix = ".spec.yml"
@@ -40,206 +40,153 @@ func CopyComponentSpecs(componentName, versionedDropPath string) (string, error)
 	return GetSHA512Hash(targetPath)
 }
 
-// This is a helper function for flattenDependencies that's used when not packaging from a manifest
-func ChecksumsWithoutManifest(versionedFlatPath string, versionedDropPath string, packageVersion string) map[string]string {
-	globExpr := filepath.Join(versionedFlatPath, fmt.Sprintf("*%s*", packageVersion))
-	if mg.Verbose() {
-		log.Printf("Finding files to copy with %s", globExpr)
-	}
-	files, err := filepath.Glob(globExpr)
-	if err != nil {
-		panic(err)
-	}
-	if mg.Verbose() {
-		log.Printf("Validating checksums for %+v", files)
-		log.Printf("--- Copying into %s: %v", versionedDropPath, files)
-	}
-
+// ChecksumsWithoutManifest is a helper function for flattenDependencies that's used when not packaging from a manifest.
+// This function will iterate over the dependencies, resolve *exactly* the package name for each dependency and platform using the passed
+// dependenciesVersion, and it will copy the extracted files contained in the rootDir of each dependency from the versionedFlatPath
+// (a directory containing all the extracted dependencies per platform) to the versionedDropPath (a drop path by platform
+// that will be used to compose the package content)
+// ChecksumsWithoutManifest will accumulate the checksums of each component spec that is copied, and return it to the caller.
+func ChecksumsWithoutManifest(platform string, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, dependencies []packaging.BinarySpec) map[string]string {
 	checksums := make(map[string]string)
-	for _, f := range files {
+
+	for _, dep := range dependencies {
+
+		if dep.PythonWheel {
+			if mg.Verbose() {
+				log.Printf(">>>>>>> Component %s/%s is a Python wheel, skipping", dep.ProjectName, dep.BinaryName)
+			}
+			continue
+		}
+
+		if !dep.SupportsPlatform(platform) {
+			if mg.Verbose() {
+				log.Printf(">>>>>>> Component %s/%s does not support platform %s, skipping", dep.ProjectName, dep.BinaryName, platform)
+			}
+			continue
+		}
+
+		atLeastOnePackageTypeSelected := false
+		for _, pkgType := range dep.PackageTypes {
+			if IsPackageTypeSelected(PackageType(pkgType)) {
+				atLeastOnePackageTypeSelected = true
+				break
+			}
+		}
+
+		if !atLeastOnePackageTypeSelected {
+			if mg.Verbose() {
+				log.Printf(">>>>>>> Component %s/%s supported package types %v do not overlap selected package types %v, skipping", dep.ProjectName, dep.BinaryName, dep.PackageTypes, SelectedPackageTypes)
+			}
+			continue
+		}
+
+		srcDir := filepath.Join(versionedFlatPath, dep.GetRootDir(dependenciesVersion, platform))
+
+		if mg.Verbose() {
+			log.Printf("Validating checksums for %+v", dep.BinaryName)
+			log.Printf("--- Copying into %s: %v", versionedDropPath, srcDir)
+		}
+
 		options := copy.Options{
 			OnSymlink: func(_ string) copy.SymlinkAction {
 				return copy.Shallow
 			},
 			Sync: true,
 		}
 		if mg.Verbose() {
-			log.Printf("> prepare to copy %s into %s ", f, versionedDropPath)
+			log.Printf("> prepare to copy %s into %s ", srcDir, versionedDropPath)
 		}
 
-		err = copy.Copy(f, versionedDropPath, options)
+		err := copy.Copy(srcDir, versionedDropPath, options)
 		if err != nil {
-			panic(err)
+			panic(fmt.Errorf("copying dependency %s files from %q to %q: %w", dep.BinaryName, srcDir, versionedDropPath, err))
 		}
 
 		// copy spec file for match
-		specName := filepath.Base(f)
-		idx := strings.Index(specName, "-"+packageVersion)
-		if idx != -1 {
-			specName = specName[:idx]
-		}
 		if mg.Verbose() {
-			log.Printf(">>>> Looking to copy spec file: [%s]", specName)
+			log.Printf(">>>> Looking to copy spec file: [%s]", dep.BinaryName)
 		}
 
-		checksum, err := CopyComponentSpecs(specName, versionedDropPath)
+		checksum, err := CopyComponentSpecs(dep.BinaryName, versionedDropPath)
 		if err != nil {
 			panic(err)
 		}
 
-		checksums[specName+ComponentSpecFileSuffix] = checksum
+		checksums[dep.BinaryName+ComponentSpecFileSuffix] = checksum
 	}
 
 	return checksums
 }
 
-// This is a helper function for flattenDependencies that's used when building from a manifest
-func ChecksumsWithManifest(requiredPackage string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build) map[string]string {
+// ChecksumsWithManifest is a helper function for flattenDependencies that's used when building from a manifest.
+// This function will iterate over the dependencies, resolve the package name for each dependency and platform using the manifest,
+// (there may be some variability there in case the manifest does not include an exact match for the expected filename),
+// and it will copy the extracted files contained in the rootDir of each dependency from the versionedFlatPath
+// (a directory containing all the extracted dependencies per platform) to the versionedDropPath (a drop path by platform
+// that will be used to compose the package content)
+// ChecksumsWithManifest will accumulate the checksums of each component spec that is copied, and return it to the caller.
+func ChecksumsWithManifest(platform string, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build, dependencies []packaging.BinarySpec) map[string]string {
 	checksums := make(map[string]string)
 	if manifestResponse == nil {
 		return checksums
 	}
 
-	// Iterate over the component projects in the manifest
-	projects := manifestResponse.Projects
-	for componentName := range projects {
-		// Iterate over the individual package files within each component project
-		for pkgName := range projects[componentName].Packages {
-			// Only care about packages that match the required package constraint (os/arch)
-			if strings.Contains(pkgName, requiredPackage) {
-				// Iterate over the external binaries that we care about for packaging agent
-				for _, spec := range manifest.ExpectedBinaries {
-					// If the individual package doesn't match the expected prefix, then continue
-					if !strings.HasPrefix(pkgName, spec.BinaryName) {
-						continue
-					}
-
-					if mg.Verbose() {
-						log.Printf(">>>>>>> Package [%s] matches requiredPackage [%s]", pkgName, requiredPackage)
-					}
-
-					// Get the version from the component based on the version in the package name
-					// This is useful in the case where it's an Independent Agent Release, where
-					// the opted-in projects will be one patch version ahead of the rest of the
-					// opted-out/previously-released projects
-					componentVersion := getComponentVersion(componentName, requiredPackage, projects[componentName])
-					if mg.Verbose() {
-						log.Printf(">>>>>>> Component [%s]/[%s] version is [%s]", componentName, requiredPackage, componentVersion)
-					}
-
-					// Combine the package name w/ the versioned flat path
-					fullPath := filepath.Join(versionedFlatPath, pkgName)
-
-					// Eliminate the file extensions to get the proper directory
-					// name that we need to copy
-					var dirToCopy string
-					if strings.HasSuffix(fullPath, ".tar.gz") {
-						dirToCopy = fullPath[:strings.LastIndex(fullPath, ".tar.gz")]
-					} else if strings.HasSuffix(fullPath, ".zip") {
-						dirToCopy = fullPath[:strings.LastIndex(fullPath, ".zip")]
-					} else {
-						dirToCopy = fullPath
-					}
-					if mg.Verbose() {
-						log.Printf(">>>>>>> Calculated directory to copy: [%s]", dirToCopy)
-					}
-
-					// Set copy options
-					options := copy.Options{
-						OnSymlink: func(_ string) copy.SymlinkAction {
-							return copy.Shallow
-						},
-						Sync: true,
-					}
-					if mg.Verbose() {
-						log.Printf("> prepare to copy %s into %s ", dirToCopy, versionedDropPath)
-					}
-
-					// Do the copy
-					err := copy.Copy(dirToCopy, versionedDropPath, options)
-					if err != nil {
-						panic(err)
-					}
-
-					// copy spec file for match
-					specName := filepath.Base(dirToCopy)
-					idx := strings.Index(specName, "-"+componentVersion)
-					if idx != -1 {
-						specName = specName[:idx]
-					}
-					if mg.Verbose() {
-						log.Printf(">>>> Looking to copy spec file: [%s]", specName)
-					}
-
-					checksum, err := CopyComponentSpecs(specName, versionedDropPath)
-					if err != nil {
-						panic(err)
-					}
-
-					checksums[specName+ComponentSpecFileSuffix] = checksum
-				}
+	// Iterate over the external binaries that we care about for packaging agent
+	for _, spec := range dependencies {
+
+		if spec.PythonWheel {
+			if mg.Verbose() {
+				log.Printf(">>>>>>> Component %s/%s is a Python wheel, skipping", spec.ProjectName, spec.BinaryName)
 			}
+			continue
 		}
-	}
 
-	return checksums
-}
+		if !spec.SupportsPlatform(platform) {
+			log.Printf(">>>>>>> Component %s/%s does not support platform %s, skipping", spec.ProjectName, spec.BinaryName, platform)
+			continue
+		}
 
-// This function is used when building with a Manifest.  In that manifest, it's possible
-// for projects in an Independent Agent Release to have different versions since the opted-in
-// ones will be one patch version higher than the opted-out/previously released projects.
-// This function tries to find the versions from the package name
-func getComponentVersion(componentName string, requiredPackage string, componentProject manifest.Project) string {
-	var componentVersion string
-	var foundIt bool
-	// Iterate over all the packages in the component project
-	for pkgName := range componentProject.Packages {
-		// Only care about the external binaries that we want to package
-		for _, spec := range manifest.ExpectedBinaries {
-			// If the given component name doesn't match the external binary component, skip
-			if componentName != spec.ProjectName {
-				continue
+		manifestPackage, err := manifest.ResolveManifestPackage(manifestResponse.Projects[spec.ProjectName], spec, dependenciesVersion, platform)
+		if err != nil {
+			if mg.Verbose() {
+				log.Printf(">>>>>>> Error resolving package for [%s/%s]", spec.BinaryName, platform)
 			}
+			continue
+		}
 
-			// Split the package name on the binary name prefix plus a dash
-			firstSplit := strings.Split(pkgName, spec.BinaryName+"-")
-			if len(firstSplit) < 2 {
-				continue
-			}
+		rootDir := spec.GetRootDir(manifestPackage.ActualVersion, platform)
 
-			// Get the second part of the first split
-			secondHalf := firstSplit[1]
-			if len(secondHalf) < 2 {
-				continue
-			}
+		// Combine the package name w/ the versioned flat path
+		fullPath := filepath.Join(versionedFlatPath, rootDir)
 
-			// Make sure the second half matches the required package
-			if strings.Contains(secondHalf, requiredPackage) {
-				// ignore packages with names where this splitting doesn't results in proper version
-				if strings.Contains(secondHalf, "docker-image") {
-					continue
-				}
-				if strings.Contains(secondHalf, "oss-") {
-					continue
-				}
-
-				// The component version should be the first entry after splitting w/ the requiredPackage
-				componentVersion = strings.Split(secondHalf, "-"+requiredPackage)[0]
-				foundIt = true
-				// break out of inner loop
-				break
-			}
+		if mg.Verbose() {
+			log.Printf(">>>>>>> Calculated directory to copy: [%s]", fullPath)
 		}
-		if foundIt {
-			// break out of outer loop
-			break
+
+		// Set copy options
+		options := copy.Options{
+			OnSymlink: func(_ string) copy.SymlinkAction {
+				return copy.Shallow
+			},
+			Sync: true,
+		}
+		if mg.Verbose() {
+			log.Printf("> prepare to copy %s into %s ", fullPath, versionedDropPath)
 		}
-	}
 
-	if componentVersion == "" {
-		errMsg := fmt.Sprintf("Unable to determine component version for [%s]", componentName)
-		panic(errMsg)
+		// Do the copy
+		err = copy.Copy(fullPath, versionedDropPath, options)
+		if err != nil {
+			panic(err)
+		}
+
+		checksum, err := CopyComponentSpecs(spec.BinaryName, versionedDropPath)
+		if err != nil {
+			panic(err)
+		}
+
+		checksums[spec.BinaryName+ComponentSpecFileSuffix] = checksum
 	}
 
-	return componentVersion
+	return checksums
 }

dev-tools/mage/crossbuild.go

@@ -30,11 +30,11 @@ const defaultCrossBuildTarget = "golangCrossBuild"
 var Platforms = BuildPlatforms.Defaults()
 
 // SelectedPackageTypes is the list of package types. If empty, all packages types
-// are considered to be selected (see isPackageTypeSelected).
+// are considered to be selected (see IsPackageTypeSelected).
 var SelectedPackageTypes []PackageType
 
 // SelectedDockerVariants is the list of docker variants. If empty, all docker variants
-// are considered to be selected (see isDockerVariantSelected).
+// are considered to be selected (see IsDockerVariantSelected).
 var SelectedDockerVariants []DockerVariant
 
 func init() {