Add manual rollback integration test

Author: pchila

testing/integration/ess/repackage.go

@@ -11,7 +11,6 @@ import (
 	"archive/zip"
 	"bytes"
 	"compress/gzip"
-	"context"
 	"errors"
 	"io"
 	"os"
@@ -25,19 +24,13 @@ import (
 
 	"github.com/elastic/elastic-agent/dev-tools/mage"
 	v1 "github.com/elastic/elastic-agent/pkg/api/v1"
-	atesting "github.com/elastic/elastic-agent/pkg/testing"
 	"github.com/elastic/elastic-agent/pkg/version"
 	agtversion "github.com/elastic/elastic-agent/version"
 )
 
-func repackageArchive(ctx context.Context, t *testing.T, startFixture *atesting.Fixture, newVersionBuildMetadata string, currentVersion *version.ParsedSemVer, newPackageContainingDir string, parsedNewVersion *version.ParsedSemVer) (*version.ParsedSemVer, error) {
-	err := startFixture.EnsurePrepared(ctx)
-	require.NoErrorf(t, err, "fixture should be prepared")
-
-	// retrieve the compressed package file location
-	srcPackage, err := startFixture.SrcPackage(ctx)
-	require.NoErrorf(t, err, "error retrieving start fixture source package")
-
+// repackageArchive will take a srcPackage elastic-agent package and create a modified copy that will present parsedNewVersion
+// in package version file, manifest and relevant metadata.
+func repackageArchive(t *testing.T, srcPackage string, newVersionBuildMetadata string, currentVersion *version.ParsedSemVer, parsedNewVersion *version.ParsedSemVer) (*version.ParsedSemVer, string, error) {
 	originalPackageFileName := filepath.Base(srcPackage)
 
 	// integration test fixtures and package names treat the version as a string including the "-SNAPSHOT" suffix
@@ -54,8 +47,8 @@ func repackageArchive(ctx context.Context, t *testing.T, startFixture *atesting.
 	// calculate the new package name
 	newPackageFileName := strings.Replace(originalPackageFileName, currentVersion.String(), versionForFixture.String(), 1)
 	t.Logf("originalPackageName: %q newPackageFileName: %q", originalPackageFileName, newPackageFileName)
-
-	newPackageAbsPath := filepath.Join(newPackageContainingDir, newPackageFileName)
+	outDir := t.TempDir()
+	newPackageAbsPath := filepath.Join(outDir, newPackageFileName)
 
 	// hack the package based on type
 	ext := filepath.Ext(originalPackageFileName)
@@ -76,9 +69,9 @@ func repackageArchive(ctx context.Context, t *testing.T, startFixture *atesting.
 	}
 
 	// Create hash file for the new package
-	err = mage.CreateSHA512File(newPackageAbsPath)
+	err := mage.CreateSHA512File(newPackageAbsPath)
 	require.NoErrorf(t, err, "error creating .sha512 for file %q", newPackageAbsPath)
-	return versionForFixture, err
+	return versionForFixture, newPackageAbsPath, err
 }
 
 func repackageTarArchive(t *testing.T, srcPackagePath string, newPackagePath string, newVersion *version.ParsedSemVer) {

testing/integration/ess/upgrade_rollback_test.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"testing"
@@ -24,6 +25,7 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/details"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/install"
+	"github.com/elastic/elastic-agent/pkg/control/v2/client"
 	"github.com/elastic/elastic-agent/pkg/control/v2/cproto"
 	atesting "github.com/elastic/elastic-agent/pkg/testing"
 	"github.com/elastic/elastic-agent/pkg/testing/define"
@@ -39,6 +41,15 @@ agent.upgrade.watcher:
   error_check.interval: 5s
 `
 
+const fastWatcherCfgWithRollbackWindow = `
+agent.upgrade:
+    watcher:
+        grace_period: 2m
+        error_check.interval: 5s
+    rollback:
+        window: 10m
+`
+
 // TestStandaloneUpgradeRollback tests the scenario where upgrading to a new version
 // of Agent fails due to the new Agent binary reporting an unhealthy status. It checks
 // that the Agent is rolled back to the previous version.
@@ -232,21 +243,26 @@ func TestStandaloneUpgradeRollbackOnRestarts(t *testing.T) {
 				require.NoError(t, err)
 
 				// Create a new package with a different version (IAR-style)
-				newPackageContainingDir := t.TempDir()
-
 				// modify the version with the "+buildYYYYMMDDHHMMSS"
 				currentVersion, err := version.ParseVersion(define.Version())
 				require.NoErrorf(t, err, "define.Version() %q is not parsable.", define.Version())
 
 				newVersionBuildMetadata := "build" + time.Now().Format("20060102150405")
 				parsedNewVersion := version.NewParsedSemVer(currentVersion.Major(), currentVersion.Minor(), currentVersion.Patch(), "", newVersionBuildMetadata)
 
-				versionForFixture, err := repackageArchive(t.Context(), t, fromFixture, newVersionBuildMetadata, currentVersion, newPackageContainingDir, parsedNewVersion)
+				err = fromFixture.EnsurePrepared(t.Context())
+				require.NoErrorf(t, err, "fixture should be prepared")
+
+				// retrieve the compressed package file location
+				srcPackage, err := fromFixture.SrcPackage(t.Context())
+				require.NoErrorf(t, err, "error retrieving start fixture source package")
+
+				versionForFixture, repackagedArchiveFile, err := repackageArchive(t, srcPackage, newVersionBuildMetadata, currentVersion, parsedNewVersion)
 				require.NoError(t, err, "error repackaging the archive built from the same commit")
 
 				// I wish I could just pass the location of the package on disk to the whole upgrade tests/fixture/fetcher code
 				// but I would have to break too much code for that, when in Rome... add more code on top of inflexible code
-				repackagedLocalFetcher := atesting.LocalFetcher(newPackageContainingDir)
+				repackagedLocalFetcher := atesting.LocalFetcher(filepath.Dir(repackagedArchiveFile))
 				toFixture, err := atesting.NewFixture(
 					t,
 					versionForFixture.String(),
@@ -323,6 +339,79 @@ func TestFleetManagedUpgradeRollbackOnRestarts(t *testing.T) {
 	}
 }
 
+// TestStandaloneUpgradeManualRollback tests the scenario where, after upgrading to a new version
+// of Agent, a manual rollback is triggered. It checks that the Agent is rolled back to the previous version.
+func TestStandaloneUpgradeManualRollback(t *testing.T) {
+	define.Require(t, define.Requirements{
+		Group: integration.Upgrade,
+		Local: false, // requires Agent installation
+		Sudo:  true,  // requires Agent installation
+	})
+
+	type fixturesSetupFunc func(t *testing.T) (from *atesting.Fixture, to *atesting.Fixture)
+	testcases := []struct {
+		name          string
+		fixturesSetup fixturesSetupFunc
+	}{
+		{
+			name: "upgrade to a repackaged agent built from the same commit",
+			fixturesSetup: func(t *testing.T) (from *atesting.Fixture, to *atesting.Fixture) {
+				// Upgrade from the current build to the same build as Independent Agent Release.
+
+				// Start from the build under test.
+				fromFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
+				require.NoError(t, err)
+
+				// Create a new package with a different version (IAR-style)
+				// modify the version with the "+buildYYYYMMDDHHMMSS"
+				currentVersion, err := version.ParseVersion(define.Version())
+				require.NoErrorf(t, err, "define.Version() %q is not parsable.", define.Version())
+
+				newVersionBuildMetadata := "build" + time.Now().Format("20060102150405")
+				parsedNewVersion := version.NewParsedSemVer(currentVersion.Major(), currentVersion.Minor(), currentVersion.Patch(), "", newVersionBuildMetadata)
+
+				err = fromFixture.EnsurePrepared(t.Context())
+				require.NoErrorf(t, err, "fixture should be prepared")
+
+				// retrieve the compressed package file location
+				srcPackage, err := fromFixture.SrcPackage(t.Context())
+				require.NoErrorf(t, err, "error retrieving start fixture source package")
+
+				versionForFixture, repackagedArchiveFile, err := repackageArchive(t, srcPackage, newVersionBuildMetadata, currentVersion, parsedNewVersion)
+				require.NoError(t, err, "error repackaging the archive built from the same commit")
+
+				repackagedLocalFetcher := atesting.LocalFetcher(filepath.Dir(repackagedArchiveFile))
+				toFixture, err := atesting.NewFixture(
+					t,
+					versionForFixture.String(),
+					atesting.WithFetcher(repackagedLocalFetcher),
+				)
+				require.NoError(t, err)
+
+				return fromFixture, toFixture
+			},
+		},
+	}
+
+	// set up start ficture with a rollback window of 1h
+	rollbackWindowConfig := `
+agent.upgrade.rollback.window: 1h
+`
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx, cancel := testcontext.WithDeadline(t, t.Context(), time.Now().Add(10*time.Minute))
+			defer cancel()
+			from, to := tc.fixturesSetup(t)
+
+			err := from.Configure(ctx, []byte(rollbackWindowConfig))
+			require.NoError(t, err, "error setting up rollback window")
+			standaloneManualRollbackTest(ctx, t, from, to)
+		})
+	}
+
+}
+
 func managedRollbackRestartTest(ctx context.Context, t *testing.T, info *define.Info, from *atesting.Fixture, to *atesting.Fixture) {
 
 	startVersionInfo, err := from.ExecVersion(ctx)
@@ -401,11 +490,29 @@ func managedRollbackRestartTest(ctx context.Context, t *testing.T, info *define.
 }
 
 func standaloneRollbackRestartTest(ctx context.Context, t *testing.T, startFixture *atesting.Fixture, endFixture *atesting.Fixture) {
+	standaloneRollbackTest(ctx, t, startFixture, endFixture, reallyFastWatcherCfg, details.ReasonWatchFailed,
+		func(t *testing.T, _ client.Client) {
+			restartAgentNTimes(t, 3, 10*time.Second)
+		})
+}
+
+func standaloneManualRollbackTest(ctx context.Context, t *testing.T, startFixture *atesting.Fixture, endFixture *atesting.Fixture) {
+	standaloneRollbackTest(ctx, t, startFixture, endFixture, fastWatcherCfgWithRollbackWindow, details.ReasonManualRollback,
+		func(t *testing.T, client client.Client) {
+			t.Logf("sending version=%s rollback=%v upgrade to agent", startFixture.Version(), true)
+			retVal, err := client.Upgrade(ctx, startFixture.Version(), true, "", false, false)
+			require.NoError(t, err, "error triggering manual rollback to version %s", startFixture.Version())
+			t.Logf("received output %s from upgrade command", retVal)
+		},
+	)
+}
+
+func standaloneRollbackTest(ctx context.Context, t *testing.T, startFixture *atesting.Fixture, endFixture *atesting.Fixture, customConfig string, rollbackReason string, rollbackTrigger func(t *testing.T, client client.Client)) {
 
 	startVersionInfo, err := startFixture.ExecVersion(ctx)
 	require.NoError(t, err, "failed to get start agent build version info")
 
-	endVersionInfo, err := startFixture.ExecVersion(ctx)
+	endVersionInfo, err := endFixture.ExecVersion(ctx)
 	require.NoError(t, err, "failed to get end agent build version info")
 
 	t.Logf("Testing Elastic Agent upgrade from %s to %s...", startFixture.Version(), endVersionInfo.Binary.String())
@@ -420,15 +527,19 @@ func standaloneRollbackRestartTest(ctx context.Context, t *testing.T, startFixtu
 	err = upgradetest.PerformUpgrade(
 		ctx, startFixture, endFixture, t,
 		upgradetest.WithPostUpgradeHook(postUpgradeHook),
-		upgradetest.WithCustomWatcherConfig(reallyFastWatcherCfg),
+		upgradetest.WithCustomWatcherConfig(customConfig),
 		upgradetest.WithDisableHashCheck(true))
 	if !errors.Is(err, ErrPostExit) {
 		require.NoError(t, err)
 	}
 
-	// A few seconds after the upgrade, deliberately restart upgraded Agent a
-	// couple of times to simulate Agent crashing.
-	restartAgentNTimes(t, 3, 10*time.Second)
+	elasticAgentClient := startFixture.Client()
+	err = elasticAgentClient.Connect(ctx)
+	require.NoError(t, err, "error connecting to installed elastic agent")
+	defer elasticAgentClient.Disconnect()
+
+	// A few seconds after the upgrade, trigger a rollback using the passed trigger
+	rollbackTrigger(t, elasticAgentClient)
 
 	// wait for the agent to be healthy and back at the start version
 	err = upgradetest.WaitHealthyAndVersion(ctx, startFixture, startVersionInfo.Binary, 2*time.Minute, 10*time.Second, t)
@@ -448,17 +559,15 @@ func standaloneRollbackRestartTest(ctx context.Context, t *testing.T, startFixtu
 	require.NoError(t, err)
 
 	if !startVersion.Less(*version.NewParsedSemVer(8, 12, 0, "", "")) {
-		client := startFixture.Client()
-		err = client.Connect(ctx)
 		require.NoError(t, err)
 
-		state, err := client.State(ctx)
+		state, err := elasticAgentClient.State(ctx)
 		require.NoError(t, err)
 
 		require.NotNil(t, state.UpgradeDetails)
 		assert.Equal(t, details.StateRollback, details.State(state.UpgradeDetails.State))
 		if !startVersion.Less(*upgradetest.Version_9_2_0_SNAPSHOT) {
-			assert.Equal(t, details.ReasonWatchFailed, state.UpgradeDetails.Metadata.Reason)
+			assert.Equal(t, rollbackReason, state.UpgradeDetails.Metadata.Reason)
 		}
 	}
 

testing/integration/ess/upgrade_standalone_same_commit_test.go

@@ -9,6 +9,7 @@ package ess
 import (
 	"context"
 	"fmt"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -82,12 +83,16 @@ func TestStandaloneUpgradeSameCommit(t *testing.T) {
 		newVersionBuildMetadata := "build" + time.Now().Format("20060102150405")
 		parsedNewVersion := version.NewParsedSemVer(currentVersion.Major(), currentVersion.Minor(), currentVersion.Patch(), "", newVersionBuildMetadata)
 
-		newPackageContainingDir := t.TempDir()
+		err = startFixture.EnsurePrepared(t.Context())
+		require.NoErrorf(t, err, "fixture should be prepared")
 
-		versionForFixture, err := repackageArchive(ctx, t, startFixture, newVersionBuildMetadata, currentVersion, newPackageContainingDir, parsedNewVersion)
+		// retrieve the compressed package file location
+		srcPackage, err := startFixture.SrcPackage(t.Context())
+		require.NoErrorf(t, err, "error retrieving start fixture source package")
 
-		// I wish I could just pass the location of the package on disk to the whole upgrade tests/fixture/fetcher code
-		// but I would have to break too much code for that, when in Rome... add more code on top of inflexible code
+		versionForFixture, repackagedArchiveFile, err := repackageArchive(t, srcPackage, newVersionBuildMetadata, currentVersion, parsedNewVersion)
+
+		newPackageContainingDir := filepath.Dir(repackagedArchiveFile)
 		repackagedLocalFetcher := atesting.LocalFetcher(newPackageContainingDir)
 
 		endFixture, err := atesting.NewFixture(t, versionForFixture.String(), atesting.WithFetcher(repackagedLocalFetcher))