improve efficiency in terragrunt generation (#1854)

* improve efficiency in terragrunt generation

Author: motatoes

backend/controllers/github.go

@@ -376,6 +376,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 	diggerYmlStr, ghService, config, projectsGraph, _, _, changedFiles, err := getDiggerConfigForPR(gh, organisationId, prLabelsStr, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
 	if err != nil {
 		log.Printf("getDiggerConfigForPR error: %v", err)
+		commentReporterManager.UpdateComment(fmt.Sprintf(":x: Error loading digger config: %v", err))
 		return fmt.Errorf("error getting digger config")
 	}
 

libs/digger_config/digger_config.go

@@ -536,6 +536,7 @@ func hydrateDiggerConfigYamlWithTerragrunt(configYaml *DiggerConfigYaml, parsing
 		parsingConfig.PreserveProjects,
 		parsingConfig.UseProjectMarkers,
 		executionOrderGroups,
+		parsingConfig.TriggerProjectsFromDirOnly,
 	)
 	if err != nil {
 		return fmt.Errorf("failed to autogenerate digger_config, error during parse: %v", err)
@@ -562,10 +563,6 @@ func hydrateDiggerConfigYamlWithTerragrunt(configYaml *DiggerConfigYaml, parsing
 		projectDir := path.Join(pathPrefix, atlantisProject.Dir)
 		atlantisProject.Autoplan.WhenModified, err = GetPatternsRelativeToRepo(projectDir, atlantisProject.Autoplan.WhenModified)
 
-		if parsingConfig.TriggerProjectsFromDirOnly {
-			atlantisProject.Autoplan.WhenModified, err = FilterPathsOutsideOfProjectPath(projectDir, atlantisProject.Autoplan.WhenModified)
-		}
-
 		if err != nil {
 			return fmt.Errorf("could not normalize patterns: %v", err)
 		}

libs/digger_config/terragrunt/atlantis/generate.go

@@ -312,8 +312,33 @@ func getDependencies(ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, g
 	}
 }
 
+func createBaseProject(dir string, workflow string, terraformVersion string, applyRequirements *[]string, autoPlan bool, dependencies []string, createProjectName bool, createWorkspace bool) *AtlantisProject {
+	project := &AtlantisProject{
+		Dir:               filepath.ToSlash(dir),
+		Workflow:          workflow,
+		TerraformVersion:  terraformVersion,
+		ApplyRequirements: applyRequirements,
+		Autoplan: AutoplanConfig{
+			Enabled:      autoPlan,
+			WhenModified: uniqueStrings(dependencies),
+		},
+	}
+
+	if createProjectName || createWorkspace {
+		projectName := projectNameFromDir(project.Dir)
+		if createProjectName {
+			project.Name = projectName
+		}
+		if createWorkspace {
+			project.Workspace = projectName
+		}
+	}
+
+	return project
+}
+
 // Creates an AtlantisProject for a directory
-func createProject(ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, gitRoot string, cascadeDependencies bool, defaultWorkflow string, defaultApplyRequirements []string, autoPlan bool, defaultTerraformVersion string, createProjectName bool, createWorkspace bool, sourcePath string) (*AtlantisProject, []string, error) {
+func createProject(ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, gitRoot string, cascadeDependencies bool, defaultWorkflow string, defaultApplyRequirements []string, autoPlan bool, defaultTerraformVersion string, createProjectName bool, createWorkspace bool, sourcePath string, triggerProjectsFromDirOnly bool) (*AtlantisProject, []string, error) {
 	options, err := options.NewTerragruntOptionsWithConfigPath(sourcePath)
 
 	var potentialProjectDependencies []string
@@ -324,6 +349,39 @@ func createProject(ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, git
 	options.RunTerragrunt = terraform.Run
 	options.Env = getEnvs()
 
+	// All dependencies depend on their own .hcl file, and any tf files in their directory
+	relativeDependencies := []string{
+		"*.hcl",
+		"*.tf*",
+	}
+
+	// Clean up the relative path to the format Atlantis expects
+	absoluteSourceDir := filepath.Dir(sourcePath) + string(filepath.Separator)
+	relativeSourceDir := strings.TrimPrefix(absoluteSourceDir, gitRoot)
+	relativeSourceDir = strings.TrimSuffix(relativeSourceDir, string(filepath.Separator))
+	if relativeSourceDir == "" {
+		relativeSourceDir = "."
+	}
+
+	if triggerProjectsFromDirOnly {
+		// TODO: Figure out easy way to make it also work with other values of gitRoot through prefix matching
+		if relativeSourceDir == "." {
+			return nil, potentialProjectDependencies, nil
+		}
+
+		project := createBaseProject(
+			relativeSourceDir,
+			defaultWorkflow,
+			defaultTerraformVersion,
+			&defaultApplyRequirements,
+			autoPlan,
+			relativeDependencies,
+			createProjectName,
+			createWorkspace,
+		)
+		return project, potentialProjectDependencies, nil
+	}
+
 	dependencies, err := getDependencies(ignoreParentTerragrunt, ignoreDependencyBlocks, gitRoot, cascadeDependencies, sourcePath, options)
 	if err != nil {
 		return nil, potentialProjectDependencies, err
@@ -334,8 +392,6 @@ func createProject(ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, git
 		return nil, potentialProjectDependencies, nil
 	}
 
-	absoluteSourceDir := filepath.Dir(sourcePath) + string(filepath.Separator)
-
 	locals, err := parseLocals(sourcePath, options, nil)
 	if err != nil {
 		return nil, potentialProjectDependencies, err
@@ -346,12 +402,6 @@ func createProject(ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, git
 		return nil, potentialProjectDependencies, nil
 	}
 
-	// All dependencies depend on their own .hcl file, and any tf files in their directory
-	relativeDependencies := []string{
-		"*.hcl",
-		"*.tf*",
-	}
-
 	// Add other dependencies based on their relative paths. We always want to output with Unix path separators
 	for _, dependencyPath := range dependencies {
 		absolutePath := dependencyPath
@@ -368,13 +418,6 @@ func createProject(ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, git
 		relativeDependencies = append(relativeDependencies, filepath.ToSlash(relativePath))
 	}
 
-	// Clean up the relative path to the format Atlantis expects
-	relativeSourceDir := strings.TrimPrefix(absoluteSourceDir, gitRoot)
-	relativeSourceDir = strings.TrimSuffix(relativeSourceDir, string(filepath.Separator))
-	if relativeSourceDir == "" {
-		relativeSourceDir = "."
-	}
-
 	workflow := defaultWorkflow
 	if locals.AtlantisWorkflow != "" {
 		workflow = locals.AtlantisWorkflow
@@ -398,16 +441,16 @@ func createProject(ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, git
 		terraformVersion = locals.TerraformVersion
 	}
 
-	project := &AtlantisProject{
-		Dir:               filepath.ToSlash(relativeSourceDir),
-		Workflow:          workflow,
-		TerraformVersion:  terraformVersion,
-		ApplyRequirements: applyRequirements,
-		Autoplan: AutoplanConfig{
-			Enabled:      resolvedAutoPlan,
-			WhenModified: uniqueStrings(relativeDependencies),
-		},
-	}
+	project := createBaseProject(
+		relativeSourceDir,
+		workflow,
+		terraformVersion,
+		applyRequirements,
+		resolvedAutoPlan,
+		relativeDependencies,
+		createProjectName,
+		createWorkspace,
+	)
 
 	projectName := projectNameFromDir(project.Dir)
 
@@ -660,7 +703,7 @@ func getAllTerragruntProjectHclFiles(projectHclFiles []string, gitRoot string) m
 	return uniqueHclFileAbsPaths
 }
 
-func Parse(gitRoot string, projectHclFiles []string, createHclProjectExternalChilds bool, autoMerge bool, parallel bool, filterPath string, createHclProjectChilds bool, ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, cascadeDependencies bool, defaultWorkflow string, defaultApplyRequirements []string, autoPlan bool, defaultTerraformVersion string, createProjectName bool, createWorkspace bool, preserveProjects bool, useProjectMarkers bool, executionOrderGroups bool) (*AtlantisConfig, map[string][]string, error) {
+func Parse(gitRoot string, projectHclFiles []string, createHclProjectExternalChilds bool, autoMerge bool, parallel bool, filterPath string, createHclProjectChilds bool, ignoreParentTerragrunt bool, ignoreDependencyBlocks bool, cascadeDependencies bool, defaultWorkflow string, defaultApplyRequirements []string, autoPlan bool, defaultTerraformVersion string, createProjectName bool, createWorkspace bool, preserveProjects bool, useProjectMarkers bool, executionOrderGroups bool, triggerProjectsFromDirOnly bool) (*AtlantisConfig, map[string][]string, error) {
 	// Ensure the gitRoot has a trailing slash and is an absolute path
 	absoluteGitRoot, err := filepath.Abs(gitRoot)
 	if err != nil {
@@ -726,7 +769,7 @@ func Parse(gitRoot string, projectHclFiles []string, createHclProjectExternalChi
 
 				errGroup.Go(func() error {
 					defer sem.Release(1)
-					project, projDeps, err := createProject(ignoreParentTerragrunt, ignoreDependencyBlocks, gitRoot, cascadeDependencies, defaultWorkflow, defaultApplyRequirements, autoPlan, defaultTerraformVersion, createProjectName, createWorkspace, terragruntPath)
+					project, projDeps, err := createProject(ignoreParentTerragrunt, ignoreDependencyBlocks, gitRoot, cascadeDependencies, defaultWorkflow, defaultApplyRequirements, autoPlan, defaultTerraformVersion, createProjectName, createWorkspace, terragruntPath, triggerProjectsFromDirOnly)
 					if err != nil {
 						return err
 					}

libs/digger_config/utils.go

@@ -5,7 +5,6 @@ import (
 	"log"
 	"path"
 	"path/filepath"
-	"strings"
 )
 
 func GetPatternsRelativeToRepo(projectPath string, patterns []string) ([]string, error) {
@@ -16,16 +15,6 @@ func GetPatternsRelativeToRepo(projectPath string, patterns []string) ([]string,
 	return res, nil
 }
 
-func FilterPathsOutsideOfProjectPath(projectPath string, patterns []string) ([]string, error) {
-	res := make([]string, 0)
-	for _, pattern := range patterns {
-		if strings.HasPrefix(pattern, projectPath) {
-			res = append(res, pattern)
-		}
-	}
-	return res, nil
-}
-
 func NormalizeFileName(fileName string) string {
 	res, err := filepath.Abs(path.Join("/", fileName))
 	if err != nil {

libs/digger_config/utils_test.go

@@ -45,13 +45,3 @@ func TestGetPatternsRelativeToRepo(t *testing.T) {
 	assert.Equal(t, "myProject/terraform/environments/devel/*.hcl", res[0])
 
 }
-
-func TestFilterPathsOutsideOfProjectPath(t *testing.T) {
-	projectDir := "staging/aws/us-east-1/k8s"
-	includePatterns := []string{"staging/aws/us-east-1/k8s/*.hcl", "staging/terragrunt-root.hcl vpc/*.tf*", "staging/aws/us-east-1/aws_region.tfvars", "staging/aws/aws_assume_role_arn.tfvars", "staging/aws/us-east-1/k8s/*.tf*"}
-	res, _ := FilterPathsOutsideOfProjectPath(projectDir, includePatterns)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "staging/aws/us-east-1/k8s/*.hcl", res[0])
-	assert.Equal(t, "staging/aws/us-east-1/k8s/*.tf*", res[1])
-
-}