merge develop into comment refactors (#1867)

* add flag to ignore all external directories per project (#1851)

* add flag to ignore all external directories per project

* revert includeparentblocks flag (#1852)

* improve efficiency in terragrunt generation (#1854)

* improve efficiency in terragrunt generation

* Update action.yml (#1856)

* handle crashes in goroutine events (#1857)

* fix/recover from webhook goroutines (#1858)

* handle crashes in goroutine events

* include stacktrace in errors

* wip generation of terraform code from application code (#1855)

* terraform code generation demo


---------

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

* fix: incorrect comment in backend/migrations (#1860)

* Update setup-opentofu to fix issues with 1.6.x downloads (#1861)

* restructure docs to have no columns (#1862)

* Create curl_bootstrap.sh

* refactor codegen parts (#1863)

* refactor codegen parts

* publish ai summaries (#1864)

* publish ai summaries

* add a header for summary comment (#1865)

* fix heading (#1866)

---------

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Co-authored-by: Aldo <[email protected]>
Co-authored-by: Hugo Samayoa <[email protected]>

Author: 4 people

action.yml

@@ -428,11 +428,9 @@ runs:
       name: cache-save
       if: ${{ always() && inputs.cache-dependencies == 'true' && steps.restore_cache.outputs.cache-hit != 'true' }}
       with:
-        path: |
-          ${{ steps.golang-env.outputs.build-cache-path }}
-          ${{ steps.golang-env.outputs.module-cache-path }}
-        key: digger-cli-cache-${{ hashFiles('.digger.go.sum') }}
-
+        path: ${{ github.workspace }}/cache
+        key: digger-cache-${{ hashFiles('**/cache') }}
+        
 branding:
   icon: globe
   color: purple

backend/controllers/github.go

@@ -10,36 +10,36 @@ import (
 	"github.com/diggerhq/digger/backend/ci_backends"
 	config2 "github.com/diggerhq/digger/backend/config"
 	"github.com/diggerhq/digger/backend/locking"
+	"github.com/diggerhq/digger/backend/middleware"
+	"github.com/diggerhq/digger/backend/models"
 	"github.com/diggerhq/digger/backend/segment"
 	"github.com/diggerhq/digger/backend/services"
+	"github.com/diggerhq/digger/backend/utils"
 	"github.com/diggerhq/digger/libs/ci"
 	"github.com/diggerhq/digger/libs/ci/generic"
+	dg_github "github.com/diggerhq/digger/libs/ci/github"
 	comment_updater "github.com/diggerhq/digger/libs/comment_utils/reporting"
+	dg_configuration "github.com/diggerhq/digger/libs/digger_config"
 	dg_locking "github.com/diggerhq/digger/libs/locking"
 	orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler"
+	"github.com/dominikbraun/graph"
+	"github.com/gin-gonic/gin"
+	"github.com/google/go-github/v61/github"
 	"github.com/google/uuid"
+	"github.com/samber/lo"
+	"golang.org/x/oauth2"
 	"gorm.io/gorm"
 	"log"
 	"math/rand"
 	"net/http"
 	"net/url"
 	"os"
-	"path"
+	"path/filepath"
 	"reflect"
+	"runtime/debug"
 	"slices"
 	"strconv"
 	"strings"
-
-	"github.com/diggerhq/digger/backend/middleware"
-	"github.com/diggerhq/digger/backend/models"
-	"github.com/diggerhq/digger/backend/utils"
-	dg_github "github.com/diggerhq/digger/libs/ci/github"
-	dg_configuration "github.com/diggerhq/digger/libs/digger_config"
-	"github.com/dominikbraun/graph"
-	"github.com/gin-gonic/gin"
-	"github.com/google/go-github/v61/github"
-	"github.com/samber/lo"
-	"golang.org/x/oauth2"
 )
 
 type IssueCommentHook func(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider) error
@@ -309,6 +309,16 @@ func handleInstallationDeletedEvent(installation *github.InstallationEvent, appI
 }
 
 func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullRequestEvent, ciBackendProvider ci_backends.CiBackendProvider, appId int64) error {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Printf("Recovered from panic in handlePullRequestEvent handler: %v", r)
+			log.Printf("\n=== PANIC RECOVERED ===\n")
+			log.Printf("Error: %v\n", r)
+			log.Printf("Stack Trace:\n%s", string(debug.Stack()))
+			log.Printf("=== END PANIC ===\n")
+		}
+	}()
+
 	installationId := *payload.Installation.ID
 	repoName := *payload.Repo.Name
 	repoOwner := *payload.Repo.Owner.Login
@@ -376,6 +386,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 	diggerYmlStr, ghService, config, projectsGraph, _, _, changedFiles, err := getDiggerConfigForPR(gh, organisationId, prLabelsStr, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
 	if err != nil {
 		log.Printf("getDiggerConfigForPR error: %v", err)
+		commentReporterManager.UpdateComment(fmt.Sprintf(":x: Error loading digger config: %v", err))
 		return fmt.Errorf("error getting digger config")
 	}
 
@@ -501,6 +512,22 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 		commentReporterManager.UpdateComment(fmt.Sprintf(":x: could not handle commentId: %v", err))
 	}
 
+
+  
+	var aiSummaryCommentId = ""
+	if config.Reporting.AiSummary {
+		aiSummaryComment, err := ghService.PublishComment(prNumber, "AI Summary will be posted here after completion")
+		if err != nil {
+			log.Printf("could not post ai summary comment: %v", err)
+			commentReporterManager.UpdateComment(fmt.Sprintf(":x: could not post ai comment summary comment id: %v", err))
+			return fmt.Errorf("could not post ai summary comment: %v", err)
+		}
+		aiSummaryCommentId = aiSummaryComment.Id
+	}
+
+	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs)
+
+  
 	placeholderComment, err := ghService.PublishComment(prNumber, "digger report placehoder")
 	if err != nil {
 		log.Printf("strconv.ParseInt error: %v", err)
@@ -509,6 +536,8 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 	}
 
 	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, &placeholderComment.Id, diggerYmlStr, 0)
+
+  
 	if err != nil {
 		log.Printf("ConvertJobsToDiggerJobs error: %v", err)
 		commentReporterManager.UpdateComment(fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
@@ -582,8 +611,11 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6
 	var dependencyGraph graph.Graph[string, dg_configuration.Project]
 
 	err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error {
-		diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
-		diggerYmlStr = string(diggerYmlBytes)
+		diggerYmlStr, err = dg_configuration.ReadDiggerYmlFileContents(dir)
+		if err != nil {
+			log.Printf("could not load digger config: %v", err)
+			return err
+		}
 		config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles)
 		if err != nil {
 			log.Printf("Error loading digger config: %v", err)
@@ -692,6 +724,16 @@ func getBatchType(jobs []orchestrator_scheduler.Job) orchestrator_scheduler.Digg
 }
 
 func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider, appId int64, postCommentHooks []IssueCommentHook) error {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Printf("Recovered from panic in handleIssueCommentEvent handler: %v", r)
+			log.Printf("\n=== PANIC RECOVERED ===\n")
+			log.Printf("Error: %v\n", r)
+			log.Printf("Stack Trace:\n%s", string(debug.Stack()))
+			log.Printf("=== END PANIC ===\n")
+		}
+	}()
+
 	installationId := *payload.Installation.ID
 	repoName := *payload.Repo.Name
 	repoOwner := *payload.Repo.Owner.Login
@@ -752,6 +794,15 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 		return fmt.Errorf("error getting digger config")
 	}
 
+	// terraform code generator
+	if os.Getenv("DIGGER_GENERATION_ENABLED") == "1" {
+		err = GenerateTerraformFromCode(payload, commentReporterManager, config, defaultBranch, ghService, repoOwner, repoName, commitSha, issueNumber, branch)
+		if err != nil {
+			log.Printf("terraform generation failed: %v", err)
+			return err
+		}
+	}
+
 	commentIdStr := strconv.FormatInt(userCommentId, 10)
 	err = ghService.CreateCommentReaction(commentIdStr, string(dg_github.GithubCommentEyesReaction))
 	if err != nil {
@@ -883,6 +934,20 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 		return fmt.Errorf("comment reporter error: %v", err)
 	}
 
+
+	var aiSummaryCommentId = ""
+	if config.Reporting.AiSummary {
+		aiSummaryComment, err := ghService.PublishComment(issueNumber, "AI Summary will be posted here after completion")
+		if err != nil {
+			log.Printf("could not post ai summary comment: %v", err)
+			commentReporterManager.UpdateComment(fmt.Sprintf(":x: could not post ai comment summary comment id: %v", err))
+			return fmt.Errorf("could not post ai summary comment: %v", err)
+		}
+		aiSummaryCommentId = aiSummaryComment.Id
+	}
+
+	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs)
+
 	placeholderComment, err := ghService.PublishComment(issueNumber, "digger report placehoder")
 	if err != nil {
 		log.Printf("strconv.ParseInt error: %v", err)
@@ -891,6 +956,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 	}
 
 	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, &placeholderComment.Id, diggerYmlStr, 0)
+
 	if err != nil {
 		log.Printf("ConvertJobsToDiggerJobs error: %v", err)
 		commentReporterManager.UpdateComment(fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
@@ -962,6 +1028,158 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 	return nil
 }
 
+func GenerateTerraformFromCode(payload *github.IssueCommentEvent, commentReporterManager utils.CommentReporterManager, config *dg_configuration.DiggerConfig, defaultBranch string, ghService *dg_github.GithubService, repoOwner string, repoName string, commitSha *string, issueNumber int, branch *string) error {
+	if strings.HasPrefix(*payload.Comment.Body, "digger generate") {
+		projectName := ci.ParseProjectName(*payload.Comment.Body)
+		if projectName == "" {
+			commentReporterManager.UpdateComment(fmt.Sprintf(":x: generate requires argument -p <project_name>"))
+			log.Printf("missing project in command: %v", *payload.Comment.Body)
+			return fmt.Errorf("generate requires argument -p <project_name>")
+		}
+
+		project := config.GetProject(projectName)
+		if project == nil {
+			commentReporterManager.UpdateComment(fmt.Sprintf("could not find project %v in digger.yml", projectName))
+			log.Printf("could not find project %v in digger.yml", projectName)
+			return fmt.Errorf("could not find project %v in digger.yml", projectName)
+		}
+
+		commentReporterManager.UpdateComment(fmt.Sprintf(":white_check_mark: Successfully loaded project"))
+
+		generationEndpoint := os.Getenv("DIGGER_GENERATION_ENDPOINT")
+		if generationEndpoint == "" {
+			commentReporterManager.UpdateComment(fmt.Sprintf(":x: server does not have generation endpoint configured, please verify"))
+			log.Printf("server does not have generation endpoint configured, please verify")
+			return fmt.Errorf("server does not have generation endpoint configured, please verify")
+		}
+		apiToken := os.Getenv("DIGGER_GENERATION_API_TOKEN")
+
+		// Get all code content from the repository at a specific commit
+		getCodeFromCommit := func(ghService *dg_github.GithubService, repoOwner, repoName string, commitSha *string, projectDir string) (string, error) {
+			const MaxPatchSize = 1024 * 1024 // 1MB limit
+
+			// Get the commit's changes compared to default branch
+			comparison, _, err := ghService.Client.Repositories.CompareCommits(
+				context.Background(),
+				repoOwner,
+				repoName,
+				defaultBranch,
+				*commitSha,
+				nil,
+			)
+			if err != nil {
+				return "", fmt.Errorf("error comparing commits: %v", err)
+			}
+
+			var appCode strings.Builder
+			for _, file := range comparison.Files {
+				if file.Patch == nil {
+					continue // Skip files without patches
+				}
+				log.Printf("Processing patch for file: %s", *file.Filename)
+				if *file.Additions > 0 {
+					lines := strings.Split(*file.Patch, "\n")
+					for _, line := range lines {
+						if strings.HasPrefix(line, "+") && !strings.HasPrefix(line, "+++") {
+							appCode.WriteString(strings.TrimPrefix(line, "+"))
+							appCode.WriteString("\n")
+						}
+					}
+				}
+				appCode.WriteString("\n")
+			}
+
+			if appCode.Len() == 0 {
+				return "", fmt.Errorf("no code changes found in commit %s. Please ensure the PR contains added or modified code", *commitSha)
+			}
+
+			return appCode.String(), nil
+		}
+
+		appCode, err := getCodeFromCommit(ghService, repoOwner, repoName, commitSha, project.Dir)
+		if err != nil {
+			commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to get code content: %v", err))
+			log.Printf("Error getting code content: %v", err)
+			return fmt.Errorf("error getting code content: %v", err)
+		}
+
+		commentReporterManager.UpdateComment(fmt.Sprintf(":white_check_mark: Successfully loaded code from commit"))
+
+		log.Printf("the app code is: %v", appCode)
+
+		commentReporterManager.UpdateComment(fmt.Sprintf("Generating terraform..."))
+		terraformCode, err := utils.GenerateTerraformCode(appCode, generationEndpoint, apiToken)
+		if err != nil {
+			commentReporterManager.UpdateComment(fmt.Sprintf(":x: could not generate terraform code: %v", err))
+			log.Printf("could not generate terraform code: %v", err)
+			return fmt.Errorf("could not generate terraform code: %v", err)
+		}
+
+		commentReporterManager.UpdateComment(fmt.Sprintf(":white_check_mark: Generated terraform"))
+
+		// comment terraform code to project dir
+		//project.Dir
+		log.Printf("terraform code is %v", terraformCode)
+
+		baseTree, _, err := ghService.Client.Git.GetTree(context.Background(), repoOwner, repoName, *commitSha, false)
+		if err != nil {
+			commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to get base tree: %v", err))
+			log.Printf("Error getting base tree: %v", err)
+			return fmt.Errorf("error getting base tree: %v", err)
+		}
+
+		// Create a new tree with the new file
+		treeEntries := []*github.TreeEntry{
+			{
+				Path:    github.String(filepath.Join(project.Dir, fmt.Sprintf("generated_%v.tf", issueNumber))),
+				Mode:    github.String("100644"),
+				Type:    github.String("blob"),
+				Content: github.String(terraformCode),
+			},
+		}
+
+		newTree, _, err := ghService.Client.Git.CreateTree(context.Background(), repoOwner, repoName, *baseTree.SHA, treeEntries)
+		if err != nil {
+			commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to create new tree: %v", err))
+			log.Printf("Error creating new tree: %v", err)
+			return fmt.Errorf("error creating new tree: %v", err)
+		}
+
+		// Create the commit
+		commitMsg := fmt.Sprintf("Add generated Terraform code for %v", projectName)
+		commit := &github.Commit{
+			Message: &commitMsg,
+			Tree:    newTree,
+			Parents: []*github.Commit{{SHA: commitSha}},
+		}
+
+		newCommit, _, err := ghService.Client.Git.CreateCommit(context.Background(), repoOwner, repoName, commit, nil)
+		if err != nil {
+			commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to commit Terraform file: %v", err))
+			log.Printf("Error committing Terraform file: %v", err)
+			return fmt.Errorf("error committing Terraform file: %v", err)
+		}
+
+		// Update the reference to point to the new commit
+		ref := &github.Reference{
+			Ref: github.String(fmt.Sprintf("refs/heads/%s", *branch)),
+			Object: &github.GitObject{
+				SHA: newCommit.SHA,
+			},
+		}
+		_, _, err = ghService.Client.Git.UpdateRef(context.Background(), repoOwner, repoName, ref, false)
+		if err != nil {
+			commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to update branch reference: %v", err))
+			log.Printf("Error updating branch reference: %v", err)
+			return fmt.Errorf("error updating branch reference: %v", err)
+		}
+
+		commentReporterManager.UpdateComment(":white_check_mark: Successfully generated and committed Terraform code")
+		return nil
+	}
+	return nil
+}
+
 func TriggerDiggerJobs(ciBackend ci_backends.CiBackend, repoFullName string, repoOwner string, repoName string, batchId *uuid.UUID, prNumber int, prService ci.PullRequestService, gh utils.GithubClientProvider) error {
 	_, err := models.DB.GetDiggerBatch(batchId)
 	if err != nil {

backend/controllers/github_test.go

@@ -731,7 +731,7 @@ func TestJobsTreeWithOneJobsAndTwoProjects(t *testing.T) {
 	graph, err := configuration.CreateProjectDependencyGraph(projects)
 	assert.NoError(t, err)
 
-	_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0)
+	_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0, "", false)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(result))
 	parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID)
@@ -760,7 +760,7 @@ func TestJobsTreeWithTwoDependantJobs(t *testing.T) {
 	projectMap["dev"] = project1
 	projectMap["prod"] = project2
 
-	_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0)
+	_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false)
 	assert.NoError(t, err)
 	assert.Equal(t, 2, len(result))
 
@@ -793,7 +793,7 @@ func TestJobsTreeWithTwoIndependentJobs(t *testing.T) {
 	projectMap["dev"] = project1
 	projectMap["prod"] = project2
 
-	_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0)
+	_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false)
 	assert.NoError(t, err)
 	assert.Equal(t, 2, len(result))
 	parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID)
@@ -838,7 +838,7 @@ func TestJobsTreeWithThreeLevels(t *testing.T) {
 	projectMap["555"] = project5
 	projectMap["666"] = project6
 
-	_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0)
+	_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false)
 	assert.NoError(t, err)
 	assert.Equal(t, 6, len(result))
 	parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["111"].DiggerJobID)