include projects api (#2006)

motatoes · web-flow · commit 5cf11a5ff644 · 2025-07-03T20:44:21.000-07:00
diff --git a/backend/bootstrap/main.go b/backend/bootstrap/main.go
@@ -229,6 +229,9 @@ func Bootstrap(templates embed.FS, diggerController controllers.DiggerController
 		reposApiGroup.GET("/", controllers.ListReposApi)
 		reposApiGroup.GET("/:repo_id/jobs", controllers.GetJobsForRepoApi)
 
+		projectsApiGroup := apiGroup.Group("/projects")
+		projectsApiGroup.GET("/", controllers.ListProjectsApi)
+
 		githubApiGroup := apiGroup.Group("/github")
 		githubApiGroup.POST("/link", controllers.LinkGithubInstallationToOrgApi)
 
diff --git a/backend/controllers/github_test.go b/backend/controllers/github_test.go
@@ -614,7 +614,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *models.Database) {
 
 	// create test project
 	projectName := "test project"
-	_, err = database.CreateProject(projectName, org, repo.RepoFullName, false, false)
+	_, err = database.CreateProject(projectName, "", org, repo.RepoFullName, false, false)
 	if err != nil {
 		panic(err)
 	}
diff --git a/backend/controllers/projects.go b/backend/controllers/projects.go
@@ -25,8 +25,50 @@ import (
 	"gorm.io/gorm"
 )
 
-func ListProjects(c *gin.Context) {
+func ListProjectsApi(c *gin.Context) {
+	// assume all exists as validated in middleware
+	organisationId := c.GetString(middleware.ORGANISATION_ID_KEY)
+	organisationSource := c.GetString(middleware.ORGANISATION_SOURCE_KEY)
 
+	var org models.Organisation
+	err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error
+	if err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			slog.Info("Organisation not found", "organisationId", organisationId, "source", organisationSource)
+			c.String(http.StatusNotFound, "Could not find organisation: "+organisationId)
+		} else {
+			slog.Error("Error fetching organisation", "organisationId", organisationId, "source", organisationSource, "error", err)
+			c.String(http.StatusInternalServerError, "Error fetching organisation")
+		}
+		return
+	}
+
+	var projects []models.Project
+
+	err = models.DB.GormDB.Preload("Organisation").
+		Where("projects.organisation_id = ?", org.ID).
+		Order("name").
+		Find(&projects).Error
+
+	if err != nil {
+		slog.Error("Error fetching projects", "organisationId", organisationId, "orgId", org.ID, "error", err)
+		c.String(http.StatusInternalServerError, "Unknown error occurred while fetching database")
+		return
+	}
+
+	marshalledRepos := make([]interface{}, 0)
+
+	for _, p := range projects {
+		marshalled := p.MapToJsonStruct()
+		marshalledRepos = append(marshalledRepos, marshalled)
+	}
+
+	slog.Info("Successfully fetched projects", "organisationId", organisationId, "orgId", org.ID, "projectCount", len(projects))
+
+	response := make(map[string]interface{})
+	response["result"] = marshalledRepos
+
+	c.JSON(http.StatusOK, response)
 }
 
 func FindProjectsForRepo(c *gin.Context) {
@@ -353,11 +395,10 @@ func ReportProjectsForRepo(c *gin.Context) {
 			)
 
 			project := models.Project{
-				Name:              request.Name,
-				ConfigurationYaml: request.ConfigurationYaml,
-				OrganisationID:    org.ID,
-				RepoFullName:      repo.RepoFullName,
-				Organisation:      org,
+				Name:           request.Name,
+				OrganisationID: org.ID,
+				RepoFullName:   repo.RepoFullName,
+				Organisation:   org,
 			}
 
 			err = models.DB.GormDB.Create(&project).Error
diff --git a/backend/go.mod b/backend/go.mod
@@ -20,9 +20,9 @@ require (
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/google/go-github/v61 v61.0.0
 	github.com/google/uuid v1.6.0
+	github.com/imdatngo/slog-gorm/v2 v2.0.0
 	github.com/ktrysmt/go-bitbucket v0.9.81
 	github.com/migueleliasweb/go-github-mock v0.0.23
-	github.com/orandin/slog-gorm v1.4.0
 	github.com/robfig/cron v1.2.0
 	github.com/samber/lo v1.39.0
 	github.com/samber/slog-gin v1.15.0
diff --git a/backend/go.sum b/backend/go.sum
@@ -1421,6 +1421,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imdatngo/slog-gorm/v2 v2.0.0 h1:SrMVqpExd3USDZjiFlJoFg3FJvZOreJX/d8N8G6PQ4w=
+github.com/imdatngo/slog-gorm/v2 v2.0.0/go.mod h1:hp2V0UOoVXwMT9YUbB/EbaXCN7g/7txxekKaalO/6fU=
 github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
 github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
 github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
@@ -1683,8 +1685,6 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ
 github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
 github.com/opencontainers/runc v1.1.9 h1:XR0VIHTGce5eWPkaPesqTBrhW2yAcaraWfsEalNwQLM=
 github.com/opencontainers/runc v1.1.9/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50=
-github.com/orandin/slog-gorm v1.4.0 h1:FgA8hJufF9/jeNSYoEXmHPPBwET2gwlF3B85JdpsTUU=
-github.com/orandin/slog-gorm v1.4.0/go.mod h1:MoZ51+b7xE9lwGNPYEhxcUtRNrYzjdcKvA8QXQQGEPA=
 github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA=
 github.com/ory/dockertest v3.3.5+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs=
 github.com/owenrumney/go-sarif v1.1.1 h1:QNObu6YX1igyFKhdzd7vgzmw7XsWN3/6NMGuDzBgXmE=
diff --git a/backend/migrations/20250704024948.sql b/backend/migrations/20250704024948.sql
@@ -0,0 +1,2 @@
+-- Modify "projects" table
+ALTER TABLE "public"."projects" ADD COLUMN "directory" text NULL;
diff --git a/backend/migrations/20250704025208.sql b/backend/migrations/20250704025208.sql
@@ -0,0 +1,2 @@
+-- Modify "projects" table
+ALTER TABLE "public"."projects" DROP COLUMN "configuration_yaml";
diff --git a/backend/migrations/atlas.sum b/backend/migrations/atlas.sum
@@ -1,4 +1,4 @@
-h1:IeQ6uHSz71Ox7tBqG6sy05KmaRMpYqhr0SeipkVSEbM=
+h1:E1MYgdCTYDdHVINjSBVtGufzvYspZC8Wur7nvPK7fN0=
 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw=
 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA=
 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw=
@@ -50,3 +50,5 @@ h1:IeQ6uHSz71Ox7tBqG6sy05KmaRMpYqhr0SeipkVSEbM=
 20250512172515.sql h1:iIZIhFq3TTyjTzsxNWv3k4FcIkXfOCdHtmHNmYqjBMM=
 20250512213729.sql h1:n8bYIXWko9xOUs+FPcG7lS3GXMVQBSygXX7Hpj20eVs=
 20250530015921.sql h1:FidRW+0ur3mCDBPgP7V/sqr2jjfmi/CFJPRpNxTmqGs=
+20250704024948.sql h1:ki8bOrfd8y7Mfq40Dz5lH6MUuQqPVah+oimeRDVHOOQ=
+20250704025208.sql h1:XRY66dphINSj53mbxl5ALmyWqebuB1swPvUuUPvNR88=
diff --git a/backend/models/orgs.go b/backend/models/orgs.go
@@ -73,14 +73,14 @@ const (
 
 type Project struct {
 	gorm.Model
-	Name              string `gorm:"uniqueIndex:idx_project_org"`
-	OrganisationID    uint   `gorm:"uniqueIndex:idx_project_org"`
-	Organisation      *Organisation
-	RepoFullName      string `gorm:"uniqueIndex:idx_project_org"`
-	ConfigurationYaml string // TODO: probably needs to be deleted
-	Status            ProjectStatus
-	IsGenerated       bool
-	IsInMainBranch    bool
+	Name           string `gorm:"uniqueIndex:idx_project_org"`
+	Directory      string
+	OrganisationID uint `gorm:"uniqueIndex:idx_project_org"`
+	Organisation   *Organisation
+	RepoFullName   string `gorm:"uniqueIndex:idx_project_org"`
+	Status         ProjectStatus
+	IsGenerated    bool
+	IsInMainBranch bool
 }
 
 func (p *Project) MapToJsonStruct() interface{} {
@@ -97,9 +97,6 @@ func (p *Project) MapToJsonStruct() interface{} {
 		OrganisationName      string `json:"organisation_name"`
 		RepoID                uint   `json:"repo_id"`
 		RepoFullName          string `json:"repo_full_name"`
-		RepoName              string `json:"repo_name"`
-		RepoOrg               string `json:"repo_org"`
-		RepoUrl               string `json:"repo_url"`
 		IsInMainBranch        bool   `json:"is_in_main_branch"`
 		IsGenerated           bool   `json:"is_generated"`
 		LastActivityTimestamp string `json:"last_activity_timestamp"`
@@ -108,6 +105,7 @@ func (p *Project) MapToJsonStruct() interface{} {
 	}{
 		Id:                    p.ID,
 		Name:                  p.Name,
+		Directory:             p.Directory,
 		OrganisationID:        p.OrganisationID,
 		OrganisationName:      p.Organisation.Name,
 		RepoFullName:          p.RepoFullName,
diff --git a/backend/models/scheduler_test.go b/backend/models/scheduler_test.go
@@ -54,7 +54,7 @@ func setupSuiteScheduler(tb testing.TB) (func(tb testing.TB), *Database) {
 	}
 
 	projectName := "test project"
-	_, err = database.CreateProject(projectName, org, repo.RepoFullName, false, false)
+	_, err = database.CreateProject(projectName, "", org, repo.RepoFullName, false, false)
 	if err != nil {
 		panic(err)
 	}
diff --git a/backend/models/setup.go b/backend/models/setup.go
@@ -1,13 +1,13 @@
 package models
 
 import (
-	"log/slog"
-	"os"
-
-	slogGorm "github.com/orandin/slog-gorm"
+	sloggorm "github.com/imdatngo/slog-gorm/v2"
 	"gorm.io/driver/postgres"
 	_ "gorm.io/driver/postgres"
 	"gorm.io/gorm"
+	"log/slog"
+	"os"
+	"time"
 )
 
 type Database struct {
@@ -20,8 +20,19 @@ var DEFAULT_ORG_NAME = "digger"
 var DB *Database
 
 func ConnectDatabase() {
+	slogger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
+
+	cfg := sloggorm.NewConfig(slogger.Handler()).
+		WithGroupKey("db").
+		WithSlowThreshold(time.Second).
+		WithIgnoreRecordNotFoundError(true)
+
+	if os.Getenv("DIGGER_LOG_LEVEL") == "DEBUG" {
+		cfg.WithTraceAll(true)
+	}
+	glogger := sloggorm.NewWithConfig(cfg)
 	database, err := gorm.Open(postgres.Open(os.Getenv("DATABASE_URL")), &gorm.Config{
-		Logger: slogGorm.New(),
+		Logger: glogger,
 	})
 	if err != nil {
 		slog.Error("Failed to connect to database", "error", err)
diff --git a/backend/models/storage.go b/backend/models/storage.go
@@ -378,7 +378,7 @@ func (db *Database) GetRepo(orgIdKey any, repoName string) (*Repo, error) {
 }
 
 func (db *Database) GetRepoByFullName(orgIdKey any, repoFullName string) (*Repo, error) {
-	slog.Info("getting repo by name",
+	slog.Info("getting repo by full name",
 		"orgId", orgIdKey,
 		"repoName", repoFullName)
 
@@ -1402,9 +1402,10 @@ func (db *Database) CreateOrganisation(name string, externalSource string, tenan
 	return org, nil
 }
 
-func (db *Database) CreateProject(name string, org *Organisation, repoFullName string, isGenerated bool, isInMainBranch bool) (*Project, error) {
+func (db *Database) CreateProject(name string, directory string, org *Organisation, repoFullName string, isGenerated bool, isInMainBranch bool) (*Project, error) {
 	project := &Project{
 		Name:           name,
+		Directory:      directory,
 		Organisation:   org,
 		RepoFullName:   repoFullName,
 		Status:         ProjectActive,
@@ -1642,15 +1643,19 @@ func (db *Database) RefreshProjectsFromRepo(orgId string, config configuration.D
 	err = db.GormDB.Transaction(func(tx *gorm.DB) error {
 		for _, dc := range config.Projects {
 			projectName := dc.Name
+			projectDirectory := dc.Dir
 			p, err := db.GetProjectByName(orgId, repoFullName, projectName)
 			if err != nil {
 				return fmt.Errorf("error retrieving project by name: %v", err)
 			}
 			if p == nil {
-				_, err := db.CreateProject(projectName, org, repoFullName, false, true)
+				_, err := db.CreateProject(projectName, projectDirectory, org, repoFullName, false, true)
 				if err != nil {
 					return fmt.Errorf("could not create project: %v", err)
 				}
+			} else {
+				p.Directory = projectDirectory
+				db.GormDB.Save(p)
 			}
 		}
 		return nil
diff --git a/backend/services/repos.go b/backend/services/repos.go
@@ -4,15 +4,13 @@ import (
 	"fmt"
 	"github.com/diggerhq/digger/backend/models"
 	utils3 "github.com/diggerhq/digger/backend/utils"
-	"github.com/diggerhq/digger/ee/drift/utils"
 	dg_configuration "github.com/diggerhq/digger/libs/digger_config"
-	utils2 "github.com/diggerhq/digger/next/utils"
 	"log/slog"
 	"strconv"
 	"strings"
 )
 
-func LoadProjectsFromGithubRepo(gh utils2.GithubClientProvider, installationId string, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string) error {
+func LoadProjectsFromGithubRepo(gh utils3.GithubClientProvider, installationId string, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string) error {
 	installationId64, err := strconv.ParseInt(installationId, 10, 64)
 	if err != nil {
 		slog.Error("failed to convert installation id %v to int64", "insallationId", installationId)
@@ -36,7 +34,8 @@ func LoadProjectsFromGithubRepo(gh utils2.GithubClientProvider, installationId s
 		return fmt.Errorf("repo not found: Org: %v | repo: %v", orgId, diggerRepoName)
 	}
 
-	_, token, err := utils.GetGithubService(gh, installationId64, repoFullName, repoOwner, repoName)
+	slog.Debug("getting github service", "installationId", installationId, "repoFullName", repoFullName, "repoOwner", repoOwner, "repoName", repoName)
+	_, token, err := utils3.GetGithubService(gh, installationId64, repoFullName, repoOwner, repoName)
 	if err != nil {
 		slog.Error("getting github service", "error", err)
 		return fmt.Errorf("error getting github service")
diff --git a/backend/tasks/runs_test.go b/backend/tasks/runs_test.go
@@ -74,7 +74,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *models.Database) {
 
 	// create test project
 	projectName := "test project"
-	_, err = database.CreateProject(projectName, org, repo.RepoFullName, false, false)
+	_, err = database.CreateProject(projectName, "", org, repo.RepoFullName, false, false)
 	if err != nil {
 		panic(err)
 	}
@@ -140,7 +140,7 @@ func TestThatRunQueueItemMovesFromQueuedToPlanningAfterPickup(t *testing.T) {
 	for i, testParam := range testParameters {
 		ciService := github2.MockCiService{}
 		batch, _ := models.DB.CreateDiggerBatch(models.DiggerVCSGithub, 123, "", "", "", 22, "", "", "", nil, 0, "", false, true, nil)
-		project, _ := models.DB.CreateProject(fmt.Sprintf("test%v", i), nil, "", false, false)
+		project, _ := models.DB.CreateProject(fmt.Sprintf("test%v", i), "", nil, "", false, false)
 		planStage, _ := models.DB.CreateDiggerRunStage(batch.ID.String())
 		applyStage, _ := models.DB.CreateDiggerRunStage(batch.ID.String())
 		diggerRun, _ := models.DB.CreateDiggerRun("", 1, testParam.InitialStatus, "sha", "", 123, 1, project.Name, "apply", &planStage.ID, &applyStage.ID)
diff --git a/ee/drift/utils/github.go b/ee/drift/utils/github.go
@@ -13,13 +13,17 @@ import (
 	"github.com/dominikbraun/graph"
 	"github.com/google/go-github/v61/github"
 	"log"
+	"log/slog"
 	net "net/http"
 	"os"
 	"path"
 	"strconv"
 )
 
 func GetGithubClient(gh utils.GithubClientProvider, installationId int64, repoFullName string) (*github.Client, *string, error) {
+	slog.Debug("Getting GitHub client",
+		"installationId", installationId,
+		"repoFullName", repoFullName)
 	repo, err := dbmodels.DB.GetRepoByInstllationIdAndRepoFullName(strconv.FormatInt(installationId, 10), repoFullName)
 	if err != nil {
 		log.Printf("Error getting repo: %v", err)
@@ -31,6 +35,7 @@ func GetGithubClient(gh utils.GithubClientProvider, installationId int64, repoFu
 }
 
 func GetGithubService(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string) (*github2.GithubService, *string, error) {
+	slog.Debug("getting github client", "installationId", installationId, "repoFullName", repoFullName)
 	ghClient, token, err := GetGithubClient(gh, installationId, repoFullName)
 	if err != nil {
 		log.Printf("Error creating github app client: %v", err)

Original file line number	Diff line number	Diff line change
`@@ -614,7 +614,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *models.Database) {`
`614`	`614`
`615`	`615`	`// create test project`
`616`	`616`	`projectName := "test project"`
`617`		`- _, err = database.CreateProject(projectName, org, repo.RepoFullName, false, false)`
	`617`	`+ _, err = database.CreateProject(projectName, "", org, repo.RepoFullName, false, false)`
`618`	`618`	`if err != nil {`
`619`	`619`	`panic(err)`
`620`	`620`	`}`
-Original file line number
+Diff line change
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/google/go-github/v61 v61.0.0
 	github.com/google/uuid v1.6.0
 +	github.com/imdatngo/slog-gorm/v2 v2.0.0
 	github.com/ktrysmt/go-bitbucket v0.9.81
 	github.com/migueleliasweb/go-github-mock v0.0.23
 -	github.com/orandin/slog-gorm v1.4.0
 	github.com/robfig/cron v1.2.0
 	github.com/samber/lo v1.39.0
 	github.com/samber/slog-gin v1.15.0
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+-- Modify "projects" table`
	`2`	`+ALTER TABLE "public"."projects" ADD COLUMN "directory" text NULL;`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+-- Modify "projects" table`
	`2`	`+ALTER TABLE "public"."projects" DROP COLUMN "configuration_yaml";`
Original file line number	Diff line number	Diff line change
`@@ -54,7 +54,7 @@ func setupSuiteScheduler(tb testing.TB) (func(tb testing.TB), *Database) {`
`54`	`54`	`}`
`55`	`55`
`56`	`56`	`projectName := "test project"`
`57`		`- _, err = database.CreateProject(projectName, org, repo.RepoFullName, false, false)`
	`57`	`+ _, err = database.CreateProject(projectName, "", org, repo.RepoFullName, false, false)`
`58`	`58`	`if err != nil {`
`59`	`59`	`panic(err)`
`60`	`60`	`}`