Skip to content

Commit e4aab7e

Browse files
SUSTAPLE117SUSTAPLE117
authored
Add Repo Metadata (#193)
1 parent 42eb8de commit e4aab7e

File tree

7 files changed

+309
-7
lines changed

7 files changed

+309
-7
lines changed

‎analyze/analyze.go‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,20 @@ type Repository interface {
2626
GetRepoIdentifier() string
2727
GetIsFork() bool
2828
BuildGitURL(baseURL string) string
29+
GetHasIssues() bool
30+
GetHasWiki() bool
31+
GetHasDiscussion() bool
32+
GetOpenIssuesCount() int
33+
GetForksCount() int
34+
GetStarsCount() int
35+
GetPrimaryLanguage() string
36+
GetSize() int
37+
GetDefaultBranch() string
38+
GetLicense() string
39+
GetIsTemplate() bool
40+
GetOrganizationID() int
41+
GetRepositoryID() int
42+
GetIsEmpty() bool
2943
}
3044

3145
type RepoBatch struct {
@@ -316,12 +330,27 @@ func (a *Analyzer) generatePackageInsights(ctx context.Context, tempDir string,
316330
}
317331

318332
pkg := &models.PackageInsights{
319-
Purl: purl.String(),
320333
LastCommitedAt: commitDate.Format(time.RFC3339),
321-
SourceGitCommitSha: commitSha,
334+
Purl: purl.String(),
322335
SourceScmType: repo.GetProviderName(),
323336
SourceGitRepo: repo.GetRepoIdentifier(),
324337
SourceGitRef: ref,
338+
SourceGitCommitSha: commitSha,
339+
OrgID: repo.GetOrganizationID(),
340+
RepoID: repo.GetRepositoryID(),
341+
RepoSize: repo.GetSize(),
342+
DefaultBranch: repo.GetDefaultBranch(),
343+
IsFork: repo.GetIsFork(),
344+
IsEmpty: repo.GetIsEmpty(),
345+
ForksCount: repo.GetForksCount(),
346+
StarsCount: repo.GetStarsCount(),
347+
IsTemplate: repo.GetIsTemplate(),
348+
HasIssues: repo.GetHasIssues(),
349+
OpenIssuesCount: repo.GetOpenIssuesCount(),
350+
HasWiki: repo.GetHasWiki(),
351+
HasDiscussions: repo.GetHasDiscussion(),
352+
PrimaryLanguage: repo.GetPrimaryLanguage(),
353+
License: repo.GetLicense(),
325354
}
326355
err = pkg.NormalizePurl()
327356
if err != nil {
@@ -339,7 +368,7 @@ func (a *Analyzer) cloneRepoToTemp(ctx context.Context, gitURL string, token str
339368
err = a.GitClient.Clone(ctx, tempDir, gitURL, token, ref)
340369
if err != nil {
341370
os.RemoveAll(tempDir) // Clean up if cloning fails
342-
return "", fmt.Errorf("failed to clone repo: %s", err)
371+
return "", fmt.Errorf("failed to clone repo: %w", err)
343372
}
344373
return tempDir, nil
345374
}

‎models/github_actions.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ type GithubActionsJob struct {
162162
Uses string `json:"uses,omitempty"`
163163
Secrets GithubActionsJobSecrets `json:"secrets,omitempty"`
164164
With GithubActionsWith `json:"with,omitempty"`
165-
Permissions GithubActionsPermissions `json:"permissions,omitempty"`
165+
Permissions GithubActionsPermissions `json:"permissions"`
166166
Needs StringList `json:"needs,omitempty"`
167167
If string `json:"if,omitempty"`
168168
RunsOn GithubActionsJobRunsOn `json:"runs_on" yaml:"runs-on"`
@@ -181,7 +181,7 @@ type GithubActionsWorkflow struct {
181181
Path string `json:"path" yaml:"-"`
182182
Name string `json:"name"`
183183
Events GithubActionsEvents `json:"events" yaml:"on"`
184-
Permissions GithubActionsPermissions `json:"permissions,omitempty"`
184+
Permissions GithubActionsPermissions `json:"permissions"`
185185
Env GithubActionsEnvs `json:"env,omitempty"`
186186
Jobs GithubActionsJobs `json:"jobs"`
187187
}

‎models/package_insights.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,22 @@ type PackageInsights struct {
2222
SourceGitRef string `json:"source_git_ref"`
2323
SourceGitCommitSha string `json:"source_git_commit_sha"`
2424

25+
OrgID int `json:"org_id"`
26+
RepoID int `json:"repo_id"`
27+
RepoSize int `json:"repo_size"`
28+
DefaultBranch string `json:"default_branch"`
29+
IsFork bool `json:"is_fork"`
30+
IsEmpty bool `json:"is_empty"`
31+
ForksCount int `json:"forks_count"`
32+
StarsCount int `json:"stars_count"`
33+
IsTemplate bool `json:"is_template"`
34+
HasIssues bool `json:"has_issues"`
35+
OpenIssuesCount int `json:"open_issues_count"`
36+
HasWiki bool `json:"has_wiki"`
37+
HasDiscussions bool `json:"has_discussions"`
38+
PrimaryLanguage string `json:"primary_language"`
39+
License string `json:"license"`
40+
2541
PackageDependencies []string `json:"package_dependencies"`
2642
BuildDependencies []string `json:"build_dependencies"`
2743

‎providers/github/client.go‎

Lines changed: 88 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,34 @@ type GithubRepository struct {
7878
IsDisabled bool `graphql:"isDisabled"`
7979
IsEmpty bool `graphql:"isEmpty"`
8080
IsTemplate bool `graphql:"isTemplate"`
81+
IsArchived bool `graphql:"isArchived"`
8182
StargazerCount int `graphql:"stargazerCount"`
8283
ForkCount int `graphql:"forkCount"`
84+
Owner struct {
85+
Organization struct {
86+
DatabaseId int `graphql:"databaseId"`
87+
} `graphql:"... on Organization"`
88+
User struct {
89+
DatabaseId int `graphql:"databaseId"`
90+
} `graphql:"... on User"`
91+
} `graphql:"owner"`
92+
DatabaseId int `graphql:"databaseId"`
93+
RepoSize int `graphql:"diskUsage"` // kilobytes
94+
DefaultBranchRef struct {
95+
Name string `graphql:"name"`
96+
} `graphql:"defaultBranchRef"`
97+
HasIssues bool `graphql:"hasIssuesEnabled"`
98+
HasWiki bool `graphql:"hasWikiEnabled"`
99+
HasDiscussions bool `graphql:"hasDiscussionsEnabled"`
100+
PrimaryLanguage struct {
101+
Name string `graphql:"name"`
102+
} `graphql:"primaryLanguage"`
103+
License struct {
104+
Name string `graphql:"name"`
105+
} `graphql:"licenseInfo"`
106+
Issues struct {
107+
TotalCount int `graphql:"totalCount"`
108+
} `graphql:"issues"`
83109
}
84110

85111
func (gh GithubRepository) GetProviderName() string {
@@ -127,6 +153,62 @@ func (gh GithubRepository) GetIsFork() bool {
127153
return gh.IsFork
128154
}
129155

156+
func (gh GithubRepository) GetHasIssues() bool {
157+
return gh.HasIssues
158+
}
159+
160+
func (gh GithubRepository) GetHasWiki() bool {
161+
return gh.HasWiki
162+
}
163+
164+
func (gh GithubRepository) GetHasDiscussion() bool {
165+
return gh.HasDiscussions
166+
}
167+
168+
func (gh GithubRepository) GetPrimaryLanguage() string {
169+
return gh.PrimaryLanguage.Name
170+
}
171+
172+
func (gh GithubRepository) GetSize() int {
173+
return gh.RepoSize
174+
}
175+
176+
func (gh GithubRepository) GetDefaultBranch() string {
177+
return gh.DefaultBranchRef.Name
178+
}
179+
180+
func (gh GithubRepository) GetLicense() string {
181+
return gh.License.Name
182+
}
183+
184+
func (gh GithubRepository) GetIsTemplate() bool {
185+
return gh.IsTemplate
186+
}
187+
188+
func (gh GithubRepository) GetOrganizationID() int {
189+
return gh.Owner.Organization.DatabaseId // even if it's a user, the organization will be filled with the same id
190+
}
191+
192+
func (gh GithubRepository) GetRepositoryID() int {
193+
return gh.DatabaseId
194+
}
195+
196+
func (gh GithubRepository) GetForksCount() int {
197+
return gh.ForkCount
198+
}
199+
200+
func (gh GithubRepository) GetStarsCount() int {
201+
return gh.StargazerCount
202+
}
203+
204+
func (gh GithubRepository) GetOpenIssuesCount() int {
205+
return gh.Issues.TotalCount
206+
}
207+
208+
func (gh GithubRepository) GetIsEmpty() bool {
209+
return gh.IsEmpty
210+
}
211+
130212
type Client struct {
131213
restClient *github.Client
132214
graphQLClient *githubv4.Client
@@ -139,14 +221,19 @@ func NewClient(ctx context.Context, token string, domain string) (*Client, error
139221
return nil, err
140222
}
141223

224+
oauth2Client := http.Client{
225+
Transport: &retryTransport{},
226+
}
227+
oauth2Context := context.WithValue(ctx, oauth2.HTTPClient, &oauth2Client)
228+
142229
var (
143230
// REST client
144231
restClient = github.NewClient(rateLimiter).WithAuthToken(token)
145232
// GraphQL client
146233
src = oauth2.StaticTokenSource(
147234
&oauth2.Token{AccessToken: token},
148235
)
149-
httpClient = oauth2.NewClient(ctx, src)
236+
httpClient = oauth2.NewClient(oauth2Context, src)
150237
graphQLClient *githubv4.Client
151238
)
152239

‎providers/github/round_tripper_rate_limit.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package github
2+
3+
import (
4+
"fmt"
5+
"net/http"
6+
)
7+
8+
type retryTransport struct{}
9+
10+
type RateLimitError struct {
11+
RetryAfter string
12+
Err error
13+
}
14+
15+
func (e *RateLimitError) Error() string {
16+
return fmt.Sprintf("retry after %s: %v", e.RetryAfter, e.Err)
17+
}
18+
19+
func (e *RateLimitError) Unwrap() error {
20+
return e.Err
21+
}
22+
23+
func (s *retryTransport) RoundTrip(r *http.Request) (*http.Response, error) {
24+
resp, err := http.DefaultTransport.RoundTrip(r)
25+
if err != nil {
26+
return nil, err
27+
}
28+
29+
if resp != nil {
30+
retryAfter := resp.Header.Get("Retry-After")
31+
if retryAfter != "" {
32+
return nil, &RateLimitError{
33+
RetryAfter: retryAfter,
34+
Err: fmt.Errorf("github graphql rate limit"),
35+
}
36+
}
37+
}
38+
39+
return resp, err
40+
}

‎providers/gitlab/client.go‎

Lines changed: 75 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,13 @@ type GitLabRepo struct {
8080
IsArchived bool
8181
StarCount int
8282
ForksCount int
83+
ID int
84+
IsEmpty bool
85+
IssuesCount int
86+
HasIssues bool
87+
HasWiki bool
88+
License string
89+
DefaultBranch string
8390
}
8491

8592
func (gl GitLabRepo) GetProviderName() string {
@@ -130,10 +137,66 @@ type Client struct {
130137
client *gitlab.Client
131138
}
132139

140+
func (gl GitLabRepo) GetHasIssues() bool {
141+
return gl.HasIssues
142+
}
143+
144+
func (gl GitLabRepo) GetHasWiki() bool {
145+
return gl.HasWiki
146+
}
147+
148+
func (gl GitLabRepo) GetHasDiscussion() bool {
149+
return false
150+
}
151+
152+
func (gl GitLabRepo) GetPrimaryLanguage() string {
153+
return ""
154+
}
155+
156+
func (gl GitLabRepo) GetSize() int {
157+
return 1337
158+
}
159+
160+
func (gl GitLabRepo) GetDefaultBranch() string {
161+
return gl.DefaultBranch
162+
}
163+
164+
func (gl GitLabRepo) GetLicense() string {
165+
return gl.License
166+
}
167+
168+
func (gl GitLabRepo) GetIsTemplate() bool {
169+
return false
170+
}
171+
172+
func (gl GitLabRepo) GetOrganizationID() int {
173+
return 1337
174+
}
175+
176+
func (gl GitLabRepo) GetRepositoryID() int {
177+
return gl.ID
178+
}
179+
180+
func (gl GitLabRepo) GetForksCount() int {
181+
return gl.ForksCount
182+
}
183+
184+
func (gl GitLabRepo) GetStarsCount() int {
185+
return gl.StarCount
186+
}
187+
188+
func (gl GitLabRepo) GetOpenIssuesCount() int {
189+
return gl.IssuesCount
190+
}
191+
192+
func (gl GitLabRepo) GetIsEmpty() bool {
193+
return gl.IsEmpty
194+
}
195+
133196
func NewClient(ctx context.Context, baseUrl string, token string) (*Client, error) {
134197
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(fmt.Sprintf("https://%s", baseUrl)))
135198
if err != nil {
136-
return nil, fmt.Errorf("failed to create gitlab client: %v", err)
199+
return nil, fmt.Errorf("failed to create gitlab client: %w", err)
137200
}
138201
return &Client{
139202
Token: token,
@@ -199,6 +262,10 @@ func projectToRepo(project *gitlab.Project) *GitLabRepo {
199262
if project.ForkedFromProject != nil {
200263
isFork = true
201264
}
265+
license := ""
266+
if project.License != nil {
267+
license = project.License.Name
268+
}
202269
return &GitLabRepo{
203270
NameWithNamespace: project.PathWithNamespace,
204271
IsPrivate: !(project.Visibility == gitlab.PublicVisibility),
@@ -207,6 +274,13 @@ func projectToRepo(project *gitlab.Project) *GitLabRepo {
207274
StarCount: project.StarCount,
208275
ForksCount: project.ForksCount,
209276
IsFork: isFork,
277+
IsEmpty: project.EmptyRepo,
278+
HasIssues: project.IssuesEnabled,
279+
ID: project.ID,
280+
IssuesCount: project.OpenIssuesCount,
281+
HasWiki: project.WikiEnabled,
282+
License: license,
283+
DefaultBranch: project.DefaultBranch,
210284
}
211285
}
212286

0 commit comments

Comments
 (0)