Skip to content

Commit 7ec9820

Browse files
SUSTAPLE117SUSTAPLE117
authored
Inventory Scanner Refactoring (#230)
1 parent 871a8db commit 7ec9820

File tree

17 files changed

+608
-516
lines changed

17 files changed

+608
-516
lines changed

‎analyze/analyze.go‎

Lines changed: 42 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ type Analyzer struct {
8787
Opa *opa.Opa
8888
}
8989

90-
func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutines *int) error {
90+
func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutines *int) ([]*models.PackageInsights, error) {
9191
provider := a.ScmClient.GetProviderName()
9292

9393
providerVersion, err := a.ScmClient.GetProviderVersion(ctx)
@@ -114,19 +114,21 @@ func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutine
114114
}
115115
goRoutineLimitSem := semaphore.NewWeighted(int64(maxGoroutines))
116116

117+
scannedPackages := make([]*models.PackageInsights, 0)
118+
117119
pkgChan := make(chan *models.PackageInsights)
118120
pkgWg := sync.WaitGroup{}
119121
pkgWg.Add(1)
120122
go func() {
121123
defer pkgWg.Done()
122124
for pkg := range pkgChan {
123-
inventory.Packages = append(inventory.Packages, pkg)
125+
scannedPackages = append(scannedPackages, pkg)
124126
}
125127
}()
126128

127129
for repoBatch := range orgReposBatches {
128130
if repoBatch.Err != nil {
129-
return fmt.Errorf("failed to get batch of repos: %w", repoBatch.Err)
131+
return scannedPackages, fmt.Errorf("failed to get batch of repos: %w", repoBatch.Err)
130132
}
131133
if repoBatch.TotalCount != 0 {
132134
bar.ChangeMax(repoBatch.TotalCount)
@@ -144,7 +146,7 @@ func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutine
144146
}
145147
if err := goRoutineLimitSem.Acquire(ctx, 1); err != nil {
146148
close(errChan)
147-
return fmt.Errorf("failed to acquire semaphore: %w", err)
149+
return scannedPackages, fmt.Errorf("failed to acquire semaphore: %w", err)
148150
}
149151

150152
reposWg.Add(1)
@@ -165,7 +167,7 @@ func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutine
165167
return
166168
}
167169

168-
scannedPkg, err := inventory.ScanPackage(ctx, pkg, tempDir)
170+
scannedPkg, err := inventory.ScanPackage(ctx, *pkg, tempDir)
169171
if err != nil {
170172
log.Error().Err(err).Str("repo", repoNameWithOwner).Msg("failed to scan package")
171173
return
@@ -192,23 +194,28 @@ func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutine
192194

193195
for err := range errChan {
194196
if err != nil {
195-
return err
197+
return scannedPackages, err
196198
}
197199
}
198200

199201
_ = bar.Finish()
200202

201-
return a.finalizeAnalysis(ctx, inventory)
203+
err = a.finalizeAnalysis(ctx, scannedPackages)
204+
if err != nil {
205+
return scannedPackages, err
206+
}
207+
208+
return scannedPackages, nil
202209
}
203210

204-
func (a *Analyzer) AnalyzeRepo(ctx context.Context, repoString string, ref string) error {
211+
func (a *Analyzer) AnalyzeRepo(ctx context.Context, repoString string, ref string) (*models.PackageInsights, error) {
205212
org, repoName, err := a.ScmClient.ParseRepoAndOrg(repoString)
206213
if err != nil {
207-
return fmt.Errorf("failed to parse repository: %w", err)
214+
return nil, fmt.Errorf("failed to parse repository: %w", err)
208215
}
209216
repo, err := a.ScmClient.GetRepo(ctx, org, repoName)
210217
if err != nil {
211-
return fmt.Errorf("failed to get repo: %w", err)
218+
return nil, fmt.Errorf("failed to get repo: %w", err)
212219
}
213220
provider := repo.GetProviderName()
214221

@@ -228,7 +235,7 @@ func (a *Analyzer) AnalyzeRepo(ctx context.Context, repoString string, ref strin
228235

229236
tempDir, err := a.cloneRepoToTemp(ctx, repo.BuildGitURL(a.ScmClient.GetProviderBaseURL()), a.ScmClient.GetToken(), ref)
230237
if err != nil {
231-
return err
238+
return nil, err
232239
}
233240
defer os.RemoveAll(tempDir)
234241

@@ -237,26 +244,31 @@ func (a *Analyzer) AnalyzeRepo(ctx context.Context, repoString string, ref strin
237244

238245
pkg, err := a.generatePackageInsights(ctx, tempDir, repo, ref)
239246
if err != nil {
240-
return err
247+
return nil, err
241248
}
242249

243-
err = inventory.AddPackage(ctx, pkg, tempDir)
250+
scannedPackage, err := inventory.ScanPackage(ctx, *pkg, tempDir)
244251
if err != nil {
245-
return err
252+
return nil, err
246253
}
247254
_ = bar.Finish()
248255

249-
return a.finalizeAnalysis(ctx, inventory)
256+
err = a.finalizeAnalysis(ctx, []*models.PackageInsights{scannedPackage})
257+
if err != nil {
258+
return nil, err
259+
}
260+
261+
return scannedPackage, nil
250262
}
251263

252-
func (a *Analyzer) AnalyzeLocalRepo(ctx context.Context, repoPath string) error {
264+
func (a *Analyzer) AnalyzeLocalRepo(ctx context.Context, repoPath string) (*models.PackageInsights, error) {
253265
org, repoName, err := a.ScmClient.ParseRepoAndOrg(repoPath)
254266
if err != nil {
255-
return fmt.Errorf("failed to parse repository: %w", err)
267+
return nil, fmt.Errorf("failed to parse repository: %w", err)
256268
}
257269
repo, err := a.ScmClient.GetRepo(ctx, org, repoName)
258270
if err != nil {
259-
return fmt.Errorf("failed to get repo: %w", err)
271+
return nil, fmt.Errorf("failed to get repo: %w", err)
260272
}
261273
provider := repo.GetProviderName()
262274

@@ -274,28 +286,28 @@ func (a *Analyzer) AnalyzeLocalRepo(ctx context.Context, repoPath string) error
274286

275287
pkg, err := a.generatePackageInsights(ctx, repoPath, repo, "")
276288
if err != nil {
277-
return err
289+
return nil, err
278290
}
279291

280-
err = inventory.AddPackage(ctx, pkg, repoPath)
292+
scannedPackage, err := inventory.ScanPackage(ctx, *pkg, repoPath)
281293
if err != nil {
282-
return err
294+
return nil, err
295+
}
296+
297+
err = a.finalizeAnalysis(ctx, []*models.PackageInsights{scannedPackage})
298+
if err != nil {
299+
return nil, err
283300
}
284301

285-
return a.finalizeAnalysis(ctx, inventory)
302+
return scannedPackage, nil
286303
}
287304

288305
type Formatter interface {
289-
Format(ctx context.Context, report *opa.FindingsResult, packages []*models.PackageInsights) error
306+
Format(ctx context.Context, packages []*models.PackageInsights) error
290307
}
291308

292-
func (a *Analyzer) finalizeAnalysis(ctx context.Context, inventory *scanner.Inventory) error {
293-
report, err := inventory.Findings(ctx)
294-
if err != nil {
295-
return err
296-
}
297-
298-
err = a.Formatter.Format(ctx, report, inventory.Packages)
309+
func (a *Analyzer) finalizeAnalysis(ctx context.Context, scannedPackages []*models.PackageInsights) error {
310+
err := a.Formatter.Format(ctx, scannedPackages)
299311
if err != nil {
300312
return err
301313
}

‎cmd/analyzeLocal.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ Example: poutine analyze_local /path/to/repo`,
3636

3737
analyzer := analyze.NewAnalyzer(localScmClient, localGitClient, formatter, config, opaClient)
3838

39-
err = analyzer.AnalyzeLocalRepo(ctx, repoPath)
39+
_, err = analyzer.AnalyzeLocalRepo(ctx, repoPath)
4040
if err != nil {
4141
return fmt.Errorf("failed to analyze repoPath %s: %w", repoPath, err)
4242
}

‎cmd/analyzeOrg.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ Note: This command will scan all repositories in the organization except those t
3131

3232
org := args[0]
3333

34-
err = analyzer.AnalyzeOrg(ctx, org, &threads)
34+
_, err = analyzer.AnalyzeOrg(ctx, org, &threads)
3535
if err != nil {
3636
return fmt.Errorf("failed to analyze org %s: %w", org, err)
3737
}

‎cmd/analyzeRepo.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ Example Scanning a remote Github Repository: poutine analyze_repo org/repo --tok
2626

2727
repo := args[0]
2828

29-
err = analyzer.AnalyzeRepo(ctx, repo, ref)
29+
_, err = analyzer.AnalyzeRepo(ctx, repo, ref)
3030
if err != nil {
3131
return fmt.Errorf("failed to analyze repo %s: %w", repo, err)
3232
}

‎formatters/json/json.go‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"github.com/boostsecurityio/poutine/models"
77
"github.com/boostsecurityio/poutine/opa"
8+
"github.com/boostsecurityio/poutine/results"
89
"io"
910
)
1011

@@ -22,11 +23,23 @@ type Format struct {
2223
format string
2324
}
2425

25-
func (f *Format) Format(ctx context.Context, report *opa.FindingsResult, packages []*models.PackageInsights) error {
26+
func (f *Format) Format(ctx context.Context, packages []*models.PackageInsights) error {
2627
var result struct {
2728
Output string `json:"output"`
2829
Error string `json:"error"`
2930
}
31+
report := &results.FindingsResult{
32+
Findings: make([]results.Finding, 0),
33+
Rules: map[string]results.Rule{},
34+
}
35+
for _, pkg := range packages {
36+
for _, finding := range pkg.FindingsResults.Findings {
37+
report.Findings = append(report.Findings, finding)
38+
}
39+
for _, rule := range pkg.FindingsResults.Rules {
40+
report.Rules[rule.Id] = rule
41+
}
42+
}
3043
err := f.opa.Eval(ctx,
3144
"data.poutine.queries.format.result",
3245
map[string]interface{}{

‎formatters/pretty/pretty.go‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,41 +3,48 @@ package pretty
33
import (
44
"context"
55
"fmt"
6+
"github.com/boostsecurityio/poutine/results"
67
"io"
78
"os"
89
"sort"
910

1011
"github.com/rs/zerolog/log"
1112

1213
"github.com/boostsecurityio/poutine/models"
13-
"github.com/boostsecurityio/poutine/opa"
1414
"github.com/olekukonko/tablewriter"
1515
)
1616

1717
type Format struct {
1818
}
1919

20-
func (f *Format) Format(ctx context.Context, report *opa.FindingsResult, packages []*models.PackageInsights) error {
20+
func (f *Format) Format(ctx context.Context, packages []*models.PackageInsights) error {
2121
failures := map[string]int{}
22-
findings := map[string][]opa.Finding{}
22+
findings := map[string][]results.Finding{}
23+
rules := map[string]results.Rule{}
2324

24-
if len(report.Findings) == 0 {
25-
log.Info().Msg("No results returned by analysis")
26-
return nil
27-
}
25+
for _, pkg := range packages {
26+
if len(pkg.FindingsResults.Findings) == 0 {
27+
log.Info().Msg("No results returned by analysis")
28+
continue
29+
}
2830

29-
for _, finding := range report.Findings {
30-
failures[finding.RuleId]++
31-
findings[finding.RuleId] = append(findings[finding.RuleId], finding)
31+
for _, finding := range pkg.FindingsResults.Findings {
32+
failures[finding.RuleId]++
33+
findings[finding.RuleId] = append(findings[finding.RuleId], finding)
34+
}
35+
36+
for _, rule := range pkg.FindingsResults.Rules {
37+
rules[rule.Id] = rule
38+
}
3239
}
3340

34-
printFindingsPerRule(os.Stdout, findings, report.Rules)
35-
printSummaryTable(os.Stdout, failures, report.Rules)
41+
printFindingsPerRule(os.Stdout, findings, rules)
42+
printSummaryTable(os.Stdout, failures, rules)
3643

3744
return nil
3845
}
3946

40-
func printFindingsPerRule(out io.Writer, results map[string][]opa.Finding, rules map[string]opa.Rule) {
47+
func printFindingsPerRule(out io.Writer, results map[string][]results.Finding, rules map[string]results.Rule) {
4148

4249
var sortedRuleIDs []string
4350
for ruleID := range rules {
@@ -106,7 +113,7 @@ func printFindingsPerRule(out io.Writer, results map[string][]opa.Finding, rules
106113
}
107114
}
108115

109-
func printSummaryTable(out io.Writer, failures map[string]int, rules map[string]opa.Rule) {
116+
func printSummaryTable(out io.Writer, failures map[string]int, rules map[string]results.Rule) {
110117
table := tablewriter.NewWriter(out)
111118
table.SetHeader([]string{"Rule ID", "Rule Name", "Failures", "Status"})
112119
table.SetColWidth(80)

‎formatters/sarif/sarif.go‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@ package sarif
33
import (
44
"context"
55
"fmt"
6+
"github.com/boostsecurityio/poutine/results"
67
"io"
78
"strings"
89

910
"github.com/boostsecurityio/poutine/docs"
1011
"github.com/boostsecurityio/poutine/models"
11-
"github.com/boostsecurityio/poutine/opa"
1212
"github.com/owenrumney/go-sarif/v2/sarif"
1313
)
1414

@@ -24,7 +24,7 @@ type Format struct {
2424
version string
2525
}
2626

27-
func (f *Format) Format(ctx context.Context, report *opa.FindingsResult, packages []*models.PackageInsights) error {
27+
func (f *Format) Format(ctx context.Context, packages []*models.PackageInsights) error {
2828
sarifReport, err := sarif.New(sarif.Version210)
2929
if err != nil {
3030
return err
@@ -35,11 +35,6 @@ func (f *Format) Format(ctx context.Context, report *opa.FindingsResult, package
3535
return parts[0]
3636
}
3737

38-
findingsByPurl := make(map[string][]opa.Finding)
39-
for _, finding := range report.Findings {
40-
findingsByPurl[finding.Purl] = append(findingsByPurl[finding.Purl], finding)
41-
}
42-
4338
docs := docs.GetPagesContent()
4439

4540
for _, pkg := range packages {
@@ -56,6 +51,11 @@ func (f *Format) Format(ctx context.Context, report *opa.FindingsResult, package
5651
WithBranch(pkg.SourceGitRef),
5752
)
5853

54+
findingsByPurl := make(map[string][]results.Finding)
55+
for _, finding := range pkg.FindingsResults.Findings {
56+
findingsByPurl[finding.Purl] = append(findingsByPurl[finding.Purl], finding)
57+
}
58+
5959
pkgFindings := findingsByPurl[pkg.Purl]
6060
for _, depPurl := range pkg.PackageDependencies {
6161
normalizedDepPurl := normalizePurl(depPurl)
@@ -65,7 +65,7 @@ func (f *Format) Format(ctx context.Context, report *opa.FindingsResult, package
6565
}
6666

6767
for _, finding := range pkgFindings {
68-
rule := report.Rules[finding.RuleId]
68+
rule := pkg.FindingsResults.Rules[finding.RuleId]
6969
ruleId := rule.Id
7070
ruleDescription := rule.Description
7171
meta := finding.Meta

‎models/package_insights.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package models
22

3-
import "fmt"
3+
import (
4+
"fmt"
5+
"github.com/boostsecurityio/poutine/results"
6+
)
47

58
type PackageInsights struct {
69
Version string `json:"version"`
@@ -46,6 +49,8 @@ type PackageInsights struct {
4649
GitlabciConfigs []GitlabciConfig `json:"gitlabci_configs"`
4750
AzurePipelines []AzurePipeline `json:"azure_pipelines"`
4851
PipelineAsCodeTekton []PipelineAsCodeTekton `json:"pipeline_as_code_tekton"`
52+
53+
FindingsResults results.FindingsResult `json:"-"`
4954
}
5055

5156
func (p *PackageInsights) GetSourceGitRepoURI() string {

0 commit comments

Comments
 (0)