API and Metrics Collection Update (#37)

eeisegn · web-flow · commit c698df30e61b · 2024-03-25T13:18:46.000Z
* SCP-198 update apis to accept /api/ and /

* added option to control OTEL host metrics
diff --git a/.idea/dictionaries/egans.xml b/.idea/dictionaries/egans.xml
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Upcoming changes...
 
+## [1.4.1] - 2024-03-22
+### Added
+- Added Telemetry option to fine tune which host level metrics are produced by default
+  - `ExtraMetrics` can be set to true/false to
+
+## [1.4.0] - 2024-03-15
+### Added
+- Added support for request routing through root
+  - So it is now possible to route through `/api/` and `/`
+  - i.e. `https://localhost/api/scan/direct` and `https://localhost/scan/api`
+
 ## [1.3.5] - 2023-12-30
 ### Added
 - Added scan file size metric
@@ -104,3 +115,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [1.3.3]: https://github.com/scanoss/api.go/compare/v1.3.2...v1.3.3
 [1.3.4]: https://github.com/scanoss/api.go/compare/v1.3.3...v1.3.4
 [1.3.5]: https://github.com/scanoss/api.go/compare/v1.3.4...v1.3.5
+[1.4.0]: https://github.com/scanoss/api.go/compare/v1.3.5...v1.4.0
+[1.4.1]: https://github.com/scanoss/api.go/compare/v1.4.0...v1.4.1
diff --git a/config/app-config-dev.json b/config/app-config-dev.json
@@ -8,7 +8,7 @@
   },
   "Scanning": {
     "ScanBinary": "./test-support/scanoss.sh",
-    "ScanningURL": "http://localhost:5443/api",
+    "ScanningURL": "http://localhost:5443",
     "TmpFileDelete": true
   }
 }
diff --git a/config/app-config-prod.json b/config/app-config-prod.json
@@ -17,6 +17,7 @@
   },
   "Telemetry": {
     "Enabled": false,
+    "ExtraMetrics": false,
     "OltpExporter": "0.0.0.0:4317"
   },
   "Scanning": {
diff --git a/pkg/config/server_config.go b/pkg/config/server_config.go
@@ -51,6 +51,7 @@ type ServerConfig struct {
 	}
 	Telemetry struct {
 		Enabled      bool   `env:"OTEL_ENABLED"`       // true/false
+		ExtraMetrics bool   `env:"OTEL_EXTRA"`         // true/false
 		OltpExporter string `env:"OTEL_EXPORTER_OLTP"` // OTEL OLTP exporter (default 0.0.0.0:4317)
 	}
 	Scanning struct {
@@ -111,6 +112,7 @@ func setServerConfigDefaults(cfg *ServerConfig) {
 	cfg.Scanning.WfpGrouping = 3   // Default number of WFPs to group into a single scan request (when Workers > 1)
 	cfg.Scanning.HPSMEnabled = true
 	cfg.Telemetry.Enabled = false
+	cfg.Telemetry.ExtraMetrics = true           // Default to sending all the extra service metrics
 	cfg.Telemetry.OltpExporter = "0.0.0.0:4317" // Default OTEL OLTP gRPC Exporter endpoint
 	cfg.Scanning.FileContents = true            // Matched File URL response enabled (true) by default
 }
diff --git a/pkg/protocol/rest/server.go b/pkg/protocol/rest/server.go
@@ -63,7 +63,7 @@ func RunServer(config *myconfig.ServerConfig, version string) error {
 		return err
 	}
 	if config.Telemetry.Enabled {
-		oltpShutdown, err := initProviders(config, version)
+		oltpShutdown, err := initProviders(config, version, config.Telemetry.ExtraMetrics)
 		if err != nil {
 			return err
 		}
@@ -82,15 +82,25 @@ func RunServer(config *myconfig.ServerConfig, version string) error {
 	router.HandleFunc("/api/", service.WelcomeMsg).Methods(http.MethodGet)
 	router.HandleFunc("/api/", service.HeadResponse).Methods(http.MethodHead)
 	router.HandleFunc("/api/health", service.HealthCheck).Methods(http.MethodGet)
+	router.HandleFunc("/health", service.HealthCheck).Methods(http.MethodGet)
 	router.HandleFunc("/api/health", service.HeadResponse).Methods(http.MethodHead)
+	router.HandleFunc("/health", service.HeadResponse).Methods(http.MethodHead)
 	router.HandleFunc("/api/health-check", service.HealthCheck).Methods(http.MethodGet)
+	router.HandleFunc("/health-check", service.HealthCheck).Methods(http.MethodGet)
 	router.HandleFunc("/api/health-check", service.HeadResponse).Methods(http.MethodHead)
+	router.HandleFunc("/health-check", service.HeadResponse).Methods(http.MethodHead)
 	router.HandleFunc("/api/metrics/{type}", service.MetricsHandler).Methods(http.MethodGet)
+	router.HandleFunc("/metrics/{type}", service.MetricsHandler).Methods(http.MethodGet)
 	router.HandleFunc("/api/file_contents/{md5}", apiService.FileContents).Methods(http.MethodGet)
+	router.HandleFunc("/file_contents/{md5}", apiService.FileContents).Methods(http.MethodGet)
 	router.HandleFunc("/api/kb/details", apiService.KBDetails).Methods(http.MethodGet)
+	router.HandleFunc("/kb/details", apiService.KBDetails).Methods(http.MethodGet)
 	router.HandleFunc("/api/license/obligations/{license}", apiService.LicenseDetails).Methods(http.MethodGet)
+	router.HandleFunc("/license/obligations/{license}", apiService.LicenseDetails).Methods(http.MethodGet)
 	router.HandleFunc("/api/scan/direct", apiService.ScanDirect).Methods(http.MethodPost)
+	router.HandleFunc("/scan/direct", apiService.ScanDirect).Methods(http.MethodPost)
 	router.HandleFunc("/api/sbom/attribution", apiService.SbomAttribution).Methods(http.MethodPost)
+	router.HandleFunc("/sbom/attribution", apiService.SbomAttribution).Methods(http.MethodPost)
 	// Setup Open Telemetry (OTEL)
 	if config.Telemetry.Enabled {
 		router.Use(otelmux.Middleware("scanoss-api"))
@@ -309,22 +319,26 @@ func loadFiltering(config *myconfig.ServerConfig) ([]string, []string, error) {
 }
 
 // initProviders sets up the OLTP Meter and Trace providers and the OLTP gRPC exporter.
-func initProviders(config *myconfig.ServerConfig, version string) (func(), error) {
+func initProviders(config *myconfig.ServerConfig, version string, extraAttributes bool) (func(), error) {
 	zlog.L.Info("Setting up Open Telemetry providers.")
 	// Setup resource for the providers
 	ctx := context.Background()
-	res, err := resource.New(ctx,
-		resource.WithFromEnv(),
-		resource.WithProcess(),
-		resource.WithTelemetrySDK(),
-		resource.WithHost(),
-		resource.WithAttributes(
-			// the service name & version used to display traces in backends
-			semconv.ServiceName(config.App.Name),
-			semconv.ServiceNamespace("scanoss-api"),
-			semconv.ServiceVersion(strings.TrimSpace(version)),
-		),
+	var opts []resource.Option
+	// Extra service level attributes to report
+	if extraAttributes {
+		opts = append(opts, resource.WithFromEnv())
+		opts = append(opts, resource.WithProcess())
+		opts = append(opts, resource.WithTelemetrySDK())
+	}
+	opts = append(opts, resource.WithHost())
+	// the service name & version used to display traces in backends
+	opts = append(opts, resource.WithAttributes(
+		semconv.ServiceName(config.App.Name),
+		semconv.ServiceNamespace("scanoss-api"),
+		semconv.ServiceVersion(strings.TrimSpace(version)),
+	),
 	)
+	res, err := resource.New(ctx, opts...)
 	if err != nil {
 		zlog.S.Errorf("Failed to create oltp resource: %v", err)
 		return nil, err
diff --git a/pkg/service/attribution_service_test.go b/pkg/service/attribution_service_test.go
@@ -117,7 +117,7 @@ func TestSbomAttribution(t *testing.T) {
 			}
 			_ = mw.Close() // close the writer before making the request
 
-			req := httptest.NewRequest(http.MethodPost, "http://localhost/api/sbom/attribution", postBody)
+			req := httptest.NewRequest(http.MethodPost, "http://localhost/sbom/attribution", postBody)
 			w := httptest.NewRecorder()
 			req.Header.Add("Content-Type", mw.FormDataContentType())
 			apiService.SbomAttribution(w, req)
diff --git a/pkg/service/filecontents_service_test.go b/pkg/service/filecontents_service_test.go
@@ -91,7 +91,7 @@ func TestFileContents(t *testing.T) {
 			}
 			myConfig.Scanning.ScanBinary = test.binary
 			myConfig.Telemetry.Enabled = test.telemetry
-			req := newReq("GET", "http://localhost/api/file_contents/{md5}", "", test.input)
+			req := newReq("GET", "http://localhost/file_contents/{md5}", "", test.input)
 			w := httptest.NewRecorder()
 			apiService.FileContents(w, req)
 			resp := w.Result()
diff --git a/pkg/service/licensedetails_service_test.go b/pkg/service/licensedetails_service_test.go
@@ -92,7 +92,7 @@ func TestLicenseDetails(t *testing.T) {
 			}
 			myConfig.Scanning.ScanBinary = test.binary
 			myConfig.Telemetry.Enabled = test.telemetry
-			req := newReq("GET", "http://localhost/api/license/obligations/{license}", "", test.input)
+			req := newReq("GET", "http://localhost/license/obligations/{license}", "", test.input)
 			w := httptest.NewRecorder()
 			apiService.LicenseDetails(w, req)
 			resp := w.Result()
diff --git a/pkg/service/scanning_service_test.go b/pkg/service/scanning_service_test.go
@@ -226,7 +226,7 @@ func TestScanDirectSingle(t *testing.T) {
 			}
 			_ = mw.Close() // close the writer before making the request
 
-			req := httptest.NewRequest(http.MethodPost, "http://localhost/api/api/scan/direct", postBody)
+			req := httptest.NewRequest(http.MethodPost, "http://localhost/scan/direct", postBody)
 			w := httptest.NewRecorder()
 			req.Header.Add("Content-Type", mw.FormDataContentType())
 			apiService.ScanDirect(w, req)
@@ -340,7 +340,7 @@ func TestScanDirectThreaded(t *testing.T) {
 			}
 			_ = mw.Close() // close the writer before making the request
 
-			req := httptest.NewRequest(http.MethodPost, "http://localhost/api/api/scan/direct", postBody)
+			req := httptest.NewRequest(http.MethodPost, "http://localhost/scan/direct", postBody)
 			w := httptest.NewRecorder()
 			req.Header.Add("Content-Type", mw.FormDataContentType())
 			apiService.ScanDirect(w, req)
@@ -433,7 +433,7 @@ func TestScanDirectSingleHPSM(t *testing.T) {
 			}
 			_ = mw.Close() // close the writer before making the request
 
-			req := httptest.NewRequest(http.MethodPost, "http://localhost/api/api/scan/direct", postBody)
+			req := httptest.NewRequest(http.MethodPost, "http://localhost/scan/direct", postBody)
 			w := httptest.NewRecorder()
 			req.Header.Add("Content-Type", mw.FormDataContentType())
 			apiService.ScanDirect(w, req)
diff --git a/pkg/service/utils_service_test.go b/pkg/service/utils_service_test.go
@@ -79,7 +79,7 @@ func TestHealthCheck(t *testing.T) {
 		t.Fatalf("an error '%s' was not expected when opening a sugared logger", err)
 	}
 	defer zlog.SyncZap()
-	req := httptest.NewRequest(http.MethodGet, "http://localhost/api/health", nil)
+	req := httptest.NewRequest(http.MethodGet, "http://localhost/health", nil)
 	w := httptest.NewRecorder()
 	HealthCheck(w, req)
 	resp := w.Result()
@@ -147,7 +147,7 @@ func TestMetricsHandler(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			req := newReq("GET", "http://localhost/api/metrics/{type}", "", test.input)
+			req := newReq("GET", "http://localhost/metrics/{type}", "", test.input)
 			w := httptest.NewRecorder()
 			MetricsHandler(w, req)
 			resp := w.Result()
@@ -200,7 +200,7 @@ func TestHeadResponse(t *testing.T) {
 		t.Fatalf("an error '%s' was not expected when opening a sugared logger", err)
 	}
 	defer zlog.SyncZap()
-	req := httptest.NewRequest(http.MethodHead, "http://localhost/api/health", nil)
+	req := httptest.NewRequest(http.MethodHead, "http://localhost/health", nil)
 	w := httptest.NewRecorder()
 	HeadResponse(w, req)
 	resp := w.Result()
diff --git a/tests/attribution_test.go b/tests/attribution_test.go
@@ -75,7 +75,7 @@ func (s *E2EAttributionSuite) TestAttribution() {
 			if err != nil {
 				s.Failf("an error was not creating multipart form data.", "error: %v", err)
 			}
-			req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%v/api/sbom/attribution", hostPort), &b)
+			req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%v/sbom/attribution", hostPort), &b)
 			if err != nil {
 				s.Failf("an error was not creating request.", "error: %v", err)
 			}
diff --git a/tests/curl_commands.txt b/tests/curl_commands.txt
@@ -2,31 +2,31 @@ This file contains a set of curl commands to exercise the SCANOSS GO API service
 
 # Scan a file
 
-curl -X POST -F 'file=@fingers.wfp' -H "flags:12540" localhost:5443/api/scan/direct > sc-new-res.json
+curl -X POST -F 'file=@fingers.wfp' -H "flags:12540" localhost:5443/scan/direct > sc-new-res.json
 
 # Scan file with context
 
-curl -X POST -F 'file=@fingers.wfp' -H 'type=identify' -H 'assets=@sbom.json'  localhost:5443/api/scan/direct > sc-context-res.json
+curl -X POST -F 'file=@fingers.wfp' -H 'type=identify' -H 'assets=@sbom.json'  localhost:5443/scan/direct > sc-context-res.json
 
 # Get file contents
 
-curl -X GET  http://localhost:5443/api/file_contents/37f7cd1e657aa3c30ece35995b4c59e5 > contents.txt
+curl -X GET  http://localhost:5443/file_contents/37f7cd1e657aa3c30ece35995b4c59e5 > contents.txt
 
 # Get SBOM Attribution
 
-curl -X POST -F 'file=@sbom.json' -H "flags:12540" localhost:5443/api/sbom/attribution > attribution.txt
+curl -X POST -F 'file=@sbom.json' -H "flags:12540" localhost:5443/sbom/attribution > attribution.txt
 
 # Get License
 
-curl -X GET  http://localhost:5443/api/license/obligations/MIT > obligations.txt
+curl -X GET  http://localhost:5443/license/obligations/MIT > obligations.txt
 
 # Service Status
 
-curl -X GET  http://localhost:5443/api/api/health-check
+curl -X GET  http://localhost:5443/health-check
 
-curl --head  http://localhost:5443/api/health-check
+curl --head  http://localhost:5443/health-check
 
 # Service Metrics
 
-curl -X GET  http://localhost:5443/api/api/metrics/all
+curl -X GET  http://localhost:5443/metrics/all
 
diff --git a/tests/file_contents_test.go b/tests/file_contents_test.go
@@ -34,7 +34,7 @@ func TestE2EContentsSuite(t *testing.T) {
 
 func (s *E2EContentsSuite) TestHappyFileContents() {
 	c := http.Client{}
-	resp, err := c.Get(fmt.Sprintf("%v/api/file_contents/37f7cd1e657aa3c30ece35995b4c59e5", hostPort))
+	resp, err := c.Get(fmt.Sprintf("%v/file_contents/37f7cd1e657aa3c30ece35995b4c59e5", hostPort))
 	if err != nil {
 		s.Failf("an error was not expected when sending request.", "error: %v", err)
 	}
diff --git a/tests/healthcheck_test.go b/tests/healthcheck_test.go
@@ -59,7 +59,7 @@ func (s *E2EHealthSuite) TestHappyWelcomeMsg() {
 
 func (s *E2EHealthSuite) TestHappyHealthcheck() {
 	c := http.Client{}
-	resp, err := c.Get(fmt.Sprintf("%v/api/health-check", hostPort))
+	resp, err := c.Get(fmt.Sprintf("%v/health-check", hostPort))
 	if err != nil {
 		s.Failf("an error was not expected when sending request.", "error: %v", err)
 	}
@@ -75,7 +75,7 @@ func (s *E2EHealthSuite) TestHappyHealthcheck() {
 	fmt.Println("Body: ", bodyStr)
 
 	// Test the HEAD call also
-	resp2, err := c.Head(fmt.Sprintf("%v/api/health-check", hostPort))
+	resp2, err := c.Head(fmt.Sprintf("%v/health-check", hostPort))
 	if err != nil {
 		s.Failf("an error was not expected when sending request.", "error: %v", err)
 	}
@@ -124,7 +124,7 @@ func (s *E2EHealthSuite) TestMetrics() {
 	}
 	for _, test := range tests {
 		s.Run(test.name, func() {
-			resp, err := c.Get(fmt.Sprintf("%v/api/metrics/%v", hostPort, test.input))
+			resp, err := c.Get(fmt.Sprintf("%v/metrics/%v", hostPort, test.input))
 			if err != nil {
 				s.Failf("an error was not expected when sending request.", "error: %v", err)
 			}
diff --git a/tests/kb_details_test.go b/tests/kb_details_test.go
@@ -34,7 +34,7 @@ func TestE2EKBDetailsSuite(t *testing.T) {
 
 func (s *E2EKBDetailsSuite) TestHappyKBDetails() {
 	c := http.Client{}
-	resp, err := c.Get(fmt.Sprintf("%v/api/kb/details", hostPort))
+	resp, err := c.Get(fmt.Sprintf("%v/kb/details", hostPort))
 	if err != nil {
 		s.Failf("an error was not expected when sending request.", "error: %v", err)
 	}
diff --git a/tests/license_details_test.go b/tests/license_details_test.go
@@ -34,7 +34,7 @@ func TestE2ELicenseSuite(t *testing.T) {
 
 func (s *E2ELicenseSuite) TestHappyLicenseObligations() {
 	c := http.Client{}
-	resp, err := c.Get(fmt.Sprintf("%v/api/license/obligations/MIT", hostPort))
+	resp, err := c.Get(fmt.Sprintf("%v/license/obligations/MIT", hostPort))
 	if err != nil {
 		s.Failf("an error was not expected when sending request.", "error: %v", err)
 	}
diff --git a/tests/scanning_test.go b/tests/scanning_test.go
@@ -103,7 +103,7 @@ func (s *E2EScanningSuite) TestScanning() {
 			if err != nil {
 				s.Failf("an error was not creating multipart form data.", "error: %v", err)
 			}
-			req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%v/api/scan/direct", hostPort), &b)
+			req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%v/scan/direct", hostPort), &b)
 			if err != nil {
 				s.Failf("an error was not creating request.", "error: %v", err)
 			}

Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,7 @@`
`8`	`8`	`},`
`9`	`9`	`"Scanning": {`
`10`	`10`	`"ScanBinary": "./test-support/scanoss.sh",`
`11`		`- "ScanningURL": "http://localhost:5443/api",`
	`11`	`+ "ScanningURL": "http://localhost:5443",`
`12`	`12`	`"TmpFileDelete": true`
`13`	`13`	`}`
`14`	`14`	`}`
Original file line number	Diff line number	Diff line change
`@@ -51,6 +51,7 @@ type ServerConfig struct {`
`51`	`51`	`}`
`52`	`52`	`Telemetry struct {`
`53`	`53`	Enabled bool `env:"OTEL_ENABLED"` // true/false
	`54`	+ ExtraMetrics bool `env:"OTEL_EXTRA"` // true/false
`54`	`55`	OltpExporter string `env:"OTEL_EXPORTER_OLTP"` // OTEL OLTP exporter (default 0.0.0.0:4317)
`55`	`56`	`}`
`56`	`57`	`Scanning struct {`
`@@ -111,6 +112,7 @@ func setServerConfigDefaults(cfg *ServerConfig) {`
`111`	`112`	`cfg.Scanning.WfpGrouping = 3 // Default number of WFPs to group into a single scan request (when Workers > 1)`
`112`	`113`	`cfg.Scanning.HPSMEnabled = true`
`113`	`114`	`cfg.Telemetry.Enabled = false`
	`115`	`+ cfg.Telemetry.ExtraMetrics = true // Default to sending all the extra service metrics`
`114`	`116`	`cfg.Telemetry.OltpExporter = "0.0.0.0:4317" // Default OTEL OLTP gRPC Exporter endpoint`
`115`	`117`	`cfg.Scanning.FileContents = true // Matched File URL response enabled (true) by default`
`116`	`118`	`}`
Original file line number	Diff line number	Diff line change
`@@ -117,7 +117,7 @@ func TestSbomAttribution(t *testing.T) {`
`117`	`117`	`}`
`118`	`118`	`_ = mw.Close() // close the writer before making the request`
`119`	`119`
`120`		`- req := httptest.NewRequest(http.MethodPost, "http://localhost/api/sbom/attribution", postBody)`
	`120`	`+ req := httptest.NewRequest(http.MethodPost, "http://localhost/sbom/attribution", postBody)`
`121`	`121`	`w := httptest.NewRecorder()`
`122`	`122`	`req.Header.Add("Content-Type", mw.FormDataContentType())`
`123`	`123`	`apiService.SbomAttribution(w, req)`
Original file line number	Diff line number	Diff line change
`@@ -91,7 +91,7 @@ func TestFileContents(t *testing.T) {`
`91`	`91`	`}`
`92`	`92`	`myConfig.Scanning.ScanBinary = test.binary`
`93`	`93`	`myConfig.Telemetry.Enabled = test.telemetry`
`94`		`- req := newReq("GET", "http://localhost/api/file_contents/{md5}", "", test.input)`
	`94`	`+ req := newReq("GET", "http://localhost/file_contents/{md5}", "", test.input)`
`95`	`95`	`w := httptest.NewRecorder()`
`96`	`96`	`apiService.FileContents(w, req)`
`97`	`97`	`resp := w.Result()`
Original file line number	Diff line number	Diff line change
`@@ -92,7 +92,7 @@ func TestLicenseDetails(t *testing.T) {`
`92`	`92`	`}`
`93`	`93`	`myConfig.Scanning.ScanBinary = test.binary`
`94`	`94`	`myConfig.Telemetry.Enabled = test.telemetry`
`95`		`- req := newReq("GET", "http://localhost/api/license/obligations/{license}", "", test.input)`
	`95`	`+ req := newReq("GET", "http://localhost/license/obligations/{license}", "", test.input)`
`96`	`96`	`w := httptest.NewRecorder()`
`97`	`97`	`apiService.LicenseDetails(w, req)`
`98`	`98`	`resp := w.Result()`