Update the loadwatcher library to the latest release for trimaran plugins

cmd/scheduler/main_test.go

@@ -274,6 +274,36 @@ profiles:
 		t.Fatal(err)
 	}
 
+	// TargetLoadPacking plugin config with Prometheus Metric Provider arguments
+	targetLoadPackingConfigWithPrometheusArgsFile := filepath.Join(tmpDir, "targetLoadPacking-with-prometheus-args.yaml")
+	if err := ioutil.WriteFile(targetLoadPackingConfigWithPrometheusArgsFile, []byte(fmt.Sprintf(`
+apiVersion: kubescheduler.config.k8s.io/v1beta3
+kind: KubeSchedulerConfiguration
+clientConnection:
+  kubeconfig: "%s"
+profiles:
+- plugins:
+    score:
+      enabled:
+      - name: TargetLoadPacking
+      disabled:
+      - name: "*"
+  pluginConfig:
+  - name: TargetLoadPacking
+    args:
+      metricProvider:
+        type: Prometheus
+        address: http://prometheus-k8s.monitoring.svc.cluster.local:9090
+        insecureSkipVerify: false
+      targetUtilization: 60 
+      defaultRequests:
+        cpu: "1000m"
+      defaultRequestsMultiplier: "1.8"
+      watcherAddress: http://deadbeef:2020
+`, configKubeconfig)), os.FileMode(0600)); err != nil {
+		t.Fatal(err)
+	}
+
 	// LoadVariationRiskBalancing plugin config with arguments
 	loadVariationRiskBalancingConfigWithArgsFile := filepath.Join(tmpDir, "loadVariationRiskBalancing-with-args.yaml")
 	if err := ioutil.WriteFile(loadVariationRiskBalancingConfigWithArgsFile, []byte(fmt.Sprintf(`
@@ -509,6 +539,24 @@ profiles:
 				},
 			},
 		},
+		{
+			name:            "single profile config - TargetLoadPacking with prometheus metric provider args",
+			flags:           []string{"--config", targetLoadPackingConfigWithPrometheusArgsFile},
+			registryOptions: []app.Option{app.WithPlugin(targetloadpacking.Name, targetloadpacking.New)},
+			wantPlugins: map[string]*config.Plugins{
+				"default-scheduler": {
+					QueueSort:  defaults.ExpandedPluginsV1beta3.QueueSort,
+					Bind:       defaults.ExpandedPluginsV1beta3.Bind,
+					PreFilter:  defaults.ExpandedPluginsV1beta3.PreFilter,
+					Filter:     defaults.ExpandedPluginsV1beta3.Filter,
+					PostFilter: defaults.ExpandedPluginsV1beta3.PostFilter,
+					PreScore:   defaults.ExpandedPluginsV1beta3.PreScore,
+					Score:      config.PluginSet{Enabled: []config.Plugin{{Name: targetloadpacking.Name, Weight: 1}}},
+					Reserve:    defaults.ExpandedPluginsV1beta3.Reserve,
+					PreBind:    defaults.ExpandedPluginsV1beta3.PreBind,
+				},
+			},
+		},
 		{
 			name:            "single profile config - LoadVariationRiskBalancing with args",
 			flags:           []string{"--config", loadVariationRiskBalancingConfigWithArgsFile},

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/google/go-cmp v0.5.5
 	github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.0.12
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/paypal/load-watcher v0.2.1
+	github.com/paypal/load-watcher v0.2.2
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.7.0
 	gonum.org/v1/gonum v0.6.2

go.sum

@@ -536,8 +536,8 @@ github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTm
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
-github.com/paypal/load-watcher v0.2.1 h1:qHcSfbaSyL85Uc3FJ+Jyv+XDChZGGgezKeCXja/sC3E=
-github.com/paypal/load-watcher v0.2.1/go.mod h1:MMCDf8aXF5k+K2q6AtMOBZItCvZ3oFAk+zNO4OAtp0w=
+github.com/paypal/load-watcher v0.2.2 h1:tKCy3Ts8LnihFSLKmyrOODrdt9ulbRXe0MFbxe9U2Bk=
+github.com/paypal/load-watcher v0.2.2/go.mod h1:MMCDf8aXF5k+K2q6AtMOBZItCvZ3oFAk+zNO4OAtp0w=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=

manifests/trimaran/scheduler-config.yaml

@@ -12,4 +12,12 @@ profiles:
       - name: NodeResourcesBalancedAllocation
       - name: NodeResourcesLeastAllocated
       enabled:
-      - name: TargetLoadPacking
+      - name: TargetLoadPacking
+  pluginConfig:
+    - name: TargetLoadPacking
+      args:
+        defaultRequestsMultiplier: "1"
+        targetUtilization: 40
+        metricProvider:
+          type: Prometheus
+          address: "http://replace_me_with_prometheus_server:9090"

pkg/apis/config/scheme/scheme_test.go

@@ -86,6 +86,7 @@ profiles:
       metricProvider:
         type: Prometheus
         address: http://prometheus-k8s.monitoring.svc.cluster.local:9090
+        insecureSkipVerify: false
       safeVarianceMargin: 1.0
       safeVarianceSensitivity: 1.0
       watcherAddress: http://deadbeef:2020
@@ -126,8 +127,9 @@ profiles:
 								DefaultRequestsMultiplier: "1.8",
 								WatcherAddress:            "http://deadbeef:2020",
 								MetricProvider: config.MetricProviderSpec{
-									Type:    config.Prometheus,
-									Address: "http://prometheus-k8s.monitoring.svc.cluster.local:9090",
+									Type:               config.Prometheus,
+									Address:            "http://prometheus-k8s.monitoring.svc.cluster.local:9090",
+									InsecureSkipVerify: true,
 								},
 							},
 						},
@@ -138,8 +140,9 @@ profiles:
 								SafeVarianceSensitivity: v1beta2.DefaultSafeVarianceSensitivity,
 								WatcherAddress:          "http://deadbeef:2020",
 								MetricProvider: config.MetricProviderSpec{
-									Type:    config.Prometheus,
-									Address: "http://prometheus-k8s.monitoring.svc.cluster.local:9090",
+									Type:               config.Prometheus,
+									Address:            "http://prometheus-k8s.monitoring.svc.cluster.local:9090",
+									InsecureSkipVerify: false,
 								},
 							},
 						},
@@ -389,8 +392,9 @@ func TestCodecsEncodePluginConfig(t *testing.T) {
 									SafeVarianceSensitivity: v1beta2.DefaultSafeVarianceSensitivity,
 									WatcherAddress:          "http://deadbeef:2020",
 									MetricProvider: config.MetricProviderSpec{
-										Type:    config.Prometheus,
-										Address: "http://prometheus-k8s.monitoring.svc.cluster.local:9090",
+										Type:               config.Prometheus,
+										Address:            "http://prometheus-k8s.monitoring.svc.cluster.local:9090",
+										InsecureSkipVerify: false,
 									},
 								},
 							},
@@ -448,6 +452,7 @@ profiles:
       kind: TargetLoadPackingArgs
       metricProvider:
         address: http://prometheus-k8s.monitoring.svc.cluster.local:9090
+        insecureSkipVerify: false
         token: ""
         type: Prometheus
       targetUtilization: 60
@@ -458,6 +463,7 @@ profiles:
       kind: LoadVariationRiskBalancingArgs
       metricProvider:
         address: http://prometheus-k8s.monitoring.svc.cluster.local:9090
+        insecureSkipVerify: false
         token: ""
         type: Prometheus
       safeVarianceMargin: 1

pkg/apis/config/types.go

@@ -78,6 +78,8 @@ type MetricProviderSpec struct {
 	Address string
 	// The authentication token of the metric provider
 	Token string
+	// Whether to enable the InsureSkipVerify options for https requests on Metric Providers.
+	InsecureSkipVerify bool
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/apis/config/v1beta2/defaults.go

@@ -62,8 +62,12 @@ var (
 	DefaultSafeVarianceMargin = 1.0
 	// DefaultSafeVarianceSensitivity is one
 	DefaultSafeVarianceSensitivity = 1.0
+
+	// Defaults for MetricProviderSpec
 	// DefaultMetricProviderType is the Kubernetes metrics server
 	DefaultMetricProviderType = KubernetesMetricsServer
+	// DefaultInsecureSkipVerify is whether to skip the certificate verification
+	DefaultInsecureSkipVerify = true
 
 	defaultResourceSpec = []schedulerconfigv1beta2.ResourceSpec{
 		{Name: string(v1.ResourceCPU), Weight: 1},
@@ -107,6 +111,9 @@ func SetDefaultTargetLoadPackingArgs(args *TargetLoadPackingArgs) {
 	if args.WatcherAddress == nil && args.MetricProvider.Type == "" {
 		args.MetricProvider.Type = MetricProviderType(DefaultMetricProviderType)
 	}
+	if args.MetricProvider.Type == Prometheus && args.MetricProvider.InsecureSkipVerify == nil {
+		args.MetricProvider.InsecureSkipVerify = &DefaultInsecureSkipVerify
+	}
 }
 
 // SetDefaultLoadVariationRiskBalancingArgs sets the default parameters for LoadVariationRiskBalancing plugin
@@ -120,6 +127,9 @@ func SetDefaultLoadVariationRiskBalancingArgs(args *LoadVariationRiskBalancingAr
 	if args.SafeVarianceSensitivity == nil || *args.SafeVarianceSensitivity < 0 {
 		args.SafeVarianceSensitivity = &DefaultSafeVarianceSensitivity
 	}
+	if args.MetricProvider.Type == Prometheus && args.MetricProvider.InsecureSkipVerify == nil {
+		args.MetricProvider.InsecureSkipVerify = &DefaultInsecureSkipVerify
+	}
 }
 
 // SetDefaultsNodeResourceTopologyMatchArgs sets the default parameters for NodeResourceTopologyMatch plugin.

pkg/apis/config/v1beta2/types.go

@@ -78,6 +78,8 @@ type MetricProviderSpec struct {
 	Address *string `json:"address,omitempty"`
 	// The authentication token of the metric provider
 	Token *string `json:"token,omitempty"`
+	// Whether to enable the InsureSkipVerify options for https requests on Prometheus Metric Provider.
+	InsecureSkipVerify *bool `json:"insecureSkipVerify,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/apis/config/v1beta3/defaults.go

@@ -62,8 +62,12 @@ var (
 	DefaultSafeVarianceMargin = 1.0
 	// DefaultSafeVarianceSensitivity is one
 	DefaultSafeVarianceSensitivity = 1.0
+
+	// Defaults for MetricProviderSpec
 	// DefaultMetricProviderType is the Kubernetes metrics server
 	DefaultMetricProviderType = KubernetesMetricsServer
+	// DefaultInsecureSkipVerify is whether to skip the certificate verification
+	DefaultInsecureSkipVerify = true
 
 	defaultResourceSpec = []schedulerconfigv1beta3.ResourceSpec{
 		{Name: string(v1.ResourceCPU), Weight: 1},
@@ -107,6 +111,9 @@ func SetDefaultTargetLoadPackingArgs(args *TargetLoadPackingArgs) {
 	if args.WatcherAddress == nil && args.MetricProvider.Type == "" {
 		args.MetricProvider.Type = MetricProviderType(DefaultMetricProviderType)
 	}
+	if args.MetricProvider.Type == Prometheus && args.MetricProvider.InsecureSkipVerify == nil {
+		args.MetricProvider.InsecureSkipVerify = &DefaultInsecureSkipVerify
+	}
 }
 
 // SetDefaultLoadVariationRiskBalancingArgs sets the default parameters for LoadVariationRiskBalancing plugin
@@ -120,6 +127,9 @@ func SetDefaultLoadVariationRiskBalancingArgs(args *LoadVariationRiskBalancingAr
 	if args.SafeVarianceSensitivity == nil || *args.SafeVarianceSensitivity < 0 {
 		args.SafeVarianceSensitivity = &DefaultSafeVarianceSensitivity
 	}
+	if args.MetricProvider.Type == Prometheus && args.MetricProvider.InsecureSkipVerify == nil {
+		args.MetricProvider.InsecureSkipVerify = &DefaultInsecureSkipVerify
+	}
 }
 
 // SetDefaultsNodeResourceTopologyMatchArgs sets the default parameters for NodeResourceTopologyMatch plugin.

pkg/apis/config/v1beta3/types.go

@@ -78,6 +78,8 @@ type MetricProviderSpec struct {
 	Address *string `json:"address,omitempty"`
 	// The authentication token of the metric provider
 	Token *string `json:"token,omitempty"`
+	// Whether to enable the InsureSkipVerify options for https requests on Prometheus Metric Provider.
+	InsecureSkipVerify *bool `json:"insecureSkipVerify,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/trimaran/README.md

@@ -66,7 +66,24 @@ profiles:
       safeVarianceMargin: 1
       safeVarianceSensitivity: 2
 ```
+### Configure Prometheus Metric Provider under different environments
+1. Invalid self-signed SSL connection error for the Prometheus metric queries
+   The Prometheus metric queries may have invalid self-signed SSL connection error when the cluster 
+environment disables the skipInsecureVerify option for HTTPs. In this case, you can configure 
+`insecureSkipVerify: true` for `metricProvider` to skip the SSL verification.
+```
+ args:
+   metricProvider:
+     type: Prometheus
+     address: http://prometheus-k8s.monitoring.svc.cluster.local:9090
+     insecureSkipVerify: true
+```
 
-## A note on multiple plugins
+2. OpenShift Prometheus authentication without tokens.
+   The OpenShift clusters disallow non-verified clients to access its Prometheus metrics. To run the 
+Trimaran plugin on OpenShift, you need to set an environment variable `ENABLE_OPENSHIFT_AUTH=true` for 
+your trimaran scheduler deployment when run [load-watcher](https://github.com/paypal/load-watcher/blob/master/README.md) 
+as a library.
 
+## A note on multiple plugins
 The Trimaran plugins have different, potentially conflicting, objectives. Thus, it is recommended not to enable them concurrently. As such, they are designed to each have its own load-watcher.

pkg/trimaran/loadvariationriskbalancing/collector.go

@@ -67,9 +67,10 @@ func newCollector(obj runtime.Object) (*Collector, error) {
 		client, _ = loadwatcherapi.NewServiceClient(args.WatcherAddress)
 	} else {
 		opts := watcher.MetricsProviderOpts{
-			Name:      string(args.MetricProvider.Type),
-			Address:   args.MetricProvider.Address,
-			AuthToken: args.MetricProvider.Token,
+			Name:               string(args.MetricProvider.Type),
+			Address:            args.MetricProvider.Address,
+			AuthToken:          args.MetricProvider.Token,
+			InsecureSkipVerify: args.MetricProvider.InsecureSkipVerify,
 		}
 		client, _ = loadwatcherapi.NewLibraryClient(opts)
 	}

pkg/trimaran/targetloadpacking/targetloadpacking.go

@@ -83,7 +83,12 @@ func New(obj runtime.Object, handle framework.Handle) (framework.Plugin, error)
 	if args.WatcherAddress != "" {
 		client, err = loadwatcherapi.NewServiceClient(args.WatcherAddress)
 	} else {
-		opts := watcher.MetricsProviderOpts{string(args.MetricProvider.Type), args.MetricProvider.Address, args.MetricProvider.Token}
+		opts := watcher.MetricsProviderOpts{
+			Name:               string(args.MetricProvider.Type),
+			Address:            args.MetricProvider.Address,
+			AuthToken:          args.MetricProvider.Token,
+			InsecureSkipVerify: args.MetricProvider.InsecureSkipVerify,
+		}
 		client, err = loadwatcherapi.NewLibraryClient(opts)
 	}
 	if err != nil {