UPSTREAM: <carry>: allows for switching KS to talk to Kube API over localhost

to force KS to use localhost set the following flag in kubescheduler (oc edit kubescheduler cluster)

unsupportedConfigOverrides:
  arguments:
    unsupported-kube-api-over-localhost::
    - "true"

UPSTREAM: <carry>: allows for switching KS to talk to Kube API over localhost-squash to other

This commit is addendum to 04eabe5
to stop using cc and start relying on scheduler config options

OpenShift-Rebase-Source: aa9dde2

UPSTREAM: <carry>: allows for switching KS to talk to Kube API over localhost

Author: p0lyn0mial

cmd/kube-scheduler/app/config/config.go

@@ -57,6 +57,9 @@ type Config struct {
 	// value, the pod will be moved from unschedulablePods to backoffQ or activeQ.
 	// If this value is empty, the default value (5min) will be used.
 	PodMaxInUnschedulablePodsDuration time.Duration
+
+	// OpenShiftContext is additional context that we need to launch the kube-scheduler for openshift
+	OpenShiftContext OpenShiftContext
 }
 
 type completedConfig struct {

cmd/kube-scheduler/app/config/patch.go

@@ -0,0 +1,15 @@
+package config
+
+import (
+	"k8s.io/client-go/transport"
+
+	"github.com/openshift/library-go/pkg/monitor/health"
+)
+
+// OpenShiftContext is additional context that we need to launch the kube-scheduler for openshift.
+// Basically, this holds our additional config information.
+type OpenShiftContext struct {
+	UnsupportedKubeAPIOverPreferredHost bool
+	PreferredHostRoundTripperWrapperFn  transport.WrapperFunc
+	PreferredHostHealthMonitor          *health.Prober
+}

cmd/kube-scheduler/app/options/options.go

@@ -49,6 +49,8 @@ import (
 	kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
 	"k8s.io/kubernetes/pkg/scheduler/apis/config/validation"
 	netutils "k8s.io/utils/net"
+
+	libgorestclient "github.com/openshift/library-go/pkg/config/client"
 )
 
 // Options has all the params needed to run a Scheduler
@@ -74,6 +76,9 @@ type Options struct {
 
 	// Flags hold the parsed CLI flags.
 	Flags *cliflag.NamedFlagSets
+
+	// OpenShiftContext is additional context that we need to launch the kube-scheduler for openshift.
+	OpenShiftContext schedulerappconfig.OpenShiftContext
 }
 
 // NewOptions returns default scheduler app options.
@@ -183,6 +188,7 @@ func (o *Options) initFlags() {
 	fs.StringVar(&o.ConfigFile, "config", o.ConfigFile, "The path to the configuration file.")
 	fs.StringVar(&o.WriteConfigTo, "write-config-to", o.WriteConfigTo, "If set, write the configuration values to this file and exit.")
 	fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
+	fs.BoolVar(&o.OpenShiftContext.UnsupportedKubeAPIOverPreferredHost, "unsupported-kube-api-over-localhost", false, "when set makes KS prefer talking to localhost kube-apiserver (when available) instead of an LB")
 
 	o.SecureServing.AddFlags(nfs.FlagSet("secure serving"))
 	o.Authentication.AddFlags(nfs.FlagSet("authentication"))
@@ -225,6 +231,10 @@ func (o *Options) ApplyTo(logger klog.Logger, c *schedulerappconfig.Config) erro
 	if err != nil {
 		return err
 	}
+	if c.OpenShiftContext.PreferredHostRoundTripperWrapperFn != nil {
+		libgorestclient.DefaultServerName(kubeConfig)
+		kubeConfig.Wrap(c.OpenShiftContext.PreferredHostRoundTripperWrapperFn)
+	}
 	c.KubeConfig = kubeConfig
 
 	if err := o.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil {
@@ -273,6 +283,7 @@ func (o *Options) Config(ctx context.Context) (*schedulerappconfig.Config, error
 	}
 
 	c := &schedulerappconfig.Config{}
+	c.OpenShiftContext = o.OpenShiftContext
 	if err := o.ApplyTo(logger, c); err != nil {
 		return nil, err
 	}

cmd/kube-scheduler/app/options/patch.go

@@ -0,0 +1,11 @@
+package options
+
+import (
+	"k8s.io/klog/v2"
+
+	kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
+)
+
+func LoadKubeSchedulerConfiguration(logger klog.Logger, file string) (*kubeschedulerconfig.KubeSchedulerConfiguration, error) {
+	return LoadConfigFromFile(logger, file)
+}

cmd/kube-scheduler/app/patch.go

@@ -0,0 +1,72 @@
+package app
+
+import (
+	"time"
+
+	"k8s.io/klog/v2"
+
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/kubernetes/cmd/kube-scheduler/app/options"
+
+	libgorestclient "github.com/openshift/library-go/pkg/config/client"
+	"github.com/openshift/library-go/pkg/monitor/health"
+)
+
+func setUpPreferredHostForOpenShift(logger klog.Logger, kubeSchedulerOptions *options.Options) error {
+	if !kubeSchedulerOptions.OpenShiftContext.UnsupportedKubeAPIOverPreferredHost {
+		return nil
+	}
+
+	master := kubeSchedulerOptions.Master
+	var kubeConfig string
+
+	// We cannot load component config anymore as the options are not being initialized.
+	// if there was no kubeconfig specified we won't be able to get cluster info.
+	// in that case try to load the configuration and read kubeconfig directly from it if it was provided.
+	if len(kubeSchedulerOptions.ConfigFile) > 0 {
+		cfg, err := options.LoadKubeSchedulerConfiguration(logger, kubeSchedulerOptions.ConfigFile)
+		if err != nil {
+			return err
+		}
+		kubeConfig = cfg.ClientConnection.Kubeconfig
+	}
+
+	config, err := clientcmd.BuildConfigFromFlags(master, kubeConfig)
+	if err != nil {
+		return err
+	}
+	libgorestclient.DefaultServerName(config)
+
+	targetProvider := health.StaticTargetProvider{"localhost:6443"}
+	kubeSchedulerOptions.OpenShiftContext.PreferredHostHealthMonitor, err = health.New(targetProvider, createRestConfigForHealthMonitor(config))
+	if err != nil {
+		return err
+	}
+	kubeSchedulerOptions.OpenShiftContext.PreferredHostHealthMonitor.
+		WithHealthyProbesThreshold(3).
+		WithUnHealthyProbesThreshold(5).
+		WithProbeInterval(5 * time.Second).
+		WithProbeResponseTimeout(2 * time.Second).
+		WithMetrics(health.Register(legacyregistry.MustRegister))
+
+	kubeSchedulerOptions.OpenShiftContext.PreferredHostRoundTripperWrapperFn = libgorestclient.NewPreferredHostRoundTripper(func() string {
+		healthyTargets, _ := kubeSchedulerOptions.OpenShiftContext.PreferredHostHealthMonitor.Targets()
+		if len(healthyTargets) == 1 {
+			return healthyTargets[0]
+		}
+		return ""
+	})
+
+	kubeSchedulerOptions.Authentication.WithCustomRoundTripper(kubeSchedulerOptions.OpenShiftContext.PreferredHostRoundTripperWrapperFn)
+	kubeSchedulerOptions.Authorization.WithCustomRoundTripper(kubeSchedulerOptions.OpenShiftContext.PreferredHostRoundTripperWrapperFn)
+	return nil
+}
+
+func createRestConfigForHealthMonitor(restConfig *rest.Config) *rest.Config {
+	restConfigCopy := *restConfig
+	rest.AddUserAgent(&restConfigCopy, "kube-scheduler-health-monitor")
+
+	return &restConfigCopy
+}

cmd/kube-scheduler/app/server.go

@@ -137,6 +137,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op
 		cancel()
 	}()
 
+	logger := klog.FromContext(ctx)
+	if err := setUpPreferredHostForOpenShift(logger, opts); err != nil {
+		return err
+	}
+
 	cc, sched, err := Setup(ctx, opts, registryOptions...)
 	if err != nil {
 		return err
@@ -155,6 +160,11 @@ func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched *
 
 	logger.Info("Golang settings", "GOGC", os.Getenv("GOGC"), "GOMAXPROCS", os.Getenv("GOMAXPROCS"), "GOTRACEBACK", os.Getenv("GOTRACEBACK"))
 
+	// start the localhost health monitor early so that it can be used by the LE client
+	if cc.OpenShiftContext.PreferredHostHealthMonitor != nil {
+		go cc.OpenShiftContext.PreferredHostHealthMonitor.Run(ctx)
+	}
+
 	// Configz registration.
 	if cz, err := configz.New("componentconfig"); err == nil {
 		cz.Set(cc.ComponentConfig)