UPSTREAM: <carry>: kube-controller-manager: allow running bare kube-controller-manager

UPSTREAM: <carry>: (squash) kube-controller-manager: allow running bare kube-controller-manager

UPSTREAM: <carry>: kube-controller-manager: allow running bare kube-controller-manager

openshift-rebase(v1.24):source=18bbb151dd9

openshift-rebase(v1.24):source=18bbb151dd9

openshift-rebase(v1.24):source=18bbb151dd9

UPSTREAM: <carry>: (squash) remove egressnetworkpolicies from gc ignored resources

egressnetworkpolicies should not be in garbage collector ignored
resources, so users can delete them using "--cascade=foreground" flag.

Signed-off-by: Flavio Fernandes <[email protected]>

openshift-rebase(v1.24):source=771b4b56597

Author: deads2k

cmd/kube-controller-manager/app/apps.go

@@ -33,7 +33,10 @@ import (
 )
 
 func startDaemonSetController(ctx context.Context, controllerContext ControllerContext) (controller.Interface, bool, error) {
-	dsc, err := daemon.NewDaemonSetsController(
+	dsc, err := daemon.NewNodeSelectorAwareDaemonSetsController(
+		controllerContext.OpenShiftContext.OpenShiftDefaultProjectNodeSelector,
+		controllerContext.OpenShiftContext.KubeDefaultProjectNodeSelector,
+		controllerContext.InformerFactory.Core().V1().Namespaces(),
 		controllerContext.InformerFactory.Apps().V1().DaemonSets(),
 		controllerContext.InformerFactory.Apps().V1().ControllerRevisions(),
 		controllerContext.InformerFactory.Core().V1().Pods(),

cmd/kube-controller-manager/app/config/config.go

@@ -26,6 +26,8 @@ import (
 
 // Config is the main context object for the controller manager.
 type Config struct {
+	OpenShiftContext OpenShiftContext
+
 	ComponentConfig kubectrlmgrconfig.KubeControllerManagerConfiguration
 
 	SecureServing *apiserver.SecureServingInfo

cmd/kube-controller-manager/app/config/patch.go

@@ -0,0 +1,9 @@
+package config
+
+// OpenShiftContext is additional context that we need to launch the kube-controller-manager for openshift.
+// Basically, this holds our additional config information.
+type OpenShiftContext struct {
+	OpenShiftConfig                     string
+	OpenShiftDefaultProjectNodeSelector string
+	KubeDefaultProjectNodeSelector      string
+}

cmd/kube-controller-manager/app/controllermanager.go

@@ -66,7 +66,6 @@ import (
 	"k8s.io/controller-manager/pkg/informerfactory"
 	"k8s.io/controller-manager/pkg/leadermigration"
 	"k8s.io/klog/v2"
-	kubefeatures "k8s.io/kubernetes/pkg/features"
 
 	"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
 	"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
@@ -135,6 +134,10 @@ controller, and serviceaccounts controller.`,
 				return err
 			}
 
+			if err := ShimForOpenShift(s, c); err != nil {
+				fmt.Fprintf(os.Stderr, "%v\n", err)
+				return err
+			}
 			return Run(c.Complete(), wait.NeverStop)
 		},
 		Args: func(cmd *cobra.Command, args []string) error {
@@ -320,6 +323,8 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {
 
 // ControllerContext defines the context object for controller
 type ControllerContext struct {
+	OpenShiftContext config.OpenShiftContext
+
 	// ClientBuilder will provide a client for this controller to use
 	ClientBuilder clientbuilder.ControllerClientBuilder
 
@@ -497,7 +502,12 @@ func GetAvailableResources(clientBuilder clientbuilder.ControllerClientBuilder)
 // the shared-informers client and token controller.
 func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clientBuilder clientbuilder.ControllerClientBuilder, stop <-chan struct{}) (ControllerContext, error) {
 	versionedClient := rootClientBuilder.ClientOrDie("shared-informers")
-	sharedInformers := informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)())
+	var sharedInformers informers.SharedInformerFactory
+	if InformerFactoryOverride == nil {
+		sharedInformers = informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)())
+	} else {
+		sharedInformers = InformerFactoryOverride
+	}
 
 	metadataClient := metadata.NewForConfigOrDie(rootClientBuilder.ConfigOrDie("metadata-informers"))
 	metadataInformers := metadatainformer.NewSharedInformerFactory(metadataClient, ResyncPeriod(s)())
@@ -528,6 +538,7 @@ func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clien
 	}
 
 	ctx := ControllerContext{
+		OpenShiftContext:                s.OpenShiftContext,
 		ClientBuilder:                   clientBuilder,
 		InformerFactory:                 sharedInformers,
 		ObjectOrMetadataInformerFactory: informerfactory.NewInformerFactory(sharedInformers, metadataInformers),
@@ -646,11 +657,10 @@ func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController
 		controllerContext.InformerFactory.Core().V1().ServiceAccounts(),
 		controllerContext.InformerFactory.Core().V1().Secrets(),
 		c.rootClientBuilder.ClientOrDie("tokens-controller"),
-		serviceaccountcontroller.TokensControllerOptions{
+		applyOpenShiftServiceServingCertCA(serviceaccountcontroller.TokensControllerOptions{
 			TokenGenerator: tokenGenerator,
 			RootCA:         rootCA,
-			AutoGenerate:   !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.LegacyServiceAccountTokenNoAutoGeneration),
-		},
+		}),
 	)
 	if err != nil {
 		return nil, true, fmt.Errorf("error creating Tokens controller: %v", err)

cmd/kube-controller-manager/app/options/options.go

@@ -94,6 +94,7 @@ type KubeControllerManagerOptions struct {
 	Master                      string
 	Kubeconfig                  string
 	ShowHiddenMetricsForVersion string
+	OpenShiftContext            kubecontrollerconfig.OpenShiftContext
 }
 
 // NewKubeControllerManagerOptions creates a new KubeControllerManagerOptions with a default config.
@@ -261,6 +262,11 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy
 	fs := fss.FlagSet("misc")
 	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
 	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
+	var dummy string
+	fs.MarkDeprecated("insecure-experimental-approve-all-kubelet-csrs-for-group", "This flag does nothing.")
+	fs.StringVar(&dummy, "insecure-experimental-approve-all-kubelet-csrs-for-group", "", "This flag does nothing.")
+	fs.StringVar(&s.OpenShiftContext.OpenShiftConfig, "openshift-config", s.OpenShiftContext.OpenShiftConfig, "indicates that this process should be compatible with openshift start master")
+	fs.MarkHidden("openshift-config")
 	utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))
 
 	return fss
@@ -360,6 +366,9 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) e
 			return err
 		}
 	}
+
+	c.OpenShiftContext = s.OpenShiftContext
+
 	return nil
 }
 

cmd/kube-controller-manager/app/patch.go

@@ -0,0 +1,84 @@
+package app
+
+import (
+	"io/ioutil"
+	"path"
+
+	"k8s.io/apimachinery/pkg/util/json"
+	kyaml "k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/client-go/informers"
+	"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
+	"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
+)
+
+var InformerFactoryOverride informers.SharedInformerFactory
+
+func ShimForOpenShift(controllerManagerOptions *options.KubeControllerManagerOptions, controllerManager *config.Config) error {
+	if len(controllerManager.OpenShiftContext.OpenShiftConfig) == 0 {
+		return nil
+	}
+
+	// TODO this gets removed when no longer take flags and no longer build a recycler template
+	openshiftConfig, err := getOpenShiftConfig(controllerManager.OpenShiftContext.OpenShiftConfig)
+	if err != nil {
+		return err
+	}
+
+	// TODO this should be replaced by using a flex volume to inject service serving cert CAs into pods instead of adding it to the sa token
+	if err := applyOpenShiftServiceServingCertCAFunc(path.Dir(controllerManager.OpenShiftContext.OpenShiftConfig), openshiftConfig); err != nil {
+		return err
+	}
+
+	// skip GC on some openshift resources
+	// TODO this should be replaced by discovery information in some way
+	if err := applyOpenShiftGCConfig(controllerManager); err != nil {
+		return err
+	}
+
+	if err := applyOpenShiftConfigDefaultProjectSelector(controllerManagerOptions, openshiftConfig); err != nil {
+		return err
+	}
+
+	// Overwrite the informers, because we have our custom generic informers for quota.
+	// TODO update quota to create its own informer like garbage collection
+	if informers, err := newInformerFactory(controllerManager.Kubeconfig); err != nil {
+		return err
+	} else {
+		InformerFactoryOverride = informers
+	}
+
+	return nil
+}
+
+func getOpenShiftConfig(configFile string) (map[string]interface{}, error) {
+	configBytes, err := ioutil.ReadFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+	jsonBytes, err := kyaml.ToJSON(configBytes)
+	if err != nil {
+		return nil, err
+	}
+	config := map[string]interface{}{}
+	if err := json.Unmarshal(jsonBytes, &config); err != nil {
+		return nil, err
+	}
+
+	return config, nil
+}
+
+func applyOpenShiftConfigDefaultProjectSelector(controllerManagerOptions *options.KubeControllerManagerOptions, openshiftConfig map[string]interface{}) error {
+	projectConfig, ok := openshiftConfig["projectConfig"]
+	if !ok {
+		return nil
+	}
+
+	castProjectConfig := projectConfig.(map[string]interface{})
+	defaultNodeSelector, ok := castProjectConfig["defaultNodeSelector"]
+	if !ok {
+		return nil
+	}
+	controllerManagerOptions.OpenShiftContext.OpenShiftDefaultProjectNodeSelector = defaultNodeSelector.(string)
+
+	return nil
+}

cmd/kube-controller-manager/app/patch_gc.go

@@ -0,0 +1,39 @@
+package app
+
+import (
+	gcconfig "k8s.io/kubernetes/pkg/controller/garbagecollector/config"
+
+	"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
+)
+
+func applyOpenShiftGCConfig(controllerManager *config.Config) error {
+	// TODO make this configurable or discoverable.  This is going to prevent us from running the stock GC controller
+	// IF YOU ADD ANYTHING TO THIS LIST, MAKE SURE THAT YOU UPDATE THEIR STRATEGIES TO PREVENT GC FINALIZERS
+	controllerManager.ComponentConfig.GarbageCollectorController.GCIgnoredResources = append(controllerManager.ComponentConfig.GarbageCollectorController.GCIgnoredResources,
+		// explicitly disabled from GC for now - not enough value to track them
+		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "rolebindingrestrictions"},
+		gcconfig.GroupResource{Group: "network.openshift.io", Resource: "clusternetworks"},
+		gcconfig.GroupResource{Group: "network.openshift.io", Resource: "hostsubnets"},
+		gcconfig.GroupResource{Group: "network.openshift.io", Resource: "netnamespaces"},
+		gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthclientauthorizations"},
+		gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthclients"},
+		gcconfig.GroupResource{Group: "quota.openshift.io", Resource: "clusterresourcequotas"},
+		gcconfig.GroupResource{Group: "user.openshift.io", Resource: "groups"},
+		gcconfig.GroupResource{Group: "user.openshift.io", Resource: "identities"},
+		gcconfig.GroupResource{Group: "user.openshift.io", Resource: "users"},
+		gcconfig.GroupResource{Group: "image.openshift.io", Resource: "images"},
+
+		// virtual resource
+		gcconfig.GroupResource{Group: "project.openshift.io", Resource: "projects"},
+		// virtual and unwatchable resource, surfaced via rbac.authorization.k8s.io objects
+		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "clusterroles"},
+		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "clusterrolebindings"},
+		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "roles"},
+		gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "rolebindings"},
+		// these resources contain security information in their names, and we don't need to track them
+		gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthaccesstokens"},
+		gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthauthorizetokens"},
+	)
+
+	return nil
+}