Introduce --reconcile flag to kOps

Kubernetes 1.31 now stops nodes joining a cluster if the minor version
of the node is greater than the minor version of the control plane.

The addition of the instance-group-roles flag to update means that we
can now update / rolling-update the control plane first.  However, we
must now issue four commands:

* Update control plane
* Rolling update control plane
* Update nodes
* Rolling update nodes

This adds a flag to automate this process.  It is implemented by
executing those 4 steps in sequence.

Author: justinsb

cmd/kops/reconcile_cluster.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"k8s.io/kops/cmd/kops/util"
+	"k8s.io/kops/pkg/apis/kops"
+)
+
+func ReconcileCluster(ctx context.Context, f *util.Factory, out io.Writer, c *UpdateClusterOptions) error {
+	fmt.Fprintf(out, "Updating control plane configuration\n")
+	{
+		opt := *c
+		opt.Reconcile = false // Prevent infinite loop
+		opt.InstanceGroupRoles = []string{
+			string(kops.InstanceGroupRoleAPIServer),
+			string(kops.InstanceGroupRoleControlPlane),
+		}
+		if _, err := RunUpdateCluster(ctx, f, out, &opt); err != nil {
+			return err
+		}
+	}
+
+	fmt.Fprintf(out, "Doing rolling-update for control plane\n")
+	{
+		opt := &RollingUpdateOptions{}
+		opt.InitDefaults()
+		opt.ClusterName = c.ClusterName
+		opt.InstanceGroupRoles = []string{
+			string(kops.InstanceGroupRoleAPIServer),
+			string(kops.InstanceGroupRoleControlPlane),
+		}
+		opt.Yes = c.Yes
+		if err := RunRollingUpdateCluster(ctx, f, out, opt); err != nil {
+			return err
+		}
+	}
+
+	fmt.Fprintf(out, "Updating node configuration\n")
+	{
+		opt := *c
+		opt.Reconcile = false // Prevent infinite loop
+		// TODO: or just do all roles?
+		opt.InstanceGroupRoles = []string{
+			string(kops.InstanceGroupRoleBastion),
+			string(kops.InstanceGroupRoleNode),
+		}
+		if _, err := RunUpdateCluster(ctx, f, out, &opt); err != nil {
+			return err
+		}
+	}
+
+	fmt.Fprintf(out, "Doing rolling-update for nodes\n")
+	{
+		opt := &RollingUpdateOptions{}
+		opt.InitDefaults()
+		opt.ClusterName = c.ClusterName
+		// TODO: or just do all roles?
+		opt.InstanceGroupRoles = []string{
+			string(kops.InstanceGroupRoleBastion),
+			string(kops.InstanceGroupRoleNode),
+		}
+		opt.Yes = c.Yes
+		if err := RunRollingUpdateCluster(ctx, f, out, opt); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cmd/kops/update_cluster.go

@@ -103,6 +103,9 @@ type UpdateClusterOptions struct {
 	// The goal is that the cluster can keep running even during more disruptive
 	// infrastructure changes.
 	Prune bool
+
+	// Reconcile is true if we should reconcile the cluster by rolling the control plane and nodes sequentially
+	Reconcile bool
 }
 
 func (o *UpdateClusterOptions) InitDefaults() {
@@ -117,6 +120,7 @@ func (o *UpdateClusterOptions) InitDefaults() {
 	o.CreateKubecfg = true
 
 	o.Prune = false
+	o.Reconcile = false
 
 	o.RunTasksOptions.InitDefaults()
 }
@@ -193,6 +197,13 @@ type UpdateClusterResults struct {
 }
 
 func RunUpdateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *UpdateClusterOptions) (*UpdateClusterResults, error) {
+	if c.Reconcile {
+		if !c.Yes {
+			return nil, fmt.Errorf("--reconcile is only supported with --yes")
+		}
+		return nil, ReconcileCluster(ctx, f, out, c)
+	}
+
 	results := &UpdateClusterResults{}
 
 	isDryrun := false