delete everything from install plan, wait for CRD deletion

go.mod

@@ -9,9 +9,10 @@ require (
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
-	k8s.io/api v0.18.4
-	k8s.io/apiextensions-apiserver v0.18.2
-	k8s.io/apimachinery v0.18.4
-	k8s.io/client-go v0.18.4
-	sigs.k8s.io/controller-runtime v0.6.0
+	k8s.io/api v0.18.6
+	k8s.io/apiextensions-apiserver v0.18.6
+	k8s.io/apimachinery v0.18.6
+	k8s.io/client-go v0.18.6
+	sigs.k8s.io/controller-runtime v0.6.2
+	sigs.k8s.io/yaml v1.2.0
 )

internal/pkg/action/constants.go

@@ -0,0 +1,6 @@
+package action
+
+const (
+	csvKind = "ClusterServiceVersion"
+	crdKind = "CustomResourceDefinition"
+)

internal/pkg/action/operator_install.go

@@ -379,7 +379,7 @@ func (i *OperatorInstall) getCSV(ctx context.Context, ip *v1alpha1.InstallPlan)
 		Namespace: i.config.Namespace,
 	}
 	for _, s := range ip.Status.Plan {
-		if s.Resource.Kind == "ClusterServiceVersion" {
+		if s.Resource.Kind == csvKind {
 			csvKey.Name = s.Resource.Name
 		}
 	}

internal/pkg/action/operator_uninstall.go

@@ -3,14 +3,20 @@ package action
 import (
 	"context"
 	"fmt"
+	"strings"
+	"time"
 
 	v1 "github.com/operator-framework/api/pkg/operators/v1"
 	"github.com/operator-framework/api/pkg/operators/v1alpha1"
 	"github.com/spf13/pflag"
-	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/yaml"
 
 	"github.com/joelanford/kubectl-operator/internal/pkg/log"
 )
@@ -59,28 +65,44 @@ func (u *OperatorUninstall) Run(ctx context.Context) error {
 		return fmt.Errorf("operator package %q not found", u.Package)
 	}
 
+	var crds, csvs, others []controllerutil.Object
+	if sub.Status.InstallPlanRef != nil {
+		ipKey := types.NamespacedName{
+			Namespace: sub.Status.InstallPlanRef.Namespace,
+			Name:      sub.Status.InstallPlanRef.Name,
+		}
+		var err error
+		crds, csvs, others, err = u.getInstallPlanResources(ctx, ipKey)
+		if err != nil {
+			return fmt.Errorf("get install plan resources: %v", err)
+		}
+	}
+
 	if err := u.config.Client.Delete(ctx, sub); err != nil {
 		return fmt.Errorf("delete subscription %q: %v", sub.Name, err)
 	}
 	log.Printf("subscription %q deleted", sub.Name)
 
-	if sub.Status.CurrentCSV != "" && sub.Status.CurrentCSV != sub.Status.InstalledCSV {
-		if err := u.deleteCSVandCRDs(ctx, sub.Status.CurrentCSV, true); err != nil {
+	if u.DeleteCRDs {
+		if err := u.deleteCRDs(ctx, crds); err != nil {
 			return err
 		}
 	}
-	if sub.Status.InstalledCSV != "" {
-		if err := u.deleteCSVandCRDs(ctx, sub.Status.InstalledCSV, false); err != nil {
-			return err
-		}
+
+	if err := u.deleteObjects(ctx, false, csvs); err != nil {
+		return err
+	}
+
+	if err := u.deleteObjects(ctx, false, others); err != nil {
+		return err
 	}
 
 	if u.DeleteOperatorGroup {
-		csvs := v1alpha1.ClusterServiceVersionList{}
-		if err := u.config.Client.List(ctx, &csvs, client.InNamespace(u.config.Namespace)); err != nil {
+		subs := v1alpha1.SubscriptionList{}
+		if err := u.config.Client.List(ctx, &subs, client.InNamespace(u.config.Namespace)); err != nil {
 			return fmt.Errorf("list clusterserviceversions: %v", err)
 		}
-		if len(csvs.Items) == 0 {
+		if len(subs.Items) == 0 {
 			ogs := v1.OperatorGroupList{}
 			if err := u.config.Client.List(ctx, &ogs, client.InNamespace(u.config.Namespace)); err != nil {
 				return fmt.Errorf("list operatorgroups: %v", err)
@@ -98,34 +120,78 @@ func (u *OperatorUninstall) Run(ctx context.Context) error {
 	return nil
 }
 
-func (u *OperatorUninstall) deleteCSVandCRDs(ctx context.Context, csvName string, ignoreNotFound bool) error {
-	csvKey := types.NamespacedName{
-		Name:      csvName,
-		Namespace: u.config.Namespace,
-	}
-	csv := v1alpha1.ClusterServiceVersion{}
-	if err := u.config.Client.Get(ctx, csvKey, &csv); err != nil {
-		if !apierrors.IsNotFound(err) || !ignoreNotFound {
-			return fmt.Errorf("get clusterserviceversion %q: %v", csvName, err)
-		}
+func (u *OperatorUninstall) deleteCRDs(ctx context.Context, crds []controllerutil.Object) error {
+	if err := u.deleteObjects(ctx, true, crds); err != nil {
+		return err
 	}
+	return nil
+}
 
-	if u.DeleteCRDs {
-		ownedCRDs := csv.Spec.CustomResourceDefinitions.Owned
-		for _, ownedCRD := range ownedCRDs {
-			crd := apiextv1.CustomResourceDefinition{}
-			crd.SetName(ownedCRD.Name)
-			if err := u.config.Client.Delete(ctx, &crd); err != nil {
-				return fmt.Errorf("delete crd %q: %v", ownedCRD.Name, err)
+func (u *OperatorUninstall) deleteObjects(ctx context.Context, waitForDelete bool, objs []controllerutil.Object) error {
+	for _, obj := range objs {
+		obj := obj
+		lowerKind := strings.ToLower(obj.GetObjectKind().GroupVersionKind().Kind)
+		if err := u.config.Client.Delete(ctx, obj); err != nil && !apierrors.IsNotFound(err) {
+			return fmt.Errorf("delete %s %q: %v", lowerKind, obj.GetName(), err)
+		} else if err == nil {
+			log.Printf("%s %q deleted", lowerKind, obj.GetName())
+		}
+		if waitForDelete {
+			key, err := client.ObjectKeyFromObject(obj)
+			if err != nil {
+				return fmt.Errorf("get %s key: %v", lowerKind, err)
+			}
+			if err := wait.PollImmediateUntil(250*time.Millisecond, func() (bool, error) {
+				if err := u.config.Client.Get(ctx, key, obj); apierrors.IsNotFound(err) {
+					return true, nil
+				} else if err != nil {
+					return false, err
+				}
+				return false, nil
+			}, ctx.Done()); err != nil {
+				return fmt.Errorf("wait for %s deleted: %v", lowerKind, err)
 			}
-			log.Printf("crd %q deleted", ownedCRD.Name)
 		}
 	}
+	return nil
+}
 
-	if err := u.config.Client.Delete(ctx, &csv); err != nil {
-		return fmt.Errorf("delete csv %q: %v", csvName, err)
+func (u *OperatorUninstall) getInstallPlanResources(ctx context.Context, installPlanKey types.NamespacedName) (crds, csvs, others []controllerutil.Object, err error) {
+	installPlan := &v1alpha1.InstallPlan{}
+	if err := u.config.Client.Get(ctx, installPlanKey, installPlan); err != nil {
+		return nil, nil, nil, fmt.Errorf("get install plan: %v", err)
 	}
-	log.Printf("csv %q deleted", csvName)
 
-	return nil
+	for _, step := range installPlan.Status.Plan {
+		if step.Status != v1alpha1.StepStatusCreated {
+			continue
+		}
+		obj := &unstructured.Unstructured{Object: map[string]interface{}{}}
+		lowerKind := strings.ToLower(step.Resource.Kind)
+		if err := yaml.Unmarshal([]byte(step.Resource.Manifest), &obj.Object); err != nil {
+			return nil, nil, nil, fmt.Errorf("parse %s manifest %q: %v", lowerKind, step.Resource.Name, err)
+		}
+		obj.SetGroupVersionKind(schema.GroupVersionKind{
+			Group:   step.Resource.Group,
+			Version: step.Resource.Version,
+			Kind:    step.Resource.Kind,
+		})
+
+		// TODO(joelanford): This seems necessary for service accounts tied to
+		//    cluster roles and cluster role bindings because the SA namespace
+		//    is not set in the manifest in this case.
+		//    See: https://github.com/operator-framework/operator-lifecycle-manager/blob/c9405d035bc50d9aa290220cb8d75b0402e72707/pkg/controller/registry/resolver/rbac.go#L133
+		if step.Resource.Kind == "ServiceAccount" && obj.GetNamespace() == "" {
+			obj.SetNamespace(installPlanKey.Namespace)
+		}
+		switch step.Resource.Kind {
+		case crdKind:
+			crds = append(crds, obj)
+		case csvKind:
+			csvs = append(csvs, obj)
+		default:
+			others = append(others, obj)
+		}
+	}
+	return crds, csvs, others, nil
 }

internal/pkg/action/operator_upgrade.go

@@ -113,7 +113,7 @@ func (u *OperatorUpgrade) getCSV(ctx context.Context, ip *v1alpha1.InstallPlan)
 		Namespace: u.config.Namespace,
 	}
 	for _, s := range ip.Status.Plan {
-		if s.Resource.Kind == "ClusterServiceVersion" {
+		if s.Resource.Kind == csvKind {
 			csvKey.Name = s.Resource.Name
 		}
 	}