Replication: Add e2e tests for workloads-related rbac objects

Signed-off-by: David Festal <[email protected]>

Author: davidfestal

test/e2e/reconciler/cache/replication_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/kcp-dev/logicalcluster/v3"
 	"github.com/stretchr/testify/require"
 
+	rbacv1 "k8s.io/api/rbac/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -64,6 +65,10 @@ var scenarios = []testScenario{
 	{"TestReplicateAPIResourceSchemaNegative", replicateAPIResourceSchemaNegativeScenario},
 	{"TestReplicateWorkspaceType", replicateWorkspaceTypeScenario},
 	{"TestReplicateWorkspaceTypeNegative", replicateWorkspaceTypeNegativeScenario},
+	{"TestReplicateWorkloadsClusterRole", replicateWorkloadsClusterRoleScenario},
+	{"TestReplicateWorkloadsClusterRoleNegative", replicateWorkloadsClusterRoleNegativeScenario},
+	{"TestReplicateWorkloadsClusterRoleBinding", replicateWorkloadsClusterRoleBindingScenario},
+	{"TestReplicateWorkloadsClusterRoleBindingNegative", replicateWorkloadsClusterRoleBindingNegativeScenario},
 }
 
 // disruptiveScenarios contains a list of scenarios that will be run in a private environment
@@ -678,14 +683,23 @@ func (b *replicateResourceScenario) verifyResourceReplicationHelper(ctx context.
 		}
 		unstructured.RemoveNestedField(originalResource.Object, "metadata", "resourceVersion")
 		unstructured.RemoveNestedField(cachedResource.Object, "metadata", "resourceVersion")
+
+		// TODO(davidfestal): find out why the generation is not equal, specially for rbacv1.
+		// Is it a characteristic of all built-in KCP resources (which are not backed by CRDs) ?
+		// Issue opened: https://github.com/kcp-dev/kcp/issues/2935
+		if b.gvr.Group == rbacv1.SchemeGroupVersion.Group {
+			unstructured.RemoveNestedField(originalResource.Object, "metadata", "generation")
+			unstructured.RemoveNestedField(cachedResource.Object, "metadata", "generation")
+		}
+
 		unstructured.RemoveNestedField(cachedResource.Object, "metadata", "annotations", genericapirequest.AnnotationKey)
 		if cachedStatus, ok := cachedResource.Object["status"]; ok && cachedStatus == nil || (cachedStatus != nil && len(cachedStatus.(map[string]interface{})) == 0) {
 			// TODO: worth investigating:
 			// for some reason cached resources have an empty status set whereas the original resources don't
 			unstructured.RemoveNestedField(cachedResource.Object, "status")
 		}
 		if diff := cmp.Diff(cachedResource.Object, originalResource.Object); len(diff) > 0 {
-			return false, fmt.Sprintf("replicated %s root|%s/%s is different from the original", b.gvr, cluster, cachedResourceMeta.GetName())
+			return false, fmt.Sprintf("replicated %s root|%s/%s is different from the original: %s", b.gvr, cluster, cachedResourceMeta.GetName(), diff)
 		}
 		return true, ""
 	}, wait.ForeverTestTimeout, 100*time.Millisecond)
@@ -732,3 +746,181 @@ func createCacheClientConfigForEnvironment(ctx context.Context, t *testing.T, kc
 	require.NoError(t, err)
 	return cacheServerRestConfig
 }
+
+// replicateWorkloadsClusterRoleScenario tests if a ClusterRole related to workloads API is propagated to the cache server.
+// The test exercises creation, modification and removal of the Shard object.
+func replicateWorkloadsClusterRoleScenario(ctx context.Context, t *testing.T, server framework.RunningServer, kcpShardClusterDynamicClient kcpdynamic.ClusterInterface, cacheKcpClusterDynamicClient kcpdynamic.ClusterInterface) {
+	t.Helper()
+	replicateResource(ctx,
+		t,
+		server,
+		kcpShardClusterDynamicClient,
+		cacheKcpClusterDynamicClient,
+		"",
+		"ClusterRole",
+		rbacv1.SchemeGroupVersion.WithResource("clusterroles"),
+		&rbacv1.ClusterRole{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: withPseudoRandomSuffix("syncer"),
+			},
+			Rules: []rbacv1.PolicyRule{
+				{
+					Verbs:         []string{"sync"},
+					APIGroups:     []string{"workload.kcp.io"},
+					Resources:     []string{"synctargets"},
+					ResourceNames: []string{"asynctarget"},
+				},
+			},
+		},
+		nil,
+	)
+}
+
+// replicateWorkloadsClusterRoleNegativeScenario checks if modified or even deleted cached ClusterRole (related to workloads API) will be reconciled to match the original object.
+func replicateWorkloadsClusterRoleNegativeScenario(ctx context.Context, t *testing.T, server framework.RunningServer, kcpShardClusterDynamicClient kcpdynamic.ClusterInterface, cacheKcpClusterDynamicClient kcpdynamic.ClusterInterface) {
+	t.Helper()
+	replicateResourceNegative(
+		ctx,
+		t,
+		server,
+		kcpShardClusterDynamicClient,
+		cacheKcpClusterDynamicClient,
+		"",
+		"ClusterRole",
+		rbacv1.SchemeGroupVersion.WithResource("clusterroles"),
+		&rbacv1.ClusterRole{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: withPseudoRandomSuffix("syncer"),
+			},
+			Rules: []rbacv1.PolicyRule{
+				{
+					Verbs:         []string{"sync"},
+					APIGroups:     []string{"workload.kcp.io"},
+					Resources:     []string{"synctargets"},
+					ResourceNames: []string{"asynctarget"},
+				},
+			},
+		},
+		nil,
+	)
+}
+
+// replicateWorkloadsClusterRoleBindingScenario tests if a ClusterRoleBinding related to workloads API is propagated to the cache server.
+// The test exercises creation, modification and removal of the Shard object.
+func replicateWorkloadsClusterRoleBindingScenario(ctx context.Context, t *testing.T, server framework.RunningServer, kcpShardClusterDynamicClient kcpdynamic.ClusterInterface, cacheKcpClusterDynamicClient kcpdynamic.ClusterInterface) {
+	t.Helper()
+
+	clusterRole := &rbacv1.ClusterRole{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: withPseudoRandomSuffix("syncer"),
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				Verbs:         []string{"sync"},
+				APIGroups:     []string{"workload.kcp.io"},
+				Resources:     []string{"synctargets"},
+				ResourceNames: []string{"asynctarget"},
+			},
+		},
+	}
+
+	orgPath, _ := framework.NewOrganizationFixture(t, server)
+	_, ws := framework.NewWorkspaceFixture(t, server, orgPath, framework.WithRootShard())
+	clusterName := logicalcluster.Name(ws.Spec.Cluster)
+
+	t.Logf("Create additional ClusterRole %s on the root shard for replication", clusterRole.Name)
+	clusterRoleGVR := rbacv1.SchemeGroupVersion.WithResource("clusterroles")
+	clusterRoleUnstr, err := toUnstructured(clusterRole, "ClusterRole", clusterRoleGVR)
+	require.NoError(t, err)
+	_, err = kcpShardClusterDynamicClient.Resource(clusterRoleGVR).Cluster(clusterName.Path()).Create(ctx, clusterRoleUnstr, metav1.CreateOptions{})
+	require.NoError(t, err)
+
+	replicateResource(ctx,
+		t,
+		server,
+		kcpShardClusterDynamicClient,
+		cacheKcpClusterDynamicClient,
+		clusterName,
+		"ClusterRoleBinding",
+		rbacv1.SchemeGroupVersion.WithResource("clusterrolebindings"),
+		&rbacv1.ClusterRoleBinding{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: withPseudoRandomSuffix("syncer"),
+			},
+			RoleRef: rbacv1.RoleRef{
+				APIGroup: rbacv1.SchemeGroupVersion.Group,
+				Kind:     "ClusterRole",
+				Name:     clusterRole.Name,
+			},
+			Subjects: []rbacv1.Subject{
+				{
+					Kind:      "ServiceAccount",
+					APIGroup:  "",
+					Name:      "kcp-syncer-0000",
+					Namespace: "kcp-syncer-namespace",
+				},
+			},
+		},
+		nil,
+	)
+}
+
+// replicateWorkloadsClusterRoleNegativeScenario checks if modified or even deleted cached ClusterRole (related to workloads API) will be reconciled to match the original object.
+func replicateWorkloadsClusterRoleBindingNegativeScenario(ctx context.Context, t *testing.T, server framework.RunningServer, kcpShardClusterDynamicClient kcpdynamic.ClusterInterface, cacheKcpClusterDynamicClient kcpdynamic.ClusterInterface) {
+	t.Helper()
+
+	clusterRole := &rbacv1.ClusterRole{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: withPseudoRandomSuffix("syncer"),
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				Verbs:         []string{"sync"},
+				APIGroups:     []string{"workload.kcp.io"},
+				Resources:     []string{"synctargets"},
+				ResourceNames: []string{"asynctarget"},
+			},
+		},
+	}
+
+	orgPath, _ := framework.NewOrganizationFixture(t, server)
+	_, ws := framework.NewWorkspaceFixture(t, server, orgPath, framework.WithRootShard())
+	clusterName := logicalcluster.Name(ws.Spec.Cluster)
+
+	t.Logf("Create additional ClusterRole %s on the root shard for replication", clusterRole.Name)
+	clusterRoleGVR := rbacv1.SchemeGroupVersion.WithResource("clusterroles")
+	clusterRoleUnstr, err := toUnstructured(clusterRole, "ClusterRole", clusterRoleGVR)
+	require.NoError(t, err)
+	_, err = kcpShardClusterDynamicClient.Resource(clusterRoleGVR).Cluster(clusterName.Path()).Create(ctx, clusterRoleUnstr, metav1.CreateOptions{})
+	require.NoError(t, err)
+
+	replicateResourceNegative(
+		ctx,
+		t,
+		server,
+		kcpShardClusterDynamicClient,
+		cacheKcpClusterDynamicClient,
+		clusterName,
+		"ClusterRoleBinding",
+		rbacv1.SchemeGroupVersion.WithResource("clusterrolebindings"),
+		&rbacv1.ClusterRoleBinding{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: withPseudoRandomSuffix("syncer"),
+			},
+			RoleRef: rbacv1.RoleRef{
+				APIGroup: rbacv1.SchemeGroupVersion.Group,
+				Kind:     "ClusterRole",
+				Name:     clusterRole.Name,
+			},
+			Subjects: []rbacv1.Subject{
+				{
+					Kind:      "ServiceAccount",
+					APIGroup:  "",
+					Name:      "kcp-syncer-0000",
+					Namespace: "kcp-syncer-namespace",
+				},
+			},
+		},
+		nil,
+	)
+}