aws · gyuho · Mar 11, 2020 · Mar 11, 2020 · Mar 11, 2020
diff --git a/eks/appmesh/appmesh.go b/eks/appmesh/appmesh.go
@@ -0,0 +1,162 @@
+package appmesh
+
+import (
+	"github.com/aws/aws-k8s-tester/eksconfig"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface"
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"os"
+	"strings"
+	"time"
+)
+
+// Config defines AppMesh configuration.
+type Config struct {
+	Logger *zap.Logger
+	Stopc  chan struct{}
+	Sig    chan os.Signal
+
+	EKSConfig *eksconfig.Config
+	K8SClient k8sClientSetGetter
+	CFNAPI    cloudformationiface.CloudFormationAPI
+}
+
+type k8sClientSetGetter interface {
+	KubernetesClientSet() *clientset.Clientset
+}
+
+// Tester defines AppMesh tester
+type Tester interface {
+	// Installs AppMesh controller/injector
+	Create() error
+
+	// Clean up AppMesh controller/injector
+	Delete() error
+}
+
+func NewTester(cfg Config) (Tester, error) {
+	return &tester{
+		cfg: cfg,
+	}, nil
+}
+
+type tester struct {
+	cfg Config
+}
+
+func (ts *tester) Create() error {
+	if ts.cfg.EKSConfig.AddOnAppMesh.Created {
+		ts.cfg.Logger.Info("skipping create AddOnAppMesh")
+		return nil
+	}
+
+	ts.cfg.EKSConfig.AddOnAppMesh.Created = true
+	ts.cfg.EKSConfig.Sync()
+	createStart := time.Now()
+
+	defer func() {
+		ts.cfg.EKSConfig.AddOnAppMesh.CreateTook = time.Since(createStart)
+		ts.cfg.EKSConfig.AddOnAppMesh.CreateTookString = ts.cfg.EKSConfig.AddOnAppMesh.CreateTook.String()
+		ts.cfg.EKSConfig.Sync()
+	}()
+
+	if err := ts.createAppMeshAddOnCFNStack(); err != nil {
+		return err
+	}
+	if err := ts.createNamespace(); err != nil {
+		return err
+	}
+	if err := ts.installController(); err != nil {
+		return err
+	}
+	if err := ts.installInjector(); err != nil {
+		return err
+	}
+	return ts.cfg.EKSConfig.Sync()
+}
+
+func (ts *tester) Delete() error {
+	if !ts.cfg.EKSConfig.AddOnAppMesh.Created {
+		ts.cfg.Logger.Info("skipping delete AddOnAppMesh")
+		return nil
+	}
+
+	deleteStart := time.Now()
+	defer func() {
+		ts.cfg.EKSConfig.AddOnAppMesh.DeleteTook = time.Since(deleteStart)
+		ts.cfg.EKSConfig.AddOnAppMesh.DeleteTookString = ts.cfg.EKSConfig.AddOnAppMesh.DeleteTook.String()
+		ts.cfg.EKSConfig.Sync()
+	}()
+
+	var errs []string
+	if err := ts.uninstallInjector(); err != nil {
+		errs = append(errs, err.Error())
+	}
+	if err := ts.uninstallController(); err != nil {
+		errs = append(errs, err.Error())
+	}
+	if err := ts.deleteNamespace(); err != nil {
+		errs = append(errs, err.Error())
+	}
+	if err := ts.deleteAppMeshAddOnCFNStack(); err != nil {
+		errs = append(errs, err.Error())
+	}
+
+	if len(errs) > 0 {
+		return errors.New(strings.Join(errs, ", "))
+	}
+
+	ts.cfg.EKSConfig.AddOnAppMesh.Created = false
+	return ts.cfg.EKSConfig.Sync()
+}
+
+func (ts *tester) createNamespace() error {
+	ts.cfg.Logger.Info("creating namespace", zap.String("namespace", ts.cfg.EKSConfig.AddOnAppMesh.Namespace))
+	_, err := ts.cfg.K8SClient.KubernetesClientSet().
+		CoreV1().
+		Namespaces().
+		Create(&v1.Namespace{
+			TypeMeta: metav1.TypeMeta{
+				APIVersion: "v1",
+				Kind:       "Namespace",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name: ts.cfg.EKSConfig.AddOnAppMesh.Namespace,
+				Labels: map[string]string{
+					"name": ts.cfg.EKSConfig.AddOnAppMesh.Namespace,
+				},
+			},
+		})
+	if err != nil {
+		return err
+	}
+	ts.cfg.Logger.Info("created namespace", zap.String("namespace", ts.cfg.EKSConfig.AddOnAppMesh.Namespace))
+	return ts.cfg.EKSConfig.Sync()
+}
+
+func (ts *tester) deleteNamespace() error {
+	ts.cfg.Logger.Info("deleting namespace", zap.String("namespace", ts.cfg.EKSConfig.AddOnAppMesh.Namespace))
+	foreground := metav1.DeletePropagationForeground
+	err := ts.cfg.K8SClient.KubernetesClientSet().
+		CoreV1().
+		Namespaces().
+		Delete(
+			ts.cfg.EKSConfig.AddOnAppMesh.Namespace,
+			&metav1.DeleteOptions{
+				GracePeriodSeconds: aws.Int64(0),
+				PropagationPolicy:  &foreground,
+			},
+		)
+	if err != nil {
+		// ref. https://github.com/aws/aws-k8s-tester/issues/79
+		if !strings.Contains(err.Error(), ` not found`) {
+			return err
+		}
+	}
+	ts.cfg.Logger.Info("deleted namespace", zap.Error(err))
+	return ts.cfg.EKSConfig.Sync()
+}
diff --git a/eks/appmesh/cfn_stack.go b/eks/appmesh/cfn_stack.go
@@ -0,0 +1,175 @@
+package appmesh
+
+import (
+	"context"
+	"fmt"
+	awscfn "github.com/aws/aws-k8s-tester/pkg/aws/cloudformation"
+	"github.com/aws/aws-k8s-tester/version"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/cloudformation"
+	"go.uber.org/zap"
+	"os"
+	"time"
+)
+
+const addOnCFNStackTemplate = `
+---
+AWSTemplateFormatVersion: "2010-09-09"
+Description: "Amazon EKS AppMesh Controller AddOn stack"
+
+Parameters:
+  AppMeshControllerPolicyName:
+    Description: The policy name for AppMesh Controller
+    Type: String
+  ManagedNodeGroupRoleName:
+    Description: The name of the node instance role
+    Type: String
+Resources:
+  AppMeshControllerPolicy:
+    Metadata:
+      Comment: Minimal policy to allow worker node instance profile that allows the AppMesh Controller to make calls to AWS APIs on your behalf
+    Type: AWS::IAM::Policy
+    Properties:
+      PolicyName: !Ref AppMeshControllerPolicyName
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Effect: Allow
+            Action:
+              - appmesh:*
+              - servicediscovery:CreateService
+              - servicediscovery:GetService
+              - servicediscovery:RegisterInstance
+              - servicediscovery:DeregisterInstance
+              - servicediscovery:ListInstances
+              - servicediscovery:ListNamespaces
+              - servicediscovery:ListServices
+              - route53:GetHealthCheck
+              - route53:CreateHealthCheck
+              - route53:UpdateHealthCheck
+              - route53:ChangeResourceRecordSets
+              - route53:DeleteHealthCheck
+            Resource: "*"
+      Roles:
+        - !Ref ManagedNodeGroupRoleName
+`
+
+// createAppMeshAddOnCFNStack creates the cfn stack needed for AppMesh addOn.
+func (ts *tester) createAppMeshAddOnCFNStack() error {
+	if ts.cfg.EKSConfig.AddOnAppMesh.AddOnCFNStackARN != "" {
+		ts.cfg.Logger.Info("already created AppMesh Controller AddOn CFN stack, ignoring")
+		return nil
+	}
+
+	ts.cfg.Logger.Info("creating AppMesh Controller AddOn CFN stack")
+
+	stackName := ts.cfg.EKSConfig.Name + "-app-mesh-addOn"
+	policyName := ts.cfg.EKSConfig.Name + "-app-mesh-policy"
+	stackInput := &cloudformation.CreateStackInput{
+		StackName:    aws.String(stackName),
+		Capabilities: aws.StringSlice([]string{"CAPABILITY_NAMED_IAM"}),
+		OnFailure:    aws.String(cloudformation.OnFailureDelete),
+		TemplateBody: aws.String(addOnCFNStackTemplate),
+		Tags: awscfn.NewTags(map[string]string{
+			"Kind":                   "aws-k8s-tester",
+			"Name":                   ts.cfg.EKSConfig.Name,
+			"aws-k8s-tester-version": version.ReleaseVersion,
+		}),
+		Parameters: []*cloudformation.Parameter{
+			{
+				ParameterKey:   aws.String("AppMeshControllerPolicyName"),
+				ParameterValue: aws.String(policyName),
+			},
+			{
+				ParameterKey:   aws.String("ManagedNodeGroupRoleName"),
+				ParameterValue: aws.String(ts.cfg.EKSConfig.AddOnManagedNodeGroups.RoleName),
+			},
+		},
+	}
+
+	stackOutput, err := ts.cfg.CFNAPI.CreateStack(stackInput)
+	if err != nil {
+		return err
+	}
+	ts.cfg.EKSConfig.AddOnAppMesh.AddOnCFNStackARN = aws.StringValue(stackOutput.StackId)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	ch := awscfn.Poll(
+		ctx,
+		ts.cfg.Stopc,
+		ts.cfg.Sig,
+		ts.cfg.Logger,
+		ts.cfg.CFNAPI,
+		ts.cfg.EKSConfig.AddOnAppMesh.AddOnCFNStackARN,
+		cloudformation.ResourceStatusCreateComplete,
+		25*time.Second,
+		10*time.Second,
+	)
+	var st awscfn.StackStatus
+	for st = range ch {
+		if st.Error != nil {
+			cancel()
+			ts.cfg.EKSConfig.RecordStatus(fmt.Sprintf("failed to wait for AppMesh Controller AddOn CFN stack creation (%v)", st.Error))
+			ts.cfg.Logger.Error("polling error", zap.Error(st.Error))
+		}
+	}
+	cancel()
+	if st.Error != nil {
+		return st.Error
+	}
+
+	ts.cfg.Logger.Info("created AppMesh Controller AddOn CFN stack",
+		zap.String("add-on-stack-arn", ts.cfg.EKSConfig.AddOnAppMesh.AddOnCFNStackARN),
+		zap.String("policy-name", policyName),
+	)
+
+	return ts.cfg.EKSConfig.Sync()
+}
+
+// deleteAppMeshAddOnCFNStack deletes the cfn stack needed for AppMesh addOn.
+func (ts *tester) deleteAppMeshAddOnCFNStack() error {
+	if ts.cfg.EKSConfig.AddOnAppMesh.AddOnCFNStackARN == "" {
+		ts.cfg.Logger.Info("empty AppMesh Controller AddOn CFN stack, no need to delete")
+		return nil
+	}
+
+	ts.cfg.Logger.Info("deleting AppMesh Controller AddOn CFN stack",
+		zap.String("add-on-stack-arn", ts.cfg.EKSConfig.AddOnAppMesh.AddOnCFNStackARN),
+	)
+
+	_, err := ts.cfg.CFNAPI.DeleteStack(&cloudformation.DeleteStackInput{
+		StackName: aws.String(ts.cfg.EKSConfig.AddOnAppMesh.AddOnCFNStackARN),
+	})
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	ch := awscfn.Poll(
+		ctx,
+		make(chan struct{}),  // do not exit on stop
+		make(chan os.Signal), // do not exit on stop
+		ts.cfg.Logger,
+		ts.cfg.CFNAPI,
+		ts.cfg.EKSConfig.AddOnAppMesh.AddOnCFNStackARN,
+		cloudformation.ResourceStatusDeleteComplete,
+		25*time.Second,
+		10*time.Second,
+	)
+	var st awscfn.StackStatus
+	for st = range ch {
+		if st.Error != nil {
+			cancel()
+			ts.cfg.EKSConfig.RecordStatus(fmt.Sprintf("failed to wait for AppMesh Controller AddOn CFN stack  deletion (%v)", st.Error))
+			ts.cfg.Logger.Error("polling error", zap.Error(st.Error))
+		}
+	}
+	cancel()
+	if st.Error != nil {
+		return st.Error
+	}
+	ts.cfg.Logger.Info("AppMesh Controller AddOn CFN stack",
+		zap.String("add-on-stack-arn", ts.cfg.EKSConfig.AddOnAppMesh.AddOnCFNStackARN),
+	)
+	ts.cfg.EKSConfig.AddOnAppMesh.AddOnCFNStackARN = ""
+
+	return ts.cfg.EKSConfig.Sync()
+}