From c3187296cc076c7f9581a53f5643bfffdf663dc1 Mon Sep 17 00:00:00 2001 From: Jerad C Date: Mon, 7 Nov 2022 13:54:08 -0600 Subject: [PATCH] refactor test suite --- Makefile | 2 +- test/reconciler/asg_lifecycle_v1.go | 134 + test/reconciler/asg_lifecycle_v2.go | 134 + test/reconciler/asg_v1_err.go | 85 + test/reconciler/asg_v1_err_ne_400.go | 87 + test/reconciler/asg_v2_err.go | 85 + test/reconciler/asg_v2_err_ne_400.go | 86 + test/reconciler/complete_asg_lifecycle.go | 94 + test/reconciler/cordon_err.go | 85 + test/reconciler/cordon_node.go | 108 + test/reconciler/drain_err.go | 87 + test/reconciler/drain_node.go | 107 + test/reconciler/ec2_res_empty_dns.go | 91 + test/reconciler/ec2_res_no_dns.go | 91 + test/reconciler/ec2_res_no_inst.go | 87 + test/reconciler/empty_msg.go | 64 + test/reconciler/empty_sqs_queue.go | 50 + test/reconciler/get_ec2_res_err.go | 85 + test/reconciler/get_node_err.go | 85 + test/reconciler/get_sqs_msg.go | 85 + test/reconciler/get_sqs_msg_err.go | 60 + test/reconciler/get_terminator_err.go | 66 + test/{ => reconciler/mock}/asgclient.go | 2 +- test/{ => reconciler/mock}/ec2client.go | 2 +- test/reconciler/mock/infrastructure.go | 398 +++ test/{ => reconciler/mock}/kubeclient.go | 2 +- test/{ => reconciler/mock}/sqsclient.go | 2 +- test/reconciler/msg_no_body.go | 62 + test/reconciler/msg_parse_error.go | 66 + test/reconciler/multiple_msgs.go | 197 ++ test/reconciler/no_ec2_res.go | 85 + test/reconciler/rebalance_rec.go | 72 + test/reconciler/scheduled_change.go | 168 + test/reconciler/spot_itn.go | 72 + test/reconciler/sqs_msg_delete_err.go | 76 + test/reconciler/state_change.go | 208 ++ .../suite_test.go} | 2 +- test/reconciler/terminator_evt_cfg.go | 546 ++++ test/reconciler/terminator_node_selector.go | 135 + test/reconciler/terminator_not_found.go | 50 + test/reconciler/terminator_webhook_cfg.go | 252 ++ test/reconciler/unrecognized_msg.go | 69 + test/reconciliation_test.go | 2735 ----------------- 43 files changed, 4318 insertions(+), 2741 deletions(-) create mode 100644 test/reconciler/asg_lifecycle_v1.go create mode 100644 test/reconciler/asg_lifecycle_v2.go create mode 100644 test/reconciler/asg_v1_err.go create mode 100644 test/reconciler/asg_v1_err_ne_400.go create mode 100644 test/reconciler/asg_v2_err.go create mode 100644 test/reconciler/asg_v2_err_ne_400.go create mode 100644 test/reconciler/complete_asg_lifecycle.go create mode 100644 test/reconciler/cordon_err.go create mode 100644 test/reconciler/cordon_node.go create mode 100644 test/reconciler/drain_err.go create mode 100644 test/reconciler/drain_node.go create mode 100644 test/reconciler/ec2_res_empty_dns.go create mode 100644 test/reconciler/ec2_res_no_dns.go create mode 100644 test/reconciler/ec2_res_no_inst.go create mode 100644 test/reconciler/empty_msg.go create mode 100644 test/reconciler/empty_sqs_queue.go create mode 100644 test/reconciler/get_ec2_res_err.go create mode 100644 test/reconciler/get_node_err.go create mode 100644 test/reconciler/get_sqs_msg.go create mode 100644 test/reconciler/get_sqs_msg_err.go create mode 100644 test/reconciler/get_terminator_err.go rename test/{ => reconciler/mock}/asgclient.go (98%) rename test/{ => reconciler/mock}/ec2client.go (98%) create mode 100644 test/reconciler/mock/infrastructure.go rename test/{ => reconciler/mock}/kubeclient.go (98%) rename test/{ => reconciler/mock}/sqsclient.go (99%) create mode 100644 test/reconciler/msg_no_body.go create mode 100644 test/reconciler/msg_parse_error.go create mode 100644 test/reconciler/multiple_msgs.go create mode 100644 test/reconciler/no_ec2_res.go create mode 100644 test/reconciler/rebalance_rec.go create mode 100644 test/reconciler/scheduled_change.go create mode 100644 test/reconciler/spot_itn.go create mode 100644 test/reconciler/sqs_msg_delete_err.go create mode 100644 test/reconciler/state_change.go rename test/{app_integ_suite_test.go => reconciler/suite_test.go} (97%) create mode 100644 test/reconciler/terminator_evt_cfg.go create mode 100644 test/reconciler/terminator_node_selector.go create mode 100644 test/reconciler/terminator_not_found.go create mode 100644 test/reconciler/terminator_webhook_cfg.go create mode 100644 test/reconciler/unrecognized_msg.go delete mode 100644 test/reconciliation_test.go diff --git a/Makefile b/Makefile index 0f2a4ced..ff918430 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ GUM = $(BIN_DIR)/gum GH = $(BIN_DIR)/gh GOLICENSES = $(BIN_DIR)/go-licenses HELM_BASE_OPTS ?= --set aws.region=${AWS_REGION},serviceAccount.name=${SERVICE_ACCOUNT_NAME},serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn=${SERVICE_ACCOUNT_ROLE_ARN} -GINKGO_BASE_OPTS ?= --coverpkg $(shell head -n 1 $(PROJECT_DIR)/go.mod | cut -s -d ' ' -f 2)/pkg/... +GINKGO_BASE_OPTS ?= -r --coverpkg $(shell head -n 1 $(PROJECT_DIR)/go.mod | cut -s -d ' ' -f 2)/pkg/... KODATA = \ cmd/controller/kodata/HEAD \ cmd/controller/kodata/refs \ diff --git a/test/reconciler/asg_lifecycle_v1.go b/test/reconciler/asg_lifecycle_v1.go new file mode 100644 index 00000000..8e9dd7d1 --- /dev/null +++ b/test/reconciler/asg_lifecycle_v1.go @@ -0,0 +1,134 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the SQS queue contains an ASG Lifecycle Notification v1", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + }) + + JustBeforeEach(func() { + result, err = infra.Reconcile() + }) + + When("the lifecycle transition is termination", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.autoscaling", + "detail-type": "EC2 Instance-terminate Lifecycle Action", + "version": "1", + "detail": { + "EC2InstanceId": "%s", + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" + } + }`, infra.InstanceIDs[1])), + }) + + infra.CreatePendingASGLifecycleAction(infra.InstanceIDs[1]) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons and drains only the targeted node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + Expect(infra.DrainedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("completes the ASG lifecycle action", func() { + Expect(infra.ASGLifecycleActions).To( + SatisfyAll( + HaveKeyWithValue(infra.InstanceIDs[1], Equal(mock.StateComplete)), + HaveLen(1), + ), + ) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("the lifecycle transition is not termination", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.autoscaling", + "detail-type": "EC2 Instance-terminate Lifecycle Action", + "version": "1", + "detail": { + "EC2InstanceId": "%s", + "LifecycleTransition": "test:INVALID" + } + }`, infra.InstanceIDs[1])), + }) + + infra.CreatePendingASGLifecycleAction(infra.InstanceIDs[1]) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("does not cordon or drain any nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("does not complete the ASG lifecycle action", func() { + Expect(infra.ASGLifecycleActions).To( + SatisfyAll( + HaveKeyWithValue(infra.InstanceIDs[1], Equal(mock.StatePending)), + HaveLen(1), + ), + ) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) + }) +}) diff --git a/test/reconciler/asg_lifecycle_v2.go b/test/reconciler/asg_lifecycle_v2.go new file mode 100644 index 00000000..47dd8184 --- /dev/null +++ b/test/reconciler/asg_lifecycle_v2.go @@ -0,0 +1,134 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the SQS queue contains an ASG Lifecycle Notification v2", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + }) + + JustBeforeEach(func() { + result, err = infra.Reconcile() + }) + + When("the lifecycle transition is termination", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.autoscaling", + "detail-type": "EC2 Instance-terminate Lifecycle Action", + "version": "2", + "detail": { + "EC2InstanceId": "%s", + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" + } + }`, infra.InstanceIDs[1])), + }) + + infra.CreatePendingASGLifecycleAction(infra.InstanceIDs[1]) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons and drains only the targeted node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + Expect(infra.DrainedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("completes the ASG lifecycle action", func() { + Expect(infra.ASGLifecycleActions).To( + SatisfyAll( + HaveKeyWithValue(infra.InstanceIDs[1], Equal(mock.StateComplete)), + HaveLen(1), + ), + ) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("the lifecycle transition is not termination", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.autoscaling", + "detail-type": "EC2 Instance-terminate Lifecycle Action", + "version": "2", + "detail": { + "EC2InstanceId": "%s", + "LifecycleTransition": "test:INVALID" + } + }`, infra.InstanceIDs[1])), + }) + + infra.CreatePendingASGLifecycleAction(infra.InstanceIDs[1]) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("does not cordon or drain any nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("does not complete the ASG lifecycle action", func() { + Expect(infra.ASGLifecycleActions).To( + SatisfyAll( + HaveKeyWithValue(infra.InstanceIDs[1], Equal(mock.StatePending)), + HaveLen(1), + ), + ) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) + }) +}) diff --git a/test/reconciler/asg_v1_err.go b/test/reconciler/asg_v1_err.go new file mode 100644 index 00000000..7a0d2163 --- /dev/null +++ b/test/reconciler/asg_v1_err.go @@ -0,0 +1,85 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "errors" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + awsrequest "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("completing an ASG Lifecycle Action (v1) fails", func() { + const errMsg = "test error" + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.autoscaling", + "detail-type": "EC2 Instance-terminate Lifecycle Action", + "version": "1", + "detail": { + "EC2InstanceId": "%s", + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" + } + }`, infra.InstanceIDs[1])), + }) + + infra.CompleteASGLifecycleActionFunc = func(_ aws.Context, _ *autoscaling.CompleteLifecycleActionInput, _ ...awsrequest.Option) (*autoscaling.CompleteLifecycleActionOutput, error) { + return nil, errors.New(errMsg) + } + + result, err = infra.Reconcile() + }) + + It("does not requeue the request", func() { + Expect(result).To(BeZero()) + }) + + It("returns an error", func() { + Expect(err).To(MatchError(ContainSubstring(errMsg))) + }) + + It("cordons and drains only the targeted node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + Expect(infra.DrainedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) +}) diff --git a/test/reconciler/asg_v1_err_ne_400.go b/test/reconciler/asg_v1_err_ne_400.go new file mode 100644 index 00000000..dbf7ea6c --- /dev/null +++ b/test/reconciler/asg_v1_err_ne_400.go @@ -0,0 +1,87 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "errors" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + awsrequest "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + + When("the request to complete the ASG Lifecycle Action (v1) fails with a status != 400", func() { + const errMsg = "test error" + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.autoscaling", + "detail-type": "EC2 Instance-terminate Lifecycle Action", + "version": "1", + "detail": { + "EC2InstanceId": "%s", + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" + } + }`, infra.InstanceIDs[1])), + }) + + infra.CompleteASGLifecycleActionFunc = func(_ aws.Context, _ *autoscaling.CompleteLifecycleActionInput, _ ...awsrequest.Option) (*autoscaling.CompleteLifecycleActionOutput, error) { + return nil, awserr.NewRequestFailure(awserr.New("", errMsg, errors.New(errMsg)), 404, "") + } + + result, err = infra.Reconcile() + }) + + It("does not requeue the request", func() { + Expect(result).To(BeZero()) + }) + + It("returns an error", func() { + Expect(err).To(MatchError(ContainSubstring(errMsg))) + }) + + It("cordons and drains only the targeted node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + Expect(infra.DrainedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) +}) diff --git a/test/reconciler/asg_v2_err.go b/test/reconciler/asg_v2_err.go new file mode 100644 index 00000000..e6c75ba6 --- /dev/null +++ b/test/reconciler/asg_v2_err.go @@ -0,0 +1,85 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "errors" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + awsrequest "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("completing an ASG Lifecycle Action (v2) fails", func() { + const errMsg = "test error" + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.autoscaling", + "detail-type": "EC2 Instance-terminate Lifecycle Action", + "version": "2", + "detail": { + "EC2InstanceId": "%s", + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" + } + }`, infra.InstanceIDs[1])), + }) + + infra.CompleteASGLifecycleActionFunc = func(_ aws.Context, _ *autoscaling.CompleteLifecycleActionInput, _ ...awsrequest.Option) (*autoscaling.CompleteLifecycleActionOutput, error) { + return nil, errors.New(errMsg) + } + + result, err = infra.Reconcile() + }) + + It("does not requeue the request", func() { + Expect(result).To(BeZero()) + }) + + It("returns an error", func() { + Expect(err).To(MatchError(ContainSubstring(errMsg))) + }) + + It("cordons and drains only the targeted node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + Expect(infra.DrainedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) +}) diff --git a/test/reconciler/asg_v2_err_ne_400.go b/test/reconciler/asg_v2_err_ne_400.go new file mode 100644 index 00000000..07e29dcc --- /dev/null +++ b/test/reconciler/asg_v2_err_ne_400.go @@ -0,0 +1,86 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "errors" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + awsrequest "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the request to complete the ASG Lifecycle Action (v2) fails with a status != 400", func() { + const errMsg = "test error" + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.autoscaling", + "detail-type": "EC2 Instance-terminate Lifecycle Action", + "version": "2", + "detail": { + "EC2InstanceId": "%s", + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" + } + }`, infra.InstanceIDs[1])), + }) + + infra.CompleteASGLifecycleActionFunc = func(_ aws.Context, _ *autoscaling.CompleteLifecycleActionInput, _ ...awsrequest.Option) (*autoscaling.CompleteLifecycleActionOutput, error) { + return nil, awserr.NewRequestFailure(awserr.New("", errMsg, errors.New(errMsg)), 404, "") + } + + result, err = infra.Reconcile() + }) + + It("does not requeue the request", func() { + Expect(result).To(BeZero()) + }) + + It("returns an error", func() { + Expect(err).To(MatchError(ContainSubstring(errMsg))) + }) + + It("cordons and drains only the targeted node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + Expect(infra.DrainedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) +}) diff --git a/test/reconciler/complete_asg_lifecycle.go b/test/reconciler/complete_asg_lifecycle.go new file mode 100644 index 00000000..9d2314a7 --- /dev/null +++ b/test/reconciler/complete_asg_lifecycle.go @@ -0,0 +1,94 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + awsrequest "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("completing an ASG Complete Lifecycle Action", func() { + const ( + autoScalingGroupName = "testAutoScalingGroupName" + lifecycleActionResult = "CONTINUE" + lifecycleHookName = "testLifecycleHookName" + lifecycleActionToken = "testLifecycleActionToken" + ) + var ( + infra *mock.Infrastructure + input *autoscaling.CompleteLifecycleActionInput + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.autoscaling", + "detail-type": "EC2 Instance-terminate Lifecycle Action", + "version": "1", + "detail": { + "AutoScalingGroupName": "%s", + "EC2InstanceId": "%s", + "LifecycleActionToken": "%s", + "LifecycleHookName": "%s", + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" + } + }`, autoScalingGroupName, infra.InstanceIDs[1], lifecycleActionToken, lifecycleHookName)), + }) + + defaultCompleteASGLifecycleActionFunc := infra.CompleteASGLifecycleActionFunc + infra.CompleteASGLifecycleActionFunc = func(ctx aws.Context, in *autoscaling.CompleteLifecycleActionInput, options ...awsrequest.Option) (*autoscaling.CompleteLifecycleActionOutput, error) { + input = in + return defaultCompleteASGLifecycleActionFunc(ctx, in, options...) + } + + infra.Reconcile() + }) + + It("sends the expected input values", func() { + Expect(input).ToNot(BeNil()) + + Expect(input.AutoScalingGroupName).ToNot(BeNil()) + Expect(*input.AutoScalingGroupName).To(Equal(autoScalingGroupName)) + + Expect(input.LifecycleActionResult).ToNot(BeNil()) + Expect(*input.LifecycleActionResult).To(Equal(lifecycleActionResult)) + + Expect(input.LifecycleHookName).ToNot(BeNil()) + Expect(*input.LifecycleHookName).To(Equal(lifecycleHookName)) + + Expect(input.LifecycleActionToken).ToNot(BeNil()) + Expect(*input.LifecycleActionToken).To(Equal(lifecycleActionToken)) + + Expect(input.InstanceId).ToNot(BeNil()) + Expect(*input.InstanceId).To(Equal(infra.InstanceIDs[1])) + }) + }) +}) diff --git a/test/reconciler/cordon_err.go b/test/reconciler/cordon_err.go new file mode 100644 index 00000000..256da8d4 --- /dev/null +++ b/test/reconciler/cordon_err.go @@ -0,0 +1,85 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "errors" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + v1 "k8s.io/api/core/v1" + kubectl "k8s.io/kubectl/pkg/drain" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("cordoning a node fails", func() { + const errMsg = "test error" + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + + infra.CordonFunc = func(_ *kubectl.Helper, _ *v1.Node, _ bool) error { + return errors.New(errMsg) + } + + result, err = infra.Reconcile() + }) + + It("does not requeue the request", func() { + Expect(result).To(BeZero()) + }) + + It("returns an error", func() { + Expect(err).To(MatchError(ContainSubstring(errMsg))) + }) + + It("does not cordon or drain any nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) +}) diff --git a/test/reconciler/cordon_node.go b/test/reconciler/cordon_node.go new file mode 100644 index 00000000..e3a998f1 --- /dev/null +++ b/test/reconciler/cordon_node.go @@ -0,0 +1,108 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v1 "k8s.io/api/core/v1" + kubectl "k8s.io/kubectl/pkg/drain" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("cordoning a node", func() { + const ( + force = true + gracePeriodSeconds = 31 + ignoreAllDaemonSets = true + deleteEmptyDirData = true + ) + var ( + helper *kubectl.Helper + timeout time.Duration + infra *mock.Infrastructure + ) + + BeforeEach(func() { + timeout = 42 * time.Second + + infra = mock.NewInfrastructure() + terminator, found := infra.Terminators[infra.TerminatorNamespaceName] + Expect(found).To(BeTrue()) + + terminator.Spec.Drain.DeleteEmptyDirData = deleteEmptyDirData + terminator.Spec.Drain.Force = force + terminator.Spec.Drain.GracePeriodSeconds = gracePeriodSeconds + terminator.Spec.Drain.IgnoreAllDaemonSets = ignoreAllDaemonSets + terminator.Spec.Drain.TimeoutSeconds = int(timeout.Seconds()) + + defaultCordonFunc := infra.CordonFunc + infra.CordonFunc = func(h *kubectl.Helper, node *v1.Node, desired bool) error { + helper = h + return defaultCordonFunc(h, node, desired) + } + + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + + infra.Reconcile() + }) + + It("sends the input values from the terminator", func() { + Expect(helper).ToNot(BeNil()) + + Expect(helper).To(SatisfyAll( + HaveField("DeleteEmptyDirData", Equal(deleteEmptyDirData)), + HaveField("Force", Equal(force)), + HaveField("GracePeriodSeconds", Equal(gracePeriodSeconds)), + HaveField("IgnoreAllDaemonSets", Equal(ignoreAllDaemonSets)), + HaveField("Timeout", Equal(timeout)), + )) + }) + + It("sends additional input values", func() { + Expect(helper).ToNot(BeNil()) + + Expect(helper).To(SatisfyAll( + HaveField("Client", Not(BeNil())), + HaveField("Ctx", Not(BeNil())), + HaveField("Out", Not(BeNil())), + HaveField("ErrOut", Not(BeNil())), + )) + }) + }) +}) diff --git a/test/reconciler/drain_err.go b/test/reconciler/drain_err.go new file mode 100644 index 00000000..e8fdd24b --- /dev/null +++ b/test/reconciler/drain_err.go @@ -0,0 +1,87 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "errors" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + kubectl "k8s.io/kubectl/pkg/drain" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("draining a node fails", func() { + const errMsg = "test error" + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + + infra.DrainFunc = func(_ *kubectl.Helper, _ string) error { + return errors.New(errMsg) + } + + result, err = infra.Reconcile() + }) + + It("does not requeue the request", func() { + Expect(result).To(BeZero()) + }) + + It("returns an error", func() { + Expect(err).To(MatchError(ContainSubstring(errMsg))) + }) + + It("cordons the target node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("does not drain the target node", func() { + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) +}) diff --git a/test/reconciler/drain_node.go b/test/reconciler/drain_node.go new file mode 100644 index 00000000..2e9b9607 --- /dev/null +++ b/test/reconciler/drain_node.go @@ -0,0 +1,107 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + kubectl "k8s.io/kubectl/pkg/drain" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("draining a node", func() { + const ( + force = true + gracePeriodSeconds = 31 + ignoreAllDaemonSets = true + deleteEmptyDirData = true + ) + var ( + infra *mock.Infrastructure + helper *kubectl.Helper + timeout time.Duration + ) + + BeforeEach(func() { + timeout = 42 * time.Second + + infra = mock.NewInfrastructure() + terminator, found := infra.Terminators[infra.TerminatorNamespaceName] + Expect(found).To(BeTrue()) + + terminator.Spec.Drain.DeleteEmptyDirData = deleteEmptyDirData + terminator.Spec.Drain.Force = force + terminator.Spec.Drain.GracePeriodSeconds = gracePeriodSeconds + terminator.Spec.Drain.IgnoreAllDaemonSets = ignoreAllDaemonSets + terminator.Spec.Drain.TimeoutSeconds = int(timeout.Seconds()) + + defaultDrainFunc := infra.DrainFunc + infra.DrainFunc = func(h *kubectl.Helper, nodeName string) error { + helper = h + return defaultDrainFunc(h, nodeName) + } + + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + + infra.Reconcile() + }) + + It("sends the input values from the terminator", func() { + Expect(helper).ToNot(BeNil()) + + Expect(helper).To(SatisfyAll( + HaveField("DeleteEmptyDirData", Equal(deleteEmptyDirData)), + HaveField("Force", Equal(force)), + HaveField("GracePeriodSeconds", Equal(gracePeriodSeconds)), + HaveField("IgnoreAllDaemonSets", Equal(ignoreAllDaemonSets)), + HaveField("Timeout", Equal(timeout)), + )) + }) + + It("sends additional values", func() { + Expect(helper).ToNot(BeNil()) + + Expect(helper).To(SatisfyAll( + HaveField("Client", Not(BeNil())), + HaveField("Ctx", Not(BeNil())), + HaveField("Out", Not(BeNil())), + HaveField("ErrOut", Not(BeNil())), + )) + }) + }) +}) diff --git a/test/reconciler/ec2_res_empty_dns.go b/test/reconciler/ec2_res_empty_dns.go new file mode 100644 index 00000000..7f1013d9 --- /dev/null +++ b/test/reconciler/ec2_res_empty_dns.go @@ -0,0 +1,91 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + awsrequest "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the EC2 reservation's instance's PrivateDnsName empty", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + + infra.DescribeEC2InstancesFunc = func(_ aws.Context, _ *ec2.DescribeInstancesInput, _ ...awsrequest.Option) (*ec2.DescribeInstancesOutput, error) { + return &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{ + { + Instances: []*ec2.Instance{ + {PrivateDnsName: aws.String("")}, + }, + }, + }, + }, nil + } + + result, err = infra.Reconcile() + }) + + It("does not requeue the request", func() { + Expect(result).To(BeZero()) + }) + + It("returns an error", func() { + Expect(err).To(HaveOccurred()) + }) + + It("does not cordon or drain any nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) +}) diff --git a/test/reconciler/ec2_res_no_dns.go b/test/reconciler/ec2_res_no_dns.go new file mode 100644 index 00000000..09511ae8 --- /dev/null +++ b/test/reconciler/ec2_res_no_dns.go @@ -0,0 +1,91 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + awsrequest "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the EC2 reservation's instance has no PrivateDnsName", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + + infra.DescribeEC2InstancesFunc = func(_ aws.Context, _ *ec2.DescribeInstancesInput, _ ...awsrequest.Option) (*ec2.DescribeInstancesOutput, error) { + return &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{ + { + Instances: []*ec2.Instance{ + {PrivateDnsName: nil}, + }, + }, + }, + }, nil + } + + result, err = infra.Reconcile() + }) + + It("does not requeue the request", func() { + Expect(result).To(BeZero()) + }) + + It("returns an error", func() { + Expect(err).To(HaveOccurred()) + }) + + It("does not cordon or drain any nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) +}) diff --git a/test/reconciler/ec2_res_no_inst.go b/test/reconciler/ec2_res_no_inst.go new file mode 100644 index 00000000..ed3b9f92 --- /dev/null +++ b/test/reconciler/ec2_res_no_inst.go @@ -0,0 +1,87 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + awsrequest "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the EC2 reservation contains no instances", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + + infra.DescribeEC2InstancesFunc = func(_ aws.Context, _ *ec2.DescribeInstancesInput, _ ...awsrequest.Option) (*ec2.DescribeInstancesOutput, error) { + return &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{ + {Instances: []*ec2.Instance{}}, + }, + }, nil + } + + result, err = infra.Reconcile() + }) + + It("does not requeue the request", func() { + Expect(result).To(BeZero()) + }) + + It("returns an error", func() { + Expect(err).To(HaveOccurred()) + }) + + It("does not cordon or drain any nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) +}) diff --git a/test/reconciler/empty_msg.go b/test/reconciler/empty_msg.go new file mode 100644 index 00000000..41e7f66e --- /dev/null +++ b/test/reconciler/empty_msg.go @@ -0,0 +1,64 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the SQS queue contains an empty message", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(""), + }) + + result, err = infra.Reconcile() + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("does not cordon or drain any nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) +}) diff --git a/test/reconciler/empty_sqs_queue.go b/test/reconciler/empty_sqs_queue.go new file mode 100644 index 00000000..9c854936 --- /dev/null +++ b/test/reconciler/empty_sqs_queue.go @@ -0,0 +1,50 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" +) + +var _ = Describe("Reconciliation", func() { + When("the SQS queue is empty", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + result, err = infra.Reconcile() + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("does not cordon or drain any nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + }) +}) diff --git a/test/reconciler/get_ec2_res_err.go b/test/reconciler/get_ec2_res_err.go new file mode 100644 index 00000000..c3d3e384 --- /dev/null +++ b/test/reconciler/get_ec2_res_err.go @@ -0,0 +1,85 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "errors" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + awsrequest "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("there is an error getting the EC2 reservation for the instance ID", func() { + const errMsg = "test error" + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + + infra.DescribeEC2InstancesFunc = func(_ aws.Context, _ *ec2.DescribeInstancesInput, _ ...awsrequest.Option) (*ec2.DescribeInstancesOutput, error) { + return nil, errors.New(errMsg) + } + + result, err = infra.Reconcile() + }) + + It("does not requeue the request", func() { + Expect(result).To(BeZero()) + }) + + It("returns an error", func() { + Expect(err).To(MatchError(ContainSubstring(errMsg))) + }) + + It("does not cordon or drain any nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) +}) diff --git a/test/reconciler/get_node_err.go b/test/reconciler/get_node_err.go new file mode 100644 index 00000000..ac1228ab --- /dev/null +++ b/test/reconciler/get_node_err.go @@ -0,0 +1,85 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "context" + "errors" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v1 "k8s.io/api/core/v1" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("there is an error getting the cluster node for a node name", func() { + const errMsg = "test error" + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + + defaultKubeGetFunc := infra.KubeGetFunc + infra.KubeGetFunc = func(ctx context.Context, key client.ObjectKey, object client.Object) error { + switch object.(type) { + case *v1.Node: + return errors.New(errMsg) + default: + return defaultKubeGetFunc(ctx, key, object) + } + } + + result, err = infra.Reconcile() + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("does not cordon or drain any nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + }) +}) diff --git a/test/reconciler/get_sqs_msg.go b/test/reconciler/get_sqs_msg.go new file mode 100644 index 00000000..b4e00896 --- /dev/null +++ b/test/reconciler/get_sqs_msg.go @@ -0,0 +1,85 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + awsrequest "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("getting messages from a terminator's SQS queue", func() { + const ( + maxNumberOfMessages = int64(10) + visibilityTimeoutSeconds = int64(20) + waitTimeSeconds = int64(20) + ) + var ( + attributeNames = []string{sqs.MessageSystemAttributeNameSentTimestamp} + messageAttributeNames = []string{sqs.QueueAttributeNameAll} + input *sqs.ReceiveMessageInput + infra *mock.Infrastructure + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + terminator, found := infra.Terminators[infra.TerminatorNamespaceName] + Expect(found).To(BeTrue()) + + terminator.Spec.SQS.QueueURL = mock.QueueURL + + defaultReceiveSQSMessageFunc := infra.ReceiveSQSMessageFunc + infra.ReceiveSQSMessageFunc = func(ctx aws.Context, in *sqs.ReceiveMessageInput, options ...awsrequest.Option) (*sqs.ReceiveMessageOutput, error) { + input = in + return defaultReceiveSQSMessageFunc(ctx, in, options...) + } + + infra.Reconcile() + }) + + It("sends the input values from the terminator", func() { + Expect(input).ToNot(BeNil()) + + for i, attrName := range input.AttributeNames { + Expect(attrName).ToNot(BeNil()) + Expect(*attrName).To(Equal(attributeNames[i])) + } + for i, attrName := range input.MessageAttributeNames { + Expect(attrName).ToNot(BeNil()) + Expect(*attrName).To(Equal(messageAttributeNames[i])) + } + + Expect(input.MaxNumberOfMessages).ToNot(BeNil()) + Expect(*input.MaxNumberOfMessages).To(Equal(maxNumberOfMessages)) + + Expect(input.QueueUrl).ToNot(BeNil()) + Expect(*input.QueueUrl).To(Equal(mock.QueueURL)) + + Expect(input.VisibilityTimeout).ToNot(BeNil()) + Expect(*input.VisibilityTimeout).To(Equal(visibilityTimeoutSeconds)) + + Expect(input.WaitTimeSeconds).ToNot(BeNil()) + Expect(*input.WaitTimeSeconds).To(Equal(waitTimeSeconds)) + }) + }) +}) diff --git a/test/reconciler/get_sqs_msg_err.go b/test/reconciler/get_sqs_msg_err.go new file mode 100644 index 00000000..80a546b8 --- /dev/null +++ b/test/reconciler/get_sqs_msg_err.go @@ -0,0 +1,60 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + awsrequest "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("there is an error getting SQS messages", func() { + const errMsg = "test error" + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ReceiveSQSMessageFunc = func(_ aws.Context, _ *sqs.ReceiveMessageInput, _ ...awsrequest.Option) (*sqs.ReceiveMessageOutput, error) { + return nil, errors.New(errMsg) + } + + result, err = infra.Reconcile() + }) + + It("does not requeue the request", func() { + Expect(result).To(BeZero()) + }) + + It("returns an error", func() { + Expect(err).To(MatchError(ContainSubstring(errMsg))) + }) + }) +}) diff --git a/test/reconciler/get_terminator_err.go b/test/reconciler/get_terminator_err.go new file mode 100644 index 00000000..6351d0e2 --- /dev/null +++ b/test/reconciler/get_terminator_err.go @@ -0,0 +1,66 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "context" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-node-termination-handler/api/v1alpha1" +) + +var _ = Describe("Reconciliation", func() { + When("there is an error getting the terminator", func() { + const errMsg = "test error" + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + defaultKubeGetFunc := infra.KubeGetFunc + infra.KubeGetFunc = func(ctx context.Context, key client.ObjectKey, object client.Object) error { + switch object.(type) { + case *v1alpha1.Terminator: + return errors.New(errMsg) + default: + return defaultKubeGetFunc(ctx, key, object) + } + } + + result, err = infra.Reconcile() + }) + + It("does not requeue the request", func() { + Expect(result).To(BeZero()) + }) + + It("returns an error", func() { + Expect(err).To(MatchError(ContainSubstring(errMsg))) + }) + }) +}) diff --git a/test/asgclient.go b/test/reconciler/mock/asgclient.go similarity index 98% rename from test/asgclient.go rename to test/reconciler/mock/asgclient.go index 246c2954..661018c2 100644 --- a/test/asgclient.go +++ b/test/reconciler/mock/asgclient.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package test +package mock import ( "github.com/aws/aws-sdk-go/aws" diff --git a/test/ec2client.go b/test/reconciler/mock/ec2client.go similarity index 98% rename from test/ec2client.go rename to test/reconciler/mock/ec2client.go index 58c6653c..8ce22faa 100644 --- a/test/ec2client.go +++ b/test/reconciler/mock/ec2client.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package test +package mock import ( "github.com/aws/aws-sdk-go/aws" diff --git a/test/reconciler/mock/infrastructure.go b/test/reconciler/mock/infrastructure.go new file mode 100644 index 00000000..f6d895cd --- /dev/null +++ b/test/reconciler/mock/infrastructure.go @@ -0,0 +1,398 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mock + +import ( + "context" + "fmt" + "io" + "net/http" + "reflect" + "time" + + . "github.com/onsi/gomega" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + v1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + kubectl "k8s.io/kubectl/pkg/drain" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/api/v1alpha1" + "github.com/aws/aws-node-termination-handler/pkg/event" + asgterminateeventv1 "github.com/aws/aws-node-termination-handler/pkg/event/asgterminate/v1" + asgterminateeventv2 "github.com/aws/aws-node-termination-handler/pkg/event/asgterminate/v2" + rebalancerecommendationeventv0 "github.com/aws/aws-node-termination-handler/pkg/event/rebalancerecommendation/v0" + scheduledchangeeventv1 "github.com/aws/aws-node-termination-handler/pkg/event/scheduledchange/v1" + spotinterruptioneventv1 "github.com/aws/aws-node-termination-handler/pkg/event/spotinterruption/v1" + statechangeeventv1 "github.com/aws/aws-node-termination-handler/pkg/event/statechange/v1" + "github.com/aws/aws-node-termination-handler/pkg/logging" + "github.com/aws/aws-node-termination-handler/pkg/node" + kubectlcordondrain "github.com/aws/aws-node-termination-handler/pkg/node/cordondrain/kubectl" + nodename "github.com/aws/aws-node-termination-handler/pkg/node/name" + "github.com/aws/aws-node-termination-handler/pkg/sqsmessage" + "github.com/aws/aws-node-termination-handler/pkg/terminator" + terminatoradapter "github.com/aws/aws-node-termination-handler/pkg/terminator/adapter" + "github.com/aws/aws-node-termination-handler/pkg/webhook" + + "github.com/aws/aws-sdk-go/aws" + awsrequest "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/sqs" +) + +type ( + Infrastructure struct { + // Input variables + // These variables are assigned default values during test setup but may be + // modified to represent different cluster states or AWS service responses. + + // Terminators currently in the cluster. + Terminators map[types.NamespacedName]*v1alpha1.Terminator + // Nodes currently in the cluster. + Nodes map[types.NamespacedName]*v1.Node + // Maps an EC2 instance id to an ASG lifecycle action state value. + ASGLifecycleActions map[EC2InstanceID]State + // Maps an EC2 instance id to the corresponding reservation for a node + // in the cluster. + EC2Reservations map[EC2InstanceID]*ec2.Reservation + // Maps a queue URL to a list of messages waiting to be fetched. + SQSQueues map[SQSQueueURL][]*sqs.Message + + // Output variables + // These variables may be modified during reconciliation and should be + // used to verify the resulting cluster state. + + // A lookup table for nodes that were cordoned. + CordonedNodes map[NodeName]bool + // A lookup table for nodes that were drained. + DrainedNodes map[NodeName]bool + + // Other variables + + // Names of all nodes currently in cluster. + NodeNames []NodeName + // Instance IDs for all nodes currently in cluster. + InstanceIDs []EC2InstanceID + // Requests sent to the configured webhook. + WebhookRequests []*http.Request + + // Name of default terminator. + TerminatorNamespaceName types.NamespacedName + // Default inputs to .Reconciler.Reconcile() + Ctx context.Context + Request reconcile.Request + + // The reconciler instance under test. + Reconciler terminator.Reconciler + + // Stubs + // Default implementations interract with the backing variables listed + // above. A test may put in place alternate behavior when needed. + CompleteASGLifecycleActionFunc CompleteASGLifecycleActionFunc + DescribeEC2InstancesFunc DescribeEC2InstancesFunc + KubeGetFunc KubeGetFunc + ReceiveSQSMessageFunc ReceiveSQSMessageFunc + DeleteSQSMessageFunc DeleteSQSMessageFunc + CordonFunc kubectlcordondrain.CordonFunc + DrainFunc kubectlcordondrain.DrainFunc + WebhookSendFunc webhook.HttpSendFunc + } + + EC2InstanceID = string + SQSQueueURL = string + NodeName = string + + State string +) + +const ( + QueueURL = "http://fake-queue.sqs.aws" + + StatePending = State("pending") + StateComplete = State("complete") +) + +// NewInfrastructure creates a new mock set of AWS and Kubernetes resources. +// +// Starting state: +// * One terminator configured to reach from a mock SQS queue +// * Zero nodes +// +// Tests should modify the resources as needed. +func NewInfrastructure() *Infrastructure { + infra := &Infrastructure{} + + // 1. Initialize variables. + + logger := zap.New(zapcore.NewCore( + zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()), + zapcore.AddSync(io.Discard), + zap.NewAtomicLevelAt(zap.DebugLevel), + )) + + infra.Ctx = logging.WithLogger(context.Background(), logger.Sugar()) + infra.TerminatorNamespaceName = types.NamespacedName{Namespace: "test", Name: "foo"} + infra.Request = reconcile.Request{NamespacedName: infra.TerminatorNamespaceName} + + infra.SQSQueues = map[SQSQueueURL][]*sqs.Message{QueueURL: {}} + infra.Terminators = map[types.NamespacedName]*v1alpha1.Terminator{ + // For convenience create a terminator that points to the sqs queue. + infra.TerminatorNamespaceName: { + Spec: v1alpha1.TerminatorSpec{ + SQS: v1alpha1.SQSSpec{ + QueueURL: QueueURL, + }, + }, + }, + } + infra.Nodes = map[types.NamespacedName]*v1.Node{} + infra.EC2Reservations = map[EC2InstanceID]*ec2.Reservation{} + infra.CordonedNodes = map[NodeName]bool{} + infra.DrainedNodes = map[NodeName]bool{} + + infra.NodeNames = []NodeName{} + infra.InstanceIDs = []EC2InstanceID{} + infra.ASGLifecycleActions = map[EC2InstanceID]State{} + + infra.WebhookRequests = []*http.Request{} + infra.WebhookSendFunc = func(req *http.Request) (*http.Response, error) { + infra.WebhookRequests = append(infra.WebhookRequests, req) + return &http.Response{StatusCode: 200}, nil + } + + // 2. Setup stub clients. + + infra.DescribeEC2InstancesFunc = func(ctx aws.Context, input *ec2.DescribeInstancesInput, _ ...awsrequest.Option) (*ec2.DescribeInstancesOutput, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + output := ec2.DescribeInstancesOutput{} + for _, instanceID := range input.InstanceIds { + if instanceID == nil { + continue + } + if reservation, found := infra.EC2Reservations[*instanceID]; found { + output.Reservations = append(output.Reservations, reservation) + } + } + return &output, nil + } + + ec2Client := EC2Client(func(ctx aws.Context, input *ec2.DescribeInstancesInput, options ...awsrequest.Option) (*ec2.DescribeInstancesOutput, error) { + return infra.DescribeEC2InstancesFunc(ctx, input, options...) + }) + + infra.CompleteASGLifecycleActionFunc = func(ctx aws.Context, input *autoscaling.CompleteLifecycleActionInput, _ ...awsrequest.Option) (*autoscaling.CompleteLifecycleActionOutput, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + Expect(input.InstanceId).ToNot(BeNil()) + if state, found := infra.ASGLifecycleActions[*input.InstanceId]; found { + Expect(state).ToNot(Equal(StateComplete)) + infra.ASGLifecycleActions[*input.InstanceId] = StateComplete + } + return &autoscaling.CompleteLifecycleActionOutput{}, nil + } + + asgClient := ASGClient(func(ctx aws.Context, input *autoscaling.CompleteLifecycleActionInput, options ...awsrequest.Option) (*autoscaling.CompleteLifecycleActionOutput, error) { + return infra.CompleteASGLifecycleActionFunc(ctx, input, options...) + }) + + infra.ReceiveSQSMessageFunc = func(ctx aws.Context, input *sqs.ReceiveMessageInput, options ...awsrequest.Option) (*sqs.ReceiveMessageOutput, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + Expect(input.QueueUrl).ToNot(BeNil()) + + messages, found := infra.SQSQueues[*input.QueueUrl] + Expect(found).To(BeTrue(), "SQS queue does not exist: %q", *input.QueueUrl) + + return &sqs.ReceiveMessageOutput{Messages: append([]*sqs.Message{}, messages...)}, nil + } + + infra.DeleteSQSMessageFunc = func(ctx aws.Context, input *sqs.DeleteMessageInput, options ...awsrequest.Option) (*sqs.DeleteMessageOutput, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + Expect(input.QueueUrl).ToNot(BeNil()) + + queue, found := infra.SQSQueues[*input.QueueUrl] + Expect(found).To(BeTrue(), "SQS queue does not exist: %q", *input.QueueUrl) + + updatedQueue := make([]*sqs.Message, 0, len(queue)) + for i, m := range queue { + if m.ReceiptHandle == input.ReceiptHandle { + updatedQueue = append(updatedQueue, queue[:i]...) + updatedQueue = append(updatedQueue, queue[i+1:]...) + break + } + } + infra.SQSQueues[*input.QueueUrl] = updatedQueue + + return &sqs.DeleteMessageOutput{}, nil + } + + sqsClient := SQSClient{ + ReceiveSQSMessageFunc: func(ctx aws.Context, input *sqs.ReceiveMessageInput, options ...awsrequest.Option) (*sqs.ReceiveMessageOutput, error) { + return infra.ReceiveSQSMessageFunc(ctx, input, options...) + }, + DeleteSQSMessageFunc: func(ctx aws.Context, input *sqs.DeleteMessageInput, options ...awsrequest.Option) (*sqs.DeleteMessageOutput, error) { + return infra.DeleteSQSMessageFunc(ctx, input, options...) + }, + } + + infra.KubeGetFunc = func(ctx context.Context, key client.ObjectKey, object client.Object) error { + if err := ctx.Err(); err != nil { + return err + } + + switch out := object.(type) { + case *v1.Node: + n, found := infra.Nodes[key] + if !found { + return k8serrors.NewNotFound(schema.GroupResource{}, key.String()) + } + *out = *n + + case *v1alpha1.Terminator: + t, found := infra.Terminators[key] + if !found { + return k8serrors.NewNotFound(schema.GroupResource{}, key.String()) + } + *out = *t + + default: + return fmt.Errorf("unknown type: %s", reflect.TypeOf(object).Name()) + } + return nil + } + + kubeClient := KubeClient(func(ctx context.Context, key client.ObjectKey, object client.Object) error { + return infra.KubeGetFunc(ctx, key, object) + }) + + infra.CordonFunc = func(_ *kubectl.Helper, node *v1.Node, desired bool) error { + if _, found := infra.Nodes[types.NamespacedName{Name: node.Name}]; !found { + return fmt.Errorf("node does not exist: %q", node.Name) + } + infra.CordonedNodes[node.Name] = true + return nil + } + + infra.DrainFunc = func(_ *kubectl.Helper, nodeName string) error { + if _, found := infra.Nodes[types.NamespacedName{Name: nodeName}]; !found { + return fmt.Errorf("node does not exist: %q", nodeName) + } + infra.DrainedNodes[nodeName] = true + return nil + } + + // 3. Construct the reconciler. + + eventParser := event.NewAggregatedParser( + asgterminateeventv1.Parser{ASGLifecycleActionCompleter: asgClient}, + asgterminateeventv2.Parser{ASGLifecycleActionCompleter: asgClient}, + rebalancerecommendationeventv0.Parser{}, + scheduledchangeeventv1.Parser{}, + spotinterruptioneventv1.Parser{}, + statechangeeventv1.Parser{}, + ) + + cordoner := kubectlcordondrain.CordonFunc(func(h *kubectl.Helper, n *v1.Node, d bool) error { + return infra.CordonFunc(h, n, d) + }) + + drainer := kubectlcordondrain.DrainFunc(func(h *kubectl.Helper, n string) error { + return infra.DrainFunc(h, n) + }) + + cordonDrainerBuilder := kubectlcordondrain.Builder{ + ClientSet: &kubernetes.Clientset{}, + Cordoner: cordoner, + Drainer: drainer, + } + + newHttpClientDoFunc := func(_ webhook.ProxyFunc) webhook.HttpSendFunc { + return infra.WebhookSendFunc + } + + infra.Reconciler = terminator.Reconciler{ + Name: "terminator", + RequeueInterval: time.Duration(10) * time.Second, + NodeGetterBuilder: terminatoradapter.NodeGetterBuilder{ + NodeGetter: node.Getter{KubeGetter: kubeClient}, + }, + NodeNameGetter: nodename.Getter{EC2InstancesDescriber: ec2Client}, + SQSClientBuilder: terminatoradapter.SQSMessageClientBuilder{ + SQSMessageClient: sqsmessage.Client{SQSClient: sqsClient}, + }, + SQSMessageParser: terminatoradapter.EventParser{Parser: eventParser}, + Getter: terminatoradapter.Getter{KubeGetter: kubeClient}, + CordonDrainerBuilder: terminatoradapter.CordonDrainerBuilder{ + Builder: cordonDrainerBuilder, + }, + WebhookClientBuilder: terminatoradapter.WebhookClientBuilder( + webhook.ClientBuilder(newHttpClientDoFunc).NewClient, + ), + } + + return infra +} + +// Change count of nodes in cluster. +func (m *Infrastructure) ResizeCluster(newNodeCount uint) { + for currNodeCount := uint(len(m.Nodes)); currNodeCount < newNodeCount; currNodeCount++ { + nodeName := fmt.Sprintf("node-%d", currNodeCount) + m.NodeNames = append(m.NodeNames, nodeName) + m.Nodes[types.NamespacedName{Name: nodeName}] = &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: nodeName}, + } + + instanceID := fmt.Sprintf("instance-%d", currNodeCount) + m.InstanceIDs = append(m.InstanceIDs, instanceID) + m.EC2Reservations[instanceID] = &ec2.Reservation{ + Instances: []*ec2.Instance{ + {PrivateDnsName: aws.String(nodeName)}, + }, + } + } + + m.NodeNames = m.NodeNames[:newNodeCount] + m.InstanceIDs = m.InstanceIDs[:newNodeCount] +} + +// Create an ASG lifecycle action state entry for an EC2 instance ID. +func (m *Infrastructure) CreatePendingASGLifecycleAction(instanceID EC2InstanceID) { + Expect(m.ASGLifecycleActions).ToNot(HaveKey(instanceID)) + m.ASGLifecycleActions[instanceID] = StatePending +} + +// Reconcile runs a reconciliation with the default context and request, and returns the +// result and any error that occured. +func (m *Infrastructure) Reconcile() (reconcile.Result, error) { + return m.Reconciler.Reconcile(m.Ctx, m.Request) +} diff --git a/test/kubeclient.go b/test/reconciler/mock/kubeclient.go similarity index 98% rename from test/kubeclient.go rename to test/reconciler/mock/kubeclient.go index f45418ff..f6209ff5 100644 --- a/test/kubeclient.go +++ b/test/reconciler/mock/kubeclient.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package test +package mock import ( "context" diff --git a/test/sqsclient.go b/test/reconciler/mock/sqsclient.go similarity index 99% rename from test/sqsclient.go rename to test/reconciler/mock/sqsclient.go index ee9c099a..b5fdbbfb 100644 --- a/test/sqsclient.go +++ b/test/reconciler/mock/sqsclient.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package test +package mock import ( "github.com/aws/aws-sdk-go/aws" diff --git a/test/reconciler/msg_no_body.go b/test/reconciler/msg_no_body.go new file mode 100644 index 00000000..70e0bc6c --- /dev/null +++ b/test/reconciler/msg_no_body.go @@ -0,0 +1,62 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the SQS queue contains a message with no body", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + }) + + result, err = infra.Reconcile() + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("does not cordon or drain any nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) +}) diff --git a/test/reconciler/msg_parse_error.go b/test/reconciler/msg_parse_error.go new file mode 100644 index 00000000..29caaa33 --- /dev/null +++ b/test/reconciler/msg_parse_error.go @@ -0,0 +1,66 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the SQS message cannot be parsed", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(`{ + "source": "test.suite", + "detail-type": "Mal-formed notification", + `), + }) + + result, err = infra.Reconcile() + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("does not cordon or drain any nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) +}) diff --git a/test/reconciler/multiple_msgs.go b/test/reconciler/multiple_msgs.go new file mode 100644 index 00000000..4fa443ad --- /dev/null +++ b/test/reconciler/multiple_msgs.go @@ -0,0 +1,197 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the SQS queue contains multiple messages", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(12) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], + &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.autoscaling", + "detail-type": "EC2 Instance-terminate Lifecycle Action", + "version": "1", + "detail": { + "EC2InstanceId": "%s", + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" + } + }`, infra.InstanceIDs[1])), + }, + &sqs.Message{ + ReceiptHandle: aws.String("msg-2"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.autoscaling", + "detail-type": "EC2 Instance-terminate Lifecycle Action", + "version": "2", + "detail": { + "EC2InstanceId": "%s", + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" + } + }`, infra.InstanceIDs[2])), + }, + &sqs.Message{ + ReceiptHandle: aws.String("msg-3"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Instance Rebalance Recommendation", + "version": "0", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[3])), + }, + &sqs.Message{ + ReceiptHandle: aws.String("msg-4"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.health", + "detail-type": "AWS Health Event", + "version": "1", + "detail": { + "service": "EC2", + "eventTypeCategory": "scheduledChange", + "affectedEntities": [ + {"entityValue": "%s"}, + {"entityValue": "%s"} + ] + } + }`, infra.InstanceIDs[4], infra.InstanceIDs[5])), + }, + &sqs.Message{ + ReceiptHandle: aws.String("msg-5"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[6])), + }, + &sqs.Message{ + ReceiptHandle: aws.String("msg-6"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Instance State-change Notification", + "version": "1", + "detail": { + "instance-id": "%s", + "state": "stopping" + } + }`, infra.InstanceIDs[7])), + }, + &sqs.Message{ + ReceiptHandle: aws.String("msg-7"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Instance State-change Notification", + "version": "1", + "detail": { + "instance-id": "%s", + "state": "stopped" + } + }`, infra.InstanceIDs[8])), + }, + &sqs.Message{ + ReceiptHandle: aws.String("msg-8"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Instance State-change Notification", + "version": "1", + "detail": { + "instance-id": "%s", + "state": "shutting-down" + } + }`, infra.InstanceIDs[9])), + }, + &sqs.Message{ + ReceiptHandle: aws.String("msg-9"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Instance State-change Notification", + "version": "1", + "detail": { + "instance-id": "%s", + "state": "terminated" + } + }`, infra.InstanceIDs[10])), + }, + ) + + result, err = infra.Reconcile() + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons and drains only the targeted nodes", func() { + Expect(infra.CordonedNodes).To(SatisfyAll( + HaveKey(infra.NodeNames[1]), + HaveKey(infra.NodeNames[2]), + HaveKey(infra.NodeNames[3]), + HaveKey(infra.NodeNames[4]), + HaveKey(infra.NodeNames[5]), + HaveKey(infra.NodeNames[6]), + HaveKey(infra.NodeNames[7]), + HaveKey(infra.NodeNames[8]), + HaveKey(infra.NodeNames[9]), + HaveKey(infra.NodeNames[10]), + HaveLen(10), + )) + Expect(infra.DrainedNodes).To(SatisfyAll( + HaveKey(infra.NodeNames[1]), + HaveKey(infra.NodeNames[2]), + HaveKey(infra.NodeNames[3]), + HaveKey(infra.NodeNames[4]), + HaveKey(infra.NodeNames[5]), + HaveKey(infra.NodeNames[6]), + HaveKey(infra.NodeNames[7]), + HaveKey(infra.NodeNames[8]), + HaveKey(infra.NodeNames[9]), + HaveKey(infra.NodeNames[10]), + HaveLen(10), + )) + }) + + It("deletes the messages from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) +}) diff --git a/test/reconciler/no_ec2_res.go b/test/reconciler/no_ec2_res.go new file mode 100644 index 00000000..f105f2b5 --- /dev/null +++ b/test/reconciler/no_ec2_res.go @@ -0,0 +1,85 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + awsrequest "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("there is no EC2 reservation for the instance ID", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + + infra.DescribeEC2InstancesFunc = func(_ aws.Context, _ *ec2.DescribeInstancesInput, _ ...awsrequest.Option) (*ec2.DescribeInstancesOutput, error) { + return &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{}, + }, nil + } + + result, err = infra.Reconcile() + }) + + It("does not requeue the request", func() { + Expect(result).To(BeZero()) + }) + + It("returns an error", func() { + Expect(err).To(HaveOccurred()) + }) + + It("does not cordon or drain any nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) +}) diff --git a/test/reconciler/rebalance_rec.go b/test/reconciler/rebalance_rec.go new file mode 100644 index 00000000..15375e32 --- /dev/null +++ b/test/reconciler/rebalance_rec.go @@ -0,0 +1,72 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the SQS queue contains a Rebalance Recommendation Notification", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Instance Rebalance Recommendation", + "version": "0", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + + result, err = infra.Reconcile() + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons and drains only the targeted node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + Expect(infra.DrainedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) +}) diff --git a/test/reconciler/scheduled_change.go b/test/reconciler/scheduled_change.go new file mode 100644 index 00000000..f6190a91 --- /dev/null +++ b/test/reconciler/scheduled_change.go @@ -0,0 +1,168 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the SQS queue contains a Scheduled Change Notification", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + }) + + JustBeforeEach(func() { + result, err = infra.Reconcile() + }) + + When("the service is EC2 and the event type category is scheduled change", func() { + BeforeEach(func() { + infra.ResizeCluster(4) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.health", + "detail-type": "AWS Health Event", + "version": "1", + "detail": { + "service": "EC2", + "eventTypeCategory": "scheduledChange", + "affectedEntities": [ + {"entityValue": "%s"}, + {"entityValue": "%s"} + ] + } + }`, infra.InstanceIDs[1], infra.InstanceIDs[2])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons and drains only the targeted nodes", func() { + Expect(infra.CordonedNodes).To( + SatisfyAll( + HaveKey(infra.NodeNames[1]), + HaveKey(infra.NodeNames[2]), + HaveLen(2), + ), + ) + Expect(infra.DrainedNodes).To( + SatisfyAll( + HaveKey(infra.NodeNames[1]), + HaveKey(infra.NodeNames[2]), + HaveLen(2), + ), + ) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("the service is not EC2", func() { + BeforeEach(func() { + infra.ResizeCluster(4) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.health", + "detail-type": "AWS Health Event", + "version": "1", + "detail": { + "service": "INVALID", + "eventTypeCategory": "scheduledChange", + "affectedEntities": [ + {"entityValue": "%s"}, + {"entityValue": "%s"} + ] + } + }`, infra.InstanceIDs[1], infra.InstanceIDs[2])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons and drains only the targeted nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) + + When("the event type category is not scheduled change", func() { + BeforeEach(func() { + infra.ResizeCluster(4) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.health", + "detail-type": "AWS Health Event", + "version": "1", + "detail": { + "service": "EC2", + "eventTypeCategory": "invalid", + "affectedEntities": [ + {"entityValue": "%s"}, + {"entityValue": "%s"} + ] + } + }`, infra.InstanceIDs[1], infra.InstanceIDs[2])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons and drains only the targeted nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) + }) +}) diff --git a/test/reconciler/spot_itn.go b/test/reconciler/spot_itn.go new file mode 100644 index 00000000..761c8c60 --- /dev/null +++ b/test/reconciler/spot_itn.go @@ -0,0 +1,72 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the SQS queue contains a Spot Interruption Notification", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + + result, err = infra.Reconcile() + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons and drains only the targeted node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + Expect(infra.DrainedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) +}) diff --git a/test/reconciler/sqs_msg_delete_err.go b/test/reconciler/sqs_msg_delete_err.go new file mode 100644 index 00000000..f3ec7a17 --- /dev/null +++ b/test/reconciler/sqs_msg_delete_err.go @@ -0,0 +1,76 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "context" + "errors" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + awsrequest "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("there is an error deleting an SQS message", func() { + const errMsg = "test error" + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + + infra.DeleteSQSMessageFunc = func(_ context.Context, _ *sqs.DeleteMessageInput, _ ...awsrequest.Option) (*sqs.DeleteMessageOutput, error) { + return nil, errors.New(errMsg) + } + + result, err = infra.Reconcile() + }) + + It("does not requeue the request", func() { + Expect(result).To(BeZero()) + }) + + It("returns an error", func() { + Expect(err).To(MatchError(ContainSubstring(errMsg))) + }) + }) +}) diff --git a/test/reconciler/state_change.go b/test/reconciler/state_change.go new file mode 100644 index 00000000..e2e82c9a --- /dev/null +++ b/test/reconciler/state_change.go @@ -0,0 +1,208 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the SQS queue contains a State Change Notification", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + }) + + JustBeforeEach(func() { + result, err = infra.Reconcile() + }) + + When("the state is stopping", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Instance State-change Notification", + "version": "1", + "detail": { + "instance-id": "%s", + "state": "stopping" + } + }`, infra.InstanceIDs[1])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons and drains only the targeted nodes", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + Expect(infra.DrainedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("the state is stopped", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Instance State-change Notification", + "version": "1", + "detail": { + "instance-id": "%s", + "state": "stopped" + } + }`, infra.InstanceIDs[1])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons and drains only the targeted node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + Expect(infra.DrainedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("the state is shutting-down", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Instance State-change Notification", + "version": "1", + "detail": { + "instance-id": "%s", + "state": "shutting-down" + } + }`, infra.InstanceIDs[1])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons and drains only the targeted node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + Expect(infra.DrainedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("the state is terminated", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Instance State-change Notification", + "version": "1", + "detail": { + "instance-id": "%s", + "state": "terminated" + } + }`, infra.InstanceIDs[1])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons and drains only the targeted node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + Expect(infra.DrainedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("the state is not recognized", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Instance State-change Notification", + "version": "1", + "detail": { + "instance-id": "%s", + "state": "invalid" + } + }`, infra.InstanceIDs[1])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons and drains only the targeted node", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) + }) +}) diff --git a/test/app_integ_suite_test.go b/test/reconciler/suite_test.go similarity index 97% rename from test/app_integ_suite_test.go rename to test/reconciler/suite_test.go index f215c7d4..7ddd2323 100644 --- a/test/app_integ_suite_test.go +++ b/test/reconciler/suite_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package test +package reconciler import ( "testing" diff --git a/test/reconciler/terminator_evt_cfg.go b/test/reconciler/terminator_evt_cfg.go new file mode 100644 index 00000000..994476fe --- /dev/null +++ b/test/reconciler/terminator_evt_cfg.go @@ -0,0 +1,546 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the terminator has event configuration", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + }) + + JustBeforeEach(func() { + result, err = infra.Reconcile() + }) + + When("Cordon on ASG Termination v1", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + terminator := infra.Terminators[infra.TerminatorNamespaceName] + terminator.Spec.Events.AutoScalingTermination = "Cordon" + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.autoscaling", + "detail-type": "EC2 Instance-terminate Lifecycle Action", + "version": "1", + "detail": { + "EC2InstanceId": "%s", + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" + } + }`, infra.InstanceIDs[1])), + }) + + infra.CreatePendingASGLifecycleAction(infra.InstanceIDs[1]) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons the targeted node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("does not drain the targeted node", func() { + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("completes the ASG lifecycle action", func() { + Expect(infra.ASGLifecycleActions).To( + SatisfyAll( + HaveKeyWithValue(infra.InstanceIDs[1], Equal(mock.StateComplete)), + HaveLen(1), + ), + ) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("\"No Action\" on ASG Termination v1", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + terminator := infra.Terminators[infra.TerminatorNamespaceName] + terminator.Spec.Events.AutoScalingTermination = "NoAction" + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.autoscaling", + "detail-type": "EC2 Instance-terminate Lifecycle Action", + "version": "1", + "detail": { + "EC2InstanceId": "%s", + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" + } + }`, infra.InstanceIDs[1])), + }) + + infra.CreatePendingASGLifecycleAction(infra.InstanceIDs[1]) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("does not cordon or drain the targeted node", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("completes the ASG lifecycle action", func() { + Expect(infra.ASGLifecycleActions).To( + SatisfyAll( + HaveKeyWithValue(infra.InstanceIDs[1], Equal(mock.StateComplete)), + HaveLen(1), + ), + ) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("Cordon on ASG Termination v2", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + terminator := infra.Terminators[infra.TerminatorNamespaceName] + terminator.Spec.Events.AutoScalingTermination = "Cordon" + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.autoscaling", + "detail-type": "EC2 Instance-terminate Lifecycle Action", + "version": "2", + "detail": { + "EC2InstanceId": "%s", + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" + } + }`, infra.InstanceIDs[1])), + }) + + infra.CreatePendingASGLifecycleAction(infra.InstanceIDs[1]) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons the targeted node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("does not drain the targeted node", func() { + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("completes the ASG lifecycle action", func() { + Expect(infra.ASGLifecycleActions).To( + SatisfyAll( + HaveKeyWithValue(infra.InstanceIDs[1], Equal(mock.StateComplete)), + HaveLen(1), + ), + ) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("\"No Action\" on ASG Termination v2", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + terminator := infra.Terminators[infra.TerminatorNamespaceName] + terminator.Spec.Events.AutoScalingTermination = "NoAction" + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.autoscaling", + "detail-type": "EC2 Instance-terminate Lifecycle Action", + "version": "2", + "detail": { + "EC2InstanceId": "%s", + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" + } + }`, infra.InstanceIDs[1])), + }) + + infra.CreatePendingASGLifecycleAction(infra.InstanceIDs[1]) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("does not cordon or drain the targeted node", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("completes the ASG lifecycle action", func() { + Expect(infra.ASGLifecycleActions).To( + SatisfyAll( + HaveKeyWithValue(infra.InstanceIDs[1], Equal(mock.StateComplete)), + HaveLen(1), + ), + ) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("Cordon on Rebalance Recommendation", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + terminator := infra.Terminators[infra.TerminatorNamespaceName] + terminator.Spec.Events.RebalanceRecommendation = "Cordon" + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Instance Rebalance Recommendation", + "version": "0", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons the targeted node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("does not drain the targeted node", func() { + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("\"No Action\" on Rebalance Recommendation", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + terminator := infra.Terminators[infra.TerminatorNamespaceName] + terminator.Spec.Events.RebalanceRecommendation = "NoAction" + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Instance Rebalance Recommendation", + "version": "0", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("does not cordon or drain the targeted node", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("Cordon on Scheduled Change", func() { + BeforeEach(func() { + infra.ResizeCluster(4) + + terminator := infra.Terminators[infra.TerminatorNamespaceName] + terminator.Spec.Events.ScheduledChange = "Cordon" + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.health", + "detail-type": "AWS Health Event", + "version": "1", + "detail": { + "service": "EC2", + "eventTypeCategory": "scheduledChange", + "affectedEntities": [ + {"entityValue": "%s"}, + {"entityValue": "%s"} + ] + } + }`, infra.InstanceIDs[1], infra.InstanceIDs[2])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons the targeted node", func() { + Expect(infra.CordonedNodes).To( + SatisfyAll( + HaveKey(infra.NodeNames[1]), + HaveKey(infra.NodeNames[2]), + HaveLen(2), + ), + ) + }) + + It("does not drain the targeted node", func() { + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("\"No Action\" on Scheduled Change", func() { + BeforeEach(func() { + infra.ResizeCluster(4) + + terminator := infra.Terminators[infra.TerminatorNamespaceName] + terminator.Spec.Events.ScheduledChange = "NoAction" + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.health", + "detail-type": "AWS Health Event", + "version": "1", + "detail": { + "service": "EC2", + "eventTypeCategory": "scheduledChange", + "affectedEntities": [ + {"entityValue": "%s"}, + {"entityValue": "%s"} + ] + } + }`, infra.InstanceIDs[1], infra.InstanceIDs[2])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("does not cordon or drain the targeted node", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("Cordon on Spot Interruption", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + terminator := infra.Terminators[infra.TerminatorNamespaceName] + terminator.Spec.Events.SpotInterruption = "Cordon" + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons the targeted node", func() { + Expect(infra.CordonedNodes).To( + SatisfyAll( + HaveKey(infra.NodeNames[1]), + HaveLen(1), + ), + ) + }) + + It("does not drain the targeted node", func() { + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("\"No Action\" on Spot Interruption", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + terminator := infra.Terminators[infra.TerminatorNamespaceName] + terminator.Spec.Events.SpotInterruption = "NoAction" + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("does not cordon or drain the targeted node", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("Cordon on State Change", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + terminator := infra.Terminators[infra.TerminatorNamespaceName] + terminator.Spec.Events.StateChange = "Cordon" + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Instance State-change Notification", + "version": "1", + "detail": { + "instance-id": "%s", + "state": "stopping" + } + }`, infra.InstanceIDs[1])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons the targeted node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("does not drain the targeted node", func() { + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("\"No Action\" on State Change", func() { + BeforeEach(func() { + infra.ResizeCluster(3) + + terminator := infra.Terminators[infra.TerminatorNamespaceName] + terminator.Spec.Events.StateChange = "NoAction" + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Instance State-change Notification", + "version": "1", + "detail": { + "instance-id": "%s", + "state": "stopping" + } + }`, infra.InstanceIDs[1])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("does not cordon or drain the targeted node", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + }) +}) diff --git a/test/reconciler/terminator_node_selector.go b/test/reconciler/terminator_node_selector.go new file mode 100644 index 00000000..d9ba480a --- /dev/null +++ b/test/reconciler/terminator_node_selector.go @@ -0,0 +1,135 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/types" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the terminator has a node label selector", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + }) + + JustBeforeEach(func() { + result, err = infra.Reconcile() + }) + + When("the label selector matches the target node", func() { + const labelName = "a-test-label" + const labelValue = "test-label-value" + + BeforeEach(func() { + infra.ResizeCluster(3) + + targetedNode, found := infra.Nodes[types.NamespacedName{Name: infra.NodeNames[1]}] + Expect(found).To(BeTrue()) + + targetedNode.Labels = map[string]string{labelName: labelValue} + + terminator, found := infra.Terminators[infra.TerminatorNamespaceName] + Expect(found).To(BeTrue()) + + terminator.Spec.MatchLabels = client.MatchingLabels{labelName: labelValue} + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("cordons and drains only the targeted node", func() { + Expect(infra.CordonedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + Expect(infra.DrainedNodes).To(SatisfyAll(HaveKey(infra.NodeNames[1]), HaveLen(1))) + }) + + It("deletes the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(BeEmpty()) + }) + }) + + When("the label selector does not match the target node", func() { + const labelName = "a-test-label" + const labelValue = "test-label-value" + + BeforeEach(func() { + infra.ResizeCluster(3) + + terminator, found := infra.Terminators[infra.TerminatorNamespaceName] + Expect(found).To(BeTrue()) + + terminator.Spec.MatchLabels = client.MatchingLabels{labelName: labelValue} + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, infra.InstanceIDs[1])), + }) + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("does not cordon or drain any nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) + }) +}) diff --git a/test/reconciler/terminator_not_found.go b/test/reconciler/terminator_not_found.go new file mode 100644 index 00000000..b76a40b3 --- /dev/null +++ b/test/reconciler/terminator_not_found.go @@ -0,0 +1,50 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" +) + +var _ = Describe("Reconciliation", func() { + When("the terminator cannot be found", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + delete(infra.Terminators, infra.TerminatorNamespaceName) + result, err = infra.Reconcile() + }) + + It("returns success but does not requeue the request", func() { + Expect(result, err).To(BeZero()) + }) + + It("does not cordon or drain any nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + }) +}) diff --git a/test/reconciler/terminator_webhook_cfg.go b/test/reconciler/terminator_webhook_cfg.go new file mode 100644 index 00000000..b89a0597 --- /dev/null +++ b/test/reconciler/terminator_webhook_cfg.go @@ -0,0 +1,252 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/api/v1alpha1" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the terminator has webhook configuration", func() { + const webhookURL = "http://webhook.example.aws" + var ( + webhookHeaders = []v1alpha1.HeaderSpec{{Name: "Content-Type", Value: "application/json"}} + webhookTemplate = fmt.Sprintf( + `EventID={{ .EventID }}, Kind={{ .Kind }}, InstanceID={{ .InstanceID }}, NodeName={{ .NodeName }}, StartTime={{ (.StartTime.Format "%s") }}`, + time.RFC3339, + ) + + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + }) + + JustBeforeEach(func() { + result, err = infra.Reconcile() + }) + + When("the reconciliation takes no action", func() { + BeforeEach(func() { + terminator := infra.Terminators[infra.TerminatorNamespaceName] + terminator.Spec.Webhook.URL = webhookURL + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("does not send any webhook requests", func() { + Expect(infra.WebhookRequests).To(BeEmpty()) + }) + }) + + When("the reconciliation acts on a node", func() { + const msgID = "id-123" + msgTime := time.Now().Format(time.RFC3339) + + BeforeEach(func() { + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "id": "%s", + "time": "%s", + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, msgID, msgTime, infra.InstanceIDs[1])), + }) + + terminator := infra.Terminators[infra.TerminatorNamespaceName] + terminator.Spec.Webhook.URL = webhookURL + terminator.Spec.Webhook.Headers = webhookHeaders + terminator.Spec.Webhook.Template = webhookTemplate + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("sends a webhook notification", func() { + Expect(infra.WebhookRequests).To(HaveLen(1)) + Expect(infra.WebhookRequests[0].Method).To(Equal(http.MethodPost)) + Expect(infra.WebhookRequests[0].URL.String()).To(Equal(webhookURL)) + Expect(infra.WebhookRequests[0].Header).To(SatisfyAll( + HaveLen(1), + HaveKeyWithValue("Content-Type", SatisfyAll( + HaveLen(1), + ContainElement("application/json"), + )))) + + Expect(ReadAll(infra.WebhookRequests[0].Body)).To(Equal(fmt.Sprintf( + "EventID=%s, Kind=spotInterruption, InstanceID=%s, NodeName=%s, StartTime=%s", + msgID, infra.InstanceIDs[1], infra.NodeNames[1], msgTime, + ))) + }) + }) + + When("the reconciliation acts on multiple nodes", func() { + msgIDs := []string{"msg-1", "msg-2", "msg-2"} + msgBaseTime := time.Now() + msgTimes := []string{ + msgBaseTime.Add(-1 * time.Minute).Format(time.RFC3339), + msgBaseTime.Format(time.RFC3339), + msgBaseTime.Format(time.RFC3339), + } + kinds := []string{"spotInterruption", "scheduledChange", "scheduledChange"} + + BeforeEach(func() { + infra.ResizeCluster(5) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], + &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "id": "%s", + "time": "%s", + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, msgIDs[0], msgTimes[0], infra.InstanceIDs[1])), + }, + &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "id": "%s", + "time": "%s", + "source": "aws.health", + "detail-type": "AWS Health Event", + "version": "1", + "detail": { + "service": "EC2", + "eventTypeCategory": "scheduledChange", + "affectedEntities": [ + {"entityValue": "%s"}, + {"entityValue": "%s"} + ] + } + }`, msgIDs[1], msgTimes[1], infra.InstanceIDs[2], infra.InstanceIDs[3])), + }, + ) + + terminator := infra.Terminators[infra.TerminatorNamespaceName] + terminator.Spec.Webhook.URL = webhookURL + terminator.Spec.Webhook.Headers = webhookHeaders + terminator.Spec.Webhook.Template = webhookTemplate + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("sends a webhook notification", func() { + Expect(infra.WebhookRequests).To(HaveLen(3)) + + for i := 0; i < 3; i++ { + Expect(infra.WebhookRequests[i].Method).To(Equal(http.MethodPost), "request #%d", i) + Expect(infra.WebhookRequests[i].URL.String()).To(Equal(webhookURL), "request #%d", i) + Expect(infra.WebhookRequests[i].Header).To(SatisfyAll( + HaveLen(1), + HaveKeyWithValue("Content-Type", SatisfyAll( + HaveLen(1), + ContainElement("application/json"), + ))), + "request #%d", i) + + Expect(ReadAll(infra.WebhookRequests[i].Body)).To(Equal(fmt.Sprintf( + "EventID=%s, Kind=%s, InstanceID=%s, NodeName=%s, StartTime=%s", + msgIDs[i], kinds[i], infra.InstanceIDs[i+1], infra.NodeNames[i+1], msgTimes[i], + )), + "request #%d", i, + ) + } + }) + }) + + When("there is an error sending the request", func() { + const msgID = "id-123" + msgTime := time.Now().Format(time.RFC3339) + + BeforeEach(func() { + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(fmt.Sprintf(`{ + "id": "%s", + "time": "%s", + "source": "aws.ec2", + "detail-type": "EC2 Spot Instance Interruption Warning", + "version": "1", + "detail": { + "instance-id": "%s" + } + }`, msgID, msgTime, infra.InstanceIDs[1])), + }) + + terminator := infra.Terminators[infra.TerminatorNamespaceName] + terminator.Spec.Webhook.URL = webhookURL + terminator.Spec.Webhook.Headers = webhookHeaders + terminator.Spec.Webhook.Template = webhookTemplate + + infra.WebhookSendFunc = func(_ *http.Request) (*http.Response, error) { + return nil, errors.New("test error") + } + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + }) + }) +}) + +func ReadAll(r io.Reader) (string, error) { + bs, err := ioutil.ReadAll(r) + if err != nil { + return "", err + } + return string(bs), nil +} diff --git a/test/reconciler/unrecognized_msg.go b/test/reconciler/unrecognized_msg.go new file mode 100644 index 00000000..5f503611 --- /dev/null +++ b/test/reconciler/unrecognized_msg.go @@ -0,0 +1,69 @@ +/* +Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconciler + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/aws-node-termination-handler/test/reconciler/mock" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" +) + +var _ = Describe("Reconciliation", func() { + When("the SQS queue contains an unrecognized message", func() { + var ( + infra *mock.Infrastructure + result reconcile.Result + err error + ) + + BeforeEach(func() { + infra = mock.NewInfrastructure() + infra.ResizeCluster(3) + + infra.SQSQueues[mock.QueueURL] = append(infra.SQSQueues[mock.QueueURL], &sqs.Message{ + ReceiptHandle: aws.String("msg-1"), + Body: aws.String(`{ + "source": "test.suite", + "detail-type": "Not a real notification", + "version": "1", + "detail": {} + }`), + }) + + result, err = infra.Reconcile() + }) + + It("returns success and requeues the request with the reconciler's configured interval", func() { + Expect(result, err).To(HaveField("RequeueAfter", Equal(infra.Reconciler.RequeueInterval))) + }) + + It("does not cordon or drain any nodes", func() { + Expect(infra.CordonedNodes).To(BeEmpty()) + Expect(infra.DrainedNodes).To(BeEmpty()) + }) + + It("does not delete the message from the SQS queue", func() { + Expect(infra.SQSQueues[mock.QueueURL]).To(HaveLen(1)) + }) + }) +}) diff --git a/test/reconciliation_test.go b/test/reconciliation_test.go deleted file mode 100644 index 97406d87..00000000 --- a/test/reconciliation_test.go +++ /dev/null @@ -1,2735 +0,0 @@ -/* -Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package test - -import ( - "context" - "errors" - "fmt" - "io" - "io/ioutil" - "net/http" - "reflect" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - - v1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" - kubectl "k8s.io/kubectl/pkg/drain" - - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - "github.com/aws/aws-node-termination-handler/api/v1alpha1" - "github.com/aws/aws-node-termination-handler/pkg/event" - asgterminateeventv1 "github.com/aws/aws-node-termination-handler/pkg/event/asgterminate/v1" - asgterminateeventv2 "github.com/aws/aws-node-termination-handler/pkg/event/asgterminate/v2" - rebalancerecommendationeventv0 "github.com/aws/aws-node-termination-handler/pkg/event/rebalancerecommendation/v0" - scheduledchangeeventv1 "github.com/aws/aws-node-termination-handler/pkg/event/scheduledchange/v1" - spotinterruptioneventv1 "github.com/aws/aws-node-termination-handler/pkg/event/spotinterruption/v1" - statechangeeventv1 "github.com/aws/aws-node-termination-handler/pkg/event/statechange/v1" - "github.com/aws/aws-node-termination-handler/pkg/logging" - "github.com/aws/aws-node-termination-handler/pkg/node" - kubectlcordondrain "github.com/aws/aws-node-termination-handler/pkg/node/cordondrain/kubectl" - nodename "github.com/aws/aws-node-termination-handler/pkg/node/name" - "github.com/aws/aws-node-termination-handler/pkg/sqsmessage" - "github.com/aws/aws-node-termination-handler/pkg/terminator" - terminatoradapter "github.com/aws/aws-node-termination-handler/pkg/terminator/adapter" - "github.com/aws/aws-node-termination-handler/pkg/webhook" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - awsrequest "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/autoscaling" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/sqs" -) - -type ( - EC2InstanceID = string - SQSQueueURL = string - NodeName = string - - State string -) - -const ( - StatePending = State("pending") - StateComplete = State("complete") -) - -var _ = Describe("Reconciliation", func() { - const ( - errMsg = "test error" - queueURL = "http://fake-queue.sqs.aws" - ) - - var ( - // Input variables - // These variables are assigned default values during test setup but may be - // modified to represent different cluster states or AWS service responses. - - // Terminators currently in the cluster. - terminators map[types.NamespacedName]*v1alpha1.Terminator - // Nodes currently in the cluster. - nodes map[types.NamespacedName]*v1.Node - // Maps an EC2 instance id to an ASG lifecycle action state value. - asgLifecycleActions map[EC2InstanceID]State - // Maps an EC2 instance id to the corresponding reservation for a node - // in the cluster. - ec2Reservations map[EC2InstanceID]*ec2.Reservation - // Maps a queue URL to a list of messages waiting to be fetched. - sqsQueues map[SQSQueueURL][]*sqs.Message - - // Output variables - // These variables may be modified during reconciliation and should be - // used to verify the resulting cluster state. - - // A lookup table for nodes that were cordoned. - cordonedNodes map[NodeName]bool - // A lookup table for nodes that were drained. - drainedNodes map[NodeName]bool - // Result information returned by the reconciler. - result reconcile.Result - // Error information returned by the reconciler. - err error - - // Other variables - - // Names of all nodes currently in cluster. - nodeNames []NodeName - // Instance IDs for all nodes currently in cluster. - instanceIDs []EC2InstanceID - // Change count of nodes in cluster. - resizeCluster func(nodeCount uint) - // Create an ASG lifecycle action state entry for an EC2 instance ID. - createPendingASGLifecycleAction func(EC2InstanceID) - // Requests sent to the configured webhook. - webhookRequests []*http.Request - - // Name of default terminator. - terminatorNamespaceName types.NamespacedName - // Inputs to .Reconcile() - ctx context.Context - request reconcile.Request - - // The reconciler instance under test. - reconciler terminator.Reconciler - - // Stubs - // Default implementations interract with the backing variables listed - // above. A test may put in place alternate behavior when needed. - completeASGLifecycleActionFunc CompleteASGLifecycleActionFunc - describeEC2InstancesFunc DescribeEC2InstancesFunc - kubeGetFunc KubeGetFunc - receiveSQSMessageFunc ReceiveSQSMessageFunc - deleteSQSMessageFunc DeleteSQSMessageFunc - cordonFunc kubectlcordondrain.CordonFunc - drainFunc kubectlcordondrain.DrainFunc - webhookSendFunc webhook.HttpSendFunc - ) - - When("the SQS queue is empty", func() { - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("does not cordon or drain any nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - }) - - When("the SQS queue contains an ASG Lifecycle Notification v1", func() { - When("the lifecycle transition is termination", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.autoscaling", - "detail-type": "EC2 Instance-terminate Lifecycle Action", - "version": "1", - "detail": { - "EC2InstanceId": "%s", - "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" - } - }`, instanceIDs[1])), - }) - - createPendingASGLifecycleAction(instanceIDs[1]) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons and drains only the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - Expect(drainedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("completes the ASG lifecycle action", func() { - Expect(asgLifecycleActions).To(And(HaveKeyWithValue(instanceIDs[1], Equal(StateComplete)), HaveLen(1))) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("the lifecycle transition is not termination", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.autoscaling", - "detail-type": "EC2 Instance-terminate Lifecycle Action", - "version": "1", - "detail": { - "EC2InstanceId": "%s", - "LifecycleTransition": "test:INVALID" - } - }`, instanceIDs[1])), - }) - - createPendingASGLifecycleAction(instanceIDs[1]) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("does not cordon or drain any nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("does not complete the ASG lifecycle action", func() { - Expect(asgLifecycleActions).To(And(HaveKeyWithValue(instanceIDs[1], Equal(StatePending)), HaveLen(1))) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - }) - - When("the SQS queue contains an ASG Lifecycle Notification v2", func() { - When("the lifecycle transition is termination", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.autoscaling", - "detail-type": "EC2 Instance-terminate Lifecycle Action", - "version": "2", - "detail": { - "EC2InstanceId": "%s", - "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" - } - }`, instanceIDs[1])), - }) - - createPendingASGLifecycleAction(instanceIDs[1]) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons and drains only the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - Expect(drainedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("completes the ASG lifecycle action", func() { - Expect(asgLifecycleActions).To(And(HaveKeyWithValue(instanceIDs[1], Equal(StateComplete)), HaveLen(1))) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("the lifecycle transition is not termination", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.autoscaling", - "detail-type": "EC2 Instance-terminate Lifecycle Action", - "version": "2", - "detail": { - "EC2InstanceId": "%s", - "LifecycleTransition": "test:INVALID" - } - }`, instanceIDs[1])), - }) - - createPendingASGLifecycleAction(instanceIDs[1]) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("does not cordon or drain any nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("does not complete the ASG lifecycle action", func() { - Expect(asgLifecycleActions).To(And(HaveKeyWithValue(instanceIDs[1], Equal(StatePending)), HaveLen(1))) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - }) - - When("the SQS queue contains a Rebalance Recommendation Notification", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Instance Rebalance Recommendation", - "version": "0", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons and drains only the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - Expect(drainedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("the SQS queue contains a Scheduled Change Notification", func() { - When("the service is EC2 and the event type category is scheduled change", func() { - BeforeEach(func() { - resizeCluster(4) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.health", - "detail-type": "AWS Health Event", - "version": "1", - "detail": { - "service": "EC2", - "eventTypeCategory": "scheduledChange", - "affectedEntities": [ - {"entityValue": "%s"}, - {"entityValue": "%s"} - ] - } - }`, instanceIDs[1], instanceIDs[2])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons and drains only the targeted nodes", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveKey(nodeNames[2]), HaveLen(2))) - Expect(drainedNodes).To(And(HaveKey(nodeNames[1]), HaveKey(nodeNames[2]), HaveLen(2))) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("the service is not EC2", func() { - BeforeEach(func() { - resizeCluster(4) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.health", - "detail-type": "AWS Health Event", - "version": "1", - "detail": { - "service": "INVALID", - "eventTypeCategory": "scheduledChange", - "affectedEntities": [ - {"entityValue": "%s"}, - {"entityValue": "%s"} - ] - } - }`, instanceIDs[1], instanceIDs[2])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons and drains only the targeted nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - - When("the event type category is not scheduled change", func() { - BeforeEach(func() { - resizeCluster(4) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.health", - "detail-type": "AWS Health Event", - "version": "1", - "detail": { - "service": "EC2", - "eventTypeCategory": "invalid", - "affectedEntities": [ - {"entityValue": "%s"}, - {"entityValue": "%s"} - ] - } - }`, instanceIDs[1], instanceIDs[2])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons and drains only the targeted nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - }) - - When("the SQS queue contains a Spot Interruption Notification", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons and drains only the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - Expect(drainedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("the SQS queue contains a State Change Notification", func() { - When("the state is stopping", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Instance State-change Notification", - "version": "1", - "detail": { - "instance-id": "%s", - "state": "stopping" - } - }`, instanceIDs[1])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons and drains only the targeted nodes", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - Expect(drainedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("the state is stopped", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Instance State-change Notification", - "version": "1", - "detail": { - "instance-id": "%s", - "state": "stopped" - } - }`, instanceIDs[1])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons and drains only the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - Expect(drainedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("the state is shutting-down", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Instance State-change Notification", - "version": "1", - "detail": { - "instance-id": "%s", - "state": "shutting-down" - } - }`, instanceIDs[1])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons and drains only the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - Expect(drainedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("the state is terminated", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Instance State-change Notification", - "version": "1", - "detail": { - "instance-id": "%s", - "state": "terminated" - } - }`, instanceIDs[1])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons and drains only the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - Expect(drainedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("the state is not recognized", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Instance State-change Notification", - "version": "1", - "detail": { - "instance-id": "%s", - "state": "invalid" - } - }`, instanceIDs[1])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons and drains only the targeted node", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - }) - - When("the SQS queue contains multiple messages", func() { - BeforeEach(func() { - resizeCluster(12) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], - &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.autoscaling", - "detail-type": "EC2 Instance-terminate Lifecycle Action", - "version": "1", - "detail": { - "EC2InstanceId": "%s", - "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" - } - }`, instanceIDs[1])), - }, - &sqs.Message{ - ReceiptHandle: aws.String("msg-2"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.autoscaling", - "detail-type": "EC2 Instance-terminate Lifecycle Action", - "version": "2", - "detail": { - "EC2InstanceId": "%s", - "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" - } - }`, instanceIDs[2])), - }, - &sqs.Message{ - ReceiptHandle: aws.String("msg-3"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Instance Rebalance Recommendation", - "version": "0", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[3])), - }, - &sqs.Message{ - ReceiptHandle: aws.String("msg-4"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.health", - "detail-type": "AWS Health Event", - "version": "1", - "detail": { - "service": "EC2", - "eventTypeCategory": "scheduledChange", - "affectedEntities": [ - {"entityValue": "%s"}, - {"entityValue": "%s"} - ] - } - }`, instanceIDs[4], instanceIDs[5])), - }, - &sqs.Message{ - ReceiptHandle: aws.String("msg-5"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[6])), - }, - &sqs.Message{ - ReceiptHandle: aws.String("msg-6"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Instance State-change Notification", - "version": "1", - "detail": { - "instance-id": "%s", - "state": "stopping" - } - }`, instanceIDs[7])), - }, - &sqs.Message{ - ReceiptHandle: aws.String("msg-7"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Instance State-change Notification", - "version": "1", - "detail": { - "instance-id": "%s", - "state": "stopped" - } - }`, instanceIDs[8])), - }, - &sqs.Message{ - ReceiptHandle: aws.String("msg-8"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Instance State-change Notification", - "version": "1", - "detail": { - "instance-id": "%s", - "state": "shutting-down" - } - }`, instanceIDs[9])), - }, - &sqs.Message{ - ReceiptHandle: aws.String("msg-9"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Instance State-change Notification", - "version": "1", - "detail": { - "instance-id": "%s", - "state": "terminated" - } - }`, instanceIDs[10])), - }, - ) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons and drains only the targeted nodes", func() { - Expect(cordonedNodes).To(And( - HaveKey(nodeNames[1]), - HaveKey(nodeNames[2]), - HaveKey(nodeNames[3]), - HaveKey(nodeNames[4]), - HaveKey(nodeNames[5]), - HaveKey(nodeNames[6]), - HaveKey(nodeNames[7]), - HaveKey(nodeNames[8]), - HaveKey(nodeNames[9]), - HaveKey(nodeNames[10]), - HaveLen(10), - )) - Expect(drainedNodes).To(And( - HaveKey(nodeNames[1]), - HaveKey(nodeNames[2]), - HaveKey(nodeNames[3]), - HaveKey(nodeNames[4]), - HaveKey(nodeNames[5]), - HaveKey(nodeNames[6]), - HaveKey(nodeNames[7]), - HaveKey(nodeNames[8]), - HaveKey(nodeNames[9]), - HaveKey(nodeNames[10]), - HaveLen(10), - )) - }) - - It("deletes the messages from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("the SQS queue contains an unrecognized message", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(`{ - "source": "test.suite", - "detail-type": "Not a real notification", - "version": "1", - "detail": {} - }`), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("does not cordon or drain any nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - - When("the SQS queue contains a message with no body", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("does not cordon or drain any nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - - When("the SQS queue contains an empty message", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(""), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("does not cordon or drain any nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - - When("the SQS message cannot be parsed", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(`{ - "source": "test.suite", - "detail-type": "Mal-formed notification", - `), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("does not cordon or drain any nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - - When("the terminator has event configuration", func() { - When("Cordon on ASG Termination v1", func() { - BeforeEach(func() { - resizeCluster(3) - - terminator := terminators[terminatorNamespaceName] - terminator.Spec.Events.AutoScalingTermination = "Cordon" - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.autoscaling", - "detail-type": "EC2 Instance-terminate Lifecycle Action", - "version": "1", - "detail": { - "EC2InstanceId": "%s", - "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" - } - }`, instanceIDs[1])), - }) - - createPendingASGLifecycleAction(instanceIDs[1]) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("does not drain the targeted node", func() { - Expect(drainedNodes).To(BeEmpty()) - }) - - It("completes the ASG lifecycle action", func() { - Expect(asgLifecycleActions).To(And(HaveKeyWithValue(instanceIDs[1], Equal(StateComplete)), HaveLen(1))) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("\"No Action\" on ASG Termination v1", func() { - BeforeEach(func() { - resizeCluster(3) - - terminator := terminators[terminatorNamespaceName] - terminator.Spec.Events.AutoScalingTermination = "NoAction" - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.autoscaling", - "detail-type": "EC2 Instance-terminate Lifecycle Action", - "version": "1", - "detail": { - "EC2InstanceId": "%s", - "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" - } - }`, instanceIDs[1])), - }) - - createPendingASGLifecycleAction(instanceIDs[1]) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("does not cordon or drain the targeted node", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("completes the ASG lifecycle action", func() { - Expect(asgLifecycleActions).To(And(HaveKeyWithValue(instanceIDs[1], Equal(StateComplete)), HaveLen(1))) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("Cordon on ASG Termination v2", func() { - BeforeEach(func() { - resizeCluster(3) - - terminator := terminators[terminatorNamespaceName] - terminator.Spec.Events.AutoScalingTermination = "Cordon" - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.autoscaling", - "detail-type": "EC2 Instance-terminate Lifecycle Action", - "version": "2", - "detail": { - "EC2InstanceId": "%s", - "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" - } - }`, instanceIDs[1])), - }) - - createPendingASGLifecycleAction(instanceIDs[1]) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("does not drain the targeted node", func() { - Expect(drainedNodes).To(BeEmpty()) - }) - - It("completes the ASG lifecycle action", func() { - Expect(asgLifecycleActions).To(And(HaveKeyWithValue(instanceIDs[1], Equal(StateComplete)), HaveLen(1))) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("\"No Action\" on ASG Termination v2", func() { - BeforeEach(func() { - resizeCluster(3) - - terminator := terminators[terminatorNamespaceName] - terminator.Spec.Events.AutoScalingTermination = "NoAction" - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.autoscaling", - "detail-type": "EC2 Instance-terminate Lifecycle Action", - "version": "2", - "detail": { - "EC2InstanceId": "%s", - "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" - } - }`, instanceIDs[1])), - }) - - createPendingASGLifecycleAction(instanceIDs[1]) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("does not cordon or drain the targeted node", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("completes the ASG lifecycle action", func() { - Expect(asgLifecycleActions).To(And(HaveKeyWithValue(instanceIDs[1], Equal(StateComplete)), HaveLen(1))) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("Cordon on Rebalance Recommendation", func() { - BeforeEach(func() { - resizeCluster(3) - - terminator := terminators[terminatorNamespaceName] - terminator.Spec.Events.RebalanceRecommendation = "Cordon" - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Instance Rebalance Recommendation", - "version": "0", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("does not drain the targeted node", func() { - Expect(drainedNodes).To(BeEmpty()) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("\"No Action\" on Rebalance Recommendation", func() { - BeforeEach(func() { - resizeCluster(3) - - terminator := terminators[terminatorNamespaceName] - terminator.Spec.Events.RebalanceRecommendation = "NoAction" - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Instance Rebalance Recommendation", - "version": "0", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("does not cordon or drain the targeted node", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("Cordon on Scheduled Change", func() { - BeforeEach(func() { - resizeCluster(4) - - terminator := terminators[terminatorNamespaceName] - terminator.Spec.Events.ScheduledChange = "Cordon" - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.health", - "detail-type": "AWS Health Event", - "version": "1", - "detail": { - "service": "EC2", - "eventTypeCategory": "scheduledChange", - "affectedEntities": [ - {"entityValue": "%s"}, - {"entityValue": "%s"} - ] - } - }`, instanceIDs[1], instanceIDs[2])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveKey(nodeNames[2]), HaveLen(2))) - }) - - It("does not drain the targeted node", func() { - Expect(drainedNodes).To(BeEmpty()) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("\"No Action\" on Scheduled Change", func() { - BeforeEach(func() { - resizeCluster(4) - - terminator := terminators[terminatorNamespaceName] - terminator.Spec.Events.ScheduledChange = "NoAction" - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.health", - "detail-type": "AWS Health Event", - "version": "1", - "detail": { - "service": "EC2", - "eventTypeCategory": "scheduledChange", - "affectedEntities": [ - {"entityValue": "%s"}, - {"entityValue": "%s"} - ] - } - }`, instanceIDs[1], instanceIDs[2])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("does not cordon or drain the targeted node", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("Cordon on Spot Interruption", func() { - BeforeEach(func() { - resizeCluster(3) - - terminator := terminators[terminatorNamespaceName] - terminator.Spec.Events.SpotInterruption = "Cordon" - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("does not drain the targeted node", func() { - Expect(drainedNodes).To(BeEmpty()) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("\"No Action\" on Spot Interruption", func() { - BeforeEach(func() { - resizeCluster(3) - - terminator := terminators[terminatorNamespaceName] - terminator.Spec.Events.SpotInterruption = "NoAction" - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("does not cordon or drain the targeted node", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("Cordon on State Change", func() { - BeforeEach(func() { - resizeCluster(3) - - terminator := terminators[terminatorNamespaceName] - terminator.Spec.Events.StateChange = "Cordon" - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Instance State-change Notification", - "version": "1", - "detail": { - "instance-id": "%s", - "state": "stopping" - } - }`, instanceIDs[1])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("does not drain the targeted node", func() { - Expect(drainedNodes).To(BeEmpty()) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("\"No Action\" on State Change", func() { - BeforeEach(func() { - resizeCluster(3) - - terminator := terminators[terminatorNamespaceName] - terminator.Spec.Events.StateChange = "NoAction" - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Instance State-change Notification", - "version": "1", - "detail": { - "instance-id": "%s", - "state": "stopping" - } - }`, instanceIDs[1])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("does not cordon or drain the targeted node", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - }) - - When("the terminator cannot be found", func() { - BeforeEach(func() { - delete(terminators, terminatorNamespaceName) - }) - - It("returns success but does not requeue the request", func() { - Expect(result, err).To(BeZero()) - }) - - It("does not cordon or drain any nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - }) - - When("there is an error getting the terminator", func() { - BeforeEach(func() { - defaultKubeGetFunc := kubeGetFunc - kubeGetFunc = func(ctx context.Context, key client.ObjectKey, object client.Object) error { - switch object.(type) { - case *v1alpha1.Terminator: - return errors.New(errMsg) - default: - return defaultKubeGetFunc(ctx, key, object) - } - } - }) - - It("does not requeue the request", func() { - Expect(result).To(BeZero()) - }) - - It("returns an error", func() { - Expect(err).To(MatchError(ContainSubstring(errMsg))) - }) - }) - - When("there is an error getting SQS messages", func() { - BeforeEach(func() { - receiveSQSMessageFunc = func(_ aws.Context, _ *sqs.ReceiveMessageInput, _ ...awsrequest.Option) (*sqs.ReceiveMessageOutput, error) { - return nil, errors.New(errMsg) - } - }) - - It("does not requeue the request", func() { - Expect(result).To(BeZero()) - }) - - It("returns an error", func() { - Expect(err).To(MatchError(ContainSubstring(errMsg))) - }) - }) - - When("the terminator has webhook configuration", func() { - const webhookURL = "http://webhook.example.aws" - webhookHeaders := []v1alpha1.HeaderSpec{{Name: "Content-Type", Value: "application/json"}} - webhookTemplate := fmt.Sprintf( - `EventID={{ .EventID }}, Kind={{ .Kind }}, InstanceID={{ .InstanceID }}, NodeName={{ .NodeName }}, StartTime={{ (.StartTime.Format "%s") }}`, - time.RFC3339, - ) - - When("the reconciliation takes no action", func() { - BeforeEach(func() { - terminator := terminators[terminatorNamespaceName] - terminator.Spec.Webhook.URL = webhookURL - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("does not send any webhook requests", func() { - Expect(webhookRequests).To(BeEmpty()) - }) - }) - - When("the reconciliation acts on a node", func() { - const msgID = "id-123" - msgTime := time.Now().Format(time.RFC3339) - - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "id": "%s", - "time": "%s", - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, msgID, msgTime, instanceIDs[1])), - }) - - terminator := terminators[terminatorNamespaceName] - terminator.Spec.Webhook.URL = webhookURL - terminator.Spec.Webhook.Headers = webhookHeaders - terminator.Spec.Webhook.Template = webhookTemplate - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("sends a webhook notification", func() { - Expect(webhookRequests).To(HaveLen(1)) - Expect(webhookRequests[0].Method).To(Equal(http.MethodPost)) - Expect(webhookRequests[0].URL.String()).To(Equal(webhookURL)) - Expect(webhookRequests[0].Header).To(And( - HaveLen(1), - HaveKeyWithValue("Content-Type", And( - HaveLen(1), - ContainElement("application/json"), - )))) - - Expect(ReadAll(webhookRequests[0].Body)).To(Equal(fmt.Sprintf( - "EventID=%s, Kind=spotInterruption, InstanceID=%s, NodeName=%s, StartTime=%s", - msgID, instanceIDs[1], nodeNames[1], msgTime, - ))) - }) - }) - - When("the reconciliation acts on multiple nodes", func() { - msgIDs := []string{"msg-1", "msg-2", "msg-2"} - msgBaseTime := time.Now() - msgTimes := []string{ - msgBaseTime.Add(-1 * time.Minute).Format(time.RFC3339), - msgBaseTime.Format(time.RFC3339), - msgBaseTime.Format(time.RFC3339), - } - kinds := []string{"spotInterruption", "scheduledChange", "scheduledChange"} - - BeforeEach(func() { - resizeCluster(5) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], - &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "id": "%s", - "time": "%s", - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, msgIDs[0], msgTimes[0], instanceIDs[1])), - }, - &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "id": "%s", - "time": "%s", - "source": "aws.health", - "detail-type": "AWS Health Event", - "version": "1", - "detail": { - "service": "EC2", - "eventTypeCategory": "scheduledChange", - "affectedEntities": [ - {"entityValue": "%s"}, - {"entityValue": "%s"} - ] - } - }`, msgIDs[1], msgTimes[1], instanceIDs[2], instanceIDs[3])), - }, - ) - - terminator := terminators[terminatorNamespaceName] - terminator.Spec.Webhook.URL = webhookURL - terminator.Spec.Webhook.Headers = webhookHeaders - terminator.Spec.Webhook.Template = webhookTemplate - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("sends a webhook notification", func() { - Expect(webhookRequests).To(HaveLen(3)) - - for i := 0; i < 3; i++ { - Expect(webhookRequests[i].Method).To(Equal(http.MethodPost), "request #%d", i) - Expect(webhookRequests[i].URL.String()).To(Equal(webhookURL), "request #%d", i) - Expect(webhookRequests[i].Header).To(And( - HaveLen(1), - HaveKeyWithValue("Content-Type", And( - HaveLen(1), - ContainElement("application/json"), - ))), - "request #%d", i) - - Expect(ReadAll(webhookRequests[i].Body)).To(Equal(fmt.Sprintf( - "EventID=%s, Kind=%s, InstanceID=%s, NodeName=%s, StartTime=%s", - msgIDs[i], kinds[i], instanceIDs[i+1], nodeNames[i+1], msgTimes[i], - )), - "request #%d", i, - ) - } - }) - }) - - When("there is an error sending the request", func() { - const msgID = "id-123" - msgTime := time.Now().Format(time.RFC3339) - - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "id": "%s", - "time": "%s", - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, msgID, msgTime, instanceIDs[1])), - }) - - terminator := terminators[terminatorNamespaceName] - terminator.Spec.Webhook.URL = webhookURL - terminator.Spec.Webhook.Headers = webhookHeaders - terminator.Spec.Webhook.Template = webhookTemplate - - webhookSendFunc = func(_ *http.Request) (*http.Response, error) { - return nil, errors.New("test error") - } - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - }) - }) - - When("there is an error deleting an SQS message", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - - deleteSQSMessageFunc = func(_ context.Context, _ *sqs.DeleteMessageInput, _ ...awsrequest.Option) (*sqs.DeleteMessageOutput, error) { - return nil, errors.New(errMsg) - } - }) - - It("does not requeue the request", func() { - Expect(result).To(BeZero()) - }) - - It("returns an error", func() { - Expect(err).To(MatchError(ContainSubstring(errMsg))) - }) - }) - - When("there is an error getting the EC2 reservation for the instance ID", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - - describeEC2InstancesFunc = func(_ aws.Context, _ *ec2.DescribeInstancesInput, _ ...awsrequest.Option) (*ec2.DescribeInstancesOutput, error) { - return nil, errors.New(errMsg) - } - }) - - It("does not requeue the request", func() { - Expect(result).To(BeZero()) - }) - - It("returns an error", func() { - Expect(err).To(MatchError(ContainSubstring(errMsg))) - }) - - It("does not cordon or drain any nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - - When("there is no EC2 reservation for the instance ID", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - - describeEC2InstancesFunc = func(_ aws.Context, _ *ec2.DescribeInstancesInput, _ ...awsrequest.Option) (*ec2.DescribeInstancesOutput, error) { - return &ec2.DescribeInstancesOutput{ - Reservations: []*ec2.Reservation{}, - }, nil - } - }) - - It("does not requeue the request", func() { - Expect(result).To(BeZero()) - }) - - It("returns an error", func() { - Expect(err).To(HaveOccurred()) - }) - - It("does not cordon or drain any nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - - When("the EC2 reservation contains no instances", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - - describeEC2InstancesFunc = func(_ aws.Context, _ *ec2.DescribeInstancesInput, _ ...awsrequest.Option) (*ec2.DescribeInstancesOutput, error) { - return &ec2.DescribeInstancesOutput{ - Reservations: []*ec2.Reservation{ - {Instances: []*ec2.Instance{}}, - }, - }, nil - } - }) - - It("does not requeue the request", func() { - Expect(result).To(BeZero()) - }) - - It("returns an error", func() { - Expect(err).To(HaveOccurred()) - }) - - It("does not cordon or drain any nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - - When("the EC2 reservation's instance has no PrivateDnsName", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - - describeEC2InstancesFunc = func(_ aws.Context, _ *ec2.DescribeInstancesInput, _ ...awsrequest.Option) (*ec2.DescribeInstancesOutput, error) { - return &ec2.DescribeInstancesOutput{ - Reservations: []*ec2.Reservation{ - { - Instances: []*ec2.Instance{ - {PrivateDnsName: nil}, - }, - }, - }, - }, nil - } - }) - - It("does not requeue the request", func() { - Expect(result).To(BeZero()) - }) - - It("returns an error", func() { - Expect(err).To(HaveOccurred()) - }) - - It("does not cordon or drain any nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - - When("the EC2 reservation's instance's PrivateDnsName empty", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - - describeEC2InstancesFunc = func(_ aws.Context, _ *ec2.DescribeInstancesInput, _ ...awsrequest.Option) (*ec2.DescribeInstancesOutput, error) { - return &ec2.DescribeInstancesOutput{ - Reservations: []*ec2.Reservation{ - { - Instances: []*ec2.Instance{ - {PrivateDnsName: aws.String("")}, - }, - }, - }, - }, nil - } - }) - - It("does not requeue the request", func() { - Expect(result).To(BeZero()) - }) - - It("returns an error", func() { - Expect(err).To(HaveOccurred()) - }) - - It("does not cordon or drain any nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - - When("there is an error getting the cluster node for a node name", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - - defaultKubeGetFunc := kubeGetFunc - kubeGetFunc = func(ctx context.Context, key client.ObjectKey, object client.Object) error { - switch object.(type) { - case *v1.Node: - return errors.New(errMsg) - default: - return defaultKubeGetFunc(ctx, key, object) - } - } - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("does not cordon or drain any nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - }) - - When("the terminator has a node label selector", func() { - When("the label selector matches the target node", func() { - const labelName = "a-test-label" - const labelValue = "test-label-value" - - BeforeEach(func() { - resizeCluster(3) - - targetedNode, found := nodes[types.NamespacedName{Name: nodeNames[1]}] - Expect(found).To(BeTrue()) - - targetedNode.Labels = map[string]string{labelName: labelValue} - - terminator, found := terminators[terminatorNamespaceName] - Expect(found).To(BeTrue()) - - terminator.Spec.MatchLabels = client.MatchingLabels{labelName: labelValue} - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("cordons and drains only the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - Expect(drainedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("the label selector does not match the target node", func() { - const labelName = "a-test-label" - const labelValue = "test-label-value" - - BeforeEach(func() { - resizeCluster(3) - - terminator, found := terminators[terminatorNamespaceName] - Expect(found).To(BeTrue()) - - terminator.Spec.MatchLabels = client.MatchingLabels{labelName: labelValue} - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - }) - - It("returns success and requeues the request with the reconciler's configured interval", func() { - Expect(result, err).To(HaveField("RequeueAfter", Equal(reconciler.RequeueInterval))) - }) - - It("does not cordon or drain any nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - }) - - When("cordoning a node fails", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - - cordonFunc = func(_ *kubectl.Helper, _ *v1.Node, _ bool) error { - return errors.New(errMsg) - } - }) - - It("does not requeue the request", func() { - Expect(result).To(BeZero()) - }) - - It("returns an error", func() { - Expect(err).To(MatchError(ContainSubstring(errMsg))) - }) - - It("does not cordon or drain any nodes", func() { - Expect(cordonedNodes).To(BeEmpty()) - Expect(drainedNodes).To(BeEmpty()) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("draining a node fails", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - - drainFunc = func(_ *kubectl.Helper, _ string) error { - return errors.New(errMsg) - } - }) - - It("does not requeue the request", func() { - Expect(result).To(BeZero()) - }) - - It("returns an error", func() { - Expect(err).To(MatchError(ContainSubstring(errMsg))) - }) - - It("cordons the target node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("does not drain the target node", func() { - Expect(drainedNodes).To(BeEmpty()) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("completing an ASG Lifecycle Action (v1) fails", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.autoscaling", - "detail-type": "EC2 Instance-terminate Lifecycle Action", - "version": "1", - "detail": { - "EC2InstanceId": "%s", - "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" - } - }`, instanceIDs[1])), - }) - - completeASGLifecycleActionFunc = func(_ aws.Context, _ *autoscaling.CompleteLifecycleActionInput, _ ...awsrequest.Option) (*autoscaling.CompleteLifecycleActionOutput, error) { - return nil, errors.New(errMsg) - } - }) - - It("does not requeue the request", func() { - Expect(result).To(BeZero()) - }) - - It("returns an error", func() { - Expect(err).To(MatchError(ContainSubstring(errMsg))) - }) - - It("cordons and drains only the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - Expect(drainedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("the request to complete the ASG Lifecycle Action (v1) fails with a status != 400", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.autoscaling", - "detail-type": "EC2 Instance-terminate Lifecycle Action", - "version": "1", - "detail": { - "EC2InstanceId": "%s", - "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" - } - }`, instanceIDs[1])), - }) - - completeASGLifecycleActionFunc = func(_ aws.Context, _ *autoscaling.CompleteLifecycleActionInput, _ ...awsrequest.Option) (*autoscaling.CompleteLifecycleActionOutput, error) { - return nil, awserr.NewRequestFailure(awserr.New("", errMsg, errors.New(errMsg)), 404, "") - } - }) - - It("does not requeue the request", func() { - Expect(result).To(BeZero()) - }) - - It("returns an error", func() { - Expect(err).To(MatchError(ContainSubstring(errMsg))) - }) - - It("cordons and drains only the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - Expect(drainedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - - When("completing an ASG Lifecycle Action (v2) fails", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.autoscaling", - "detail-type": "EC2 Instance-terminate Lifecycle Action", - "version": "2", - "detail": { - "EC2InstanceId": "%s", - "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" - } - }`, instanceIDs[1])), - }) - - completeASGLifecycleActionFunc = func(_ aws.Context, _ *autoscaling.CompleteLifecycleActionInput, _ ...awsrequest.Option) (*autoscaling.CompleteLifecycleActionOutput, error) { - return nil, errors.New(errMsg) - } - }) - - It("does not requeue the request", func() { - Expect(result).To(BeZero()) - }) - - It("returns an error", func() { - Expect(err).To(MatchError(ContainSubstring(errMsg))) - }) - - It("cordons and drains only the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - Expect(drainedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("deletes the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(BeEmpty()) - }) - }) - - When("the request to complete the ASG Lifecycle Action (v2) fails with a status != 400", func() { - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.autoscaling", - "detail-type": "EC2 Instance-terminate Lifecycle Action", - "version": "2", - "detail": { - "EC2InstanceId": "%s", - "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" - } - }`, instanceIDs[1])), - }) - - completeASGLifecycleActionFunc = func(_ aws.Context, _ *autoscaling.CompleteLifecycleActionInput, _ ...awsrequest.Option) (*autoscaling.CompleteLifecycleActionOutput, error) { - return nil, awserr.NewRequestFailure(awserr.New("", errMsg, errors.New(errMsg)), 404, "") - } - }) - - It("does not requeue the request", func() { - Expect(result).To(BeZero()) - }) - - It("returns an error", func() { - Expect(err).To(MatchError(ContainSubstring(errMsg))) - }) - - It("cordons and drains only the targeted node", func() { - Expect(cordonedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - Expect(drainedNodes).To(And(HaveKey(nodeNames[1]), HaveLen(1))) - }) - - It("does not delete the message from the SQS queue", func() { - Expect(sqsQueues[queueURL]).To(HaveLen(1)) - }) - }) - - When("getting messages from a terminator's SQS queue", func() { - const ( - maxNumberOfMessages = int64(10) - visibilityTimeoutSeconds = int64(20) - waitTimeSeconds = int64(20) - ) - var ( - attributeNames = []string{sqs.MessageSystemAttributeNameSentTimestamp} - messageAttributeNames = []string{sqs.QueueAttributeNameAll} - input *sqs.ReceiveMessageInput - ) - - BeforeEach(func() { - terminator, found := terminators[terminatorNamespaceName] - Expect(found).To(BeTrue()) - - terminator.Spec.SQS.QueueURL = queueURL - - defaultReceiveSQSMessageFunc := receiveSQSMessageFunc - receiveSQSMessageFunc = func(ctx aws.Context, in *sqs.ReceiveMessageInput, options ...awsrequest.Option) (*sqs.ReceiveMessageOutput, error) { - input = in - return defaultReceiveSQSMessageFunc(ctx, in, options...) - } - }) - - It("sends the input values from the terminator", func() { - Expect(input).ToNot(BeNil()) - - for i, attrName := range input.AttributeNames { - Expect(attrName).ToNot(BeNil()) - Expect(*attrName).To(Equal(attributeNames[i])) - } - for i, attrName := range input.MessageAttributeNames { - Expect(attrName).ToNot(BeNil()) - Expect(*attrName).To(Equal(messageAttributeNames[i])) - } - - Expect(input.MaxNumberOfMessages).ToNot(BeNil()) - Expect(*input.MaxNumberOfMessages).To(Equal(maxNumberOfMessages)) - - Expect(input.QueueUrl).ToNot(BeNil()) - Expect(*input.QueueUrl).To(Equal(queueURL)) - - Expect(input.VisibilityTimeout).ToNot(BeNil()) - Expect(*input.VisibilityTimeout).To(Equal(visibilityTimeoutSeconds)) - - Expect(input.WaitTimeSeconds).ToNot(BeNil()) - Expect(*input.WaitTimeSeconds).To(Equal(waitTimeSeconds)) - }) - }) - - When("cordoning a node", func() { - const ( - force = true - gracePeriodSeconds = 31 - ignoreAllDaemonSets = true - deleteEmptyDirData = true - ) - var helper *kubectl.Helper - var timeout time.Duration - - BeforeEach(func() { - timeout = 42 * time.Second - - terminator, found := terminators[terminatorNamespaceName] - Expect(found).To(BeTrue()) - - terminator.Spec.Drain.DeleteEmptyDirData = deleteEmptyDirData - terminator.Spec.Drain.Force = force - terminator.Spec.Drain.GracePeriodSeconds = gracePeriodSeconds - terminator.Spec.Drain.IgnoreAllDaemonSets = ignoreAllDaemonSets - terminator.Spec.Drain.TimeoutSeconds = int(timeout.Seconds()) - - defaultCordonFunc := cordonFunc - cordonFunc = func(h *kubectl.Helper, node *v1.Node, desired bool) error { - helper = h - return defaultCordonFunc(h, node, desired) - } - - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - }) - - It("sends the input values from the terminator", func() { - Expect(helper).ToNot(BeNil()) - - Expect(helper).To(And( - HaveField("DeleteEmptyDirData", Equal(deleteEmptyDirData)), - HaveField("Force", Equal(force)), - HaveField("GracePeriodSeconds", Equal(gracePeriodSeconds)), - HaveField("IgnoreAllDaemonSets", Equal(ignoreAllDaemonSets)), - HaveField("Timeout", Equal(timeout)), - )) - }) - - It("sends additional input values", func() { - Expect(helper).ToNot(BeNil()) - - Expect(helper).To(And( - HaveField("Client", Not(BeNil())), - HaveField("Ctx", Not(BeNil())), - HaveField("Out", Not(BeNil())), - HaveField("ErrOut", Not(BeNil())), - )) - }) - }) - - When("draining a node", func() { - const ( - force = true - gracePeriodSeconds = 31 - ignoreAllDaemonSets = true - deleteEmptyDirData = true - ) - var helper *kubectl.Helper - var timeout time.Duration - - BeforeEach(func() { - timeout = 42 * time.Second - - terminator, found := terminators[terminatorNamespaceName] - Expect(found).To(BeTrue()) - - terminator.Spec.Drain.DeleteEmptyDirData = deleteEmptyDirData - terminator.Spec.Drain.Force = force - terminator.Spec.Drain.GracePeriodSeconds = gracePeriodSeconds - terminator.Spec.Drain.IgnoreAllDaemonSets = ignoreAllDaemonSets - terminator.Spec.Drain.TimeoutSeconds = int(timeout.Seconds()) - - defaultDrainFunc := drainFunc - drainFunc = func(h *kubectl.Helper, nodeName string) error { - helper = h - return defaultDrainFunc(h, nodeName) - } - - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.ec2", - "detail-type": "EC2 Spot Instance Interruption Warning", - "version": "1", - "detail": { - "instance-id": "%s" - } - }`, instanceIDs[1])), - }) - }) - - It("sends the input values from the terminator", func() { - Expect(helper).ToNot(BeNil()) - - Expect(helper).To(And( - HaveField("DeleteEmptyDirData", Equal(deleteEmptyDirData)), - HaveField("Force", Equal(force)), - HaveField("GracePeriodSeconds", Equal(gracePeriodSeconds)), - HaveField("IgnoreAllDaemonSets", Equal(ignoreAllDaemonSets)), - HaveField("Timeout", Equal(timeout)), - )) - }) - - It("sends additional values", func() { - Expect(helper).ToNot(BeNil()) - - Expect(helper).To(And( - HaveField("Client", Not(BeNil())), - HaveField("Ctx", Not(BeNil())), - HaveField("Out", Not(BeNil())), - HaveField("ErrOut", Not(BeNil())), - )) - }) - }) - - When("completing an ASG Complete Lifecycle Action", func() { - const ( - autoScalingGroupName = "testAutoScalingGroupName" - lifecycleActionResult = "CONTINUE" - lifecycleHookName = "testLifecycleHookName" - lifecycleActionToken = "testLifecycleActionToken" - ) - var input *autoscaling.CompleteLifecycleActionInput - - BeforeEach(func() { - resizeCluster(3) - - sqsQueues[queueURL] = append(sqsQueues[queueURL], &sqs.Message{ - ReceiptHandle: aws.String("msg-1"), - Body: aws.String(fmt.Sprintf(`{ - "source": "aws.autoscaling", - "detail-type": "EC2 Instance-terminate Lifecycle Action", - "version": "1", - "detail": { - "AutoScalingGroupName": "%s", - "EC2InstanceId": "%s", - "LifecycleActionToken": "%s", - "LifecycleHookName": "%s", - "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" - } - }`, autoScalingGroupName, instanceIDs[1], lifecycleActionToken, lifecycleHookName)), - }) - - defaultCompleteASGLifecycleActionFunc := completeASGLifecycleActionFunc - completeASGLifecycleActionFunc = func(ctx aws.Context, in *autoscaling.CompleteLifecycleActionInput, options ...awsrequest.Option) (*autoscaling.CompleteLifecycleActionOutput, error) { - input = in - return defaultCompleteASGLifecycleActionFunc(ctx, in, options...) - } - }) - - It("sends the expected input values", func() { - Expect(input).ToNot(BeNil()) - - Expect(input.AutoScalingGroupName).ToNot(BeNil()) - Expect(*input.AutoScalingGroupName).To(Equal(autoScalingGroupName)) - - Expect(input.LifecycleActionResult).ToNot(BeNil()) - Expect(*input.LifecycleActionResult).To(Equal(lifecycleActionResult)) - - Expect(input.LifecycleHookName).ToNot(BeNil()) - Expect(*input.LifecycleHookName).To(Equal(lifecycleHookName)) - - Expect(input.LifecycleActionToken).ToNot(BeNil()) - Expect(*input.LifecycleActionToken).To(Equal(lifecycleActionToken)) - - Expect(input.InstanceId).ToNot(BeNil()) - Expect(*input.InstanceId).To(Equal(instanceIDs[1])) - }) - }) - - // Setup the starter state: - // * One terminator (terminatorNamedspacedName) - // * The terminator references an empty sqs queue (queueURL) - // * Zero nodes (use resizeCluster()) - // - // Tests should modify the cluster/aws service states as needed. - BeforeEach(func() { - // 1. Initialize variables. - - logger := zap.New(zapcore.NewCore( - zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()), - zapcore.AddSync(io.Discard), - zap.NewAtomicLevelAt(zap.DebugLevel), - )) - - ctx = logging.WithLogger(context.Background(), logger.Sugar()) - terminatorNamespaceName = types.NamespacedName{Namespace: "test", Name: "foo"} - request = reconcile.Request{NamespacedName: terminatorNamespaceName} - sqsQueues = map[SQSQueueURL][]*sqs.Message{queueURL: {}} - terminators = map[types.NamespacedName]*v1alpha1.Terminator{ - // For convenience create a terminator that points to the sqs queue. - terminatorNamespaceName: { - Spec: v1alpha1.TerminatorSpec{ - SQS: v1alpha1.SQSSpec{ - QueueURL: queueURL, - }, - }, - }, - } - nodes = map[types.NamespacedName]*v1.Node{} - ec2Reservations = map[EC2InstanceID]*ec2.Reservation{} - cordonedNodes = map[NodeName]bool{} - drainedNodes = map[NodeName]bool{} - - nodeNames = []NodeName{} - instanceIDs = []EC2InstanceID{} - resizeCluster = func(newNodeCount uint) { - for currNodeCount := uint(len(nodes)); currNodeCount < newNodeCount; currNodeCount++ { - nodeName := fmt.Sprintf("node-%d", currNodeCount) - nodeNames = append(nodeNames, nodeName) - nodes[types.NamespacedName{Name: nodeName}] = &v1.Node{ - ObjectMeta: metav1.ObjectMeta{Name: nodeName}, - } - - instanceID := fmt.Sprintf("instance-%d", currNodeCount) - instanceIDs = append(instanceIDs, instanceID) - ec2Reservations[instanceID] = &ec2.Reservation{ - Instances: []*ec2.Instance{ - {PrivateDnsName: aws.String(nodeName)}, - }, - } - } - - nodeNames = nodeNames[:newNodeCount] - instanceIDs = instanceIDs[:newNodeCount] - } - - asgLifecycleActions = map[EC2InstanceID]State{} - createPendingASGLifecycleAction = func(instanceID EC2InstanceID) { - Expect(asgLifecycleActions).ToNot(HaveKey(instanceID)) - asgLifecycleActions[instanceID] = StatePending - } - - webhookRequests = []*http.Request{} - webhookSendFunc = func(req *http.Request) (*http.Response, error) { - webhookRequests = append(webhookRequests, req) - return &http.Response{StatusCode: 200}, nil - } - - // 2. Setup stub clients. - - describeEC2InstancesFunc = func(ctx aws.Context, input *ec2.DescribeInstancesInput, _ ...awsrequest.Option) (*ec2.DescribeInstancesOutput, error) { - if err := ctx.Err(); err != nil { - return nil, err - } - - output := ec2.DescribeInstancesOutput{} - for _, instanceID := range input.InstanceIds { - if instanceID == nil { - continue - } - if reservation, found := ec2Reservations[*instanceID]; found { - output.Reservations = append(output.Reservations, reservation) - } - } - return &output, nil - } - - ec2Client := EC2Client(func(ctx aws.Context, input *ec2.DescribeInstancesInput, options ...awsrequest.Option) (*ec2.DescribeInstancesOutput, error) { - return describeEC2InstancesFunc(ctx, input, options...) - }) - - completeASGLifecycleActionFunc = func(ctx aws.Context, input *autoscaling.CompleteLifecycleActionInput, _ ...awsrequest.Option) (*autoscaling.CompleteLifecycleActionOutput, error) { - if err := ctx.Err(); err != nil { - return nil, err - } - Expect(input.InstanceId).ToNot(BeNil()) - if state, found := asgLifecycleActions[*input.InstanceId]; found { - Expect(state).ToNot(Equal(StateComplete)) - asgLifecycleActions[*input.InstanceId] = StateComplete - } - return &autoscaling.CompleteLifecycleActionOutput{}, nil - } - - asgClient := ASGClient(func(ctx aws.Context, input *autoscaling.CompleteLifecycleActionInput, options ...awsrequest.Option) (*autoscaling.CompleteLifecycleActionOutput, error) { - return completeASGLifecycleActionFunc(ctx, input, options...) - }) - - receiveSQSMessageFunc = func(ctx aws.Context, input *sqs.ReceiveMessageInput, options ...awsrequest.Option) (*sqs.ReceiveMessageOutput, error) { - if err := ctx.Err(); err != nil { - return nil, err - } - Expect(input.QueueUrl).ToNot(BeNil()) - - messages, found := sqsQueues[*input.QueueUrl] - Expect(found).To(BeTrue(), "SQS queue does not exist: %q", *input.QueueUrl) - - return &sqs.ReceiveMessageOutput{Messages: append([]*sqs.Message{}, messages...)}, nil - } - - deleteSQSMessageFunc = func(ctx aws.Context, input *sqs.DeleteMessageInput, options ...awsrequest.Option) (*sqs.DeleteMessageOutput, error) { - if err := ctx.Err(); err != nil { - return nil, err - } - Expect(input.QueueUrl).ToNot(BeNil()) - - queue, found := sqsQueues[*input.QueueUrl] - Expect(found).To(BeTrue(), "SQS queue does not exist: %q", *input.QueueUrl) - - updatedQueue := make([]*sqs.Message, 0, len(queue)) - for i, m := range queue { - if m.ReceiptHandle == input.ReceiptHandle { - updatedQueue = append(updatedQueue, queue[:i]...) - updatedQueue = append(updatedQueue, queue[i+1:]...) - break - } - } - sqsQueues[*input.QueueUrl] = updatedQueue - - return &sqs.DeleteMessageOutput{}, nil - } - - sqsClient := SQSClient{ - ReceiveSQSMessageFunc: func(ctx aws.Context, input *sqs.ReceiveMessageInput, options ...awsrequest.Option) (*sqs.ReceiveMessageOutput, error) { - return receiveSQSMessageFunc(ctx, input, options...) - }, - DeleteSQSMessageFunc: func(ctx aws.Context, input *sqs.DeleteMessageInput, options ...awsrequest.Option) (*sqs.DeleteMessageOutput, error) { - return deleteSQSMessageFunc(ctx, input, options...) - }, - } - - kubeGetFunc = func(ctx context.Context, key client.ObjectKey, object client.Object) error { - if err := ctx.Err(); err != nil { - return err - } - - switch out := object.(type) { - case *v1.Node: - n, found := nodes[key] - if !found { - return k8serrors.NewNotFound(schema.GroupResource{}, key.String()) - } - *out = *n - - case *v1alpha1.Terminator: - t, found := terminators[key] - if !found { - return k8serrors.NewNotFound(schema.GroupResource{}, key.String()) - } - *out = *t - - default: - return fmt.Errorf("unknown type: %s", reflect.TypeOf(object).Name()) - } - return nil - } - - kubeClient := KubeClient(func(ctx context.Context, key client.ObjectKey, object client.Object) error { - return kubeGetFunc(ctx, key, object) - }) - - cordonFunc = func(_ *kubectl.Helper, node *v1.Node, desired bool) error { - if _, found := nodes[types.NamespacedName{Name: node.Name}]; !found { - return fmt.Errorf("node does not exist: %q", node.Name) - } - cordonedNodes[node.Name] = true - return nil - } - - drainFunc = func(_ *kubectl.Helper, nodeName string) error { - if _, found := nodes[types.NamespacedName{Name: nodeName}]; !found { - return fmt.Errorf("node does not exist: %q", nodeName) - } - drainedNodes[nodeName] = true - return nil - } - - // 3. Construct the reconciler. - - eventParser := event.NewAggregatedParser( - asgterminateeventv1.Parser{ASGLifecycleActionCompleter: asgClient}, - asgterminateeventv2.Parser{ASGLifecycleActionCompleter: asgClient}, - rebalancerecommendationeventv0.Parser{}, - scheduledchangeeventv1.Parser{}, - spotinterruptioneventv1.Parser{}, - statechangeeventv1.Parser{}, - ) - - cordoner := kubectlcordondrain.CordonFunc(func(h *kubectl.Helper, n *v1.Node, d bool) error { - return cordonFunc(h, n, d) - }) - - drainer := kubectlcordondrain.DrainFunc(func(h *kubectl.Helper, n string) error { - return drainFunc(h, n) - }) - - cordonDrainerBuilder := kubectlcordondrain.Builder{ - ClientSet: &kubernetes.Clientset{}, - Cordoner: cordoner, - Drainer: drainer, - } - - newHttpClientDoFunc := func(_ webhook.ProxyFunc) webhook.HttpSendFunc { - return webhookSendFunc - } - - reconciler = terminator.Reconciler{ - Name: "terminator", - RequeueInterval: time.Duration(10) * time.Second, - NodeGetterBuilder: terminatoradapter.NodeGetterBuilder{ - NodeGetter: node.Getter{KubeGetter: kubeClient}, - }, - NodeNameGetter: nodename.Getter{EC2InstancesDescriber: ec2Client}, - SQSClientBuilder: terminatoradapter.SQSMessageClientBuilder{ - SQSMessageClient: sqsmessage.Client{SQSClient: sqsClient}, - }, - SQSMessageParser: terminatoradapter.EventParser{Parser: eventParser}, - Getter: terminatoradapter.Getter{KubeGetter: kubeClient}, - CordonDrainerBuilder: terminatoradapter.CordonDrainerBuilder{ - Builder: cordonDrainerBuilder, - }, - WebhookClientBuilder: terminatoradapter.WebhookClientBuilder( - webhook.ClientBuilder(newHttpClientDoFunc).NewClient, - ), - } - }) - - // Run the reconciliation before each test subject. - JustBeforeEach(func() { - result, err = reconciler.Reconcile(ctx, request) - }) -}) - -func ReadAll(r io.Reader) (string, error) { - bs, err := ioutil.ReadAll(r) - if err != nil { - return "", err - } - return string(bs), nil -}