From 320e5215138df3c43aecc20d3aa91e651d476ef0 Mon Sep 17 00:00:00 2001 From: Jerad C Date: Wed, 29 Jun 2022 15:29:59 -0500 Subject: [PATCH 01/13] do not hardcode namespace in helm deployment --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 19b59eae..29a522a1 100644 --- a/Makefile +++ b/Makefile @@ -105,7 +105,9 @@ $(KODATA): .PHONY: apply apply: $(KO) $(KODATA) ## Deploy the controller into the current kubernetes cluster. - helm upgrade --install dev charts/aws-node-termination-handler-2 --namespace nthv2 --create-namespace \ + helm upgrade --install dev charts/aws-node-termination-handler-2 \ + --namespace ${CLUSTER_NAMESPACE} \ + --create-namespace \ $(HELM_BASE_OPTS) \ $(HELM_OPTS) \ --set controller.image=$(shell $(KO) publish -B github.com/aws/aws-node-termination-handler/cmd/controller) \ From 5295cc267cbbc0960e65eff71579d3ccfd88be73 Mon Sep 17 00:00:00 2001 From: Jerad C Date: Thu, 30 Jun 2022 10:25:08 -0500 Subject: [PATCH 02/13] add name length advisory * Shorten EventBridge rule name suffixes --- DEVELOPMENT.md | 4 ++++ resources/queue-infrastructure.yaml | 10 +++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 003f75bd..581d44fa 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -59,6 +59,10 @@ Resources created: * `WebhookRespository` - ECR Repository for images of the Kubernetes admission webhook. ```sh +# Note: The queue-infrastructure.yaml template generates the names of EventBridge rules +# from the ClusterName and QueueName parameters. To avoid exceeding name length limits +# the combined length of ClusterName and QueueName parameters should not exceed 51 +# characters. export QUEUE_NAME= export QUEUE_STACK_NAME="${INFRASTRUCTURE_STACK_NAME}-queue-${QUEUE_NAME}" diff --git a/resources/queue-infrastructure.yaml b/resources/queue-infrastructure.yaml index 77607a6f..ed6c26d8 100644 --- a/resources/queue-infrastructure.yaml +++ b/resources/queue-infrastructure.yaml @@ -32,7 +32,7 @@ Resources: AutoScalingTerminateRule: Type: AWS::Events::Rule Properties: - Name: !Sub "nth_${ClusterName}_${QueueName}_auto-scaling-terminate" + Name: !Sub "nth_${ClusterName}_${QueueName}_asgtrm" Description: !Sub "Route instance-terminate lifecycle actions from EC2 Auto Scaling to SQS Queue, ${QueueName}." EventPattern: source: @@ -53,7 +53,7 @@ Resources: RebalanceRecommendationRule: Type: AWS::Events::Rule Properties: - Name: !Sub "nth_${ClusterName}_${QueueName}_rebalance-recommendation" + Name: !Sub "nth_${ClusterName}_${QueueName}_rebrec" Description: !Sub "Route rebalance recommendations from EC2 to SQS Queue, ${QueueName}." EventPattern: source: @@ -70,7 +70,7 @@ Resources: ScheduledChangeRule: Type: AWS::Events::Rule Properties: - Name: !Sub "nth_${ClusterName}_${QueueName}_scheduled-change" + Name: !Sub "nth_${ClusterName}_${QueueName}_schchg" Description: !Sub "Route scheduled change health events from AWS Health to SQS Queue, ${QueueName}." EventPattern: source: @@ -92,7 +92,7 @@ Resources: SpotInterruptionRule: Type: AWS::Events::Rule Properties: - Name: !Sub "nth_${ClusterName}_${QueueName}_spot-interruption" + Name: !Sub "nth_${ClusterName}_${QueueName}_sptint" Description: !Sub "Route spot interruption notices from EC2 to SQS Queue, ${QueueName}." EventPattern: source: @@ -109,7 +109,7 @@ Resources: StateChangeRule: Type: AWS::Events::Rule Properties: - Name: !Sub "nth_${ClusterName}_${QueueName}_state-change" + Name: !Sub "nth_${ClusterName}_${QueueName}_sttchg" Description: !Sub "Route state change notifications from EC2 to SQS Queue, ${QueueName}." EventPattern: source: From fa925880c020ff0f1d3cfb5d7c4779afb3a7aa8c Mon Sep 17 00:00:00 2001 From: Jerad C Date: Thu, 30 Jun 2022 10:30:34 -0500 Subject: [PATCH 03/13] update cluster version to 1.22 --- resources/eks-cluster.yaml.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/eks-cluster.yaml.tmpl b/resources/eks-cluster.yaml.tmpl index 9f6de787..505ae9dd 100644 --- a/resources/eks-cluster.yaml.tmpl +++ b/resources/eks-cluster.yaml.tmpl @@ -3,7 +3,7 @@ kind: ClusterConfig metadata: name: ${CLUSTER_NAME} region: ${AWS_REGION} - version: "1.21" + version: "1.22" tags: karpenter.sh/discovery: ${CLUSTER_NAME} managedNodeGroups: From b0ac04e755e495b5f7bc1681aa1c06ae2be57c40 Mon Sep 17 00:00:00 2001 From: Jerad C Date: Thu, 30 Jun 2022 10:49:00 -0500 Subject: [PATCH 04/13] update wording of alternate instructions --- DEVELOPMENT.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 581d44fa..9dde88d8 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -26,7 +26,7 @@ envsubst terminator-${TERMINATOR_NAME}.yaml kubectl apply -f terminator-${TERMINATOR_NAME}.yaml ``` -If you do not want to use `envsubst` you can copy the template file and substitute the referenced values. +As an alternative to using `envsubst` you can copy the template file and substitute the referenced values. ## 8. Remove deployed controller from EKS cluster From 28b7f022ffb3eb3f3cc19123adf5ca246bbeb1b4 Mon Sep 17 00:00:00 2001 From: Jerad C Date: Thu, 30 Jun 2022 11:15:18 -0500 Subject: [PATCH 05/13] add tools list to guide --- DEVELOPMENT.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 9dde88d8..52d117e1 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,5 +1,12 @@ # Setup Development Environment +**Tools used in this guide** +* [kubectl](https://docs.aws.amazon.com/eks/latest/userguide/install-kubectl.html) +* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html#getting-started-install-instructions) - version 2 is recommended +* [eksctl](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) +* [jq](https://stedolan.github.io/jq/) +* [envsubst](https://www.gnu.org/software/gettext/manual/html_node/envsubst-Invocation.html) + ## 1. Clone the repo ```sh From 052551d51c02849a503c6487f5da1aaa7eda4606 Mon Sep 17 00:00:00 2001 From: Jerad C Date: Thu, 30 Jun 2022 11:33:22 -0500 Subject: [PATCH 06/13] highlight useful make commands in dev guide * Display help message * Run test suite --- DEVELOPMENT.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 52d117e1..08e096a0 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -12,6 +12,11 @@ ```sh git clone --branch v2 https://github.com/aws/aws-node-termination-handler.git nthv2 cd nthv2 + +# Display all targets and the descriptions. +make help + +make test ``` ## 2. Specify an EKS Cluster From ce1b1fb5484d119864839b72a58421d75aed9d2f Mon Sep 17 00:00:00 2001 From: Jerad C Date: Thu, 30 Jun 2022 11:53:00 -0500 Subject: [PATCH 07/13] add optional step to display created resources --- DEVELOPMENT.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 08e096a0..5636238b 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -140,6 +140,13 @@ The `apply` target sets some Helm chart values for you based on environment vari make HELM_OPTS='--set logging.level=debug' apply ``` +### 6.2. (Optional) List all deployed resources + +```sh +kubectl api-resources --verbs=list --namespaced -o name | \ + xargs -n 1 kubectl get --show-kind --ignore-not-found --namespace "${CLUSTER_NAMESPACE}" +``` + ## 7. Define and deploy a Terminator to EKS cluster ```sh From 4756b93583d2778a16c53a3a14447754f9c34ffa Mon Sep 17 00:00:00 2001 From: Jerad C Date: Thu, 30 Jun 2022 12:04:09 -0500 Subject: [PATCH 08/13] add section for creating an IAM OIDC provider --- DEVELOPMENT.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 5636238b..f0aecc17 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -40,6 +40,12 @@ export KUBECONFIG="$PWD/kubeconfig" As an alternative to using `envsubst` you can copy the template file and substitute the referenced values. +### 2.2. Create an IAM OIDC provider + +*Note:* If you created your cluster using `resource/eks-cluster.yaml.tmpl` in the previous step then your cluster will already have an IAM OIDC provider. + +This [guide](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html) will walk you through determining whether you cluster already has an IAM OIDC provider, and how to create one if it does not already exist. + ## 3. Create Infrastructure ```sh From 96f4ca5ea31471083d34043b121abc962f569453 Mon Sep 17 00:00:00 2001 From: Jerad C Date: Fri, 1 Jul 2022 11:58:27 -0500 Subject: [PATCH 09/13] download pre-built ko --- DEVELOPMENT.md | 8 +++--- Makefile | 2 +- scripts/download-ko.sh | 57 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) create mode 100755 scripts/download-ko.sh diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index f0aecc17..d008773b 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -2,16 +2,18 @@ **Tools used in this guide** * [kubectl](https://docs.aws.amazon.com/eks/latest/userguide/install-kubectl.html) -* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html#getting-started-install-instructions) - version 2 is recommended +* [aws](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html#getting-started-install-instructions) - version 2 is recommended * [eksctl](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) * [jq](https://stedolan.github.io/jq/) * [envsubst](https://www.gnu.org/software/gettext/manual/html_node/envsubst-Invocation.html) +* [wget](https://www.gnu.org/software/wget/) +* [Go](https://go.dev/dl/) - version 1.17+ ## 1. Clone the repo ```sh -git clone --branch v2 https://github.com/aws/aws-node-termination-handler.git nthv2 -cd nthv2 +git clone --branch v2 https://github.com/aws/aws-node-termination-handler.git "${GOPATH}/src/nthv2" +cd "${GOPATH}/src/nthv2" # Display all targets and the descriptions. make help diff --git a/Makefile b/Makefile index 29a522a1..ecd6915c 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ $(GINKGO): GOBIN="$(BIN_DIR)" go install github.com/onsi/ginkgo/v2/ginkgo@v2.1.3 $(KO): - GOBIN="$(BIN_DIR)" go install github.com/google/ko@v0.9.3 + @./scripts/download-ko.sh "$(BIN_DIR)" $(SETUP_ENVTEST): GOBIN="$(BIN_DIR)" go install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20220217150738-f62a0f579d73 diff --git a/scripts/download-ko.sh b/scripts/download-ko.sh new file mode 100755 index 00000000..224bbc51 --- /dev/null +++ b/scripts/download-ko.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -euo pipefail + +usage=$(cat << EOM +usage: $(basename "$0") -h | DIR_PATH + + Download the ko binary to DIR_PATH. + + Arguments: + -h Print usage message then exit. + +EOM +) + +while getopts "h" opt; do + case $opt in + h ) echo "${usage}" + exit 0 + ;; + \? ) echo "${usage}" 1>&2 + exit 1 + ;; + esac +done + +dir_path="$1" + +if [[ -z "${dir_path}" ]]; then + echo "error: missing directory path" 1>&2 + echo 1>&2 + echo "${usage}" 1>&2 + exit 1 +fi + +if ! which wget >/dev/null ; then + echo "error: wget not installed" 1>&2 + exit 1 +fi + +version="0.11.2" +os="$(go env GOHOSTOS)" +arch="$(go env GOHOSTARCH)" + +if [[ "${arch}" == "amd64" ]]; then + arch="x86_64" +elif [[ "${arch}" == "arm" ]]; then + arch="arm64" +elif [[ "${arch}" == "386" ]]; then + arch="i386" +fi + +echo "Downloading github.com/google/ko@v${version} ..." + +cd "${dir_path}" +wget https://github.com/google/ko/releases/download/v${version}/ko_${version}_${os}_${arch}.tar.gz -O - | \ + tar xzf - ko From 630f83c160d41fbfd52081ab5d37afe713796653 Mon Sep 17 00:00:00 2001 From: Jerad C Date: Tue, 12 Jul 2022 15:34:03 -0500 Subject: [PATCH 10/13] add terminators to 'all' category --- .../templates/node.k8s.aws_terminators.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/aws-node-termination-handler-2/templates/node.k8s.aws_terminators.yaml b/charts/aws-node-termination-handler-2/templates/node.k8s.aws_terminators.yaml index f2d61e2f..888272f2 100644 --- a/charts/aws-node-termination-handler-2/templates/node.k8s.aws_terminators.yaml +++ b/charts/aws-node-termination-handler-2/templates/node.k8s.aws_terminators.yaml @@ -13,6 +13,8 @@ spec: listKind: TerminatorList plural: terminators singular: terminator + categories: + - all scope: Namespaced versions: - name: v1alpha1 From 977853a54644a78d3a54ad58130dc7699ce8baf6 Mon Sep 17 00:00:00 2001 From: Jerad C Date: Tue, 12 Jul 2022 15:36:18 -0500 Subject: [PATCH 11/13] add namespace to terminator template --- resources/terminator.yaml.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/resources/terminator.yaml.tmpl b/resources/terminator.yaml.tmpl index ef674775..4c786397 100644 --- a/resources/terminator.yaml.tmpl +++ b/resources/terminator.yaml.tmpl @@ -2,6 +2,7 @@ apiVersion: node.k8s.aws/v1alpha1 kind: Terminator metadata: name: ${TERMINATOR_NAME} + namespace: ${CLUSTER_NAMESPACE} spec: sqs: From 7e6c6facd4396c4b14a73c95fad290ffecfc8652 Mon Sep 17 00:00:00 2001 From: Jerad C Date: Tue, 19 Jul 2022 10:03:25 -0500 Subject: [PATCH 12/13] do not hardcode namespace in helm uninstall --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ecd6915c..c0f0a409 100644 --- a/Makefile +++ b/Makefile @@ -116,4 +116,4 @@ apply: $(KO) $(KODATA) ## Deploy the controller into the current kubernetes clus .PHONY: delete delete: ## Delete controller from current kubernetes cluster. - helm uninstall dev --namespace nthv2 + helm uninstall dev --namespace ${CLUSTER_NAMESPACE} From bc59ed20604dd85b25ff3c7c8eb869a2ad33dfff Mon Sep 17 00:00:00 2001 From: Jerad C Date: Tue, 19 Jul 2022 10:17:16 -0500 Subject: [PATCH 13/13] download-ko.sh creates destination directory --- scripts/download-ko.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/download-ko.sh b/scripts/download-ko.sh index 224bbc51..3839ac50 100755 --- a/scripts/download-ko.sh +++ b/scripts/download-ko.sh @@ -52,6 +52,7 @@ fi echo "Downloading github.com/google/ko@v${version} ..." +mkdir -p "${dir_path}" cd "${dir_path}" wget https://github.com/google/ko/releases/download/v${version}/ko_${version}_${os}_${arch}.tar.gz -O - | \ tar xzf - ko