master: update release-tools

release-tools/build.make

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# force the usage of /bin/bash instead of /bin/sh
+SHELL := /bin/bash
+
 .PHONY: build-% build container-% container push-% push clean test
 
 # A space-separated list of all commands in the repository, must be
@@ -63,26 +66,35 @@ endif
 # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables
 # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below.
 
-# BUILD_PLATFORMS contains a set of <os> <arch> <suffix> triplets,
+# BUILD_PLATFORMS contains a set of tuples [os arch suffix base_image addon_image]
 # separated by semicolon. An empty variable or empty entry (= just a
 # semicolon) builds for the default platform of the current Go
 # toolchain.
 BUILD_PLATFORMS =
 
 # Add go ldflags using LDFLAGS at the time of compilation.
-IMPORTPATH_LDFLAGS = -X main.version=$(REV) 
+IMPORTPATH_LDFLAGS = -X main.version=$(REV)
 EXT_LDFLAGS = -extldflags "-static"
-LDFLAGS = 
+LDFLAGS =
 FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS)
 # This builds each command (= the sub-directories of ./cmd) for the target platform(s)
 # defined by BUILD_PLATFORMS.
 $(CMDS:%=build-%): build-%: check-go-version-go
 	mkdir -p bin
-	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \
+	# os_arch_seen captures all of the $$os-$$arch seen for the current binary
+	# that we want to build, if we've seen an $$os-$$arch before it means that
+	# we don't need to build it again, this is done to avoid building
+	# the windows binary multiple times (see the default value of $$BUILD_PLATFORMS)
+	export os_arch_seen="" && echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
+		os_arch_seen_pre=$${os_arch_seen%%$$os-$$arch*}; \
+		if ! [ $${#os_arch_seen_pre} = $${#os_arch_seen} ]; then \
+			continue; \
+		fi; \
 		if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \
 			echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \
 			exit 1; \
 		fi; \
+		os_arch_seen+=";$$os-$$arch"; \
 	done
 
 $(CMDS:%=container-%): container-%: build-%
@@ -131,30 +143,46 @@ DOCKER_BUILDX_CREATE_ARGS ?=
 # the tag for the resulting multiarch image.
 $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
 	set -ex; \
-	DOCKER_CLI_EXPERIMENTAL=enabled; \
-	export DOCKER_CLI_EXPERIMENTAL; \
+	export DOCKER_CLI_EXPERIMENTAL=enabled; \
 	docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \
 	trap "docker buildx rm multiarchimage-buildertest" EXIT; \
 	dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \
 	dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \
 	if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \
 	if ! [ -f "$$dockerfile_windows" ]; then \
-		build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \
+		build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$//')"; \
 	fi; \
 	pushMultiArch () { \
 		tag=$$1; \
-		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \
+		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
+			escaped_base_image=$${base_image/:/-}; \
+			if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
 			docker buildx build --push \
-				--tag $(IMAGE_NAME):$$arch-$$os-$$tag \
+				--tag $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag \
 				--platform=$$os/$$arch \
 				--file $$(eval echo \$${dockerfile_$$os}) \
 				--build-arg binary=./bin/$*$$suffix \
 				--build-arg ARCH=$$arch \
+				--build-arg BASE_IMAGE=$$base_image \
+				--build-arg ADDON_IMAGE=$$addon_image \
 				--label revision=$(REV) \
 				.; \
 		done; \
-		images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \
+		images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
+			escaped_base_image=$${base_image/:/-}; \
+			if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
+			echo $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \
+		done); \
 		docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \
+		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
+			if [ $$os = "windows" ]; then \
+				escaped_base_image=$${base_image/:/-}; \
+				if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
+				image=$(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \
+				os_version=$$(docker manifest inspect mcr.microsoft.com/windows/$${base_image} | grep "os.version" | head -n 1 | awk '{print $$2}' | sed -e 's/"//g') || true; \
+				docker manifest annotate --os-version $$os_version $(IMAGE_NAME):$$tag $$image; \
+			fi; \
+		done; \
 		docker manifest push -p $(IMAGE_NAME):$$tag; \
 	}; \
 	if [ $(PULL_BASE_REF) = "master" ]; then \
@@ -288,4 +316,3 @@ test-spelling:
 test-boilerplate:
 	@ echo; echo "### $@:"
 	@ ./release-tools/verify-boilerplate.sh "$(pwd)"
-

release-tools/cloudbuild.yaml

@@ -27,7 +27,7 @@ steps:
   # The image must contain bash and curl. Ideally it should also contain
   # the desired version of Go (currently defined in release-tools/prow.sh),
   # but that just speeds up the build and is not required.
-  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8'
+  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20210331-c732583'
     entrypoint: ./.cloudbuild.sh
     env:
     - GIT_TAG=${_GIT_TAG}

release-tools/prow.sh

@@ -77,7 +77,10 @@ version_to_git () {
     esac
 }
 
-configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
+# the list of windows versions was matched from:
+# - https://hub.docker.com/_/microsoft-windows-nanoserver
+# - https://hub.docker.com/_/microsoft-windows-servercore
+configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
 
 # If we have a vendor directory, then use it. We must be careful to only
 # use this for "make" invocations inside the project's repo itself because
@@ -723,7 +726,7 @@ install_csi_driver () {
     fi
 }
 
-# Installs all nessesary snapshotter CRDs  
+# Installs all nessesary snapshotter CRDs
 install_snapshot_crds() {
   # Wait until volumesnapshot CRDs are in place.
   CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd"
@@ -770,7 +773,7 @@ install_snapshot_controller() {
     fi
     echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt"
 	cnt=$((cnt + 1))
-    sleep 10   
+    sleep 10
   done
 
   SNAPSHOT_CONTROLLER_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml"
@@ -839,7 +842,7 @@ install_snapshot_controller() {
     fi
     echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt"
 	cnt=$((cnt + 1))
-    sleep 10   
+    sleep 10
   done
 }
 
@@ -979,7 +982,7 @@ run_sanity () (
 kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- mkdir "\$@" && echo "\$@"
 EOF
     # Using "rm -rf" as fallback for "rmdir" is a workaround for:
-    # Node Service 
+    # Node Service
     #     should work
     # /nvme/gopath.tmp/src/github.com/kubernetes-csi/csi-test/pkg/sanity/node.go:624
     # STEP: reusing connection to CSI driver at dns:///172.17.0.2:30896
@@ -1006,6 +1009,24 @@ if ! kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" --
     exit 1
 fi
 EOF
+
+    cat >"${CSI_PROW_WORK}/checkdir_in_pod.sh" <<EOF
+#!/bin/sh
+CHECK_PATH=\$(cat <<SCRIPT
+if [ -f "\$@" ]; then
+    echo "file"
+elif [ -d "\$@" ]; then
+    echo "directory"
+elif [ -e "\$@" ]; then
+    echo "other"
+else
+    echo "not_found"
+fi
+SCRIPT
+)
+kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- /bin/sh -c "\${CHECK_PATH}"
+EOF
+
     chmod u+x "${CSI_PROW_WORK}"/*dir_in_pod.sh
 
     # This cannot run in parallel, because -csi.junitfile output
@@ -1021,6 +1042,7 @@ EOF
                      -csi.createmountpathcmd "${CSI_PROW_WORK}/mkdir_in_pod.sh" \
                      -csi.removestagingpathcmd "${CSI_PROW_WORK}/rmdir_in_pod.sh" \
                      -csi.removemountpathcmd "${CSI_PROW_WORK}/rmdir_in_pod.sh" \
+                     -csi.checkpathcmd "${CSI_PROW_WORK}/checkdir_in_pod.sh" \
 )
 
 ascii_to_xml () {
@@ -1124,7 +1146,7 @@ make_test_to_junit () {
 # version_gt 1.3.1 v1.2.0  (returns true)
 # version_gt 1.1.1 release-1.2.0  (returns false)
 # version_gt 1.2.0 1.2.2  (returns false)
-function version_gt() { 
+function version_gt() {
     versions=$(for ver in "$@"; do ver=${ver#release-}; ver=${ver#kubernetes-}; echo "${ver#v}"; done)
     greaterVersion=${1#"release-"};
     greaterVersion=${greaterVersion#"kubernetes-"};
@@ -1187,7 +1209,7 @@ main () {
                 if [ "$rbac_file_path" == "" ]; then
                     rbac_file_path="$(pwd)/deploy/kubernetes/rbac.yaml"
                 fi
-                
+
                 if [ -e "$rbac_file_path" ]; then
                     # This is one of those components which has its own RBAC rules (like external-provisioner).
                     # We are testing a locally built image and also want to test with the the current,

release-tools/verify-shellcheck.sh

@@ -84,7 +84,7 @@ done < <(find . -name "*.sh" \
 # detect if the host machine has the required shellcheck version installed
 # if so, we will use that instead.
 HAVE_SHELLCHECK=false
-if which shellcheck &>/dev/null; then
+if command -v shellcheck &>/dev/null; then
   detected_version="$(shellcheck --version | grep 'version: .*')"
   if [[ "${detected_version}" = "version: ${SHELLCHECK_VERSION}" ]]; then
     HAVE_SHELLCHECK=true