Merge pull request #4118 from AkihiroSuda/test-k8s

CI: test Kubernetes port forwarder

Author: jandubois

.github/workflows/test.yml

@@ -354,6 +354,19 @@ jobs:
         template: templates/default.yaml
     - name: "Run BATS integration tests"
       run: make bats
+    - name: Cache image used by templates/k8s.yaml
+      uses: ./.github/actions/setup_cache_for_template
+      with:
+        template: templates/k8s.yaml
+    - name: "Run BATS k8s tests"
+      # Wish we could use BATS_TEST_RETRIES=3 as an environment variable here,
+      # but bats does not seem to support it.
+      uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08  # v3.0.2
+      with:
+        timeout_minutes: 30
+        retry_on: error
+        max_attempts: 3
+        command: ./hack/bats/lib/bats-core/bin/bats --timing ./hack/bats/extras/k8s.bats
 
   colima:
     name: "Colima tests (QEMU, Linux host)"

hack/bats/extras/README.md

@@ -0,0 +1,6 @@
+# Extra tests
+
+The extra tests located in this directory are not automatically executed via `make bats`.
+
+Some tests are executed on the CI, some ones are not.
+Refer to the configuration of the GitHub Actions to see what tests are executed.

hack/bats/extras/k8s.bats

@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: Copyright The Lima Authors
+# SPDX-License-Identifier: Apache-2.0
+
+# This test verifies that a Kubernetes cluster can be started and that the single node is ready.
+
+load "../helpers/load"
+
+: "${TEMPLATE:=k8s}"
+
+# Instance names are "${NAME}-0", "${NAME}-1", ...
+NAME="k8s"
+
+get_num_nodes() {
+    local nodes=0
+    for tag in "${BATS_TEST_TAGS[@]}"; do
+        if [[ $tag =~ ^nodes:([0-9]+)$ ]]; then
+            nodes="${BASH_REMATCH[1]}"
+        fi
+    done
+    if [[ $nodes -eq 0 ]]; then
+        echo >&2 "nodes:N tag is required"
+        exit 1
+    fi
+    echo "$nodes"
+}
+
+local_setup() {
+    local nodes=$(get_num_nodes)
+    for ((i=0; i<nodes; i++)); do
+        limactl delete --force "${NAME}-$i" || :
+        limactl start --tty=false --name "${NAME}-$i" "template:${TEMPLATE}" 3>&- 4>&-
+        # NOTE: No support for multi-node clusters yet.
+    done
+    for node in $(k get node -o name); do
+	    k wait --timeout=5m --for=condition=ready "${node}"
+    done
+}
+
+local_teardown() {
+    local nodes=$(get_num_nodes)
+    for ((i=0; i<nodes; i++)); do
+        limactl delete --force "${NAME}-$i" || :
+    done
+}
+
+k() {
+    # The host home directory is not mounted in the case of k8s.
+    limactl shell --workdir=/ "${NAME}-0" -- kubectl "$@"
+}
+
+# bats test_tags=nodes:1
+@test 'Single-node' {
+    k create deployment nginx --image="${TEST_CONTAINER_IMAGES["nginx"]}"
+    k rollout status deployment nginx --timeout 60s
+    k create service nodeport nginx --node-port=31080 --tcp=80:80
+    run curl --fail --silent --show-error --retry 30 --retry-all-errors http://localhost:31080
+    assert_success
+    assert_output --partial "Welcome to nginx"
+    # TODO: support UDP
+    k delete service nginx
+    k delete deployment nginx
+}
+
+# TODO: add a test for multi-node

hack/bats/helpers/load.bash

@@ -65,3 +65,9 @@ teardown() {
 assert_output_lines_count() {
     assert_equal "${#lines[@]}" "$1"
 }
+
+# Use GHCR and ECR to avoid hitting Docker Hub rate limit.
+# NOTE: keep this list in sync with hack/test-templates.sh .
+declare -A -g TEST_CONTAINER_IMAGES=(
+    ["nginx"]="ghcr.io/stargz-containers/nginx:1.19-alpine-org"
+)

templates/README.md

@@ -57,7 +57,7 @@ Container orchestration:
 - [`faasd`](./faasd.yaml): [Faasd](https://docs.openfaas.com/deployment/edge/)
 - [`k0s`](./k0s.yaml): [k0s](https://k0sproject.io/) Zero Friction Kubernetes
 - [`k3s`](./k3s.yaml): Kubernetes via k3s
-- [`k8s`](./k8s.yaml): Kubernetes via kubeadm
+- [`k8s`](./k8s.yaml): ⭐Kubernetes via kubeadm
 - [`experimental/u7s`](./experimental/u7s.yaml): [Usernetes](https://github.com/rootless-containers/usernetes): Rootless Kubernetes
 
 Optional feature enablers: