diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index c80904cf..5bcedc6b 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -64,6 +64,12 @@ jobs: - run: make -j test-unit test-helm + - name: Upload test artifacts + uses: actions/upload-artifact@v4 + with: + name: unit-artifacts + path: _bin/artifacts + test-e2e: if: contains(github.event.pull_request.labels.*.name, 'test-e2e') runs-on: ubuntu-latest @@ -136,3 +142,16 @@ jobs: --project=machineidentitysecurity-jsci-e \ --zone=europe-west1-b \ --quiet + +# TODO: REMOVE THIS DEBUGGING! + - name: Setup upterm session + if: always() + uses: owenthereal/action-upterm@v1 + with: + limit-access-to-actor: true + + - name: Upload test artifacts + uses: actions/upload-artifact@v4 + with: + name: e2e-artifacts + path: _bin/artifacts \ No newline at end of file diff --git a/coverage_server.go b/coverage_server.go new file mode 100644 index 00000000..a59c7b0b --- /dev/null +++ b/coverage_server.go @@ -0,0 +1,70 @@ +package main + +import ( + "bytes" + "log" + "net/http" + "runtime/coverage" +) + +func startCoverageServer() { + adminMux := http.NewServeMux() + + adminMux.HandleFunc("/_debug/coverage/download", func(w http.ResponseWriter, r *http.Request) { + var buffer bytes.Buffer + + // Attempt to write the coverage counters to the buffer. + if err := coverage.WriteCounters(&buffer); err != nil { + log.Printf("Error writing coverage counters to buffer: %v", err) + // Inform the client that an internal error occurred. + http.Error(w, "Failed to generate coverage report", http.StatusInternalServerError) + return + } + + // Check if any data was written to the buffer. + if buffer.Len() == 0 { + log.Println("Coverage data is empty. No counters were written.") + } else { + log.Printf("Successfully wrote %d bytes of coverage data to the buffer.", buffer.Len()) + } + + // If successful, proceed to write the buffer's content to the actual HTTP response. + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", `attachment; filename="coverage.out"`) + + // Write the captured coverage data from the buffer to the response writer. + if _, err := w.Write(buffer.Bytes()); err != nil { + log.Printf("Error writing coverage data from buffer to response: %v", err) + } + }) + + adminMux.HandleFunc("/_debug/coverage/meta/download", func(w http.ResponseWriter, r *http.Request) { + log.Println("Received request to download coverage metadata...") + + var buffer bytes.Buffer + if err := coverage.WriteMeta(&buffer); err != nil { + log.Printf("Error writing coverage meta to buffer: %v", err) + // Inform the client that an internal error occurred. + http.Error(w, "Failed to generate coverage meta", http.StatusInternalServerError) + return + } + // Check if any data was written to the buffer. + if buffer.Len() == 0 { + log.Println("Coverage meta is empty.") + } else { + log.Printf("Successfully wrote %d bytes of coverage meta to the buffer.", buffer.Len()) + } + + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", `attachment; filename="coverage.meta"`) + if _, err := w.Write(buffer.Bytes()); err != nil { + log.Printf("Error writing coverage meta from buffer to response: %v", err) + } + }) + + go func() { + if err := http.ListenAndServe("localhost:8089", adminMux); err != nil { + log.Printf("Admin server failed: %v", err) + } + }() +} diff --git a/hack/e2e/test.sh b/hack/e2e/test.sh index dbf2195d..36bf0146 100755 --- a/hack/e2e/test.sh +++ b/hack/e2e/test.sh @@ -83,6 +83,19 @@ if ! gcloud container clusters get-credentials "${CLUSTER_NAME}"; then fi kubectl create ns venafi || true +kubectl apply -n venafi -f - </dev/null && pwd) +root_dir=$(cd "${script_dir}/../.." && pwd) +export TERM=dumb + +# Your Venafi Cloud API key. +: ${VEN_API_KEY?} +# Separate API Key for getting a pull secret. +: ${VEN_API_KEY_PULL?} +# The Venafi Cloud zone. +: ${VEN_ZONE?} +# The hostname of the Venafi API server (e.g., api.venafi.cloud). +: ${VEN_API_HOST?} +# The region of the Venafi API server (e.g., "us" or "eu"). +: ${VEN_VCP_REGION?} +# The base URL of the OCI registry (e.g., ttl.sh/some-random-uuid). +: ${OCI_BASE?} + +REMOTE_AGENT_IMAGE="${OCI_BASE}/venafi-kubernetes-agent-e2e" + +cd "${script_dir}" + +# Build and PUSH agent image and Helm chart to the anonymous registry +echo ">>> Building and pushing agent to '${REMOTE_AGENT_IMAGE}'..." +pushd "${root_dir}" +> release.env +make release \ + OCI_SIGN_ON_PUSH=false \ + oci_platforms=linux/amd64 \ + oci_preflight_image_name=${REMOTE_AGENT_IMAGE} \ + helm_chart_image_name=$OCI_BASE/charts/venafi-kubernetes-agent \ + GITHUB_OUTPUT=release.env +source release.env +popd + +AGENT_IMAGE_WITH_TAG="${REMOTE_AGENT_IMAGE}:${RELEASE_HELM_CHART_VERSION}" +echo ">>> Successfully pushed image: ${AGENT_IMAGE_WITH_TAG}" + +kubectl create ns venafi || true + +# Create pull secret for Venafi's OCI registry if it doesn't exist. +if ! kubectl get secret venafi-image-pull-secret -n venafi; then + echo ">>> Creating Venafi OCI registry pull secret..." + venctl iam service-accounts registry create \ + --api-key $VEN_API_KEY_PULL \ + --no-prompts \ + --owning-team "$(curl --fail-with-body -sS "https://${VEN_API_HOST}/v1/teams" -H "tppl-api-key: ${VEN_API_KEY_PULL}" | jq '.teams[0].id' -r)" \ + --name "venafi-kubernetes-agent-e2e-registry-${RANDOM}" \ + --scopes enterprise-cert-manager,enterprise-venafi-issuer,enterprise-approver-policy \ + | jq '{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "venafi-image-pull-secret" + }, + "type": "kubernetes.io/dockerconfigjson", + "stringData": { + ".dockerconfigjson": { + "auths": { + "\(.oci_registry)": { + "username": .username, + "password": .password + } + } + } | tostring + } + }' \ + | kubectl create -n venafi -f - +fi + +echo ">>> Generating temporary Helm values for the custom agent image..." +cat < /tmp/agent-image-values.yaml +image: + repository: ${REMOTE_AGENT_IMAGE} + tag: ${RELEASE_HELM_CHART_VERSION} + pullPolicy: IfNotPresent +EOF + +echo ">>> Applying Venafi components to the cluster..." +export VENAFI_KUBERNETES_AGENT_CLIENT_ID="not-used-but-required-by-venctl" +venctl components kubernetes apply \ + --region $VEN_VCP_REGION \ + --cert-manager \ + --venafi-enhanced-issuer \ + --approver-policy-enterprise \ + --venafi-kubernetes-agent \ + --venafi-kubernetes-agent-version "${RELEASE_HELM_CHART_VERSION}" \ + --venafi-kubernetes-agent-values-files "${script_dir}/values.venafi-kubernetes-agent.yaml" \ + --venafi-kubernetes-agent-values-files "/tmp/agent-image-values.yaml" \ + --venafi-kubernetes-agent-custom-chart-repository "oci://${OCI_BASE}/charts" + +kubectl apply -n venafi -f venafi-components.yaml + +# Configure Workload Identity Federation with Venafi Cloud +echo ">>> Configuring Workload Identity Federation..." +subject="system:serviceaccount:venafi:venafi-components" +audience="https://${VEN_API_HOST}" +issuerURL=$(kubectl get --raw /.well-known/openid-configuration | jq -r '.issuer') +openidDiscoveryURL="${issuerURL}/.well-known/openid-configuration" +jwksURI=$(curl --fail-with-body -sSL ${openidDiscoveryURL} | jq -r '.jwks_uri') + +# Create the Venafi agent service account if one does not already exist +echo ">>> Ensuring Venafi Cloud service account exists for the agent..." +while true; do + tenantID=$(curl --fail-with-body -sSL -H "tppl-api-key: $VEN_API_KEY" https://${VEN_API_HOST}/v1/serviceaccounts \ + | jq -r '.[] | select(.issuerURL==$issuerURL and .subject == $subject) | .companyId' \ + --arg issuerURL "${issuerURL}" \ + --arg subject "${subject}") + + if [[ "${tenantID}" != "" ]]; then + echo "Service account already exists." + break + fi + + echo "Service account not found, creating it..." + jq -n '{ + "name": "venafi-kubernetes-agent-e2e-agent-\($random)", + "authenticationType": "rsaKeyFederated", + "scopes": ["kubernetes-discovery-federated", "certificate-issuance"], + "subject": $subject, + "audience": $audience, + "issuerURL": $issuerURL, + "jwksURI": $jwksURI, + "owner": $owningTeamID + }' \ + --arg random "${RANDOM}" \ + --arg subject "${subject}" \ + --arg audience "${audience}" \ + --arg issuerURL "${issuerURL}" \ + --arg jwksURI "${jwksURI}" \ + --arg owningTeamID "$(curl --fail-with-body -sS "https://${VEN_API_HOST}/v1/teams" -H "tppl-api-key: $VEN_API_KEY" | jq '.teams[0].id' -r)" \ + | curl "https://${VEN_API_HOST}/v1/serviceaccounts" \ + -H "tppl-api-key: $VEN_API_KEY" \ + --fail-with-body \ + -sSL --json @- +done + +# Create the VenafiConnection resource +echo ">>> Applying VenafiConnection resource..." +kubectl apply -n venafi -f - <>> Testing certificate issuance..." +envsubst >> Waiting for agent log message confirming successful data upload..." +set +o pipefail +kubectl logs deployments/venafi-kubernetes-agent \ + --follow \ + --namespace venafi \ + | timeout 60s jq 'if .msg | test("Data sent successfully") then . | halt_error(0) end' +set -o pipefail + +# Create a unique TLS secret and verify its discovery by the agent +echo ">>> Testing discovery of a manually created TLS secret..." +commonname="venafi-kubernetes-agent-e2e.$(uuidgen | tr '[:upper:]' '[:lower:]')" +openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /tmp/tls.key -out /tmp/tls.crt -subj "/CN=$commonname" +kubectl create secret tls "$commonname" --cert=/tmp/tls.crt --key=/tmp/tls.key -o yaml --dry-run=client | kubectl apply -f - + +getCertificate() { + jq -n '{ + "expression": { + "field": "subjectCN", + "operator": "MATCH", + "value": $commonname + }, + "ordering": { + "orders": [ + { "direction": "DESC", "field": "certificatInstanceModificationDate" } + ] + }, + "paging": { "pageNumber": 0, "pageSize": 10 } + }' --arg commonname "${commonname}" \ + | curl "https://${VEN_API_HOST}/outagedetection/v1/certificatesearch?excludeSupersededInstances=true&ownershipTree=true" \ + -fsSL \ + -H "tppl-api-key: $VEN_API_KEY" \ + --json @- \ + | jq 'if .count == 0 then . | halt_error(1) end' +} + +# Wait up to 5 minutes for the certificate to appear in the Venafi inventory +echo ">>> Waiting for certificate '${commonname}' to appear in Venafi Cloud inventory..." +for ((i=0;;i++)); do + if getCertificate; then + echo "Successfully found certificate in Venafi Cloud." + exit 0; + fi; + echo "Certificate not found yet, retrying in 30 seconds..." + sleep 30; +done | timeout -v -- 5m cat + +echo "!!! Test Failed: Timed out waiting for certificate to appear in Venafi Cloud." +exit 1 \ No newline at end of file diff --git a/hack/e2e/values.venafi-kubernetes-agent.yaml b/hack/e2e/values.venafi-kubernetes-agent.yaml index 0e5c2120..e6267549 100644 --- a/hack/e2e/values.venafi-kubernetes-agent.yaml +++ b/hack/e2e/values.venafi-kubernetes-agent.yaml @@ -11,3 +11,13 @@ authentication: extraArgs: - --logging-format=json - --log-level=4 + +podSecurityContext: + fsGroup: 2000 +volumes: + - name: coverage-storage + persistentVolumeClaim: + claimName: coverage-pvc +volumeMounts: + - name: coverage-storage + mountPath: /coverage \ No newline at end of file diff --git a/main.go b/main.go index 18f19838..057f15d3 100644 --- a/main.go +++ b/main.go @@ -3,5 +3,6 @@ package main import "github.com/jetstack/preflight/cmd" func main() { + startCoverageServer() cmd.Execute() } diff --git a/make/00_mod.mk b/make/00_mod.mk index 2e08f20a..0ac3e80a 100644 --- a/make/00_mod.mk +++ b/make/00_mod.mk @@ -13,6 +13,13 @@ kind_cluster_config := $(bin_dir)/scratch/kind_cluster.yaml build_names := preflight +# HACK: The test-unit and test-e2e targets require the go binary to be built with the -cover flag set. +# This allows us to do coverage reporting for our end-to-end tests. +ifeq ($(findstring test-,$(MAKECMDGOALS)),test-) +go_preflight_flags := -cover +endif +COVERAGE_HOST_PATH := $(CURDIR)/$(bin_dir)/artifacts + go_preflight_main_dir := . go_preflight_mod_dir := . go_preflight_ldflags := \ diff --git a/make/02_mod.mk b/make/02_mod.mk index 12760fa0..0d9b4844 100644 --- a/make/02_mod.mk +++ b/make/02_mod.mk @@ -1,6 +1,14 @@ include make/test-unit.mk include make/ark/02_mod.mk +$(kind_cluster_config): make/config/kind/cluster.yaml | $(bin_dir)/scratch + @echo "--- COVERAGE_HOST_PATH is $(COVERAGE_HOST_PATH) ---" + mkdir -p $(COVERAGE_HOST_PATH) + @cat $< | \ + sed -e 's|{{KIND_IMAGES}}|$(CURDIR)/$(images_tar_dir)|g' | \ + sed -e 's|{{COVERAGE_HOST_PATH}}|$(COVERAGE_HOST_PATH)|g' \ + > $@ + GITHUB_OUTPUT ?= /dev/stderr .PHONY: release ## Publish all release artifacts (image + helm chart) @@ -51,8 +59,9 @@ shared_generate_targets += generate-crds-venconn ## Wait for it to log a message indicating successful data upload. ## See `hack/e2e/test.sh` for the full test script. ## @category Testing -test-e2e-gke: | $(NEEDS_HELM) $(NEEDS_STEP) $(NEEDS_VENCTL) - ./hack/e2e/test.sh +test-e2e-gke: | kind-cluster $(NEEDS_HELM) $(NEEDS_STEP) $(NEEDS_VENCTL) + #COVERAGE_HOST_PATH="$(COVERAGE_HOST_PATH)" ./hack/e2e/test_ci.sh + COVERAGE_HOST_PATH="$(COVERAGE_HOST_PATH)" ./hack/e2e/test.sh .PHONY: test-helm ## Run `helm unittest`. diff --git a/make/_shared/oci-build/01_mod.mk b/make/_shared/oci-build/01_mod.mk index 726ad13c..93632387 100644 --- a/make/_shared/oci-build/01_mod.mk +++ b/make/_shared/oci-build/01_mod.mk @@ -28,7 +28,8 @@ $(ko_config_path_$1:$(CURDIR)/%=%): | $(NEEDS_YQ) $(bin_dir)/scratch/image $(YQ) '.builds[0].ldflags[0] = "-s"' | \ $(YQ) '.builds[0].ldflags[1] = "-w"' | \ $(YQ) '.builds[0].ldflags[2] = "{{.Env.LDFLAGS}}"' | \ - $(YQ) '.builds[0].flags[0] = "$(go_$1_flags)"' | \ + $(YQ) '.builds[0].flags[0] = "-cover"' | \ + $(YQ) '.builds[0].flags[1] = "-covermode=atomic"' | \ $(YQ) '.builds[0].linux_capabilities = "$(oci_$1_linux_capabilities)"' \ > $(CURDIR)/$(oci_layout_path_$1).ko_config.yaml diff --git a/make/config/kind/cluster.yaml b/make/config/kind/cluster.yaml new file mode 100644 index 00000000..e5fd1ac8 --- /dev/null +++ b/make/config/kind/cluster.yaml @@ -0,0 +1,20 @@ +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +kubeadmConfigPatches: + - | + kind: ClusterConfiguration + metadata: + name: config + etcd: + local: + extraArgs: + unsafe-no-fsync: "true" + networking: + serviceSubnet: 10.0.0.0/16 +nodes: + - role: control-plane + extraMounts: + - hostPath: {{KIND_IMAGES}} + containerPath: /mounted_images + - hostPath: {{COVERAGE_HOST_PATH}} + containerPath: /coverage