MLE-17368: K8s Operator: E2E test to verify Logging with a backend(Loki) on MarkLogic Cluster (#44)

* e2e tests for logging

* e2e assesment for fluent bit logs

* added check to verify MarkLogic logs using fluent-bit filter, loki and grafana

Author: barkhachoithani

Makefile

@@ -5,6 +5,9 @@
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
 VERSION ?= 0.0.1
 
+# VERIFY_HUGE_PAGES defines if hugepages test is enabled or not for e2e test
+VERIFY_HUGE_PAGES ?= true
+
 export E2E_DOCKER_IMAGE ?= $(IMG)
 export E2E_KUSTOMIZE_VERSION ?= $(KUSTOMIZE_VERSION)
 export E2E_CONTROLLER_TOOLS_VERSION ?= $(CONTROLLER_TOOLS_VERSION)
@@ -126,6 +129,8 @@ test: manifests generate fmt vet envtest ## Run tests.
 # To run specific e2e test with label, try 	go test -v ./test/e2e -count=1 -args --labels="type=tls-multi-node"
 .PHONY: e2e-test  # Run the e2e tests against a minikube k8s instance that is spun up.
 e2e-test: 
+	@echo "=====Check Huges pages test is enabled or not for e2e test"
+ifeq ($(VERIFY_HUGE_PAGES), true)
 	@echo "=====Setting hugepages value to 1280 for hugepages-e2e test"
 	sudo sysctl -w vm.nr_hugepages=1280
 
@@ -142,6 +147,10 @@ e2e-test:
 	@echo "=====Restart minikube cluster"
 	minikube stop
 	minikube start
+else
+	@echo "=====Running e2e test without hugepages test"
+	go test -v -count=1 ./test/e2e
+endif
 
 .PHONY: e2e-setup-minikube
 e2e-setup-minikube: kustomize controller-gen build docker-build

test/e2e/2_marklogic_cluster_test.go

@@ -2,6 +2,7 @@ package e2e
 
 import (
 	"context"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"strings"
@@ -10,6 +11,7 @@ import (
 
 	databasev1alpha1 "github.com/marklogic/marklogic-kubernetes-operator/api/v1alpha1"
 	coreV1 "k8s.io/api/core/v1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -31,6 +33,7 @@ const (
 
 var (
 	replicas         = int32(1)
+	logOutput        = "[OUTPUT]\n\tname loki\n\tmatch *\n\thost loki.loki.svc.cluster.local\n\tport 3100\n\tlabels job=fluent-bit\n\thttp_user admin\n\thttp_passwd admin"
 	adminUsername    = "admin"
 	adminPassword    = "Admin@8001"
 	marklogiccluster = &databasev1alpha1.MarklogicCluster{
@@ -55,14 +58,141 @@ var (
 					IsBootstrap: true,
 				},
 			},
+			LogCollection: &databasev1alpha1.LogCollection{
+				Enabled: true,
+				Image:   "fluent/fluent-bit:3.1.1",
+				Files: databasev1alpha1.LogFilesConfig{
+					ErrorLogs:   true,
+					AccessLogs:  true,
+					RequestLogs: true,
+					CrashLogs:   true,
+					AuditLogs:   true,
+				},
+				Outputs: logOutput,
+			},
 		},
 	}
+	dashboardPayload = `{
+		"dashboard": {
+			"panels": [
+				{
+					"type": "graph",
+					"title": "Fluent Bit Logs",
+					"targets": [
+						{
+							"expr": "rate({job=\"fluent-bit\"}[5m])",
+							"legendFormat": "{{job}}"
+						}
+					]
+				}
+			],
+			"title": "Fluent Bit Dashboard"
+		},
+			"overwrite": true
+	}`
+	dashboardUID      = ""
+	dataSourcePayload = `{
+		"name": "Loki",
+		"type": "loki",
+		"url": "http://loki-gateway.loki.svc.cluster.local",
+		"access": "proxy",
+		"basicAuth": false
+	}`
+	dataSourceUID = ""
 )
 
+type DashboardResponse struct {
+	UID    string `json:"uid"`
+	Status string `json:"status"`
+}
+type DataSourceResponse struct {
+	DataSource DataSource `json:"datasource"`
+}
+type DataSource struct {
+	UID     string `json:"uid"`
+	Message string `json:"message"`
+}
+
 func TestMarklogicCluster(t *testing.T) {
 	feature := features.New("Marklogic Cluster Test")
 
-	// Assessment for MarklogicCluster creation
+	// Setup Loki and Grafana to verify Logging for Operator
+	feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+		t.Log("Setting up Loki and Grafana")
+		client := c.Client()
+		err := utils.AddHelmRepo("grafana", "https://grafana.github.io/helm-charts")
+		if err != nil {
+			t.Fatalf("Failed to add helm repo: %v", err)
+		}
+
+		err = utils.InstallHelmChart("loki", "grafana/loki", "loki", "6.6.5", "loki.yaml")
+		if err != nil {
+			t.Fatalf("Failed to install loki helm chart: %v", err)
+		}
+
+		err = utils.InstallHelmChart("grafana", "grafana/grafana", "grafana", "8.3.2")
+		if err != nil {
+			t.Fatalf("Failed to install grafana helm chart: %v", err)
+		}
+
+		podList := &corev1.PodList{}
+		if err := client.Resources().List(ctx, podList, func(lo *metav1.ListOptions) {
+			lo.FieldSelector = "metadata.namespace=" + "grafana"
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		grafanaPodName := podList.Items[0].Name
+		err = utils.WaitForPod(ctx, t, client, "grafana", grafanaPodName, 120*time.Second)
+		if err != nil {
+			t.Fatalf("Failed to wait for grafana pod creation: %v", err)
+		}
+
+		// Get Grafana admin password
+		grafanaAdminUser, grafanaAdminPassword, err := utils.GetSecretData(ctx, client, "grafana", "grafana", "admin-user", "admin-password")
+		if err != nil {
+			t.Fatalf("Failed to get Grafana admin user and password: %v", err)
+		}
+
+		//Check Grafana Health before creating datasource
+		start := time.Now()
+		timeout := 2 * time.Minute
+		grafanaURL := "http://localhost:3000"
+		for {
+			if time.Since(start) > timeout {
+				t.Fatalf("Grafana is not ready after %v", timeout)
+			}
+			curlCommand := fmt.Sprintf(`curl -s -o /dev/null -w "%%{http_code}" %s/api/health`, grafanaURL)
+			output, err := utils.ExecCmdInPod(grafanaPodName, "grafana", "grafana", curlCommand)
+			if err != nil {
+				t.Logf("Grafana is not ready yet...an error occurred: %v", err)
+			}
+			if output == "200" {
+				t.Log("Grafana is ready")
+				break
+			}
+			time.Sleep(5 * time.Second)
+		}
+
+		// Create datasource for Grafana
+		url := fmt.Sprintf("%s/api/datasources", grafanaURL)
+		curlCommand := fmt.Sprintf(`curl -X POST %s -u %s:%s -H "Content-Type: application/json" -d '%s'`, url, grafanaAdminUser, grafanaAdminPassword, dataSourcePayload)
+		output, err := utils.ExecCmdInPod(grafanaPodName, "grafana", "grafana", curlCommand)
+		if err != nil {
+			t.Fatalf("Failed to execute kubectl command grafana in pod: %v", err)
+		}
+		if !(strings.Contains(string(output), "Datasource added") && strings.Contains(string(output), "Loki")) {
+			t.Fatal("Failed to create datasource for Grafana")
+		}
+		var dataSourceResponse DataSourceResponse
+		if err := json.Unmarshal([]byte(output), &dataSourceResponse); err != nil {
+			t.Fatalf("Failed to unmarshal JSON response: %v", err)
+		}
+		dataSourceUID = dataSourceResponse.DataSource.UID
+		return ctx
+	})
+
+	// Setup for MarklogicCluster creation
 	feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
 		client := c.Client()
 		databasev1alpha1.AddToScheme(client.Resources(mlNamespace).GetScheme())
@@ -106,6 +236,86 @@ func TestMarklogicCluster(t *testing.T) {
 
 	})
 
+	// Assessment to check for logging in MarkLogic Operator
+	feature.Assess("Grafana Dashboard created", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+		client := c.Client()
+		podList := &corev1.PodList{}
+		if err := client.Resources().List(ctx, podList, func(lo *metav1.ListOptions) {
+			lo.FieldSelector = "metadata.namespace=" + "grafana"
+		}); err != nil {
+			t.Fatal(err)
+		}
+		grafanaPodName := podList.Items[0].Name
+		grafanaAdminUser, grafanaAdminPassword, err := utils.GetSecretData(ctx, client, "grafana", "grafana", "admin-user", "admin-password")
+		if err != nil {
+			t.Fatalf("Failed to get Grafana admin user and password: %v", err)
+		}
+		time.Sleep(90 * time.Second)
+		grafanaURL := "http://localhost:3000"
+		url := fmt.Sprintf("%s/api/dashboards/db", grafanaURL)
+		curlCommand := fmt.Sprintf(`curl -X POST %s -u %s:%s -H "Content-Type: application/json" -d '%s'`, url, grafanaAdminUser, grafanaAdminPassword, dashboardPayload)
+		output, err := utils.ExecCmdInPod(grafanaPodName, "grafana", "grafana", curlCommand)
+		if err != nil {
+			t.Fatalf("Failed to execute kubectl command in grafana pod: %v", err)
+		}
+		var dashboardResponse DashboardResponse
+		if err := json.Unmarshal([]byte(output), &dashboardResponse); err != nil {
+			t.Fatalf("Failed to unmarshal JSON response: %v", err)
+		}
+		dashboardUID = dashboardResponse.UID
+		if dashboardResponse.Status != "success" {
+			t.Fatal("Failed to create dashboard with loki and fluent-bit")
+		}
+
+		// Create query to verify MarkLogic logs in Grafana
+		payload := map[string]interface{}{
+			"queries": []map[string]interface{}{
+				{
+					"refId":     "A",
+					"expr":      "{job=\"fluent-bit\"} |= ``",
+					"queryType": "range",
+					"datasource": map[string]string{
+						"type": "loki",
+						"uid":  dataSourceUID,
+					},
+					"editorMode":    "builder",
+					"maxLines":      1000,
+					"legendFormat":  "",
+					"datasourceId":  1,
+					"intervalMs":    20000,
+					"maxDataPoints": 1073,
+				},
+			},
+			"from": "now-5m",
+			"to":   "now",
+		}
+
+		payloadBytes, err := json.Marshal(payload)
+		if err != nil {
+			t.Fatalf("Failed to marshal payload: %v", err)
+		}
+		queryUrl := fmt.Sprintf("%s/api/ds/query?ds_type=loki", grafanaURL)
+		curlCommand = fmt.Sprintf(`curl -X POST %s -u %s:%s -H "Content-Type: application/json" -d '%s'`, queryUrl, grafanaAdminUser, grafanaAdminPassword, payloadBytes)
+		output, err = utils.ExecCmdInPod(grafanaPodName, "grafana", "grafana", curlCommand)
+		if err != nil {
+			t.Fatalf("Failed to execute kubectl command in grafana pod: %v", err)
+		}
+		// t.Logf("Query datasource response: %s", output)
+		// Verify MarkLogic logs in Grafana using Loki and Fluent Bit
+		if !(strings.Contains(string(output), "Starting MarkLogic Server")) {
+			t.Fatal("Failed to Query datasource")
+		}
+
+		curlCommand = fmt.Sprintf(`curl -u %s:%s %s/api/dashboards/uid/%s`, grafanaAdminUser, grafanaAdminPassword, grafanaURL, dashboardUID)
+		output, err = utils.ExecCmdInPod(grafanaPodName, "grafana", "grafana", curlCommand)
+		if err != nil {
+			t.Fatalf("Failed to execute kubectl command in grafana pod: %v", err)
+		}
+		if !strings.Contains(string(output), "Fluent Bit Dashboard") {
+			t.Fatal("Failed to associate Fluent Bit as filter in Grafana dashboard")
+		}
+		return ctx
+	})
 	// Run hugepages verification tests if verifyHugePages flag is set
 	if *verifyHugePages {
 		t.Log("Running HugePages verification tests")
@@ -157,6 +367,13 @@ func TestMarklogicCluster(t *testing.T) {
 
 	// Using feature.Teardown to clean up
 	feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+		client := c.Client()
+		if err := client.Resources().Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "grafana"}}); err != nil {
+			t.Fatalf("Failed to delete namespace: %s", err)
+		}
+		if err := client.Resources().Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "loki"}}); err != nil {
+			t.Fatalf("Failed to delete namespace: %s", err)
+		}
 		return ctx
 	})
 

test/e2e/data/loki.yaml

@@ -0,0 +1,24 @@
+deploymentMode: SingleBinary
+loki:
+  auth_enabled: false
+  commonConfig:
+    replication_factor: 1
+  storage:
+    type: 'filesystem'
+  schemaConfig:
+    configs:
+    - from: "2024-01-01"
+      store: tsdb
+      index:
+        prefix: loki_index_
+        period: 24h
+      object_store: filesystem # we're storing on filesystem so there's no real persistence here.
+      schema: v13
+singleBinary:
+  replicas: 1
+read:
+  replicas: 0
+backend:
+  replicas: 0
+write:
+  replicas: 0