diff --git a/docs/tenant_crd.adoc b/docs/tenant_crd.adoc index 678d205bf69..8c242d2203f 100644 --- a/docs/tenant_crd.adoc +++ b/docs/tenant_crd.adoc @@ -1143,6 +1143,13 @@ Directs the MinIO Operator to use prometheus operator. + Tenant scrape configuration will be added to prometheus managed by the prometheus-operator. +|*`prometheusOperatorScrapeMetricsPaths`* __string array__ +|*Optional* + + + +API end point(s) to scrape metrics from PrometheusOperatorScrapeMetricsPaths +If PrometheusOperator: true and PrometheusOperatorScrapeMetricsPaths is empty, will add `/minio/v2/metrics/cluster` to the list of paths to scrape as default like before. + |*`serviceAccountName`* __string__ |*Optional* + diff --git a/helm/operator/templates/minio.min.io_tenants.yaml b/helm/operator/templates/minio.min.io_tenants.yaml index 72d7e591fc6..b6459b4bd6a 100644 --- a/helm/operator/templates/minio.min.io_tenants.yaml +++ b/helm/operator/templates/minio.min.io_tenants.yaml @@ -3682,6 +3682,10 @@ spec: type: string prometheusOperator: type: boolean + prometheusOperatorScrapeMetricsPaths: + items: + type: string + type: array readiness: properties: exec: diff --git a/pkg/apis/minio.min.io/v2/constants.go b/pkg/apis/minio.min.io/v2/constants.go index ad4354679be..61b628e4315 100644 --- a/pkg/apis/minio.min.io/v2/constants.go +++ b/pkg/apis/minio.min.io/v2/constants.go @@ -197,9 +197,6 @@ const StatefulSetPrefix = "ss" // StatefulSetLegacyPrefix by old operators const StatefulSetLegacyPrefix = "zone" -// MinIOPrometheusPathCluster is the path where MinIO tenant exposes cluster Prometheus metrics -const MinIOPrometheusPathCluster = "/minio/v2/metrics/cluster" - // MinIOPrometheusScrapeInterval defines how frequently to scrape targets. const MinIOPrometheusScrapeInterval = 30 * time.Second diff --git a/pkg/apis/minio.min.io/v2/helper.go b/pkg/apis/minio.min.io/v2/helper.go index c4f85436871..72060b91cff 100644 --- a/pkg/apis/minio.min.io/v2/helper.go +++ b/pkg/apis/minio.min.io/v2/helper.go @@ -413,6 +413,23 @@ func (t *Tenant) GenBearerToken(accessKey, secretKey string) string { return token } +// GetAccessKeyFromBearerToken parses the BearerToken with secretKey to extract accessKey +func GetAccessKeyFromBearerToken(bearerToken string, secretKey string) (string, error) { + claims := &jwt.StandardClaims{} + token, err := jwt.ParseWithClaims(bearerToken, claims, func(_ *jwt.Token) (interface{}, error) { + return []byte(secretKey), nil + }) + if err != nil { + return "", fmt.Errorf("failed to parse token: %v", err) + } + + if !token.Valid { + return "", errors.New("invalid token") + } + + return claims.Subject, nil +} + // MinIOHosts returns the domain names in ellipses format created for current Tenant func (t *Tenant) MinIOHosts() (hosts []string) { // Create the ellipses style URL diff --git a/pkg/apis/minio.min.io/v2/helper_test.go b/pkg/apis/minio.min.io/v2/helper_test.go index d8bc4cccda0..5a4c15f2b4d 100644 --- a/pkg/apis/minio.min.io/v2/helper_test.go +++ b/pkg/apis/minio.min.io/v2/helper_test.go @@ -9,6 +9,23 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +func TestTenant_GetAccessKeyFromBearerToken(t *testing.T) { + mt := Tenant{} + mt.EnsureDefaults() + accessKey := "testAccessKey" + secretKey := "testSecretKey" + bearerToken := mt.GenBearerToken(accessKey, secretKey) + + actualAccessKey, err := GetAccessKeyFromBearerToken(bearerToken, secretKey) + if err != nil { + t.Errorf("GetAccessAndSecretKeyFromBearerToken() returned an error: %v", err) + } + + if actualAccessKey != accessKey { + t.Errorf("GetAccessAndSecretKeyFromBearerToken() accessKey = %v, want %v", actualAccessKey, accessKey) + } +} + func TestEnsureDefaults(t *testing.T) { mt := Tenant{} mt.EnsureDefaults() diff --git a/pkg/apis/minio.min.io/v2/types.go b/pkg/apis/minio.min.io/v2/types.go index 115a390ff21..2c6097f4587 100644 --- a/pkg/apis/minio.min.io/v2/types.go +++ b/pkg/apis/minio.min.io/v2/types.go @@ -278,6 +278,13 @@ type TenantSpec struct { PrometheusOperator bool `json:"prometheusOperator,omitempty"` // *Optional* + // + // API end point(s) to scrape metrics from PrometheusOperatorScrapeMetricsPaths + // If PrometheusOperator: true and PrometheusOperatorScrapeMetricsPaths is empty, will add `/minio/v2/metrics/cluster` to the list of paths to scrape as default like before. + // + // +optional + PrometheusOperatorScrapeMetricsPaths []string `json:"prometheusOperatorScrapeMetricsPaths,omitempty"` + // *Optional* + + // // The https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/[Kubernetes Service Account] to use for running MinIO pods created as part of the Tenant. + // +optional ServiceAccountName string `json:"serviceAccountName,omitempty"` diff --git a/pkg/apis/minio.min.io/v2/zz_generated.deepcopy.go b/pkg/apis/minio.min.io/v2/zz_generated.deepcopy.go index cec7292ab06..a0c8bc733b3 100644 --- a/pkg/apis/minio.min.io/v2/zz_generated.deepcopy.go +++ b/pkg/apis/minio.min.io/v2/zz_generated.deepcopy.go @@ -771,6 +771,11 @@ func (in *TenantSpec) DeepCopyInto(out *TenantSpec) { *out = new(KESConfig) (*in).DeepCopyInto(*out) } + if in.PrometheusOperatorScrapeMetricsPaths != nil { + in, out := &in.PrometheusOperatorScrapeMetricsPaths, &out.PrometheusOperatorScrapeMetricsPaths + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.SideCars != nil { in, out := &in.SideCars, &out.SideCars *out = new(SideCars) diff --git a/pkg/client/applyconfiguration/minio.min.io/v2/tenantspec.go b/pkg/client/applyconfiguration/minio.min.io/v2/tenantspec.go index 74acd245bd0..e241708a104 100644 --- a/pkg/client/applyconfiguration/minio.min.io/v2/tenantspec.go +++ b/pkg/client/applyconfiguration/minio.min.io/v2/tenantspec.go @@ -27,41 +27,42 @@ import ( // TenantSpecApplyConfiguration represents a declarative configuration of the TenantSpec type for use // with apply. type TenantSpecApplyConfiguration struct { - Pools []PoolApplyConfiguration `json:"pools,omitempty"` - Image *string `json:"image,omitempty"` - ImagePullSecret *v1.LocalObjectReference `json:"imagePullSecret,omitempty"` - PodManagementPolicy *appsv1.PodManagementPolicyType `json:"podManagementPolicy,omitempty"` - Env []v1.EnvVar `json:"env,omitempty"` - ExternalCertSecret []*miniominiov2.LocalCertificateReference `json:"externalCertSecret,omitempty"` - ExternalCaCertSecret []*miniominiov2.LocalCertificateReference `json:"externalCaCertSecret,omitempty"` - ExternalClientCertSecret *LocalCertificateReferenceApplyConfiguration `json:"externalClientCertSecret,omitempty"` - ExternalClientCertSecrets []*miniominiov2.LocalCertificateReference `json:"externalClientCertSecrets,omitempty"` - Mountpath *string `json:"mountPath,omitempty"` - Subpath *string `json:"subPath,omitempty"` - RequestAutoCert *bool `json:"requestAutoCert,omitempty"` - CertExpiryAlertThreshold *int32 `json:"certExpiryAlertThreshold,omitempty"` - Liveness *v1.Probe `json:"liveness,omitempty"` - Readiness *v1.Probe `json:"readiness,omitempty"` - Startup *v1.Probe `json:"startup,omitempty"` - Lifecycle *v1.Lifecycle `json:"lifecycle,omitempty"` - Features *FeaturesApplyConfiguration `json:"features,omitempty"` - CertConfig *CertificateConfigApplyConfiguration `json:"certConfig,omitempty"` - KES *KESConfigApplyConfiguration `json:"kes,omitempty"` - PrometheusOperator *bool `json:"prometheusOperator,omitempty"` - ServiceAccountName *string `json:"serviceAccountName,omitempty"` - PriorityClassName *string `json:"priorityClassName,omitempty"` - ImagePullPolicy *v1.PullPolicy `json:"imagePullPolicy,omitempty"` - SideCars *SideCarsApplyConfiguration `json:"sideCars,omitempty"` - ExposeServices *ExposeServicesApplyConfiguration `json:"exposeServices,omitempty"` - ServiceMetadata *ServiceMetadataApplyConfiguration `json:"serviceMetadata,omitempty"` - PoolsMetadata *PoolsMetadataApplyConfiguration `json:"poolsMetadata,omitempty"` - Users []v1.LocalObjectReference `json:"users,omitempty"` - Buckets []BucketApplyConfiguration `json:"buckets,omitempty"` - Logging *LoggingApplyConfiguration `json:"logging,omitempty"` - Configuration *v1.LocalObjectReference `json:"configuration,omitempty"` - InitContainers []v1.Container `json:"initContainers,omitempty"` - AdditionalVolumes []v1.Volume `json:"additionalVolumes,omitempty"` - AdditionalVolumeMounts []v1.VolumeMount `json:"additionalVolumeMounts,omitempty"` + Pools []PoolApplyConfiguration `json:"pools,omitempty"` + Image *string `json:"image,omitempty"` + ImagePullSecret *v1.LocalObjectReference `json:"imagePullSecret,omitempty"` + PodManagementPolicy *appsv1.PodManagementPolicyType `json:"podManagementPolicy,omitempty"` + Env []v1.EnvVar `json:"env,omitempty"` + ExternalCertSecret []*miniominiov2.LocalCertificateReference `json:"externalCertSecret,omitempty"` + ExternalCaCertSecret []*miniominiov2.LocalCertificateReference `json:"externalCaCertSecret,omitempty"` + ExternalClientCertSecret *LocalCertificateReferenceApplyConfiguration `json:"externalClientCertSecret,omitempty"` + ExternalClientCertSecrets []*miniominiov2.LocalCertificateReference `json:"externalClientCertSecrets,omitempty"` + Mountpath *string `json:"mountPath,omitempty"` + Subpath *string `json:"subPath,omitempty"` + RequestAutoCert *bool `json:"requestAutoCert,omitempty"` + CertExpiryAlertThreshold *int32 `json:"certExpiryAlertThreshold,omitempty"` + Liveness *v1.Probe `json:"liveness,omitempty"` + Readiness *v1.Probe `json:"readiness,omitempty"` + Startup *v1.Probe `json:"startup,omitempty"` + Lifecycle *v1.Lifecycle `json:"lifecycle,omitempty"` + Features *FeaturesApplyConfiguration `json:"features,omitempty"` + CertConfig *CertificateConfigApplyConfiguration `json:"certConfig,omitempty"` + KES *KESConfigApplyConfiguration `json:"kes,omitempty"` + PrometheusOperator *bool `json:"prometheusOperator,omitempty"` + PrometheusOperatorScrapeMetricsPath []string `json:"prometheusOperatorScrapeMetricsPath,omitempty"` + ServiceAccountName *string `json:"serviceAccountName,omitempty"` + PriorityClassName *string `json:"priorityClassName,omitempty"` + ImagePullPolicy *v1.PullPolicy `json:"imagePullPolicy,omitempty"` + SideCars *SideCarsApplyConfiguration `json:"sideCars,omitempty"` + ExposeServices *ExposeServicesApplyConfiguration `json:"exposeServices,omitempty"` + ServiceMetadata *ServiceMetadataApplyConfiguration `json:"serviceMetadata,omitempty"` + PoolsMetadata *PoolsMetadataApplyConfiguration `json:"poolsMetadata,omitempty"` + Users []v1.LocalObjectReference `json:"users,omitempty"` + Buckets []BucketApplyConfiguration `json:"buckets,omitempty"` + Logging *LoggingApplyConfiguration `json:"logging,omitempty"` + Configuration *v1.LocalObjectReference `json:"configuration,omitempty"` + InitContainers []v1.Container `json:"initContainers,omitempty"` + AdditionalVolumes []v1.Volume `json:"additionalVolumes,omitempty"` + AdditionalVolumeMounts []v1.VolumeMount `json:"additionalVolumeMounts,omitempty"` } // TenantSpecApplyConfiguration constructs a declarative configuration of the TenantSpec type for use with @@ -260,6 +261,16 @@ func (b *TenantSpecApplyConfiguration) WithPrometheusOperator(value bool) *Tenan return b } +// WithPrometheusOperatorScrapeMetricsPath adds the given value to the PrometheusOperatorScrapeMetricsPath field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the PrometheusOperatorScrapeMetricsPath field. +func (b *TenantSpecApplyConfiguration) WithPrometheusOperatorScrapeMetricsPath(values ...string) *TenantSpecApplyConfiguration { + for i := range values { + b.PrometheusOperatorScrapeMetricsPath = append(b.PrometheusOperatorScrapeMetricsPath, values[i]) + } + return b +} + // WithServiceAccountName sets the ServiceAccountName field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the ServiceAccountName field is set to the value of the last call. diff --git a/pkg/controller/prometheus.go b/pkg/controller/prometheus.go index 1b231992d84..7af822c7f10 100644 --- a/pkg/controller/prometheus.go +++ b/pkg/controller/prometheus.go @@ -17,6 +17,8 @@ package controller import ( "context" "errors" + "reflect" + "strings" promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" promv1alpha1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1alpha1" @@ -141,7 +143,7 @@ func (c *Controller) checkAndCreatePrometheusAddlConfig(ctx context.Context, ten // If the secret is not found, create the secret if k8serrors.IsNotFound(err) { - klog.Infof("Adding MinIO tenant Prometheus scrape config") + klog.Infof("Adding MinIO tenant %s/%s Prometheus scrape config", ns, tenant.Name) scrapeCfgYaml, err := yaml.Marshal(&promCfg.ScrapeConfigs) if err != nil { return err @@ -161,23 +163,46 @@ func (c *Controller) checkAndCreatePrometheusAddlConfig(ctx context.Context, ten return err } } else { - var scrapeConfigs []configmaps.ScrapeConfig + var scrapeConfigs, expectedScrapeConfigs []configmaps.ScrapeConfig err := yaml.Unmarshal(secret.Data[miniov2.PrometheusAddlScrapeConfigKey], &scrapeConfigs) if err != nil { return err } - // Check if the scrape config is already present - hasScrapeConfig := false + // get other scrape configs for _, sc := range scrapeConfigs { - if sc.JobName == tenant.PrometheusOperatorAddlConfigJobName() { - hasScrapeConfig = true - break + if !strings.HasPrefix(sc.JobName, tenant.PrometheusOperatorAddlConfigJobName()) { + expectedScrapeConfigs = append(expectedScrapeConfigs, sc) } } - if !hasScrapeConfig { - klog.Infof("Adding MinIO tenant Prometheus scrape config") - scrapeConfigs = append(scrapeConfigs, promCfg.ScrapeConfigs...) - scrapeCfgYaml, err := yaml.Marshal(scrapeConfigs) + ignoreScrapeConfigsIndex := len(expectedScrapeConfigs) + expectedScrapeConfigs = append(expectedScrapeConfigs, promCfg.ScrapeConfigs...) + updateScrapeConfig := false + if len(scrapeConfigs) != len(expectedScrapeConfigs) { + updateScrapeConfig = true + } else { + for i := range scrapeConfigs { + // can't compare that is generated by operator + if i < ignoreScrapeConfigsIndex { + continue + } + if scrapeConfigs[i].JobName != expectedScrapeConfigs[i].JobName || + scrapeConfigs[i].MetricsPath != expectedScrapeConfigs[i].MetricsPath || + scrapeConfigs[i].Scheme != expectedScrapeConfigs[i].Scheme || + !reflect.DeepEqual(scrapeConfigs[i].TLSConfig, expectedScrapeConfigs[i].TLSConfig) || + !reflect.DeepEqual(scrapeConfigs[i].StaticConfigs, expectedScrapeConfigs[i].StaticConfigs) { + updateScrapeConfig = true + break + } + accKey, _ := miniov2.GetAccessKeyFromBearerToken(scrapeConfigs[i].BearerToken, secretKey) + if accKey != accessKey { + updateScrapeConfig = true + break + } + } + } + if updateScrapeConfig { + klog.Infof("Updating MinIO tenant %s/%s Prometheus scrape config", ns, tenant.Name) + scrapeCfgYaml, err := yaml.Marshal(expectedScrapeConfigs) if err != nil { return err } @@ -224,27 +249,20 @@ func (c *Controller) deletePrometheusAddlConfig(ctx context.Context, tenant *min return err } - var scrapeConfigs []configmaps.ScrapeConfig + var scrapeConfigs, exceptedScrapeConfigs []configmaps.ScrapeConfig err = yaml.Unmarshal(secret.Data[miniov2.PrometheusAddlScrapeConfigKey], &scrapeConfigs) if err != nil { return err } - // Check if the scrape config is present - hasScrapeConfig := false - scIndex := -1 - for i, sc := range scrapeConfigs { - if sc.JobName == tenant.PrometheusOperatorAddlConfigJobName() { - hasScrapeConfig = true - scIndex = i - break + for _, sc := range scrapeConfigs { + if !strings.HasPrefix(sc.JobName, tenant.PrometheusOperatorAddlConfigJobName()) { + exceptedScrapeConfigs = append(exceptedScrapeConfigs, sc) } } - if hasScrapeConfig { + if !reflect.DeepEqual(scrapeConfigs, exceptedScrapeConfigs) { klog.Infof("Deleting MinIO tenant Prometheus scrape config") - // Delete the config - newScrapeConfigs := append(scrapeConfigs[:scIndex], scrapeConfigs[scIndex+1:]...) // Update the secret - scrapeCfgYaml, err := yaml.Marshal(newScrapeConfigs) + scrapeCfgYaml, err := yaml.Marshal(exceptedScrapeConfigs) if err != nil { return err } diff --git a/pkg/controller/prometheus_test.go b/pkg/controller/prometheus_test.go new file mode 100644 index 00000000000..c934e90d7a2 --- /dev/null +++ b/pkg/controller/prometheus_test.go @@ -0,0 +1,301 @@ +// Copyright (C) 2025, MinIO, Inc. +// +// This code is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License, version 3, +// as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License, version 3, +// along with this program. If not, see + +package controller + +import ( + "context" + "testing" + "time" + + "github.com/minio/operator/pkg/resources/configmaps" + v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/fake" + "gopkg.in/yaml.v2" + v3 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + fake3 "k8s.io/client-go/kubernetes/fake" + + miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2" +) + +func Test_checkAndCreatePrometheusAddlConfig(t *testing.T) { + type expect struct { + afterScrapeNumber int + afterScrapePath []string + afterScrapeName []string + changedTokenNumber int + } + type arg struct { + accessKey string + secretKey string + beforeScrapes []configmaps.ScrapeConfig + paths []string + expects expect + } + type args struct { + name string + arg arg + } + testRunCheckAndCreatePrometheusAddlConfig := func(t *testing.T, arg arg) { + beforeScrapesBytes, err := yaml.Marshal(arg.beforeScrapes) + if err != nil { + t.Fatal(err) + } + testNS := "default" + prometheus := &v1.Prometheus{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPrometheus", + Namespace: testNS, + }, + Spec: v1.PrometheusSpec{ + CommonPrometheusFields: v1.CommonPrometheusFields{ + AdditionalScrapeConfigs: &v3.SecretKeySelector{ + Key: "", + LocalObjectReference: v3.LocalObjectReference{ + Name: miniov2.PrometheusAddlScrapeConfigSecret, + }, + }, + }, + }, + } + beforeSecret := &v3.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: miniov2.PrometheusAddlScrapeConfigSecret, + Namespace: testNS, + }, + Data: map[string][]byte{ + miniov2.PrometheusAddlScrapeConfigKey: beforeScrapesBytes, + }, + } + kubeclient := fake3.NewSimpleClientset(beforeSecret) + controller := Controller{ + promClient: fake.NewSimpleClientset(prometheus), + kubeClientSet: kubeclient, + } + tenant := &miniov2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testTenant", + Namespace: testNS, + }, + Spec: miniov2.TenantSpec{ + PrometheusOperatorScrapeMetricsPaths: arg.paths, + }, + } + // bearer_token generated by unix timestamp + time.Sleep(time.Second) + err = controller.checkAndCreatePrometheusAddlConfig(context.Background(), tenant, arg.accessKey, arg.secretKey) + if err != nil { + t.Fatal(err) + } + afterSecret, err := kubeclient.CoreV1().Secrets(testNS).Get(context.Background(), miniov2.PrometheusAddlScrapeConfigSecret, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + var afterScrapes []configmaps.ScrapeConfig + err = yaml.Unmarshal(afterSecret.Data[miniov2.PrometheusAddlScrapeConfigKey], &afterScrapes) + if err != nil { + t.Fatal(err) + } + if len(afterScrapes) != arg.expects.afterScrapeNumber { + t.Fatalf("Expected %d scrape configs, got %d", arg.expects.afterScrapeNumber, len(afterScrapes)) + } + for i, scrape := range afterScrapes { + if scrape.JobName != arg.expects.afterScrapeName[i] { + t.Fatalf("Expected scrape config name %s, got %s", arg.expects.afterScrapeName[i], scrape.JobName) + } + if scrape.MetricsPath != arg.expects.afterScrapePath[i] { + t.Fatalf("Expected scrape config path %s, got %s", arg.expects.afterScrapePath[i], scrape.MetricsPath) + } + } + changedTokenNumber := 0 + for _, afterScrape := range afterScrapes { + for _, scrape := range arg.beforeScrapes { + if scrape.JobName == afterScrape.JobName { + if scrape.BearerToken != afterScrape.BearerToken { + changedTokenNumber++ + } + break + } + } + } + if changedTokenNumber != arg.expects.changedTokenNumber { + t.Fatalf("Expected %d changed tokens, got %d", arg.expects.changedTokenNumber, changedTokenNumber) + } + } + tests := []args{ + { + name: "testDefault", + arg: arg{ + accessKey: "accessKey", + secretKey: "secretKey", + beforeScrapes: []configmaps.ScrapeConfig{}, + expects: expect{ + afterScrapeNumber: 1, + afterScrapePath: []string{"/minio/v2/metrics/cluster"}, + afterScrapeName: []string{"testTenant-minio-job-0"}, + changedTokenNumber: 0, + }, + }, + }, + { + name: "testFirst", + arg: arg{ + accessKey: "accessKey", + secretKey: "secretKey", + paths: []string{"/minio/v2/metrics/cluster", "/minio/metrics/v3/api"}, + beforeScrapes: []configmaps.ScrapeConfig{}, + expects: expect{ + afterScrapeNumber: 2, + afterScrapePath: []string{"/minio/v2/metrics/cluster", "/minio/metrics/v3/api"}, + afterScrapeName: []string{"testTenant-minio-job-0", "testTenant-minio-job-1"}, + changedTokenNumber: 0, + }, + }, + }, + { + name: "testAlreadyHave", + arg: arg{ + accessKey: "accessKey", + secretKey: "secretKey", + paths: []string{"/minio/v2/metrics/cluster", "/minio/metrics/v3/api"}, + beforeScrapes: []configmaps.ScrapeConfig{ + { + JobName: "testTarget", + StaticConfigs: []configmaps.StaticConfig{ + { + Targets: []string{"testTarget"}, + }, + }, + }, + }, + expects: expect{ + afterScrapeNumber: 3, + afterScrapePath: []string{"", "/minio/v2/metrics/cluster", "/minio/metrics/v3/api"}, + afterScrapeName: []string{"testTarget", "testTenant-minio-job-0", "testTenant-minio-job-1"}, + changedTokenNumber: 0, + }, + }, + }, + { + name: "testNoChange", + arg: arg{ + accessKey: "accessKey", + secretKey: "secretKey", + paths: []string{"/minio/v2/metrics/cluster", "/minio/metrics/v3/api"}, + beforeScrapes: []configmaps.ScrapeConfig{ + { + JobName: "testTarget", + StaticConfigs: []configmaps.StaticConfig{ + { + Targets: []string{"testTarget"}, + }, + }, + }, + { + JobName: "testTenant-minio-job-0", + BearerToken: (&miniov2.Tenant{}).GenBearerToken("accessKey", "secretKey"), + MetricsPath: "/minio/v2/metrics/cluster", + Scheme: "https", + TLSConfig: configmaps.TLSConfig{ + CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + }, + StaticConfigs: []configmaps.StaticConfig{ + { + Targets: []string{"minio.default.svc.cluster.local:443"}, + }, + }, + }, + { + JobName: "testTenant-minio-job-1", + BearerToken: (&miniov2.Tenant{}).GenBearerToken("accessKey", "secretKey"), + MetricsPath: "/minio/metrics/v3/api", + Scheme: "https", + TLSConfig: configmaps.TLSConfig{ + CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + }, + StaticConfigs: []configmaps.StaticConfig{ + { + Targets: []string{"minio.default.svc.cluster.local:443"}, + }, + }, + }, + }, + expects: expect{ + afterScrapeNumber: 3, + afterScrapePath: []string{"", "/minio/v2/metrics/cluster", "/minio/metrics/v3/api"}, + afterScrapeName: []string{"testTarget", "testTenant-minio-job-0", "testTenant-minio-job-1"}, + changedTokenNumber: 0, + }, + }, + }, + { + name: "testPasswordChange", + arg: arg{ + accessKey: "accessKeyChanged", + secretKey: "secretKeyChanged", + paths: []string{"/minio/v2/metrics/cluster", "/minio/metrics/v3/api"}, + beforeScrapes: []configmaps.ScrapeConfig{ + { + JobName: "testTarget", + StaticConfigs: []configmaps.StaticConfig{ + { + Targets: []string{"testTarget"}, + }, + }, + }, + { + JobName: "testTenant-minio-job-0", + BearerToken: (&miniov2.Tenant{}).GenBearerToken("accessKey", "secretKey"), + MetricsPath: "/minio/v2/metrics/cluster", + Scheme: "https", + TLSConfig: configmaps.TLSConfig{ + CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + }, + StaticConfigs: []configmaps.StaticConfig{ + { + Targets: []string{"minio.default.svc.cluster.local:443"}, + }, + }, + }, + { + JobName: "testTenant-minio-job-1", + BearerToken: (&miniov2.Tenant{}).GenBearerToken("accessKey", "secretKey"), + MetricsPath: "/minio/metrics/v3/api", + Scheme: "https", + TLSConfig: configmaps.TLSConfig{ + CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + }, + StaticConfigs: []configmaps.StaticConfig{ + { + Targets: []string{"minio.default.svc.cluster.local:443"}, + }, + }, + }, + }, + expects: expect{ + afterScrapeNumber: 3, + afterScrapePath: []string{"", "/minio/v2/metrics/cluster", "/minio/metrics/v3/api"}, + afterScrapeName: []string{"testTarget", "testTenant-minio-job-0", "testTenant-minio-job-1"}, + changedTokenNumber: 2, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testRunCheckAndCreatePrometheusAddlConfig(t, tt.arg) + }) + } +} diff --git a/pkg/resources/configmaps/prometheus.go b/pkg/resources/configmaps/prometheus.go index 82b1c5c6171..071c9195608 100644 --- a/pkg/resources/configmaps/prometheus.go +++ b/pkg/resources/configmaps/prometheus.go @@ -27,11 +27,13 @@ type globalConfig struct { EvaluationInterval time.Duration `yaml:"evaluation_interval"` } -type staticConfig struct { +// StaticConfig contains the static configuration for prometheus +type StaticConfig struct { Targets []string `yaml:"targets"` } -type tlsConfig struct { +// TLSConfig contains the tls configuration for prometheus +type TLSConfig struct { CAFile string `yaml:"ca_file"` } @@ -41,8 +43,8 @@ type ScrapeConfig struct { BearerToken string `yaml:"bearer_token"` MetricsPath string `yaml:"metrics_path"` Scheme string `yaml:"scheme"` - TLSConfig tlsConfig `yaml:"tls_config"` - StaticConfigs []staticConfig `yaml:"static_configs"` + TLSConfig TLSConfig `yaml:"tls_config"` + StaticConfigs []StaticConfig `yaml:"static_configs"` } // PrometheusConfig contains the prometheus configuration @@ -80,22 +82,29 @@ func GetPrometheusConfig(t *miniov2.Tenant, accessKey, secretKey string) *Promet ScrapeInterval: miniov2.MinIOPrometheusScrapeInterval, EvaluationInterval: 30 * time.Second, }, - ScrapeConfigs: []ScrapeConfig{ - { - JobName: t.PrometheusConfigJobName(), - BearerToken: bearerToken, - MetricsPath: miniov2.MinIOPrometheusPathCluster, - Scheme: minioScheme, - TLSConfig: tlsConfig{ - CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", - }, - StaticConfigs: []staticConfig{ - { - Targets: []string{minioTargets}, - }, + ScrapeConfigs: []ScrapeConfig{}, + } + + if len(t.Spec.PrometheusOperatorScrapeMetricsPaths) == 0 { + t.Spec.PrometheusOperatorScrapeMetricsPaths = []string{"/minio/v2/metrics/cluster"} + } + + for index, scrape := range t.Spec.PrometheusOperatorScrapeMetricsPaths { + promConfig.ScrapeConfigs = append(promConfig.ScrapeConfigs, ScrapeConfig{ + JobName: fmt.Sprintf("%s-%d", t.PrometheusOperatorAddlConfigJobName(), index), + BearerToken: bearerToken, + MetricsPath: scrape, + Scheme: minioScheme, + TLSConfig: TLSConfig{ + CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + }, + StaticConfigs: []StaticConfig{ + { + Targets: []string{minioTargets}, }, }, - }, + }) } + return promConfig } diff --git a/resources/base/crds/minio.min.io_tenants.yaml b/resources/base/crds/minio.min.io_tenants.yaml index 72d7e591fc6..b6459b4bd6a 100644 --- a/resources/base/crds/minio.min.io_tenants.yaml +++ b/resources/base/crds/minio.min.io_tenants.yaml @@ -3682,6 +3682,10 @@ spec: type: string prometheusOperator: type: boolean + prometheusOperatorScrapeMetricsPaths: + items: + type: string + type: array readiness: properties: exec: