diff --git a/examples/simple/deploy/01_simple-csi-driver.yaml b/examples/simple/deploy/01_simple-csi-driver.yaml index 4fca9b8..4b35006 100644 --- a/examples/simple/deploy/01_simple-csi-driver.yaml +++ b/examples/simple/deploy/01_simple-csi-driver.yaml @@ -59,7 +59,7 @@ spec: allowPrivilegeEscalation: false capabilities: { drop: [ "ALL" ] } readOnlyRootFilesystem: true - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.14.0 args: - --v=5 - --csi-address=/plugin/csi.sock @@ -94,6 +94,10 @@ spec: fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix://plugin/csi.sock + ports: + - containerPort: 9402 + name: http-metrics + protocol: TCP volumeMounts: - name: plugin-dir mountPath: /plugin diff --git a/examples/simple/deploy/02_example-app.yaml b/examples/simple/deploy/02_example-app.yaml index a63ea12..fdcd203 100644 --- a/examples/simple/deploy/02_example-app.yaml +++ b/examples/simple/deploy/02_example-app.yaml @@ -48,7 +48,7 @@ spec: runAsUser: 2000 containers: - name: my-frontend - image: busybox:1.35.0 + image: busybox:1.36.1 volumeMounts: - mountPath: "/tls" name: tls diff --git a/examples/simple/go.mod b/examples/simple/go.mod index 9121b61..831a0fe 100644 --- a/examples/simple/go.mod +++ b/examples/simple/go.mod @@ -7,6 +7,9 @@ replace github.com/cert-manager/csi-lib => ../../ require ( github.com/cert-manager/cert-manager v1.18.2 github.com/cert-manager/csi-lib v0.0.0-00010101000000-000000000000 + github.com/go-logr/logr v1.4.3 + github.com/prometheus/client_golang v1.22.0 + golang.org/x/sync v0.15.0 k8s.io/client-go v0.34.1 k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 @@ -23,7 +26,6 @@ require ( github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-asn1-ber/asn1-ber v1.5.6 // indirect github.com/go-ldap/ldap/v3 v3.4.8 // indirect - github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect @@ -43,7 +45,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect diff --git a/examples/simple/go.sum b/examples/simple/go.sum index e5652e8..d935dc9 100644 --- a/examples/simple/go.sum +++ b/examples/simple/go.sum @@ -198,6 +198,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/examples/simple/main.go b/examples/simple/main.go index d2feb08..9c753d3 100644 --- a/examples/simple/main.go +++ b/examples/simple/main.go @@ -28,6 +28,7 @@ import ( "flag" "fmt" "net" + "net/http" "net/url" "strings" "time" @@ -35,7 +36,11 @@ import ( cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" cmclient "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned" + "github.com/cert-manager/cert-manager/pkg/client/informers/externalversions" "github.com/cert-manager/cert-manager/pkg/util/pki" + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "golang.org/x/sync/errgroup" "k8s.io/client-go/rest" "k8s.io/klog/v2/klogr" "k8s.io/utils/clock" @@ -43,6 +48,7 @@ import ( "github.com/cert-manager/csi-lib/driver" "github.com/cert-manager/csi-lib/manager" "github.com/cert-manager/csi-lib/metadata" + "github.com/cert-manager/csi-lib/metrics" "github.com/cert-manager/csi-lib/storage" ) @@ -104,13 +110,29 @@ func main() { store.FSGroupVolumeAttributeKey = FsGroupKey - d, err := driver.New(context.Background(), *endpoint, log, driver.Options{ + cmClient := cmclient.NewForConfigOrDie(restConfig) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + certRequestInformerFactory := externalversions.NewSharedInformerFactory(cmClient, 5*time.Second) + certRequestInformer := certRequestInformerFactory.Certmanager().V1().CertificateRequests() + metricsHandler := metrics.New(*nodeID, &log, prometheus.NewRegistry(), store, certRequestInformer.Lister()) + + go func() { + err := startMetricsServer(ctx, log, metricsHandler, certRequestInformerFactory) + if err != nil { + panic("failed to setup metrics server: " + err.Error()) + } + }() + + d, err := driver.New(ctx, *endpoint, log, driver.Options{ DriverName: "csi.cert-manager.io", DriverVersion: "v0.0.1", NodeID: *nodeID, Store: store, Manager: manager.NewManagerOrDie(manager.Options{ - Client: cmclient.NewForConfigOrDie(restConfig), + Client: cmClient, MetadataReader: store, Clock: clock.RealClock{}, Log: &log, @@ -119,6 +141,7 @@ func main() { GenerateRequest: generateRequest, SignRequest: signRequest, WriteKeypair: (&writer{store: store}).writeKeypair, + Metrics: metricsHandler, }), }) if err != nil { @@ -351,3 +374,50 @@ func keyUsagesFromAttributes(usagesCSV string) []cmapi.KeyUsage { return keyUsages } + +// startMetricsServer starts a server listening on port 9402, until the supplied context is cancelled, +// after which the server will gracefully shutdown (within 5 seconds). +func startMetricsServer( + rootCtx context.Context, + logger logr.Logger, + metricsHandler *metrics.Metrics, + certRequestInformerFactory externalversions.SharedInformerFactory, +) error { + g, ctx := errgroup.WithContext(rootCtx) + + listenConfig := &net.ListenConfig{} + metricsLn, err := listenConfig.Listen(ctx, "tcp", ":9402") + if err != nil { + return err + } + metricsServer := &http.Server{ + Addr: metricsLn.Addr().String(), + ReadTimeout: 8 * time.Second, + WriteTimeout: 8 * time.Second, + MaxHeaderBytes: 1 << 20, // 1 MiB + Handler: metricsHandler.DefaultHandler(), + } + + g.Go(func() error { + certRequestInformerFactory.Start(ctx.Done()) + certRequestInformerFactory.WaitForCacheSync(ctx.Done()) + return nil + }) + g.Go(func() error { + <-rootCtx.Done() + // allow a timeout for graceful shutdown + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // nolint: contextcheck + return metricsServer.Shutdown(shutdownCtx) + }) + g.Go(func() error { + logger.Info("starting metrics server", "address", metricsLn.Addr()) + if err := metricsServer.Serve(metricsLn); err != http.ErrServerClosed { + return err + } + return nil + }) + return g.Wait() +} diff --git a/go.mod b/go.mod index 97261d7..3a3a6cc 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/container-storage-interface/spec v1.11.0 github.com/go-logr/logr v1.4.3 github.com/kubernetes-csi/csi-lib-utils v0.22.0 + github.com/prometheus/client_golang v1.22.0 github.com/stretchr/testify v1.11.1 google.golang.org/grpc v1.75.0 k8s.io/apimachinery v0.34.1 @@ -17,12 +18,15 @@ require ( ) require ( + github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-asn1-ber/asn1-ber v1.5.6 // indirect + github.com/go-ldap/ldap/v3 v3.4.8 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect @@ -34,6 +38,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -41,7 +46,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect @@ -54,6 +58,7 @@ require ( go.uber.org/zap v1.27.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.39.0 // indirect golang.org/x/net v0.41.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect golang.org/x/sys v0.33.0 // indirect diff --git a/go.sum b/go.sum index 1107f5b..e5652e8 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= +github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI= +github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -17,6 +21,11 @@ github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-asn1-ber/asn1-ber v1.5.6 h1:CYsqysemXfEaQbyrLJmdsCRuufHoLa3P/gGWGl5TDrM= +github.com/go-asn1-ber/asn1-ber v1.5.6/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ= +github.com/go-ldap/ldap/v3 v3.4.8/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -44,8 +53,25 @@ github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgY github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= +github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= +github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= +github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= +github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= +github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -100,15 +126,22 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= @@ -134,12 +167,28 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= @@ -147,15 +196,35 @@ golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKl golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= @@ -164,6 +233,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -185,6 +256,8 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= diff --git a/manager/manager.go b/manager/manager.go index 869693d..5698932 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -47,6 +47,7 @@ import ( internalapi "github.com/cert-manager/csi-lib/internal/api" internalapiutil "github.com/cert-manager/csi-lib/internal/api/util" "github.com/cert-manager/csi-lib/metadata" + "github.com/cert-manager/csi-lib/metrics" "github.com/cert-manager/csi-lib/storage" ) @@ -89,6 +90,9 @@ type Options struct { // RenewalBackoffConfig configures the exponential backoff applied to certificate renewal failures. RenewalBackoffConfig *wait.Backoff + + // Metrics is used for exposing Prometheus metrics + Metrics *metrics.Metrics } // NewManager constructs a new manager used to manage volumes containing @@ -241,6 +245,7 @@ func NewManager(opts Options) (*Manager, error) { metadataReader: opts.MetadataReader, clock: opts.Clock, log: *opts.Log, + metrics: opts.Metrics, generatePrivateKey: opts.GeneratePrivateKey, generateRequest: opts.GenerateRequest, @@ -375,6 +380,9 @@ type Manager struct { // No thread safety is added around this field, and it MUST NOT be used for any implementation logic. // It should not be used full-stop :). doNotUse_CallOnEachIssue func() + + // metrics is used to expose Prometheus + metrics *metrics.Metrics } // issue will step through the entire issuance flow for a volume. @@ -387,6 +395,11 @@ func (m *Manager) issue(ctx context.Context, volumeID string) error { log := m.log.WithValues("volume_id", volumeID) log.Info("Processing issuance") + // Increase issue count + if m.metrics != nil { + m.metrics.IncrementIssueCallCountTotal(m.nodeNameHash, volumeID) + } + if err := m.cleanupStaleRequests(ctx, log, volumeID); err != nil { return fmt.Errorf("cleaning up stale requests: %w", err) } @@ -756,6 +769,10 @@ func (m *Manager) ManageVolumeImmediate(ctx context.Context, volumeID string) (m // If issuance fails, immediately return without retrying so the caller can decide // how to proceed depending on the context this method was called within. if err := m.issue(ctx, volumeID); err != nil { + // Increase issue error count + if m.metrics != nil { + m.metrics.IncrementIssueErrorCountTotal(m.nodeNameHash, volumeID) + } return true, err } } @@ -835,6 +852,10 @@ func (m *Manager) startRenewalRoutine(volumeID string) (started bool) { defer issueCancel() if err := m.issue(issueCtx, volumeID); err != nil { log.Error(err, "Failed to issue certificate, retrying after applying exponential backoff") + // Increase issue error count + if m.metrics != nil { + m.metrics.IncrementIssueErrorCountTotal(m.nodeNameHash, volumeID) + } return false, nil } return true, nil diff --git a/metrics/certificaterequest_collector.go b/metrics/certificaterequest_collector.go new file mode 100644 index 0000000..6433f5b --- /dev/null +++ b/metrics/certificaterequest_collector.go @@ -0,0 +1,249 @@ +/* +Copyright 2025 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "time" + + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + cmlisters "github.com/cert-manager/cert-manager/pkg/client/listers/certmanager/v1" + "github.com/prometheus/client_golang/prometheus" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + + internalapi "github.com/cert-manager/csi-lib/internal/api" + internalapiutil "github.com/cert-manager/csi-lib/internal/api/util" + "github.com/cert-manager/csi-lib/storage" +) + +var ( + certRequestReadyConditionStatuses = [...]cmmeta.ConditionStatus{cmmeta.ConditionTrue, cmmeta.ConditionFalse, cmmeta.ConditionUnknown} + certRequestReadyStatusMetric = prometheus.NewDesc("certmanager_csi_certificate_request_ready_status", "The ready status of the certificate request.", []string{"name", "namespace", "condition", "issuer_name", "issuer_kind", "issuer_group"}, nil) + certRequestExpirationTimestampSeconds = prometheus.NewDesc("certmanager_csi_certificate_request_expiration_timestamp_seconds", "The timestamp after which the certificate request expires, expressed in Unix Epoch Time.", []string{"name", "namespace", "issuer_name", "issuer_kind", "issuer_group"}, nil) + certRequestRenewalTimestampSeconds = prometheus.NewDesc("certmanager_csi_certificate_request_renewal_timestamp_seconds", "The timestamp after which the certificate request should be renewed, expressed in Unix Epoch Time.", []string{"name", "namespace", "issuer_name", "issuer_kind", "issuer_group"}, nil) + managedVolumeCountTotal = prometheus.NewDesc("certmanager_csi_managed_volume_count_total", "The total number of managed volumes by the csi driver.", []string{"node"}, nil) + managedCertRequestCountTotal = prometheus.NewDesc("certmanager_csi_managed_certificate_request_count_total", "The total number of managed certificate requests by the csi driver.", []string{"node"}, nil) +) + +type CertificateRequestCollector struct { + nodeNameHash string + metadataReader storage.MetadataReader + certificateRequestLister cmlisters.CertificateRequestLister + certificateRequestReadyStatusMetric *prometheus.Desc + certificateRequestExpirationTimestampSeconds *prometheus.Desc + certificateRequestRenewalTimestampSeconds *prometheus.Desc + managedVolumeCountTotal *prometheus.Desc + managedCertificateRequestCountTotal *prometheus.Desc +} + +func NewCertificateRequestCollector(nodeNameHash string, metadataReader storage.MetadataReader, certificateRequestLister cmlisters.CertificateRequestLister) prometheus.Collector { + return &CertificateRequestCollector{ + nodeNameHash: nodeNameHash, + metadataReader: metadataReader, + certificateRequestLister: certificateRequestLister, + certificateRequestReadyStatusMetric: certRequestReadyStatusMetric, + certificateRequestExpirationTimestampSeconds: certRequestExpirationTimestampSeconds, + certificateRequestRenewalTimestampSeconds: certRequestRenewalTimestampSeconds, + managedVolumeCountTotal: managedVolumeCountTotal, + managedCertificateRequestCountTotal: managedCertRequestCountTotal, + } +} + +func (cc *CertificateRequestCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- cc.certificateRequestReadyStatusMetric + ch <- cc.certificateRequestExpirationTimestampSeconds + ch <- cc.certificateRequestRenewalTimestampSeconds + ch <- cc.managedVolumeCountTotal + ch <- cc.managedCertificateRequestCountTotal +} + +func (cc *CertificateRequestCollector) Collect(ch chan<- prometheus.Metric) { + // Get the certificate requests from the lister, filtered by node selector + nodeSelector := labels.NewSelector() + req, err := labels.NewRequirement(internalapi.NodeIDHashLabelKey, selection.Equals, []string{cc.nodeNameHash}) + if err != nil { + return + } + nodeSelector = nodeSelector.Add(*req) + certRequestList, err := cc.certificateRequestLister.List(nodeSelector) + if err != nil { + return + } + cc.updateManagedCertificateRequestCount(len(certRequestList), ch) + + // Get the next issuance time map from the metadata reader + nextIssuanceTimeMap, err := cc.getNextIssuanceTimeMapFromMetadata() + if err != nil { + return + } + cc.updateManagedVolumeCount(len(nextIssuanceTimeMap), ch) // each volume has one nextIssuanceTime entry + + for _, cr := range certRequestList { + cc.updateCertificateRequestReadyStatus(cr, ch) + cc.updateCertificateRequestExpiry(cr, ch) + cc.updateCertificateRequestRenewalTime(cr, nextIssuanceTimeMap, ch) + } +} + +func (cc *CertificateRequestCollector) updateCertificateRequestReadyStatus(cr *cmapi.CertificateRequest, ch chan<- prometheus.Metric) { + setMetric := func(cr *cmapi.CertificateRequest, ch chan<- prometheus.Metric, status cmmeta.ConditionStatus) { + for _, condition := range certRequestReadyConditionStatuses { + value := 0.0 + + if status == condition { + value = 1.0 + } + + metric := prometheus.MustNewConstMetric( + cc.certificateRequestReadyStatusMetric, prometheus.GaugeValue, + value, + cr.Name, + cr.Namespace, + string(condition), + cr.Spec.IssuerRef.Name, + cr.Spec.IssuerRef.Kind, + cr.Spec.IssuerRef.Group, + ) + + ch <- metric + } + } + + for _, st := range cr.Status.Conditions { + if st.Type == cmapi.CertificateRequestConditionReady { + setMetric(cr, ch, st.Status) + return + } + } + + setMetric(cr, ch, cmmeta.ConditionUnknown) +} + +func (cc *CertificateRequestCollector) updateCertificateRequestExpiry(cr *cmapi.CertificateRequest, ch chan<- prometheus.Metric) { + expiryTime := 0.0 + + if cr.Status.Certificate != nil { + notAfter, err := getCertNotAfterTime(cr.Status.Certificate) + if err != nil { + return + } + expiryTime = float64(notAfter.Unix()) + } + + metric := prometheus.MustNewConstMetric( + cc.certificateRequestExpirationTimestampSeconds, + prometheus.GaugeValue, + expiryTime, + cr.Name, + cr.Namespace, + cr.Spec.IssuerRef.Name, + cr.Spec.IssuerRef.Kind, + cr.Spec.IssuerRef.Group, + ) + + ch <- metric +} + +// updateCertificateRequestRenewalTime updates the renewal time metric for the given certificate request. +// The renewal time is the time at which the volume should be renewed. +// Note: there might be multiple certificate requests for a volume depending on the MaxRequestsPerVolume value, +// but only the latest one will be stored in the nextIssuanceTimeMap. +func (cc *CertificateRequestCollector) updateCertificateRequestRenewalTime(cr *cmapi.CertificateRequest, nextIssuanceTimeMap map[string]time.Time, ch chan<- prometheus.Metric) { + renewalTime := 0.0 + + if len(cr.Labels) != 0 { + if nextIssuanceTime, ok := nextIssuanceTimeMap[cr.Labels[internalapi.VolumeIDHashLabelKey]]; ok { + renewalTime = float64(nextIssuanceTime.Unix()) + } + } + + metric := prometheus.MustNewConstMetric( + cc.certificateRequestRenewalTimestampSeconds, + prometheus.GaugeValue, + renewalTime, + cr.Name, + cr.Namespace, + cr.Spec.IssuerRef.Name, + cr.Spec.IssuerRef.Kind, + cr.Spec.IssuerRef.Group, + ) + + ch <- metric +} + +// getCertNotAfterTime returns the NotAfter time of the issued certificate. +// It expects the certificate to be encoded in PEM format. +func getCertNotAfterTime(certBytes []byte) (time.Time, error) { + block, _ := pem.Decode(certBytes) + if block == nil { + return time.Time{}, fmt.Errorf("invalid PEM data: could not decode certificate") + } + crt, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return time.Time{}, fmt.Errorf("parsing issued certificate: %w", err) + } + + return crt.NotAfter, nil +} + +// getNextIssuanceTimeMapFromMetadata returns a map of volume ID hashes to the next issuance time. +// The map is keyed by the volume ID hash. +// The next issuance time is the time at which the volume should be renewed. +func (cc *CertificateRequestCollector) getNextIssuanceTimeMapFromMetadata() (map[string]time.Time, error) { + volumeIDs, err := cc.metadataReader.ListVolumes() + if err != nil { + return nil, fmt.Errorf("listing volumes: %w", err) + } + + nextIssuanceTimeMap := make(map[string]time.Time, len(volumeIDs)) + for _, id := range volumeIDs { + volumeMetadata, err := cc.metadataReader.ReadMetadata(id) + if err != nil { + return nil, err + } + if volumeMetadata.NextIssuanceTime != nil { + nextIssuanceTimeMap[internalapiutil.HashIdentifier(id)] = *volumeMetadata.NextIssuanceTime + } + } + return nextIssuanceTimeMap, nil +} + +func (cc *CertificateRequestCollector) updateManagedVolumeCount(count int, ch chan<- prometheus.Metric) { + metric := prometheus.MustNewConstMetric( + cc.managedVolumeCountTotal, + prometheus.CounterValue, + float64(count), + cc.nodeNameHash, + ) + + ch <- metric +} + +func (cc *CertificateRequestCollector) updateManagedCertificateRequestCount(count int, ch chan<- prometheus.Metric) { + metric := prometheus.MustNewConstMetric( + cc.managedCertificateRequestCountTotal, + prometheus.CounterValue, + float64(count), + cc.nodeNameHash, + ) + + ch <- metric +} diff --git a/metrics/certificaterequest_test.go b/metrics/certificaterequest_test.go new file mode 100644 index 0000000..de38e27 --- /dev/null +++ b/metrics/certificaterequest_test.go @@ -0,0 +1,450 @@ +/* +Copyright 2024 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "strings" + "testing" + "time" + + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned/fake" + "github.com/cert-manager/cert-manager/pkg/client/informers/externalversions" + testcrypto "github.com/cert-manager/cert-manager/test/unit/crypto" + "github.com/cert-manager/cert-manager/test/unit/gen" + "github.com/go-logr/logr/testr" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + internalapi "github.com/cert-manager/csi-lib/internal/api" + internalapiutil "github.com/cert-manager/csi-lib/internal/api/util" + "github.com/cert-manager/csi-lib/metadata" + "github.com/cert-manager/csi-lib/storage" +) + +const expiryMetadata = ` + # HELP certmanager_csi_certificate_request_expiration_timestamp_seconds The timestamp after which the certificate request expires, expressed in Unix Epoch Time. + # TYPE certmanager_csi_certificate_request_expiration_timestamp_seconds gauge +` + +const renewalTimeMetadata = ` + # HELP certmanager_csi_certificate_request_renewal_timestamp_seconds The timestamp after which the certificate request should be renewed, expressed in Unix Epoch Time. + # TYPE certmanager_csi_certificate_request_renewal_timestamp_seconds gauge +` + +const readyMetadata = ` + # HELP certmanager_csi_certificate_request_ready_status The ready status of the certificate request. + # TYPE certmanager_csi_certificate_request_ready_status gauge +` + +func TestCertificateRequestMetrics(t *testing.T) { + testNodeName := "test-node-name" + testVolumeID := "test-vol-id" + + // private key to be used to generate X509 certificate + privKey := testcrypto.MustCreatePEMPrivateKey(t) + certTemplate := &cmapi.Certificate{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns", Name: "test-cert"}, + Spec: cmapi.CertificateSpec{ + CommonName: "test.example.com", + }, + } + notBefore := time.Unix(0, 0) + notAfter := time.Unix(100, 0) + testCert := testcrypto.MustCreateCertWithNotBeforeAfter(t, privKey, certTemplate, notBefore, notAfter) + renew := time.Unix(50, 0) + + type testT struct { + cr *cmapi.CertificateRequest + meta metadata.Metadata + expectedExpiry, expectedReady, expectedRenewalTime string + } + tests := map[string]testT{ + "certificate with expiry and ready status": { + cr: gen.CertificateRequest("test-certificate-request", + gen.SetCertificateRequestNamespace("test-ns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionTrue, + }), + gen.SetCertificateRequestCertificate(testCert), + ), + + expectedExpiry: ` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 100 +`, + expectedReady: ` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 1 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + expectedRenewalTime: ` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + }, + "certificate with no expiry and no status should give an expiry of 0 and Unknown status": { + cr: gen.CertificateRequest("test-certificate-request", + gen.SetCertificateRequestNamespace("test-ns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + ), + + expectedExpiry: ` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + expectedReady: ` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 1 +`, + expectedRenewalTime: ` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + }, + "certificate with expiry and status False should give an expiry and False status": { + cr: gen.CertificateRequest("test-certificate-request", + gen.SetCertificateRequestNamespace("test-ns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionFalse, + }), + gen.SetCertificateRequestCertificate(testCert), + ), + + expectedExpiry: ` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 100 +`, + expectedReady: ` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 1 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + expectedRenewalTime: ` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + }, + "certificate with expiry and status Unknown should give an expiry and Unknown status": { + cr: gen.CertificateRequest("test-certificate-request", + gen.SetCertificateRequestNamespace("test-ns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionUnknown, + }), + gen.SetCertificateRequestCertificate(testCert), + ), + + expectedExpiry: ` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 100 +`, + expectedReady: ` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 1 +`, + expectedRenewalTime: ` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + }, + "certificate with expiry and ready status and renew before": { + cr: gen.CertificateRequest("test-certificate-request", + gen.SetCertificateRequestNamespace("test-ns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionTrue, + }), + gen.SetCertificateRequestCertificate(testCert), + ), + meta: metadata.Metadata{ + VolumeID: testVolumeID, + NextIssuanceTime: &renew, + }, + + expectedExpiry: ` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 100 +`, + expectedReady: ` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 1 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + expectedRenewalTime: ` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 50 +`, + }, + } + for n, test := range tests { + t.Run(n, func(t *testing.T) { + testLog := testr.New(t) + + fakeClient := fake.NewSimpleClientset() + factory := externalversions.NewSharedInformerFactory(fakeClient, 0) + certRequestInformer := factory.Certmanager().V1().CertificateRequests() + test.cr.Labels = map[string]string{ + internalapi.NodeIDHashLabelKey: internalapiutil.HashIdentifier(testNodeName), + internalapi.VolumeIDHashLabelKey: internalapiutil.HashIdentifier(testVolumeID), + } + err := certRequestInformer.Informer().GetIndexer().Add(test.cr) + assert.NoError(t, err) + fakeMetadata := storage.NewMemoryFS() + fakeMetadata.RegisterMetadata(test.meta) + m := New(testNodeName, &testLog, prometheus.NewRegistry(), fakeMetadata, certRequestInformer.Lister()) + + if err := testutil.CollectAndCompare(m.certificateRequestCollector, + strings.NewReader(expiryMetadata+test.expectedExpiry), + "certmanager_csi_certificate_request_expiration_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + if err := testutil.CollectAndCompare(m.certificateRequestCollector, + strings.NewReader(renewalTimeMetadata+test.expectedRenewalTime), + "certmanager_csi_certificate_request_renewal_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + if err := testutil.CollectAndCompare(m.certificateRequestCollector, + strings.NewReader(readyMetadata+test.expectedReady), + "certmanager_csi_certificate_request_ready_status", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + err = certRequestInformer.Informer().GetIndexer().Delete(test.cr) + assert.NoError(t, err) + }) + } +} + +func TestCertificateRequestCache(t *testing.T) { + testNodeName := "test-node-name" + testNodeNameHash := internalapiutil.HashIdentifier(testNodeName) + + // private key to be used to generate X509 certificate + privKey := testcrypto.MustCreatePEMPrivateKey(t) + certTemplate := &cmapi.Certificate{ + ObjectMeta: metav1.ObjectMeta{Namespace: "testns", Name: "test"}, + Spec: cmapi.CertificateSpec{ + CommonName: "test.example.com", + }, + } + notBefore := time.Unix(0, 0) + notAfter1, notAfter2, notAfter3 := + time.Unix(100, 0), time.Unix(200, 0), time.Unix(300, 0) + renew1, renew2, renew3 := + time.Unix(50, 0), time.Unix(150, 0), time.Unix(250, 0) + + cr1 := gen.CertificateRequest("cr1", + gen.SetCertificateRequestNamespace("testns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionUnknown, + }), + gen.SetCertificateRequestCertificate( + testcrypto.MustCreateCertWithNotBeforeAfter(t, privKey, certTemplate, notBefore, notAfter1)), + ) + cr2 := gen.CertificateRequest("cr2", + gen.SetCertificateRequestNamespace("testns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionTrue, + }), + gen.SetCertificateRequestCertificate( + testcrypto.MustCreateCertWithNotBeforeAfter(t, privKey, certTemplate, notBefore, notAfter2)), + ) + cr3 := gen.CertificateRequest("cr3", + gen.SetCertificateRequestNamespace("testns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionFalse, + }), + gen.SetCertificateRequestCertificate( + testcrypto.MustCreateCertWithNotBeforeAfter(t, privKey, certTemplate, notBefore, notAfter3)), + ) + + cr1.Labels = map[string]string{ + internalapi.NodeIDHashLabelKey: testNodeNameHash, + internalapi.VolumeIDHashLabelKey: internalapiutil.HashIdentifier("vol-1"), + } + cr2.Labels = map[string]string{ + internalapi.NodeIDHashLabelKey: testNodeNameHash, + internalapi.VolumeIDHashLabelKey: internalapiutil.HashIdentifier("vol-2"), + } + cr3.Labels = map[string]string{ + internalapi.NodeIDHashLabelKey: testNodeNameHash, + internalapi.VolumeIDHashLabelKey: internalapiutil.HashIdentifier("vol-3"), + } + + fakeMetadata := storage.NewMemoryFS() + fakeMetadata.RegisterMetadata(metadata.Metadata{ + VolumeID: "vol-1", NextIssuanceTime: &renew1, + }) + fakeMetadata.RegisterMetadata(metadata.Metadata{ + VolumeID: "vol-2", NextIssuanceTime: &renew2, + }) + fakeMetadata.RegisterMetadata(metadata.Metadata{ + VolumeID: "vol-3", NextIssuanceTime: &renew3, + }) + + fakeClient := fake.NewSimpleClientset() + factory := externalversions.NewSharedInformerFactory(fakeClient, 0) + certRequestInformer := factory.Certmanager().V1().CertificateRequests() + + err := certRequestInformer.Informer().GetIndexer().Add(cr1) + assert.NoError(t, err) + err = certRequestInformer.Informer().GetIndexer().Add(cr2) + assert.NoError(t, err) + err = certRequestInformer.Informer().GetIndexer().Add(cr3) + assert.NoError(t, err) + + testLog := testr.New(t) + m := New(testNodeName, &testLog, prometheus.NewRegistry(), fakeMetadata, certRequestInformer.Lister()) + + // Check all three metrics exist + if err := testutil.CollectAndCompare(m.certificateRequestCollector, + strings.NewReader(readyMetadata+` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr2",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 1 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr2",namespace="testns"} 1 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 1 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr2",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 0 +`), + "certmanager_csi_certificate_request_ready_status", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + if err := testutil.CollectAndCompare(m.certificateRequestCollector, + strings.NewReader(expiryMetadata+` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 100 + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr2",namespace="testns"} 200 + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 300 +`), + "certmanager_csi_certificate_request_expiration_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + if err := testutil.CollectAndCompare(m.certificateRequestCollector, + strings.NewReader(renewalTimeMetadata+` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 50 + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr2",namespace="testns"} 150 + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 250 +`), + "certmanager_csi_certificate_request_renewal_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + // Remove second certificate and check not exists + err = certRequestInformer.Informer().GetIndexer().Delete(cr2) + assert.NoError(t, err) + + if err := testutil.CollectAndCompare(m.certificateRequestCollector, + strings.NewReader(readyMetadata+` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 1 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 1 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 0 +`), + "certmanager_csi_certificate_request_ready_status", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + if err := testutil.CollectAndCompare(m.certificateRequestCollector, + strings.NewReader(expiryMetadata+` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 100 + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 300 +`), + "certmanager_csi_certificate_request_expiration_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + if err := testutil.CollectAndCompare(m.certificateRequestCollector, + strings.NewReader(renewalTimeMetadata+` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 50 + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 250 +`), + "certmanager_csi_certificate_request_renewal_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + // Remove all Certificates (even is already removed) and observe no Certificates + err = certRequestInformer.Informer().GetIndexer().Delete(cr1) + assert.NoError(t, err) + err = certRequestInformer.Informer().GetIndexer().Delete(cr2) + assert.NoError(t, err) + err = certRequestInformer.Informer().GetIndexer().Delete(cr3) + assert.NoError(t, err) + + if testutil.CollectAndCount(m.certificateRequestCollector, "certmanager_csi_certificate_request_ready_status") != 0 { + t.Errorf("unexpected collecting result") + } + if testutil.CollectAndCount(m.certificateRequestCollector, "certmanager_csi_certificate_request_expiration_timestamp_seconds") != 0 { + t.Errorf("unexpected collecting result") + } + if testutil.CollectAndCount(m.certificateRequestCollector, "certmanager_csi_certificate_request_renewal_timestamp_seconds") != 0 { + t.Errorf("unexpected collecting result") + } +} diff --git a/metrics/metrics.go b/metrics/metrics.go new file mode 100644 index 0000000..983b48f --- /dev/null +++ b/metrics/metrics.go @@ -0,0 +1,124 @@ +/* +Copyright 2024 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "net/http" + + cmlisters "github.com/cert-manager/cert-manager/pkg/client/listers/certmanager/v1" + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + internalapiutil "github.com/cert-manager/csi-lib/internal/api/util" + "github.com/cert-manager/csi-lib/storage" +) + +const ( + // Namespace is the namespace for csi-lib metric names + namespace = "certmanager" + subsystem = "csi" +) + +// Metrics is designed to be a shared object for updating the metrics exposed by csi-lib +type Metrics struct { + log logr.Logger + registry *prometheus.Registry + + driverIssueCallCountTotal *prometheus.CounterVec + driverIssueErrorCountTotal *prometheus.CounterVec + certificateRequestCollector prometheus.Collector +} + +// New creates a Metrics struct and populates it with prometheus metric types. +func New( + nodeId string, + logger *logr.Logger, + registry *prometheus.Registry, + metadataReader storage.MetadataReader, + certificateRequestLister cmlisters.CertificateRequestLister, +) *Metrics { + var ( + // driverIssueCallCountTotal is a Prometheus counter for the number of issue() calls made by the driver. + driverIssueCallCountTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "driver_issue_call_count_total", + Help: "The number of issue() calls made by the driver.", + }, + []string{"node", "volume"}, + ) + + // driverIssueErrorCountTotal is a Prometheus counter for the number of errors encountered + // during the driver issue() calls. + driverIssueErrorCountTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "driver_issue_error_count_total", + Help: "The number of errors encountered during the driver issue() calls.", + }, + []string{"node", "volume"}, + ) + ) + + // Create server and register Prometheus metrics handler + m := &Metrics{ + log: logger.WithName("metrics"), + registry: registry, + + driverIssueCallCountTotal: driverIssueCallCountTotal, + driverIssueErrorCountTotal: driverIssueErrorCountTotal, + certificateRequestCollector: NewCertificateRequestCollector( + internalapiutil.HashIdentifier(nodeId), + metadataReader, + certificateRequestLister, + ), + } + + m.registry.MustRegister( + driverIssueCallCountTotal, + driverIssueErrorCountTotal, + m.certificateRequestCollector, + ) + + return m +} + +// DefaultHandler returns a default prometheus metrics HTTP handler +func (m *Metrics) DefaultHandler() http.Handler { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{})) + + return mux +} + +func (m *Metrics) SetupCertificateRequestCollector(nodeId string, metadataReader storage.MetadataReader, certificateRequestLister cmlisters.CertificateRequestLister) { + m.certificateRequestCollector = NewCertificateRequestCollector(internalapiutil.HashIdentifier(nodeId), metadataReader, certificateRequestLister) + m.registry.MustRegister(m.certificateRequestCollector) +} + +// IncrementIssueCallCountTotal will increase the issue call counter for the driver. +func (m *Metrics) IncrementIssueCallCountTotal(nodeNameHash, volumeID string) { + m.driverIssueCallCountTotal.WithLabelValues(nodeNameHash, volumeID).Inc() +} + +// IncrementIssueErrorCountTotal will increase count of errors during issue call of the driver. +func (m *Metrics) IncrementIssueErrorCountTotal(nodeNameHash, volumeID string) { + m.driverIssueErrorCountTotal.WithLabelValues(nodeNameHash, volumeID).Inc() +} diff --git a/test/driver/driver_testing.go b/test/driver/driver_testing.go index 165a204..ddc1fc3 100644 --- a/test/driver/driver_testing.go +++ b/test/driver/driver_testing.go @@ -36,6 +36,7 @@ import ( "github.com/cert-manager/csi-lib/driver" "github.com/cert-manager/csi-lib/manager" "github.com/cert-manager/csi-lib/metadata" + "github.com/cert-manager/csi-lib/metrics" "github.com/cert-manager/csi-lib/storage" ) @@ -45,6 +46,7 @@ type Options struct { Log *logr.Logger Client cmclient.Interface Mounter mount.Interface + Metrics *metrics.Metrics NodeID string MaxRequestsPerVolume int @@ -110,6 +112,7 @@ func Run(t *testing.T, opts Options) (Options, csi.NodeClient, func()) { Clock: opts.Clock, Log: opts.Log, NodeID: opts.NodeID, + Metrics: opts.Metrics, MaxRequestsPerVolume: opts.MaxRequestsPerVolume, GeneratePrivateKey: opts.GeneratePrivateKey, GenerateRequest: opts.GenerateRequest, diff --git a/test/integration/metrics_test.go b/test/integration/metrics_test.go new file mode 100644 index 0000000..f99209e --- /dev/null +++ b/test/integration/metrics_test.go @@ -0,0 +1,284 @@ +/* +Copyright 2025 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "crypto" + "crypto/x509" + "fmt" + "io" + "net" + "net/http" + "os" + "strings" + "testing" + "time" + + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned/fake" + "github.com/cert-manager/cert-manager/pkg/client/informers/externalversions" + testcrypto "github.com/cert-manager/cert-manager/test/unit/crypto" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/go-logr/logr/testr" + "github.com/prometheus/client_golang/prometheus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + fakeclock "k8s.io/utils/clock/testing" + + "github.com/cert-manager/csi-lib/manager" + "github.com/cert-manager/csi-lib/metadata" + "github.com/cert-manager/csi-lib/metrics" + "github.com/cert-manager/csi-lib/storage" + testdriver "github.com/cert-manager/csi-lib/test/driver" + testutil "github.com/cert-manager/csi-lib/test/util" +) + +var ( + testMetrics = func(ctx context.Context, metricsEndpoint, expectedOutput string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, metricsEndpoint, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + output, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + trimmedOutput := strings.SplitN(string(output), "# HELP go_gc_duration_seconds", 2)[0] + if strings.TrimSpace(trimmedOutput) != strings.TrimSpace(expectedOutput) { + return fmt.Errorf("got unexpected metrics output\nexp:\n%s\ngot:\n%s\n", + expectedOutput, trimmedOutput) + } + + return nil + } + + waitForMetrics = func(t *testing.T, ctx context.Context, metricsEndpoint, expectedOutput string) { + var lastErr error + err := wait.PollUntilContextCancel(ctx, time.Millisecond*100, true, func(ctx context.Context) (done bool, err error) { + if err := testMetrics(ctx, metricsEndpoint, expectedOutput); err != nil { + lastErr = err + return false, nil + } + + return true, nil + }) + if err != nil { + t.Fatalf("%s: failed to wait for expected metrics to be exposed: %s", err, lastErr) + } + } +) + +func TestMetricsServer(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + testLog := testr.New(t) + testNamespace := "test-ns" + testNodeId := "test-node" + + // Build metrics handler, and start metrics server with a random available port + store := storage.NewMemoryFS() + fakeClient := fake.NewSimpleClientset() + // client-go imposes a minimum resync period of 1 second, so that is the lowest we can go + // https://github.com/kubernetes/client-go/blob/5a019202120ab4dd7dfb3788e5cb87269f343ebe/tools/cache/shared_informer.go#L575 + factory := externalversions.NewSharedInformerFactory(fakeClient, time.Second) + certRequestInformer := factory.Certmanager().V1().CertificateRequests() + metricsHandler := metrics.New(testNodeId, &testLog, prometheus.NewRegistry(), store, certRequestInformer.Lister()) + factory.Start(ctx.Done()) + factory.WaitForCacheSync(ctx.Done()) + + // listenConfig + listenConfig := &net.ListenConfig{} + metricsLn, err := listenConfig.Listen(ctx, "tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + metricsServer := &http.Server{ + Addr: metricsLn.Addr().String(), + ReadTimeout: 8 * time.Second, + WriteTimeout: 8 * time.Second, + MaxHeaderBytes: 1 << 20, // 1 MiB + Handler: metricsHandler.DefaultHandler(), + } + + errCh := make(chan error) + go func() { + defer close(errCh) + testLog.Info("starting metrics server", "address", metricsLn.Addr()) + if err := metricsServer.Serve(metricsLn); err != http.ErrServerClosed { + errCh <- err + } + }() + defer func() { + // allow a timeout for graceful shutdown + shutdownCtx, cancel := context.WithTimeout(ctx, time.Second*5) + defer cancel() + + if err := metricsServer.Shutdown(shutdownCtx); err != nil { + t.Fatal(err) + } + err := <-errCh + if err != nil { + t.Fatal(err) + } + }() + + // Build and start the driver + clock := fakeclock.NewFakeClock(time.Now()) + opts, cl, stop := testdriver.Run(t, testdriver.Options{ + Store: store, + Clock: clock, + Metrics: metricsHandler, + Client: fakeClient, + NodeID: testNodeId, + Log: &testLog, + GeneratePrivateKey: func(meta metadata.Metadata) (crypto.PrivateKey, error) { + return nil, nil + }, + GenerateRequest: func(meta metadata.Metadata) (*manager.CertificateRequestBundle, error) { + return &manager.CertificateRequestBundle{ + Namespace: testNamespace, + IssuerRef: cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }, + }, nil + }, + SignRequest: func(meta metadata.Metadata, key crypto.PrivateKey, request *x509.CertificateRequest) (csr []byte, err error) { + return []byte{}, nil + }, + WriteKeypair: func(meta metadata.Metadata, key crypto.PrivateKey, chain []byte, ca []byte) error { + store.WriteFiles(meta, map[string][]byte{ + "ca": ca, + "cert": chain, + }) + nextIssuanceTime := time.Unix(200, 0) + meta.NextIssuanceTime = &nextIssuanceTime + return store.WriteMetadata(meta.VolumeID, meta) + }, + }) + defer stop() + + // Should expose no additional metrics + metricsEndpoint := fmt.Sprintf("http://%s/metrics", metricsServer.Addr) + waitForMetrics(t, ctx, metricsEndpoint, `# HELP certmanager_csi_managed_certificate_request_count_total The total number of managed certificate requests by the csi driver. +# TYPE certmanager_csi_managed_certificate_request_count_total counter +certmanager_csi_managed_certificate_request_count_total{node="f56fd9f8b"} 0 +# HELP certmanager_csi_managed_volume_count_total The total number of managed volumes by the csi driver. +# TYPE certmanager_csi_managed_volume_count_total counter +certmanager_csi_managed_volume_count_total{node="f56fd9f8b"} 0 +`) + + // Create a self-signed Certificate and wait for it to be issued + privKey := testcrypto.MustCreatePEMPrivateKey(t) + certTemplate := &cmapi.Certificate{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: "test"}, + Spec: cmapi.CertificateSpec{ + CommonName: "test.example.com", + }, + } + notBefore, notAfter := time.Unix(0, 0), time.Unix(300, 0) // renewal time will be 200 + selfSignedCertBytesWithValidity := testcrypto.MustCreateCertWithNotBeforeAfter(t, privKey, certTemplate, notBefore, notAfter) + go testutil.IssueOneRequest(ctx, t, opts.Client, testNamespace, selfSignedCertBytesWithValidity, []byte("ca bytes")) + + // Spin up a test pod + tmpDir, err := os.MkdirTemp("", "*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + _, err = cl.NodePublishVolume(ctx, &csi.NodePublishVolumeRequest{ + VolumeId: "test-vol", + VolumeContext: map[string]string{ + "csi.storage.k8s.io/ephemeral": "true", + "csi.storage.k8s.io/pod.name": "the-pod-name", + "csi.storage.k8s.io/pod.namespace": testNamespace, + }, + TargetPath: tmpDir, + Readonly: true, + }) + if err != nil { + t.Fatal(err) + } + + // Get the CSR name + req, err := testutil.WaitAndGetOneCertificateRequestInNamespace(ctx, opts.Client, testNamespace) + if err != nil { + t.Fatal(err) + } + + // Should expose that CertificateRequest as ready with expiry and renewal time + // node="f56fd9f8b" is the hash value of "test-node" + expectedOutputTemplate := `# HELP certmanager_csi_certificate_request_expiration_timestamp_seconds The timestamp after which the certificate request expires, expressed in Unix Epoch Time. +# TYPE certmanager_csi_certificate_request_expiration_timestamp_seconds gauge +certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-cr-name",namespace="test-ns"} 300 +# HELP certmanager_csi_certificate_request_ready_status The ready status of the certificate request. +# TYPE certmanager_csi_certificate_request_ready_status gauge +certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-cr-name",namespace="test-ns"} 0 +certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-cr-name",namespace="test-ns"} 1 +certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-cr-name",namespace="test-ns"} 0 +# HELP certmanager_csi_certificate_request_renewal_timestamp_seconds The timestamp after which the certificate request should be renewed, expressed in Unix Epoch Time. +# TYPE certmanager_csi_certificate_request_renewal_timestamp_seconds gauge +certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-cr-name",namespace="test-ns"} 200 +# HELP certmanager_csi_driver_issue_call_count_total The number of issue() calls made by the driver. +# TYPE certmanager_csi_driver_issue_call_count_total counter +certmanager_csi_driver_issue_call_count_total{node="f56fd9f8b",volume="test-vol"} 1 +# HELP certmanager_csi_managed_certificate_request_count_total The total number of managed certificate requests by the csi driver. +# TYPE certmanager_csi_managed_certificate_request_count_total counter +certmanager_csi_managed_certificate_request_count_total{node="f56fd9f8b"} 1 +# HELP certmanager_csi_managed_volume_count_total The total number of managed volumes by the csi driver. +# TYPE certmanager_csi_managed_volume_count_total counter +certmanager_csi_managed_volume_count_total{node="f56fd9f8b"} 1 +` + waitForMetrics(t, ctx, metricsEndpoint, strings.ReplaceAll(expectedOutputTemplate, "test-cr-name", req.Name)) + + // Delete the test pod + _, err = cl.NodeUnpublishVolume(ctx, &csi.NodeUnpublishVolumeRequest{ + VolumeId: "test-vol", + TargetPath: tmpDir, + }) + if err != nil { + t.Fatal(err) + } + err = opts.Client.CertmanagerV1().CertificateRequests(testNamespace).Delete(ctx, req.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatal(err) + } + + // Should expose no CertificateRequest and only metrics counters + waitForMetrics(t, ctx, metricsEndpoint, `# HELP certmanager_csi_driver_issue_call_count_total The number of issue() calls made by the driver. +# TYPE certmanager_csi_driver_issue_call_count_total counter +certmanager_csi_driver_issue_call_count_total{node="f56fd9f8b",volume="test-vol"} 1 +# HELP certmanager_csi_managed_certificate_request_count_total The total number of managed certificate requests by the csi driver. +# TYPE certmanager_csi_managed_certificate_request_count_total counter +certmanager_csi_managed_certificate_request_count_total{node="f56fd9f8b"} 0 +# HELP certmanager_csi_managed_volume_count_total The total number of managed volumes by the csi driver. +# TYPE certmanager_csi_managed_volume_count_total counter +certmanager_csi_managed_volume_count_total{node="f56fd9f8b"} 0 +`) + +} diff --git a/test/util/testutil.go b/test/util/testutil.go index d405e4a..6d9dd95 100644 --- a/test/util/testutil.go +++ b/test/util/testutil.go @@ -29,7 +29,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" ) -func waitAndGetOneCertificateRequestInNamespace(ctx context.Context, client cmclient.Interface, ns string) (*cmapi.CertificateRequest, error) { +func WaitAndGetOneCertificateRequestInNamespace(ctx context.Context, client cmclient.Interface, ns string) (*cmapi.CertificateRequest, error) { var req *cmapi.CertificateRequest if err := wait.PollUntilContextCancel(ctx, time.Millisecond*50, true, func(ctx context.Context) (done bool, err error) { reqs, err := client.CertmanagerV1().CertificateRequests(ns).List(ctx, metav1.ListOptions{}) @@ -53,7 +53,7 @@ func waitAndGetOneCertificateRequestInNamespace(ctx context.Context, client cmcl func IssueOneRequest(ctx context.Context, t *testing.T, client cmclient.Interface, namespace string, cert, ca []byte) { if err := func() error { - req, err := waitAndGetOneCertificateRequestInNamespace(ctx, client, namespace) + req, err := WaitAndGetOneCertificateRequestInNamespace(ctx, client, namespace) if err != nil { return err } @@ -80,7 +80,7 @@ func IssueOneRequest(ctx context.Context, t *testing.T, client cmclient.Interfac func SetCertificateRequestConditions(ctx context.Context, t *testing.T, client cmclient.Interface, namespace string, conditions ...cmapi.CertificateRequestCondition) { if err := func() error { - req, err := waitAndGetOneCertificateRequestInNamespace(ctx, client, namespace) + req, err := WaitAndGetOneCertificateRequestInNamespace(ctx, client, namespace) if err != nil { return err }