diff --git a/cmd/csi-resizer/main.go b/cmd/csi-resizer/main.go index 7e7e8883f..c1e378732 100644 --- a/cmd/csi-resizer/main.go +++ b/cmd/csi-resizer/main.go @@ -36,6 +36,7 @@ import ( "github.com/kubernetes-csi/external-resizer/pkg/controller" "github.com/kubernetes-csi/external-resizer/pkg/resizer" "github.com/kubernetes-csi/external-resizer/pkg/util" + csitrans "k8s.io/csi-translation-lib" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" @@ -127,6 +128,17 @@ func main() { } klog.V(2).Infof("CSI driver name: %q", driverName) + translator := csitrans.New() + if translator.IsMigratedCSIDriverByName(driverName) { + metricsManager = metrics.NewCSIMetricsManagerWithOptions(driverName, metrics.WithMigration()) + migratedCsiClient, err := csi.New(*csiAddress, *timeout, metricsManager) + if err != nil { + klog.Fatal(err.Error()) + } + csiClient.CloseConnection() + csiClient = migratedCsiClient + } + csiResizer, err := resizer.NewResizerFromClient( csiClient, *timeout, diff --git a/go.mod b/go.mod index 5eceb61d3..bd3838590 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/googleapis/gnostic v0.5.3 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/imdario/mergo v0.3.11 // indirect - github.com/kubernetes-csi/csi-lib-utils v0.9.0 + github.com/kubernetes-csi/csi-lib-utils v0.9.1 github.com/prometheus/client_golang v1.9.0 // indirect golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 // indirect golang.org/x/net v0.0.0-20201216054612-986b41b23924 // indirect diff --git a/go.sum b/go.sum index 6a2330ad0..2d75bbae3 100644 --- a/go.sum +++ b/go.sum @@ -303,8 +303,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kubernetes-csi/csi-lib-utils v0.9.0 h1:TbuDmxoVqM+fvVkzG/7sShyX/8jUln0ElLHuETcsQJI= -github.com/kubernetes-csi/csi-lib-utils v0.9.0/go.mod h1:8E2jVUX9j3QgspwHXa6LwyN7IHQDjW9jX3kwoWnSC+M= +github.com/kubernetes-csi/csi-lib-utils v0.9.1 h1:sGq6ifVujfMSkfTsMZip44Ttv8SDXvsBlFk9GdYl/b8= +github.com/kubernetes-csi/csi-lib-utils v0.9.1/go.mod h1:8E2jVUX9j3QgspwHXa6LwyN7IHQDjW9jX3kwoWnSC+M= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= diff --git a/pkg/csi/client.go b/pkg/csi/client.go index 0f50f8daf..7b7bb18d6 100644 --- a/pkg/csi/client.go +++ b/pkg/csi/client.go @@ -49,6 +49,9 @@ type Client interface { // Expand expands the volume to a new size at least as big as requestBytes. // It returns the new size and whether the volume need expand operation on the node. Expand(ctx context.Context, volumeID string, requestBytes int64, secrets map[string]string, capability *csi.VolumeCapability) (int64, bool, error) + + //CloseConnection closes the gRPC connection established by the client + CloseConnection() } // New creates a new CSI client. @@ -134,3 +137,7 @@ func (c *client) Expand( } return resp.CapacityBytes, resp.NodeExpansionRequired, nil } + +func (c *client) CloseConnection() { + c.conn.Close() +} diff --git a/pkg/csi/mock_client.go b/pkg/csi/mock_client.go index 1852f498e..18341e7ad 100644 --- a/pkg/csi/mock_client.go +++ b/pkg/csi/mock_client.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/kubernetes-csi/csi-lib-utils/connection" ) func NewMockClient( @@ -28,6 +29,7 @@ type MockClient struct { supportsPluginControllerService bool expandCalled int expansionFailed bool + checkMigratedLabel bool usedSecrets map[string]string usedCapability *csi.VolumeCapability } @@ -52,6 +54,10 @@ func (c *MockClient) SetExpansionFailed() { c.expansionFailed = true } +func (c *MockClient) SetCheckMigratedLabel() { + c.checkMigratedLabel = true +} + func (c *MockClient) Expand( ctx context.Context, volumeID string, @@ -63,6 +69,15 @@ func (c *MockClient) Expand( c.expandCalled++ return requestBytes, c.supportsNodeResize, fmt.Errorf("expansion failed") } + if c.checkMigratedLabel { + additionalInfo := ctx.Value(connection.AdditionalInfoKey) + additionalInfoVal := additionalInfo.(connection.AdditionalInfo) + migrated := additionalInfoVal.Migrated + if migrated != "true" { + err := fmt.Errorf("Expected value of migrated label: true, Actual value: %s", migrated) + return requestBytes, c.supportsNodeResize, err + } + } c.expandCalled++ c.usedSecrets = secrets c.usedCapability = capability @@ -81,3 +96,7 @@ func (c *MockClient) GetCapability() *csi.VolumeCapability { func (c *MockClient) GetSecrets() map[string]string { return c.usedSecrets } + +func (c *MockClient) CloseConnection() { + +} diff --git a/pkg/resizer/csi_resizer.go b/pkg/resizer/csi_resizer.go index df09534d4..95a22963d 100644 --- a/pkg/resizer/csi_resizer.go +++ b/pkg/resizer/csi_resizer.go @@ -20,9 +20,11 @@ import ( "context" "errors" "fmt" + "strconv" "time" csilib "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/kubernetes-csi/csi-lib-utils/connection" "github.com/kubernetes-csi/external-resizer/pkg/csi" "github.com/kubernetes-csi/external-resizer/pkg/util" v1 "k8s.io/api/core/v1" @@ -125,6 +127,7 @@ func (r *csiResizer) Resize(pv *v1.PersistentVolume, requestSize resource.Quanti var volumeID string var source *v1.CSIPersistentVolumeSource var pvSpec v1.PersistentVolumeSpec + var migrated bool if pv.Spec.CSI != nil { // handle CSI volume source = pv.Spec.CSI @@ -138,6 +141,7 @@ func (r *csiResizer) Resize(pv *v1.PersistentVolume, requestSize resource.Quanti if err != nil { return oldSize, false, fmt.Errorf("failed to translate persistent volume: %v", err) } + migrated = true source = csiPV.Spec.CSI pvSpec = csiPV.Spec volumeID = source.VolumeHandle @@ -167,8 +171,10 @@ func (r *csiResizer) Resize(pv *v1.PersistentVolume, requestSize resource.Quanti } ctx, cancel := timeoutCtx(r.timeout) + resizeCtx := context.WithValue(ctx, connection.AdditionalInfoKey, connection.AdditionalInfo{Migrated: strconv.FormatBool(migrated)}) + defer cancel() - newSizeBytes, nodeResizeRequired, err := r.client.Expand(ctx, volumeID, requestSize.Value(), secrets, capability) + newSizeBytes, nodeResizeRequired, err := r.client.Expand(resizeCtx, volumeID, requestSize.Value(), secrets, capability) if err != nil { return oldSize, nodeResizeRequired, err } diff --git a/pkg/resizer/csi_resizer_test.go b/pkg/resizer/csi_resizer_test.go index 978dc7b35..39f4f1dd3 100644 --- a/pkg/resizer/csi_resizer_test.go +++ b/pkg/resizer/csi_resizer_test.go @@ -153,6 +153,7 @@ func TestResizeMigratedPV(t *testing.T) { t.Run(tc.name, func(t *testing.T) { driverName := tc.driverName client := csi.NewMockClient(driverName, true, true, true) + client.SetCheckMigratedLabel() k8sClient, informerFactory := fakeK8s() resizer, err := NewResizerFromClient(client, 0, k8sClient, informerFactory, driverName) if err != nil { diff --git a/vendor/github.com/kubernetes-csi/csi-lib-utils/connection/connection.go b/vendor/github.com/kubernetes-csi/csi-lib-utils/connection/connection.go index ac07d59be..fbd8d37b5 100644 --- a/vendor/github.com/kubernetes-csi/csi-lib-utils/connection/connection.go +++ b/vendor/github.com/kubernetes-csi/csi-lib-utils/connection/connection.go @@ -150,7 +150,7 @@ func connect( return nil, errors.New("OnConnectionLoss callback only supported for unix:// addresses") } - klog.Infof("Connecting to %s", address) + klog.V(5).Infof("Connecting to %s", address) // Connect in background. var conn *grpc.ClientConn @@ -191,6 +191,13 @@ type ExtendedCSIMetricsManager struct { metrics.CSIMetricsManager } +type AdditionalInfo struct { + Migrated string +} +type AdditionalInfoKeyType struct{} + +var AdditionalInfoKey AdditionalInfoKeyType + // RecordMetricsClientInterceptor is a gPRC unary interceptor for recording metrics for CSI operations // in a gRPC client. func (cmm ExtendedCSIMetricsManager) RecordMetricsClientInterceptor( @@ -203,11 +210,35 @@ func (cmm ExtendedCSIMetricsManager) RecordMetricsClientInterceptor( start := time.Now() err := invoker(ctx, method, req, reply, cc, opts...) duration := time.Since(start) - cmm.RecordMetrics( + + var cmmBase metrics.CSIMetricsManager + cmmBase = cmm + if cmm.HaveAdditionalLabel(metrics.LabelMigrated) { + // record migration status + additionalInfo := ctx.Value(AdditionalInfoKey) + migrated := "false" + if additionalInfo != nil { + additionalInfoVal, ok := additionalInfo.(AdditionalInfo) + if !ok { + klog.Errorf("Failed to record migrated status, cannot convert additional info %v", additionalInfo) + return err + } + migrated = additionalInfoVal.Migrated + } + cmmv, metricsErr := cmm.WithLabelValues(map[string]string{metrics.LabelMigrated: migrated}) + if metricsErr != nil { + klog.Errorf("Failed to record migrated status, error: %v", metricsErr) + } else { + cmmBase = cmmv + } + } + // Record the default metric + cmmBase.RecordMetrics( method, /* operationName */ err, /* operationErr */ duration, /* operationDuration */ ) + return err } diff --git a/vendor/github.com/kubernetes-csi/csi-lib-utils/metrics/metrics.go b/vendor/github.com/kubernetes-csi/csi-lib-utils/metrics/metrics.go index 495e1ff85..3baab8ca7 100644 --- a/vendor/github.com/kubernetes-csi/csi-lib-utils/metrics/metrics.go +++ b/vendor/github.com/kubernetes-csi/csi-lib-utils/metrics/metrics.go @@ -47,6 +47,9 @@ const ( labelGrpcStatusCode = "grpc_status_code" unknownCSIDriverName = "unknown-driver" + // LabelMigrated is the Label that indicate whether this is a CSI migration operation + LabelMigrated = "migrated" + // CSI Operation Latency with status code total - Histogram Metric operationsLatencyMetricName = "operations_seconds" operationsLatencyHelp = "Container Storage Interface operation duration with gRPC error code status total" @@ -83,6 +86,10 @@ type CSIMetricsManager interface { // and then accumulates values. WithLabelValues(labels map[string]string) (CSIMetricsManager, error) + // HaveAdditionalLabel can be used to check if the additional label + // value is defined in the metrics manager + HaveAdditionalLabel(name string) bool + // SetDriverName is called to update the CSI driver name. This should be done // as soon as possible, otherwise metrics recorded by this manager will be // recorded with an "unknown-driver" driver_name. @@ -149,6 +156,13 @@ func WithLabels(labels map[string]string) MetricsManagerOption { } } +// WithMigration adds the migrated field to the current metrics label +func WithMigration() MetricsManagerOption { + return func(cmm *csiMetricsManager) { + cmm.additionalLabelNames = append(cmm.additionalLabelNames, LabelMigrated) + } +} + // WithProcessStartTime controlls whether process_start_time_seconds is registered // in the registry of the metrics manager. It's enabled by default out of convenience // (no need to do anything special in most sidecars) but should be disabled in more @@ -316,7 +330,7 @@ func (cmmv *csiMetricsManagerWithValues) WithLabelValues(labels map[string]strin } // Now add all new values. for name, value := range labels { - if !extended.haveAdditionalLabel(name) { + if !extended.HaveAdditionalLabel(name) { return nil, fmt.Errorf("label %q was not defined via WithLabelNames", name) } if v, ok := extended.additionalValues[name]; ok { @@ -327,7 +341,7 @@ func (cmmv *csiMetricsManagerWithValues) WithLabelValues(labels map[string]strin return extended, nil } -func (cmm *csiMetricsManager) haveAdditionalLabel(name string) bool { +func (cmm *csiMetricsManager) HaveAdditionalLabel(name string) bool { for _, n := range cmm.additionalLabelNames { if n == name { return true diff --git a/vendor/modules.txt b/vendor/modules.txt index a02a37430..98fedbe33 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -54,7 +54,7 @@ github.com/hashicorp/golang-lru/simplelru github.com/imdario/mergo # github.com/json-iterator/go v1.1.10 github.com/json-iterator/go -# github.com/kubernetes-csi/csi-lib-utils v0.9.0 +# github.com/kubernetes-csi/csi-lib-utils v0.9.1 ## explicit github.com/kubernetes-csi/csi-lib-utils/connection github.com/kubernetes-csi/csi-lib-utils/leaderelection