WIP: cross-namespace snapshot populator for csi-hostpath driver

Makefile

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 CMDS_DIR=example
-CMDS=hello-populator
+CMDS=hello-populator csi-hostpath-transfer
 all: build
 
 include release-tools/build.make

example/csi-hostpath-transfer/Dockerfile

@@ -0,0 +1,7 @@
+FROM alpine
+LABEL maintainers="Kubernetes Authors"
+LABEL description="csi hostpath transfer"
+ARG binary=./bin/csi-hostpath-transfer
+
+COPY ${binary} csi-hostpath-transfer
+ENTRYPOINT ["/csi-hostpath-transfer"]

example/csi-hostpath-transfer/README.md

@@ -0,0 +1,179 @@
+
+# Cross-namespace Snapshot Populator for csi-hostpath Driver
+
+This example demonstrates a populator implementation for populating volume from cross-namespace snapshot.
+This populator only works for csi-hostpath driver.
+
+## How To
+
+1. Deploy k8s with AnyVolumeDataSource feature gate enabled (It is enabled by default in k8s 1.24 or later)
+
+ex) In k/k directory:
+
+- k8s 1.23 or prior:
+
+```bash
+FEATURE_GATES=AnyVolumeDataSource=true hack/local-up-cluster.sh
+```
+
+- k8s 1.24 or later:
+
+```bash
+hack/local-up-cluster.sh
+```
+
+2. Deploy volumeSnapshot controller and csi-hostpath driver
+
+```bash
+kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml
+kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml
+kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml
+
+kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml
+kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml
+
+cd /tmp
+git clone --depth=1 https://github.com/kubernetes-csi/csi-driver-host-path.git
+cd csi-driver-host-path
+./deploy/kubernetes-latest/deploy.sh
+```
+
+3. Prepare for using this feature
+
+- Deploy VolumePopulator  CRD and volume-data-source-validator
+
+```bash
+kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/volume-data-source-validator/master/client/config/crd/populator.storage.k8s.io_volumepopulators.yaml
+kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/volume-data-source-validator/master/deploy/kubernetes/rbac-data-source-validator.yaml
+kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/volume-data-source-validator/master/deploy/kubernetes/setup-data-source-validator.yaml
+```
+
+- Deploy ReferencePolicy CRD and VolumeSnapshotLink CRD
+
+```bash
+kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/master/config/crd/experimental/gateway.networking.k8s.io_referencepolicies.yaml
+curl -L https://raw.githubusercontent.com/mkimuram/lib-volume-populator/csi-hostpath-transfer/example/csi-hostpath-transfer/crd.yaml | sed 's|controller-gen.kubebuilder.io/version: v0.7.0|api-approved.kubernetes.io: https://github.com/kubernetes/enhancements/pull/3295|' | kubectl apply -f -
+```
+
+- Build and deploiy csi-hostpath-transfer
+
+```bash
+cd /tmp
+git clone --depth=1 -b csi-hostpath-transfer https://github.com/mkimuram/lib-volume-populator.git
+cd lib-volume-populator
+make container-csi-hostpath-transfer
+kubectl apply -f example/csi-hostpath-transfer/deploy.yaml
+```
+
+Depending on the environment, the container image above may need to be imported.
+```bash
+ctr images list
+ctr image export /tmp/csi-hostpath-transfer.tar docker.io/library/csi-hostpath-transfer:latest
+ctr -n=k8s.io images import /tmp/csi-hostpath-transfer.tar
+```
+
+4. Create StorageClass, PVC, and VolumeSnapshot by the examples in the csi-hostpath repo
+
+Note that touching /var/lib/csi-hostpath-data/${volumeHandle}/dummydata for the original PV is done, before taking snapshot. So, this file should be included in the volume restored from the snapshot.
+
+```bash
+cd /tmp/csi-driver-host-path
+kubectl apply -f examples/csi-storageclass.yaml
+kubectl apply -f examples/csi-pvc.yaml
+
+kubectl get pvc,pv
+NAME                            STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS      AGE
+persistentvolumeclaim/csi-pvc   Bound    pvc-43b57144-0666-41f0-9554-cff9a42daeb5   1Gi        RWO            csi-hostpath-sc   5s
+
+NAME                                                        CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM             STORAGECLASS      REASON   AGE
+persistentvolume/pvc-43b57144-0666-41f0-9554-cff9a42daeb5   1Gi        RWO            Delete           Bound    default/csi-pvc   csi-hostpath-sc            4s
+
+
+touch /var/lib/csi-hostpath-data/$(kubectl get pv $(kubectl get pvc csi-pvc -o jsonpath='{.spec.volumeName}') -o jsonpath='{.spec.csi.volumeHandle}')/dummydata
+
+
+cat examples/csi-snapshot-v1beta1.yaml | sed 's|v1beta1|v1|' | kubectl apply -f -
+
+kubectl get volumesnapshot,volumesnapshotcontent
+NAME                                                       READYTOUSE   SOURCEPVC   SOURCESNAPSHOTCONTENT   RESTORESIZE   SNAPSHOTCLASS            SNAPSHOTCONTENT                                    CREATIONTIME   AGE
+volumesnapshot.snapshot.storage.k8s.io/new-snapshot-demo   true         csi-pvc                             1Gi           csi-hostpath-snapclass   snapcontent-20514895-2c63-4f1c-8f43-990208ae8f87   5s             5s
+
+NAME                                                                                             READYTOUSE   RESTORESIZE   DELETIONPOLICY   DRIVER                VOLUMESNAPSHOTCLASS      VOLUMESNAPSHOT      VOLUMESNAPSHOTNAMESPACE   AGE
+volumesnapshotcontent.snapshot.storage.k8s.io/snapcontent-20514895-2c63-4f1c-8f43-990208ae8f87   true         1073741824    Delete           hostpath.csi.k8s.io   csi-hostpath-snapclass   new-snapshot-demo   default                   5s
+```
+
+5. Test creating PVC in ns1 namespace from VolumeSnapshot in default namespace 
+
+- Create ReferencePolicy, VolumeSnapshotLink, and PersistentVolumeClaim
+
+```bash
+kubectl create ns ns1
+
+cat << EOF | kubectl apply -f -
+apiVersion: gateway.networking.k8s.io/v1alpha2
+kind: ReferencePolicy
+metadata:
+  name: bar
+  namespace: default
+spec:
+  from:
+  - group: snapshot.storage.k8s.io
+    kind: VolumeSnapshotLink
+    namespace: ns1
+  to:
+  - group: snapshot.storage.k8s.io
+    kind: VolumeSnapshot
+    name: new-snapshot-demo
+EOF
+
+cat << EOF | kubectl apply -f -
+apiVersion: snapshot.storage.k8s.io/v1alpha1
+kind: VolumeSnapshotLink
+metadata:
+  name: foolink
+  namespace: ns1
+spec:
+  source:
+    name: new-snapshot-demo
+    namespace: default
+EOF
+
+cat << EOF | kubectl apply -f -
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: foo-pvc
+  namespace: ns1
+spec:
+  storageClassName: csi-hostpath-sc
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+  dataSourceRef:
+    apiGroup: snapshot.storage.k8s.io
+    kind: VolumeSnapshotLink
+    name: foolink
+  volumeMode: Filesystem
+EOF
+```
+
+- Confirm that the PVC and PV are created from the volumeSnapshot
+
+```bash
+kubectl get pvc,pv -n ns1 
+NAME                            STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS      AGE
+persistentvolumeclaim/foo-pvc   Bound    pvc-bbd237a9-4d92-4b10-b597-7154909e88a4   1Gi        RWO            csi-hostpath-sc   2m33s
+
+NAME                                                        CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM             STORAGECLASS      REASON   AGE
+persistentvolume/pvc-43b57144-0666-41f0-9554-cff9a42daeb5   1Gi        RWO            Delete           Bound    default/csi-pvc   csi-hostpath-sc            3m51s
+persistentvolume/pvc-bbd237a9-4d92-4b10-b597-7154909e88a4   1Gi        RWO            Delete           Bound    ns1/foo-pvc       csi-hostpath-sc            26s
+```
+
+- Confirm that that the PV contains the dummydata
+
+```bash
+ls -l /var/lib/csi-hostpath-data/$(kubectl get pv $(kubectl get pvc foo-pvc -n ns1 -o jsonpath='{.spec.volumeName}') -o jsonpath='{.spec.csi.volumeHandle}')
+-rw-r--r--. 1 root root 0 Nov  2 21:25 dummydata
+```

example/csi-hostpath-transfer/crd.yaml

@@ -0,0 +1,72 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  name: volumesnapshotlinks.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotLink
+    listKind: VolumeSnapshotLinkList
+    plural: volumesnapshotlinks
+    singular: volumesnapshotlink
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: 'VolumeSnapshotLink is a link to VolumeSnapshot that may exist
+          in different namespace. If namespace is different, the reference needs to
+          be allowed by using networking.gateway.k8s.io ReferencePolicy. TODO: consider
+          moving this object to kubernetes-csi/external-snapshotter repo.'
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: spec defines the desired characteristics of a snapshot requested
+              by a user.
+            properties:
+              source:
+                description: source specifies where a snapshot will be created from.
+                  This field is immutable after creation. Required.
+                properties:
+                  name:
+                    description: Name specifies the name of the VolumeSnapshot object
+                      to reference to.
+                    type: string
+                  namespace:
+                    description: Namespace specifies the namespace of the VolumeSnapshot
+                      object to reference to.
+                    type: string
+                required:
+                - name
+                - namespace
+                type: object
+            required:
+            - source
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

example/csi-hostpath-transfer/deploy.yaml

@@ -0,0 +1,89 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: csi-hostpath-transfer
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-hostpath-transfer-account
+  namespace: csi-hostpath-transfer
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-hostpath-transfer-role
+rules:
+  - apiGroups: [""]
+    resources: [persistentvolumes]
+    verbs: [get, list, watch, patch]
+  - apiGroups: [""]
+    resources: [persistentvolumeclaims]
+    verbs: [get, list, watch, patch, create, delete]
+  - apiGroups: [""]
+    resources: [pods]
+    verbs: [get, list, watch, create, delete]
+  - apiGroups: [storage.k8s.io]
+    resources: [storageclasses]
+    verbs: [get, list, watch]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: [volumesnapshotlinks]
+    verbs: [get, list, watch]
+  - apiGroups: ["gateway.networking.k8s.io"]
+    resources: ["referencepolicies"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-hostpath-transfer-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-hostpath-transfer-account
+    namespace: csi-hostpath-transfer
+roleRef:
+  kind: ClusterRole
+  name: csi-hostpath-transfer-role
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: csi-hostpath-transfer-populator
+  namespace: csi-hostpath-transfer
+spec:
+  selector:
+    matchLabels:
+      app: csi-hostpath-transfer
+  template:
+    metadata:
+      labels:
+        app: csi-hostpath-transfer
+    spec:
+      serviceAccount: csi-hostpath-transfer-account
+      containers:
+        - name: csi-hostpath-transfer
+          image: csi-hostpath-transfer:latest
+          imagePullPolicy: IfNotPresent
+          args:
+            - --mode=controller
+            - --image-name=csi-hostpath-transfer:latest
+            - --http-endpoint=:8080
+          ports:
+            - containerPort: 8080
+              name: http-endpoint
+              protocol: TCP
+---
+kind: VolumePopulator
+apiVersion: populator.storage.k8s.io/v1beta1
+metadata:
+  name: csi-hostpath-transfer
+sourceKind:
+  group: "snapshot.storage.k8s.io"
+  kind: VolumeSnapshotLink