From 303848b67e31a28ba2d24916b785a892de5932ee Mon Sep 17 00:00:00 2001
From: almahmoud <amahmou4@jhu.edu>
Date: Sun, 7 Jul 2019 17:22:53 +0200
Subject: [PATCH] Allowing mounts from multiple PVCs across different
 namespaces

---
 .gitlab-ci.yml                       | 113 ++++++++++++++-------------
 deployments/docker/Dockerfile        |   2 +-
 pkg/cvmfs/cvmfsconf.go               |   2 +-
 pkg/cvmfs/nodeserver.go              |  32 +++++++-
 test/01-validation-cern.yaml         |  73 +++++++++++++++++
 test/02-validation-cern-multins.yaml |  65 +++++++++++++++
 6 files changed, 228 insertions(+), 59 deletions(-)
 create mode 100644 test/01-validation-cern.yaml
 create mode 100644 test/02-validation-cern-multins.yaml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 26bcaec2..3802cb5a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,6 +7,7 @@ stages:
 
 before_script:
   - export CLUSTER_NAME=cvmfs-validation-$(echo $CI_COMMIT_SHA | head -c 6)
+  - export BASE_CLUSTER_TEMPLATE="kubernetes-1.17.2-3"
   - export OS_AUTH_URL="https://keystone.cern.ch/main/v3"
   - export OS_IDENTITY_API_VERSION="3"
   - export OS_USERNAME="svcbuild"
@@ -19,84 +20,88 @@ before_script:
 
 build:
   stage: build
-  image: gitlab-registry.cern.ch/cloud/ciadm
+  image: gitlab-registry.cern.ch/cloud/ciadm:v0.3.6
   script:
-    - rpm --import https://mirror.go-repo.io/centos/RPM-GPG-KEY-GO-REPO
-    - curl -s https://mirror.go-repo.io/centos/go-repo.repo | tee /etc/yum.repos.d/go-repo.repo
-    - echo 'priority=1' >> /etc/yum.repos.d/go-repo.repo
-    - yum install -y gcc git golang make
-    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.cern.ch
     - make
     - make test
   artifacts:
-    expire_in: '10 days'
+    expire_in: '30 minutes'
     paths:
-      - _output/csi-cvmfsplugin
+      - bin/csi-cvmfsplugin
   except:
     - qa
     - master
-  tags:
-    - docker-privileged
 
-build image:
+buildimg:
   stage: image
-  image: gitlab-registry.cern.ch/cloud/ciadm
-  script:
-    - cp _output/csi-cvmfsplugin deploy/docker
-    - yum install -y make
-    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.cern.ch
-    - docker build -t gitlab-registry.cern.ch/cloud-infrastructure/cvmfs-csi/cvmfsplugin:$CI_COMMIT_SHA deploy/docker
-    - docker push gitlab-registry.cern.ch/cloud-infrastructure/cvmfs-csi/cvmfsplugin:$CI_COMMIT_SHA
+  variables:
+    TO: "${CI_REGISTRY_IMAGE}/cvmfs-csi:${CI_COMMIT_SHORT_SHA}"
+    CONTEXT_DIR: "./"
+    DOCKER_FILE: "deployments/docker/Dockerfile"
+  tags:
+    - docker-image-build
+  script: "echo"
   dependencies:
     - build
+  only:
+    - branches
   except:
     - master
-    - qa
-  tags:
-    - docker-privileged
 
-test k8s:
+testk8s:
   stage: test
-  image: gitlab-registry.cern.ch/cloud/ciadm
-  script:
-    - openstack coe cluster create $CLUSTER_NAME --cluster-template kubernetes-preview --keypair lxplus --node-count 1 --master-count 1 --flavor m2.small --labels influx_grafana_dashboard_enabled=True --labels cephfs_csi_enabled=True --labels manila_enabled=True --labels kube_tag=v1.14.1 --labels kube_csi_enabled=True --labels kube_csi_version=cern-csi-1.0-1 --labels heat_container_agent_tag=stein-dev-1 --labels cgroup_driver=cgroupfs --labels container_infra_prefix=gitlab-registry.cern.ch/cloud/atomic-system-containers/ --labels flannel_backend=vxlan --labels cvmfs_csi_version=$CI_COMMIT_SHA --labels admission_control_list=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,Priority --labels ingress_controller=traefik --labels manila_version=v0.3.0 --labels cvmfs_csi_enabled=True --labels cvmfs_tag=qa --labels cephfs_csi_version=cern-csi-1.0-1
-    - sleep 10
-    - STATUS=$(openstack coe cluster show $CLUSTER_NAME -c status | grep 'status ' | awk '{print $4}')
-    - while [ "$STATUS" != "CREATE_COMPLETE" ] && [ "$STATUS" != "CREATE_FAILED" ]; do STATUS=$(openstack coe cluster show $CLUSTER_NAME -c status | grep 'status ' | awk '{print $4}'); done
-    - openstack coe cluster show $CLUSTER_NAME
-    - openstack coe cluster config $CLUSTER_NAME
-    - export KUBECONFIG=config
-    - kubectl create --validate=false -f magnum-cvmfs.yaml
-    - until kubectl rollout status deployment/cvmfs; do kubectl get all; kubectl describe pvc; done
-    - POD=$(kubectl get pod | grep cvmfs | awk '{print $1}')
-    - kubectl exec $POD -c atlas cat /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/README.INSTALL
-    - kubectl exec $POD -c cms cat /cvmfs/cms.cern.ch/README
+  image: gitlab-registry.cern.ch/cloud/ciadm:v0.3.6
+  script: |
+    LABELS=$(openstack coe cluster template show -f json $BASE_CLUSTER_TEMPLATE | jq --raw-output '.labels | to_entries | map("--labels \(.key)=\(.value)") | join(" ")')
+    openstack coe cluster create $CLUSTER_NAME --cluster-template $BASE_CLUSTER_TEMPLATE --keypair lxplus --node-count 1 --master-count 1 --flavor m2.small $LABELS
+    sleep 10
+    STATUS=$(openstack coe cluster show $CLUSTER_NAME -c status -f value)
+    while [ "${STATUS}" != "CREATE_COMPLETE" ] && [ "${STATUS}" != "CREATE_FAILED" ]
+    do
+      STATUS=$(openstack coe cluster show $CLUSTER_NAME -c status -f value)
+      echo "Current cluster status ... $STATUS $(date)"
+      sleep 10
+    done
+
+    openstack coe cluster show $CLUSTER_NAME
+    openstack coe cluster config $CLUSTER_NAME
+    export KUBECONFIG=config
+
+    for m in $(ls test)
+    do
+      kubectl create --validate=false -f test/${m}
+    done
+    for m in $(ls test)
+    do
+      kubectl wait --for=condition=complete --timeout=600s -l ci=true --all-namespaces job
+    done
   dependencies:
-    - build
+    - buildimg
   except:
     - master
     - qa
 
 cleanup test clusters:
   stage: cleanup
-  image: gitlab-registry.cern.ch/cloud/ciadm
-  script:
-     - openstack coe cluster show ${CLUSTER_NAME}; if [ $? -eq 0 ]; then openstack coe cluster delete ${CLUSTER_NAME} || true; fi
-     - sleep 5
-     - while openstack coe cluster show ${CLUSTER_NAME}; [ $? -eq 0 ]; do sleep 3; done
+  image: gitlab-registry.cern.ch/cloud/ciadm:v0.3.6
+  script: |
+    STATUS=$(openstack coe cluster show $CLUSTER_NAME -c status -f value)
+    while true
+    do
+      if [ "${STATUS}" != "DELETE_IN_PROGRESS" ]
+      then
+        openstack coe cluster delete $CLUSTER_NAME || true
+      fi
+      sleep 10
+      if STATUS=$(openstack coe cluster show $CLUSTER_NAME -c status -f value)
+      then
+        echo "Current cluster status ... $STATUS $(date)"
+      else
+        echo "done"
+        break
+      fi
+    done
   when: always
   except:
     - master
     - qa
-
-deploy tag:
-  stage: deploy
-  image: gitlab-registry.cern.ch/cloud/ciadm
-  script:
-    - docker pull gitlab-registry.cern.ch/cloud-infrastructure/cvmfs-csi/cvmfsplugin:$CI_COMMIT_SHA
-    - docker tag gitlab-registry.cern.ch/cloud-infrastructure/cvmfs-csi/cvmfsplugin:$CI_COMMIT_SHA gitlab-registry.cern.ch/cloud-infrastructure/cvmfs-csi/cvmfsplugin:$CI_COMMIT_TAG
-    - docker push gitlab-registry.cern.ch/cloud-infrastructure/cvmfs-csi/cvmfsplugin:$CI_COMMIT_TAG
-  only:
-    - tags
-  tags:
-    - docker-privileged
diff --git a/deployments/docker/Dockerfile b/deployments/docker/Dockerfile
index fc628bd9..44f3375b 100644
--- a/deployments/docker/Dockerfile
+++ b/deployments/docker/Dockerfile
@@ -6,6 +6,6 @@ RUN yum install -y http://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-l
 
 COPY configs/default.local /etc/cvmfs
 
-COPY _dist/linux-amd64/csi-cvmfsplugin_linux_amd64 /csi-cvmfsplugin
+COPY bin/csi-cvmfsplugin /csi-cvmfsplugin
 RUN chmod +x /csi-cvmfsplugin
 ENTRYPOINT ["/csi-cvmfsplugin"]
diff --git a/pkg/cvmfs/cvmfsconf.go b/pkg/cvmfs/cvmfsconf.go
index 02e4ee1f..0df7d917 100644
--- a/pkg/cvmfs/cvmfsconf.go
+++ b/pkg/cvmfs/cvmfsconf.go
@@ -41,7 +41,7 @@ CVMFS_CACHE_BASE={{cacheBase .VolumeId}}
 CVMFS_ROOT_HASH={{.Hash}}
 CVMFS_AUTO_UPDATE=no
 {{else if .Tag}}
-CVFMFS_REPOSITORY_TAG={{.Tag}}
+CVMFS_REPOSITORY_TAG={{.Tag}}
 {{end}}`
 
 var (
diff --git a/pkg/cvmfs/nodeserver.go b/pkg/cvmfs/nodeserver.go
index 2b1c3444..fb151977 100644
--- a/pkg/cvmfs/nodeserver.go
+++ b/pkg/cvmfs/nodeserver.go
@@ -48,6 +48,32 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
 		return nil, status.Error(codes.InvalidArgument, err.Error())
 	}
 
+
+	base := "/cvmfs/"+string(volOptions.Repository)
+
+
+	if err = createMountPoint(base); err != nil {
+		glog.Errorf("failed to create staging mount point at %s for volume %s: %v", base, volId, err)
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	// Check if the volume is already mounted
+
+	isMnt, err := isMountPoint(base)
+
+	if err != nil {
+		glog.Errorf("stat failed: %v", err)
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	if !isMnt {
+		// It's not, mount now
+		if err = mountCvmfs(volOptions, volId, base); err != nil {
+			glog.Errorf("failed to mount volume %s to %s: %v", volId, base, err)
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+	}
+
 	if err = createMountPoint(stagingTargetPath); err != nil {
 		glog.Errorf("failed to create staging mount point at %s for volume %s: %v", stagingTargetPath, volId, err)
 		return nil, status.Error(codes.Internal, err.Error())
@@ -71,7 +97,7 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
 
 	// Check if the volume is already mounted
 
-	isMnt, err := isMountPoint(stagingTargetPath)
+	isMnt, err = isMountPoint(stagingTargetPath)
 
 	if err != nil {
 		glog.Errorf("stat failed: %v", err)
@@ -85,8 +111,8 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
 
 	// It's not, mount now
 
-	if err = mountCvmfs(volOptions, volId, stagingTargetPath); err != nil {
-		glog.Errorf("failed to mount volume %s to %s: %v", volId, stagingTargetPath, err)
+	if err = bindMount(base, stagingTargetPath); err != nil {
+		glog.Errorf("failed to bind-mount %s to %s: %v", base, stagingTargetPath, err)
 		return nil, status.Error(codes.Internal, err.Error())
 	}
 
diff --git a/test/01-validation-cern.yaml b/test/01-validation-cern.yaml
new file mode 100644
index 00000000..0a0a98a7
--- /dev/null
+++ b/test/01-validation-cern.yaml
@@ -0,0 +1,73 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: csi-cvmfs-atlas
+provisioner: cvmfs.csi.cern.ch
+parameters:
+  repository: atlas.cern.ch
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: csi-cvmfs-cms
+provisioner: cvmfs.csi.cern.ch
+parameters:
+  repository: cms.cern.ch
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: csi-cvmfs-atlas-pvc
+spec:
+  accessModes:
+  - ReadOnlyMany
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: csi-cvmfs-atlas
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: csi-cvmfs-cms-pvc
+spec:
+  accessModes:
+  - ReadOnlyMany
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: csi-cvmfs-cms
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: validation-cern
+  labels:
+    app: validation-cern
+    ci: "true"
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+       - name: atlas
+         image: busybox
+         command: ["/bin/sh", "-c", "cat /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/README.INSTALL"]
+         volumeMounts:
+           - mountPath: /cvmfs/atlas.cern.ch
+             name: atlas-pvc
+       - name: cms
+         image: busybox
+         command: ["/bin/sh", "-c", "cat /cvmfs/cms.cern.ch/README"]
+         volumeMounts:
+           - mountPath: /cvmfs/cms.cern.ch
+             name: cms-pvc
+      volumes:
+      - name: atlas-pvc
+        persistentVolumeClaim:
+          claimName: csi-cvmfs-atlas-pvc
+          readOnly: true
+      - name: cms-pvc
+        persistentVolumeClaim:
+          claimName: csi-cvmfs-cms-pvc
+          readOnly: true
diff --git a/test/02-validation-cern-multins.yaml b/test/02-validation-cern-multins.yaml
new file mode 100644
index 00000000..fb4f1771
--- /dev/null
+++ b/test/02-validation-cern-multins.yaml
@@ -0,0 +1,65 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: multins
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: csi-cvmfs-atlas-pvc
+  namespace: multins
+spec:
+  accessModes:
+  - ReadOnlyMany
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: csi-cvmfs-atlas
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: csi-cvmfs-cms-pvc
+  namespace: multins
+spec:
+  accessModes:
+  - ReadOnlyMany
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: csi-cvmfs-cms
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: validation-cern-multins
+  namespace: multins
+  labels:
+    app: validation-cern-multins
+    ci: "true"
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+       - name: atlas
+         image: busybox
+         command: ["/bin/sh", "-c", "cat /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/README.INSTALL"]
+         volumeMounts:
+           - mountPath: /cvmfs/atlas.cern.ch
+             name: atlas-pvc
+       - name: cms
+         image: busybox
+         command: ["/bin/sh", "-c", "cat /cvmfs/cms.cern.ch/README"]
+         volumeMounts:
+           - mountPath: /cvmfs/cms.cern.ch
+             name: cms-pvc
+      volumes:
+      - name: atlas-pvc
+        persistentVolumeClaim:
+          claimName: csi-cvmfs-atlas-pvc
+          readOnly: true
+      - name: cms-pvc
+        persistentVolumeClaim:
+          claimName: csi-cvmfs-cms-pvc
+          readOnly: true