Skip to content

Commit 64110dd

Browse files
rochaportorochaporto
authored
Merge pull request #7 from cernops/9248multins
Allowing mounts from multiple PVCs across different namespaces
2 parents 4dcfe43 + 303848b commit 64110dd

File tree

6 files changed

+228
-59
lines changed

6 files changed

+228
-59
lines changed

‎.gitlab-ci.yml‎

Lines changed: 59 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ stages:
77

88
before_script:
99
- export CLUSTER_NAME=cvmfs-validation-$(echo $CI_COMMIT_SHA | head -c 6)
10+
- export BASE_CLUSTER_TEMPLATE="kubernetes-1.17.2-3"
1011
- export OS_AUTH_URL="https://keystone.cern.ch/main/v3"
1112
- export OS_IDENTITY_API_VERSION="3"
1213
- export OS_USERNAME="svcbuild"
@@ -19,84 +20,88 @@ before_script:
1920

2021
build:
2122
stage: build
22-
image: gitlab-registry.cern.ch/cloud/ciadm
23+
image: gitlab-registry.cern.ch/cloud/ciadm:v0.3.6
2324
script:
24-
- rpm --import https://mirror.go-repo.io/centos/RPM-GPG-KEY-GO-REPO
25-
- curl -s https://mirror.go-repo.io/centos/go-repo.repo | tee /etc/yum.repos.d/go-repo.repo
26-
- echo 'priority=1' >> /etc/yum.repos.d/go-repo.repo
27-
- yum install -y gcc git golang make
28-
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.cern.ch
2925
- make
3026
- make test
3127
artifacts:
32-
expire_in: '10 days'
28+
expire_in: '30 minutes'
3329
paths:
34-
- _output/csi-cvmfsplugin
30+
- bin/csi-cvmfsplugin
3531
except:
3632
- qa
3733
- master
38-
tags:
39-
- docker-privileged
4034

41-
build image:
35+
buildimg:
4236
stage: image
43-
image: gitlab-registry.cern.ch/cloud/ciadm
44-
script:
45-
- cp _output/csi-cvmfsplugin deploy/docker
46-
- yum install -y make
47-
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.cern.ch
48-
- docker build -t gitlab-registry.cern.ch/cloud-infrastructure/cvmfs-csi/cvmfsplugin:$CI_COMMIT_SHA deploy/docker
49-
- docker push gitlab-registry.cern.ch/cloud-infrastructure/cvmfs-csi/cvmfsplugin:$CI_COMMIT_SHA
37+
variables:
38+
TO: "${CI_REGISTRY_IMAGE}/cvmfs-csi:${CI_COMMIT_SHORT_SHA}"
39+
CONTEXT_DIR: "./"
40+
DOCKER_FILE: "deployments/docker/Dockerfile"
41+
tags:
42+
- docker-image-build
43+
script: "echo"
5044
dependencies:
5145
- build
46+
only:
47+
- branches
5248
except:
5349
- master
54-
- qa
55-
tags:
56-
- docker-privileged
5750

58-
test k8s:
51+
testk8s:
5952
stage: test
60-
image: gitlab-registry.cern.ch/cloud/ciadm
61-
script:
62-
- openstack coe cluster create $CLUSTER_NAME --cluster-template kubernetes-preview --keypair lxplus --node-count 1 --master-count 1 --flavor m2.small --labels influx_grafana_dashboard_enabled=True --labels cephfs_csi_enabled=True --labels manila_enabled=True --labels kube_tag=v1.14.1 --labels kube_csi_enabled=True --labels kube_csi_version=cern-csi-1.0-1 --labels heat_container_agent_tag=stein-dev-1 --labels cgroup_driver=cgroupfs --labels container_infra_prefix=gitlab-registry.cern.ch/cloud/atomic-system-containers/ --labels flannel_backend=vxlan --labels cvmfs_csi_version=$CI_COMMIT_SHA --labels admission_control_list=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,Priority --labels ingress_controller=traefik --labels manila_version=v0.3.0 --labels cvmfs_csi_enabled=True --labels cvmfs_tag=qa --labels cephfs_csi_version=cern-csi-1.0-1
63-
- sleep 10
64-
- STATUS=$(openstack coe cluster show $CLUSTER_NAME -c status | grep 'status ' | awk '{print $4}')
65-
- while [ "$STATUS" != "CREATE_COMPLETE" ] && [ "$STATUS" != "CREATE_FAILED" ]; do STATUS=$(openstack coe cluster show $CLUSTER_NAME -c status | grep 'status ' | awk '{print $4}'); done
66-
- openstack coe cluster show $CLUSTER_NAME
67-
- openstack coe cluster config $CLUSTER_NAME
68-
- export KUBECONFIG=config
69-
- kubectl create --validate=false -f magnum-cvmfs.yaml
70-
- until kubectl rollout status deployment/cvmfs; do kubectl get all; kubectl describe pvc; done
71-
- POD=$(kubectl get pod | grep cvmfs | awk '{print $1}')
72-
- kubectl exec $POD -c atlas cat /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/README.INSTALL
73-
- kubectl exec $POD -c cms cat /cvmfs/cms.cern.ch/README
53+
image: gitlab-registry.cern.ch/cloud/ciadm:v0.3.6
54+
script: |
55+
LABELS=$(openstack coe cluster template show -f json $BASE_CLUSTER_TEMPLATE | jq --raw-output '.labels | to_entries | map("--labels \(.key)=\(.value)") | join(" ")')
56+
openstack coe cluster create $CLUSTER_NAME --cluster-template $BASE_CLUSTER_TEMPLATE --keypair lxplus --node-count 1 --master-count 1 --flavor m2.small $LABELS
57+
sleep 10
58+
STATUS=$(openstack coe cluster show $CLUSTER_NAME -c status -f value)
59+
while [ "${STATUS}" != "CREATE_COMPLETE" ] && [ "${STATUS}" != "CREATE_FAILED" ]
60+
do
61+
STATUS=$(openstack coe cluster show $CLUSTER_NAME -c status -f value)
62+
echo "Current cluster status ... $STATUS $(date)"
63+
sleep 10
64+
done
65+
66+
openstack coe cluster show $CLUSTER_NAME
67+
openstack coe cluster config $CLUSTER_NAME
68+
export KUBECONFIG=config
69+
70+
for m in $(ls test)
71+
do
72+
kubectl create --validate=false -f test/${m}
73+
done
74+
for m in $(ls test)
75+
do
76+
kubectl wait --for=condition=complete --timeout=600s -l ci=true --all-namespaces job
77+
done
7478
dependencies:
75-
- build
79+
- buildimg
7680
except:
7781
- master
7882
- qa
7983

8084
cleanup test clusters:
8185
stage: cleanup
82-
image: gitlab-registry.cern.ch/cloud/ciadm
83-
script:
84-
- openstack coe cluster show ${CLUSTER_NAME}; if [ $? -eq 0 ]; then openstack coe cluster delete ${CLUSTER_NAME} || true; fi
85-
- sleep 5
86-
- while openstack coe cluster show ${CLUSTER_NAME}; [ $? -eq 0 ]; do sleep 3; done
86+
image: gitlab-registry.cern.ch/cloud/ciadm:v0.3.6
87+
script: |
88+
STATUS=$(openstack coe cluster show $CLUSTER_NAME -c status -f value)
89+
while true
90+
do
91+
if [ "${STATUS}" != "DELETE_IN_PROGRESS" ]
92+
then
93+
openstack coe cluster delete $CLUSTER_NAME || true
94+
fi
95+
sleep 10
96+
if STATUS=$(openstack coe cluster show $CLUSTER_NAME -c status -f value)
97+
then
98+
echo "Current cluster status ... $STATUS $(date)"
99+
else
100+
echo "done"
101+
break
102+
fi
103+
done
87104
when: always
88105
except:
89106
- master
90107
- qa
91-
92-
deploy tag:
93-
stage: deploy
94-
image: gitlab-registry.cern.ch/cloud/ciadm
95-
script:
96-
- docker pull gitlab-registry.cern.ch/cloud-infrastructure/cvmfs-csi/cvmfsplugin:$CI_COMMIT_SHA
97-
- docker tag gitlab-registry.cern.ch/cloud-infrastructure/cvmfs-csi/cvmfsplugin:$CI_COMMIT_SHA gitlab-registry.cern.ch/cloud-infrastructure/cvmfs-csi/cvmfsplugin:$CI_COMMIT_TAG
98-
- docker push gitlab-registry.cern.ch/cloud-infrastructure/cvmfs-csi/cvmfsplugin:$CI_COMMIT_TAG
99-
only:
100-
- tags
101-
tags:
102-
- docker-privileged

‎deployments/docker/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,6 @@ RUN yum install -y http://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-l
66

77
COPY configs/default.local /etc/cvmfs
88

9-
COPY _dist/linux-amd64/csi-cvmfsplugin_linux_amd64 /csi-cvmfsplugin
9+
COPY bin/csi-cvmfsplugin /csi-cvmfsplugin
1010
RUN chmod +x /csi-cvmfsplugin
1111
ENTRYPOINT ["/csi-cvmfsplugin"]

‎pkg/cvmfs/cvmfsconf.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ CVMFS_CACHE_BASE={{cacheBase .VolumeId}}
4141
CVMFS_ROOT_HASH={{.Hash}}
4242
CVMFS_AUTO_UPDATE=no
4343
{{else if .Tag}}
44-
CVFMFS_REPOSITORY_TAG={{.Tag}}
44+
CVMFS_REPOSITORY_TAG={{.Tag}}
4545
{{end}}`
4646

4747
var (

‎pkg/cvmfs/nodeserver.go‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,32 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
4848
return nil, status.Error(codes.InvalidArgument, err.Error())
4949
}
5050

51+
52+
base := "/cvmfs/"+string(volOptions.Repository)
53+
54+
55+
if err = createMountPoint(base); err != nil {
56+
glog.Errorf("failed to create staging mount point at %s for volume %s: %v", base, volId, err)
57+
return nil, status.Error(codes.Internal, err.Error())
58+
}
59+
60+
// Check if the volume is already mounted
61+
62+
isMnt, err := isMountPoint(base)
63+
64+
if err != nil {
65+
glog.Errorf("stat failed: %v", err)
66+
return nil, status.Error(codes.Internal, err.Error())
67+
}
68+
69+
if !isMnt {
70+
// It's not, mount now
71+
if err = mountCvmfs(volOptions, volId, base); err != nil {
72+
glog.Errorf("failed to mount volume %s to %s: %v", volId, base, err)
73+
return nil, status.Error(codes.Internal, err.Error())
74+
}
75+
}
76+
5177
if err = createMountPoint(stagingTargetPath); err != nil {
5278
glog.Errorf("failed to create staging mount point at %s for volume %s: %v", stagingTargetPath, volId, err)
5379
return nil, status.Error(codes.Internal, err.Error())
@@ -71,7 +97,7 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
7197

7298
// Check if the volume is already mounted
7399

74-
isMnt, err := isMountPoint(stagingTargetPath)
100+
isMnt, err = isMountPoint(stagingTargetPath)
75101

76102
if err != nil {
77103
glog.Errorf("stat failed: %v", err)
@@ -85,8 +111,8 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
85111

86112
// It's not, mount now
87113

88-
if err = mountCvmfs(volOptions, volId, stagingTargetPath); err != nil {
89-
glog.Errorf("failed to mount volume %s to %s: %v", volId, stagingTargetPath, err)
114+
if err = bindMount(base, stagingTargetPath); err != nil {
115+
glog.Errorf("failed to bind-mount %s to %s: %v", base, stagingTargetPath, err)
90116
return nil, status.Error(codes.Internal, err.Error())
91117
}
92118

‎test/01-validation-cern.yaml‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
apiVersion: storage.k8s.io/v1
2+
kind: StorageClass
3+
metadata:
4+
name: csi-cvmfs-atlas
5+
provisioner: cvmfs.csi.cern.ch
6+
parameters:
7+
repository: atlas.cern.ch
8+
---
9+
apiVersion: storage.k8s.io/v1
10+
kind: StorageClass
11+
metadata:
12+
name: csi-cvmfs-cms
13+
provisioner: cvmfs.csi.cern.ch
14+
parameters:
15+
repository: cms.cern.ch
16+
---
17+
apiVersion: v1
18+
kind: PersistentVolumeClaim
19+
metadata:
20+
name: csi-cvmfs-atlas-pvc
21+
spec:
22+
accessModes:
23+
- ReadOnlyMany
24+
resources:
25+
requests:
26+
storage: 1Gi
27+
storageClassName: csi-cvmfs-atlas
28+
---
29+
apiVersion: v1
30+
kind: PersistentVolumeClaim
31+
metadata:
32+
name: csi-cvmfs-cms-pvc
33+
spec:
34+
accessModes:
35+
- ReadOnlyMany
36+
resources:
37+
requests:
38+
storage: 1Gi
39+
storageClassName: csi-cvmfs-cms
40+
---
41+
apiVersion: batch/v1
42+
kind: Job
43+
metadata:
44+
name: validation-cern
45+
labels:
46+
app: validation-cern
47+
ci: "true"
48+
spec:
49+
template:
50+
spec:
51+
restartPolicy: Never
52+
containers:
53+
- name: atlas
54+
image: busybox
55+
command: ["/bin/sh", "-c", "cat /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/README.INSTALL"]
56+
volumeMounts:
57+
- mountPath: /cvmfs/atlas.cern.ch
58+
name: atlas-pvc
59+
- name: cms
60+
image: busybox
61+
command: ["/bin/sh", "-c", "cat /cvmfs/cms.cern.ch/README"]
62+
volumeMounts:
63+
- mountPath: /cvmfs/cms.cern.ch
64+
name: cms-pvc
65+
volumes:
66+
- name: atlas-pvc
67+
persistentVolumeClaim:
68+
claimName: csi-cvmfs-atlas-pvc
69+
readOnly: true
70+
- name: cms-pvc
71+
persistentVolumeClaim:
72+
claimName: csi-cvmfs-cms-pvc
73+
readOnly: true

‎test/02-validation-cern-multins.yaml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
apiVersion: v1
2+
kind: Namespace
3+
metadata:
4+
name: multins
5+
---
6+
apiVersion: v1
7+
kind: PersistentVolumeClaim
8+
metadata:
9+
name: csi-cvmfs-atlas-pvc
10+
namespace: multins
11+
spec:
12+
accessModes:
13+
- ReadOnlyMany
14+
resources:
15+
requests:
16+
storage: 1Gi
17+
storageClassName: csi-cvmfs-atlas
18+
---
19+
apiVersion: v1
20+
kind: PersistentVolumeClaim
21+
metadata:
22+
name: csi-cvmfs-cms-pvc
23+
namespace: multins
24+
spec:
25+
accessModes:
26+
- ReadOnlyMany
27+
resources:
28+
requests:
29+
storage: 1Gi
30+
storageClassName: csi-cvmfs-cms
31+
---
32+
apiVersion: batch/v1
33+
kind: Job
34+
metadata:
35+
name: validation-cern-multins
36+
namespace: multins
37+
labels:
38+
app: validation-cern-multins
39+
ci: "true"
40+
spec:
41+
template:
42+
spec:
43+
restartPolicy: Never
44+
containers:
45+
- name: atlas
46+
image: busybox
47+
command: ["/bin/sh", "-c", "cat /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/README.INSTALL"]
48+
volumeMounts:
49+
- mountPath: /cvmfs/atlas.cern.ch
50+
name: atlas-pvc
51+
- name: cms
52+
image: busybox
53+
command: ["/bin/sh", "-c", "cat /cvmfs/cms.cern.ch/README"]
54+
volumeMounts:
55+
- mountPath: /cvmfs/cms.cern.ch
56+
name: cms-pvc
57+
volumes:
58+
- name: atlas-pvc
59+
persistentVolumeClaim:
60+
claimName: csi-cvmfs-atlas-pvc
61+
readOnly: true
62+
- name: cms-pvc
63+
persistentVolumeClaim:
64+
claimName: csi-cvmfs-cms-pvc
65+
readOnly: true

0 commit comments

Comments
 (0)