diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..76fb5a2 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,14 @@ +{ + "permissions": { + "allow": [ + "WebFetch(domain:github.com)", + "Bash(grep:*)", + "Bash(make:*)" + ] + }, + "enabledMcpjsonServers": [ + "playwright", + "taskmaster-ai", + "deepwiki" + ] +} \ No newline at end of file diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..6b070fc --- /dev/null +++ b/.mcp.json @@ -0,0 +1,24 @@ +{ + "mcpServers": { + "playwright": { + "command": "npx", + "args": [ + "@playwright/mcp@latest", + "--headless" + ] + }, + "taskmaster-ai": { + "command": "npx", + "args": [ + "-y", + "--package=task-master-ai", + "task-master-ai" + ], + "env": {} + }, + "deepwiki": { + "type": "sse", + "url": "https://mcp.deepwiki.com/sse" + } + } +} diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..3fe645c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,128 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +This is a Container Storage Interface (CSI) driver for MooseFS, a distributed file system. The driver enables Kubernetes to provision and manage persistent volumes backed by MooseFS storage. + +## Build Commands + +**Compile the binary:** +```bash +make compile +``` + +**Build development Docker image:** +```bash +make build-dev +``` + +**Build production Docker image:** +```bash +make build-prod +``` + +**Cross-platform build with buildx:** +```bash +make dev-buildx # For development +make prod-buildx # For production +``` + +**Clean build artifacts:** +```bash +make clean +``` + +## Driver Architecture + +The CSI driver operates in two distinct modes, each running as separate processes: + +### Controller Service (driver/controller.go) +- Handles volume lifecycle operations: create, delete, expand +- Manages volume publishing/unpublishing to nodes +- Runs as a single instance in the cluster +- Maintains a MooseFS mount at `/mnt/controller` for volume operations +- Supports dynamic provisioning with quotas and static provisioning + +### Node Service (driver/node.go) +- Handles volume mounting/unmounting on individual nodes +- Manages multiple MooseFS client connections (pool-based) +- Each node runs one instance with configurable mount point count +- Mount points: `/mnt/${nodeId}[_${mountId}]` + +### Core Components + +**mfsHandler (driver/mfs_handler.go):** +- Abstraction layer for MooseFS operations +- Handles mfsmount/mfsumount commands +- Manages quotas via mfsgetquota/mfssetquota +- Creates volume directories and mount volumes + +**Service Interface (driver/service.go):** +- gRPC server setup and request routing +- Common driver initialization and logging +- CSI endpoint management (Unix sockets) + +## Key Configuration + +The driver is configured via `deploy/csi-moosefs-config.yaml`: +- `master_host`: MooseFS master server address +- `master_port`: MooseFS master port (default: 9421) +- `k8s_root_dir`: Root directory on MooseFS for all volumes +- `mount_count`: Number of pre-created MooseFS clients per node + +## Volume Management + +**Dynamic Provisioning:** +- Volumes created in `${k8s_root_dir}/${driver_working_dir}/volumes/` +- Quotas enforced via MooseFS quota system +- Support for volume expansion (increase only) + +**Static Provisioning:** +- Mount any MooseFS directory using `mfsSubDir` parameter +- No quota management for static volumes + +**Mount Volumes:** +- Special volumes for mounting existing MooseFS directories +- Created in `${k8s_root_dir}/${driver_working_dir}/mount_volumes/` + +## Dependencies + +- Go 1.14+ +- github.com/container-storage-interface/spec v1.5.0 (updated for fsGroup support) +- google.golang.org/grpc v1.36.0 +- github.com/sirupsen/logrus v1.8.0 +- gopkg.in/natefinch/lumberjack.v2 v2.0.0 (log rotation) + +## Deployment Structure + +The driver uses two main manifests: +- `deploy/csi-moosefs-config.yaml`: Configuration values +- `deploy/csi-moosefs.yaml`: Pod specs for controller and node services + +Version compatibility is maintained between Kubernetes versions and driver releases (see README). + +## FSGroup Support + +The driver implements full Kubernetes CSI fsGroup support to ensure proper UID/GID handling: + +**Features:** +- `VOLUME_MOUNT_GROUP` node capability advertised +- `fsGroupPolicy: File` configured in CSIDriver spec +- Automatic application of fsGroup permissions during NodePublishVolume +- Volume directories get group ownership set to fsGroup with 0775 permissions + +**Implementation Details:** +- fsGroup extracted from `VolumeCapability.MountVolume.VolumeMountGroup` +- Applied via `applyFSGroupPermissions()` in driver/mfs_handler.go:345 +- Permissions changed using `os.Chown()` and `os.Chmod()` syscalls +- Non-fatal: permission errors logged but don't fail the mount operation + +**Usage:** +Set fsGroup in Pod SecurityContext - the driver will automatically apply it: +```yaml +spec: + securityContext: + fsGroup: 1000 +``` \ No newline at end of file diff --git a/cmd/moosefs-csi-plugin/moosefs-csi b/cmd/moosefs-csi-plugin/moosefs-csi new file mode 100755 index 0000000..37de0c7 Binary files /dev/null and b/cmd/moosefs-csi-plugin/moosefs-csi differ diff --git a/deploy/csi-moosefs.yaml b/deploy/csi-moosefs.yaml index 69b6fb9..83731ec 100644 --- a/deploy/csi-moosefs.yaml +++ b/deploy/csi-moosefs.yaml @@ -6,6 +6,7 @@ metadata: spec: attachRequired: true podInfoOnMount: false + fsGroupPolicy: File --- kind: StorageClass diff --git a/driver/mfs_handler.go b/driver/mfs_handler.go index 80a4aa8..b709204 100644 --- a/driver/mfs_handler.go +++ b/driver/mfs_handler.go @@ -25,6 +25,7 @@ import ( "path" "strconv" "strings" + "syscall" "gopkg.in/natefinch/lumberjack.v2" ) @@ -269,17 +270,30 @@ func (mnt *mfsHandler) MountMfs() error { } func (mnt *mfsHandler) BindMount(mfsSource string, target string, options ...string) error { + return mnt.BindMountWithFSGroup(mfsSource, target, nil, options...) +} + +func (mnt *mfsHandler) BindMountWithFSGroup(mfsSource string, target string, fsGroup *int64, options ...string) error { mounter := Mounter{} source := mnt.HostPathTo(mfsSource) - log.Infof("BindMount - source: %s, target: %s, options: %v", source, target, options) + log.Infof("BindMountWithFSGroup - source: %s, target: %s, fsGroup: %v, options: %v", source, target, fsGroup, options) + if isMounted, err := mounter.IsMounted(target); err != nil { return err } else if !isMounted { if err := mounter.Mount(source, target, fsType, append(options, "bind")...); err != nil { return err } + + // Apply fsGroup permissions if specified + if fsGroup != nil { + if err := mnt.applyFSGroupPermissions(source, *fsGroup); err != nil { + log.Errorf("BindMountWithFSGroup - Failed to apply fsGroup permissions: %v", err) + // Don't fail the mount for permission errors, log and continue + } + } } else { - log.Infof("BindMount - target %s is already mounted", target) + log.Infof("BindMountWithFSGroup - target %s is already mounted", target) } return nil } @@ -323,3 +337,37 @@ func (mnt *mfsHandler) HostPluginDataPath() string { func (mnt *mfsHandler) HostPathTo(to string) string { return path.Join(mnt.hostMountPath, to) } + +// applyFSGroupPermissions applies the specified fsGroup ownership to the volume directory +// This function sets the group ownership of the volume root directory to the fsGroup +// and ensures it's group-writable (0775 permissions) +func (mnt *mfsHandler) applyFSGroupPermissions(volumePath string, fsGroup int64) error { + log.Infof("applyFSGroupPermissions - path: %s, fsGroup: %d", volumePath, fsGroup) + + // Get current file info + fileInfo, err := os.Stat(volumePath) + if err != nil { + return fmt.Errorf("failed to stat volume path %s: %v", volumePath, err) + } + + // Get current uid (should remain unchanged) + stat, ok := fileInfo.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("failed to get syscall.Stat_t for %s", volumePath) + } + currentUID := int(stat.Uid) + + // Change ownership to current uid and specified fsGroup + if err := os.Chown(volumePath, currentUID, int(fsGroup)); err != nil { + return fmt.Errorf("failed to chown %s to %d:%d: %v", volumePath, currentUID, fsGroup, err) + } + + // Set permissions to 0775 (owner+group writable, others readable) + // This ensures the fsGroup can write to the volume + if err := os.Chmod(volumePath, 0775); err != nil { + return fmt.Errorf("failed to chmod %s to 0775: %v", volumePath, err) + } + + log.Infof("applyFSGroupPermissions - successfully applied fsGroup %d to %s", fsGroup, volumePath) + return nil +} diff --git a/driver/node.go b/driver/node.go index bfc0312..4fe210d 100644 --- a/driver/node.go +++ b/driver/node.go @@ -19,6 +19,7 @@ package driver import ( "context" "math/rand" + "strconv" "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" @@ -40,6 +41,7 @@ var nodeCapabilities = []csi.NodeServiceCapability_RPC_Type{ //csi.NodeServiceCapability_RPC_GET_VOLUME_STATS, // csi.NodeServiceCapability_RPC_VOLUME_CONDITION, // csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, + csi.NodeServiceCapability_RPC_VOLUME_MOUNT_GROUP, } func NewNodeService(mfsmaster string, mfsmaster_port int, rootPath, pluginDataPath, nodeId, mfsMountOptions string, mountPointsCount int) (*NodeService, error) { @@ -87,10 +89,22 @@ func (ns *NodeService) NodePublishVolume(ctx context.Context, req *csi.NodePubli if req.GetReadonly() { options = append(options, "ro") } + + // Extract fsGroup from VolumeCapability if provided + var fsGroup *int64 + if mountVol := req.VolumeCapability.GetMount(); mountVol != nil && mountVol.VolumeMountGroup != "" { + if gid, err := strconv.ParseInt(mountVol.VolumeMountGroup, 10, 64); err == nil { + fsGroup = &gid + log.Infof("NodePublishVolume - Parsed fsGroup: %d", gid) + } else { + log.Errorf("NodePublishVolume - Failed to parse fsGroup '%s': %v", mountVol.VolumeMountGroup, err) + } + } + if handler, err := ns.pickHandler(req.GetVolumeContext(), req.GetPublishContext()); err != nil { return nil, err } else { - if err := handler.BindMount(source, target, options...); err != nil { + if err := handler.BindMountWithFSGroup(source, target, fsGroup, options...); err != nil { return nil, status.Error(codes.Internal, err.Error()) } } diff --git a/examples/fsgroup-test/README.md b/examples/fsgroup-test/README.md new file mode 100644 index 0000000..6be8afa --- /dev/null +++ b/examples/fsgroup-test/README.md @@ -0,0 +1,64 @@ +# FSGroup Support Test + +This example demonstrates the MooseFS CSI driver's fsGroup support, which automatically applies proper UID/GID permissions to volumes. + +## What This Tests + +- Pod running as non-root user (UID 1000) +- fsGroup set to 1000 in Pod SecurityContext +- CSI driver automatically applies group ownership to volume +- Container can write to the volume without permission errors + +## Usage + +1. Apply the PVC: + ```bash + kubectl apply -f pvc.yaml + ``` + +2. Apply the Pod: + ```bash + kubectl apply -f pod.yaml + ``` + +3. Verify the Pod can write to the volume: + ```bash + kubectl exec test-fsgroup-pod -- touch /data/test-file + kubectl exec test-fsgroup-pod -- ls -la /data/ + ``` + +## Expected Results + +- The volume directory should have group ownership set to 1000 +- The container running as UID 1000 should be able to create files +- Files created should be owned by UID 1000 and GID 1000 + +## Verification Commands + +Check permissions on the volume: +```bash +kubectl exec test-fsgroup-pod -- ls -ld /data +``` + +Test write access: +```bash +kubectl exec test-fsgroup-pod -- echo "Hello World" > /data/test.txt +kubectl exec test-fsgroup-pod -- cat /data/test.txt +``` + +## Cleanup + +```bash +kubectl delete -f pod.yaml +kubectl delete -f pvc.yaml +``` + +## How It Works + +1. **CSI Driver Configuration**: `fsGroupPolicy: File` in CSIDriver spec enables fsGroup support +2. **Node Capability**: `VOLUME_MOUNT_GROUP` capability tells Kubernetes the driver handles fsGroup +3. **Permission Application**: During `NodePublishVolume`, the driver: + - Extracts fsGroup from `VolumeCapability.MountVolume.VolumeMountGroup` + - Calls `applyFSGroupPermissions()` to set group ownership + - Changes directory permissions to 0775 (group writable) +4. **Result**: Pod processes can write to the volume even when running as non-root \ No newline at end of file diff --git a/examples/fsgroup-test/pod.yaml b/examples/fsgroup-test/pod.yaml new file mode 100644 index 0000000..de2e40b --- /dev/null +++ b/examples/fsgroup-test/pod.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-fsgroup-pod +spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + containers: + - name: test-container + image: busybox:latest + command: ["sleep", "3600"] + volumeMounts: + - name: test-volume + mountPath: /data + securityContext: + allowPrivilegeEscalation: false + volumes: + - name: test-volume + persistentVolumeClaim: + claimName: test-fsgroup-pvc \ No newline at end of file diff --git a/examples/fsgroup-test/pvc.yaml b/examples/fsgroup-test/pvc.yaml new file mode 100644 index 0000000..1716519 --- /dev/null +++ b/examples/fsgroup-test/pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: test-fsgroup-pvc +spec: + accessModes: + - ReadWriteMany + storageClassName: moosefs-storage + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/go.mod b/go.mod index 45ed505..ea9190e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/moosefs/moosefs-csi go 1.14 require ( - github.com/container-storage-interface/spec v1.4.0 + github.com/container-storage-interface/spec v1.5.0 github.com/golang/protobuf v1.4.3 github.com/sirupsen/logrus v1.8.0 golang.org/x/net v0.0.0-20200320220750-118fecf932d8 // indirect diff --git a/go.sum b/go.sum index 8117b06..296d50f 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/container-storage-interface/spec v1.4.0 h1:ozAshSKxpJnYUfmkpZCTYyF/4MYeYlhdXbAvPvfGmkg= -github.com/container-storage-interface/spec v1.4.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= +github.com/container-storage-interface/spec v1.5.0 h1:lvKxe3uLgqQeVQcrnL2CPQKISoKjTJxojEs9cBk+HXo= +github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -83,6 +83,7 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=