qemu: Add virtiofs support on Linux

This adds support for using virtiofs to mount filesystems on Linux
hosts, via QEMU's vhost-user-fs-pci device + the Rust implementation of
virtiofsd. In a simple "benchmark" running sha256sum on a copy of the
Windows 11 ARM64 VHDK (because it's a large file I randomly had lying
around):

- reverse-sshfs took ~21s
- 9p took ~13-15s
- virtiofs took ~6-7s

(For comparison, running it directly on the host system took ~5s.)

This is marked as "experimental" because it has undergone testing
by...me and relies on additional tools installed other than just QEMU.

Unfortunately, this does *not* include support for DAX, because that's
not merged into upstream QEMU yet, making it rather difficult to test.

Ref. #20.

Signed-off-by: Ryan Gonzalez <[email protected]>

Author: refi64

docs/experimental.md

@@ -3,10 +3,11 @@
 The following features are experimental and subject to change:
 
 - `mountType: 9p`
+- `mountType: virtiofs` on Linux
 - `vmType: vz` and relevant configurations (`mountType: virtiofs`, `rosetta`, `[]networks.vzNAT`)
 - `arch: riscv64`
 - `video.display: vnc` and relevant configuration (`video.vnc.display`)
-- `mode: user-v2` in `networks.yml` and relevant configuration in `lima.yaml` 
+- `mode: user-v2` in `networks.yml` and relevant configuration in `lima.yaml`
 - `audio.device`
 
 The following commands are experimental and subject to change:

docs/mount.md

@@ -88,19 +88,21 @@ The "9p" mount type requires Lima v0.10.0 or later.
 > **Warning**
 > "virtiofs" mode is experimental
 
-| :zap: Requirement | Lima >= 0.14, macOS >= 13.0 |
-|-------------------|-----------------------------|
+| :zap: Requirement | Lima >= 0.14, macOS >= 13.0 | Lima >= 0.17.0, Linux, QEMU 4.2.0+, virtiofsd (Rust version) |
+|-------------------|-----------------------------| ------------------------------------------------------------ |
 
-The "virtiofs" mount type is implemented by using apple Virtualization.Framework shared directory (uses virtio-fs) device. 
+The "virtiofs" mount type is implemented via the virtio-fs device by using apple Virtualization.Framework shared directory on macOS and virtiofsd on Linux.
 Linux guest kernel must enable the CONFIG_VIRTIO_FS support for this support.
 
 An example configuration:
 ```yaml
-vmType: "vz"
+vmType: "vz"  # only for macOS; Linux uses 'qemu'
 mountType: "virtiofs"
 mounts:
 - location: "~"
 ```
 
 #### Caveats
-- The "virtiofs" mount type is supported only on macOS 13 or above with `vmType: vz` config. See also [`vmtype.md`](./vmtype.md).
+- For macOS, the "virtiofs" mount type is supported only on macOS 13 or above with `vmType: vz` config. See also [`vmtype.md`](./vmtype.md).
+- For Linux, the "virtiofs" mount type requires the [Rust version of virtiofsd](https://gitlab.com/virtio-fs/virtiofsd).
+  Using the version from QEMU (usually packaged as `qemu-virtiofsd`) will *not* work, as it requires root access to run.

examples/README.md

@@ -44,6 +44,7 @@ Container orchestration:
 Optional feature enablers:
 - [`vmnet.yaml`](./vmnet.yaml): ⭐enable [`vmnet.framework`](../docs/network.md)
 - [`experimental/9p.yaml`](experimental/9p.yaml): [experimental] use 9p mount type
+- [`experimental/virtiofs-linux.yaml`](experimental/9p.yaml): [experimental] use virtiofs mount type for Linux
 - [`experimental/riscv64.yaml`](experimental/riscv64.yaml): [experimental] RISC-V
 - [`experimental/net-user-v2.yaml`](experimental/net-user-v2.yaml): [experimental] user-v2 network
   to enable VM-to-VM communication without root privilege

examples/experimental/virtiofs-linux.yaml

@@ -0,0 +1,26 @@
+# This example requires Lima v0.17.0 or later, running on Linux with:
+# - QEMU v4.2.0 or later.
+# - virtiofsd's Rust implementation: https://gitlab.com/virtio-fs/virtiofsd
+#   The QEMU version (qemu-virtiofsd) will *not* work, as it requires root access
+#   for all operations.
+images:
+# Try to use release-yyyyMMdd image if available. Note that release-yyyyMMdd will be removed after several months.
+- location: "https://cloud-images.ubuntu.com/releases/23.04/release-20230502/ubuntu-23.04-server-cloudimg-amd64.img"
+  arch: "x86_64"
+  digest: "sha256:13965c84c65cbab0b34326ac34ac0c47a88030f9dff80e6391e56cb9077cadd0"
+- location: "https://cloud-images.ubuntu.com/releases/23.04/release-20230502/ubuntu-23.04-server-cloudimg-arm64.img"
+  arch: "aarch64"
+  digest: "sha256:76a0fc791ed48ea8d0325463e2748e06aa3836292df1178ee4af8daf12a643bf"
+# Fallback to the latest release image.
+# Hint: run `limactl prune` to invalidate the cache
+- location: "https://cloud-images.ubuntu.com/releases/23.04/release/ubuntu-23.04-server-cloudimg-amd64.img"
+  arch: "x86_64"
+- location: "https://cloud-images.ubuntu.com/releases/23.04/release/ubuntu-23.04-server-cloudimg-arm64.img"
+  arch: "aarch64"
+
+mounts:
+- location: "~"
+- location: "/tmp/lima"
+  writable: true
+
+mountType: "virtiofs"

hack/test-example.sh

@@ -54,6 +54,9 @@ case "$NAME" in
 "9p")
 	CHECKS["snapshot-online"]=""
 	;;
+"virtiofs-linux")
+	CHECKS["snapshot-online"]=""
+	;;
 "vmnet")
 	CHECKS["vmnet"]=1
 	;;

pkg/limayaml/defaults.go

@@ -31,6 +31,8 @@ const (
 	Default9pMsize           string = "128KiB"
 	Default9pCacheForRO      string = "fscache"
 	Default9pCacheForRW      string = "mmap"
+
+	DefaultVirtiofsQueueSize int = 1024
 )
 
 func defaultContainerdArchives() []File {
@@ -510,6 +512,9 @@ func FillDefault(y, d, o *LimaYAML, filePath string) {
 			if mount.NineP.Cache != nil {
 				mounts[i].NineP.Cache = mount.NineP.Cache
 			}
+			if mount.Virtiofs.QueueSize != nil {
+				mounts[i].Virtiofs.QueueSize = mount.Virtiofs.QueueSize
+			}
 			if mount.Writable != nil {
 				mounts[i].Writable = mount.Writable
 			}
@@ -543,6 +548,9 @@ func FillDefault(y, d, o *LimaYAML, filePath string) {
 		if mount.NineP.Msize == nil {
 			mounts[i].NineP.Msize = pointer.String(Default9pMsize)
 		}
+		if mount.Virtiofs.QueueSize == nil {
+			mounts[i].Virtiofs.QueueSize = pointer.Int(DefaultVirtiofsQueueSize)
+		}
 		if mount.Writable == nil {
 			mount.Writable = pointer.Bool(false)
 		}

pkg/limayaml/defaults_test.go

@@ -186,6 +186,7 @@ func TestFillDefault(t *testing.T) {
 	expect.Mounts[0].NineP.ProtocolVersion = pointer.String(Default9pProtocolVersion)
 	expect.Mounts[0].NineP.Msize = pointer.String(Default9pMsize)
 	expect.Mounts[0].NineP.Cache = pointer.String(Default9pCacheForRO)
+	expect.Mounts[0].Virtiofs.QueueSize = pointer.Int(DefaultVirtiofsQueueSize)
 	// Only missing Mounts field is Writable, and the default value is also the null value: false
 
 	expect.MountType = pointer.String(NINEP)
@@ -369,6 +370,7 @@ func TestFillDefault(t *testing.T) {
 	expect.Mounts[0].NineP.ProtocolVersion = pointer.String(Default9pProtocolVersion)
 	expect.Mounts[0].NineP.Msize = pointer.String(Default9pMsize)
 	expect.Mounts[0].NineP.Cache = pointer.String(Default9pCacheForRO)
+	expect.Mounts[0].Virtiofs.QueueSize = pointer.Int(DefaultVirtiofsQueueSize)
 	expect.HostResolver.Hosts = map[string]string{
 		"default": d.HostResolver.Hosts["default"],
 	}
@@ -494,6 +496,9 @@ func TestFillDefault(t *testing.T) {
 					Msize:           pointer.String("8KiB"),
 					Cache:           pointer.String("none"),
 				},
+				Virtiofs: Virtiofs{
+					QueueSize: pointer.Int(2048),
+				},
 			},
 		},
 		Provision: []Provision{
@@ -569,6 +574,7 @@ func TestFillDefault(t *testing.T) {
 	expect.Mounts[0].NineP.ProtocolVersion = pointer.String("9p2000")
 	expect.Mounts[0].NineP.Msize = pointer.String("8KiB")
 	expect.Mounts[0].NineP.Cache = pointer.String("none")
+	expect.Mounts[0].Virtiofs.QueueSize = pointer.Int(2048)
 
 	expect.MountType = pointer.String(NINEP)
 

pkg/limayaml/limayaml.go

@@ -80,11 +80,12 @@ type Image struct {
 type Disk = string
 
 type Mount struct {
-	Location   string `yaml:"location" json:"location"` // REQUIRED
-	MountPoint string `yaml:"mountPoint,omitempty" json:"mountPoint,omitempty"`
-	Writable   *bool  `yaml:"writable,omitempty" json:"writable,omitempty"`
-	SSHFS      SSHFS  `yaml:"sshfs,omitempty" json:"sshfs,omitempty"`
-	NineP      NineP  `yaml:"9p,omitempty" json:"9p,omitempty"`
+	Location   string   `yaml:"location" json:"location"` // REQUIRED
+	MountPoint string   `yaml:"mountPoint,omitempty" json:"mountPoint,omitempty"`
+	Writable   *bool    `yaml:"writable,omitempty" json:"writable,omitempty"`
+	SSHFS      SSHFS    `yaml:"sshfs,omitempty" json:"sshfs,omitempty"`
+	NineP      NineP    `yaml:"9p,omitempty" json:"9p,omitempty"`
+	Virtiofs   Virtiofs `yaml:"virtiofs,omitempty" json:"virtiofs,omitempty"`
 }
 
 type SFTPDriver = string
@@ -107,6 +108,10 @@ type NineP struct {
 	Cache           *string `yaml:"cache,omitempty" json:"cache,omitempty"`
 }
 
+type Virtiofs struct {
+	QueueSize *int `yaml:"queueSize,omitempty" json:"queueSize,omitempty"`
+}
+
 type SSH struct {
 	LocalPort *int `yaml:"localPort,omitempty" json:"localPort,omitempty"`
 

pkg/limayaml/validate.go

@@ -159,6 +159,14 @@ func Validate(y LimaYAML, warn bool) error {
 		return fmt.Errorf("field `mountType` must be %q or %q or %q, got %q", REVSSHFS, NINEP, VIRTIOFS, *y.MountType)
 	}
 
+	if warn && runtime.GOOS != "linux" {
+		for i, mount := range y.Mounts {
+			if mount.Virtiofs.QueueSize != nil {
+				logrus.Warnf("field mounts[%d].virtiofs.queueSize is only supported on Linux", i)
+			}
+		}
+	}
+
 	// y.Firmware.LegacyBIOS is ignored for aarch64, but not a fatal error.
 
 	for i, p := range y.Provision {
@@ -442,6 +450,9 @@ func warnExperimental(y LimaYAML) {
 	if *y.MountType == NINEP {
 		logrus.Warn("`mountType: 9p` is experimental")
 	}
+	if *y.MountType == VIRTIOFS && runtime.GOOS == "linux" {
+		logrus.Warn("`mountType: virtiofs` on Linux is experimental")
+	}
 	if *y.VMType == VZ {
 		logrus.Warn("`vmType: vz` is experimental")
 	}

pkg/qemu/qemu.go

@@ -2,6 +2,7 @@ package qemu
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -478,6 +479,12 @@ func Cmdline(cfg Config) (string, []string, error) {
 	memBytes = adjustMemBytesDarwinARM64HVF(memBytes, accel, features)
 	args = appendArgsIfNoConflict(args, "-m", strconv.Itoa(int(memBytes>>20)))
 
+	if *y.MountType == limayaml.VIRTIOFS {
+		args = appendArgsIfNoConflict(args, "-object",
+			fmt.Sprintf("memory-backend-file,id=virtiofs-shm,size=%s,mem-path=/dev/shm,share=on", strconv.Itoa(int(memBytes))))
+		args = appendArgsIfNoConflict(args, "-numa", "node,memdev=virtiofs-shm")
+	}
+
 	// CPU
 	cpu := y.CPUType[*y.Arch]
 	if runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
@@ -775,7 +782,7 @@ func Cmdline(cfg Config) (string, []string, error) {
 
 	// We also want to enable vsock here, but QEMU does not support vsock for macOS hosts
 
-	if *y.MountType == limayaml.NINEP {
+	if *y.MountType == limayaml.NINEP || *y.MountType == limayaml.VIRTIOFS {
 		for i, f := range y.Mounts {
 			tag := fmt.Sprintf("mount%d", i)
 			location, err := localpathutil.Expand(f.Location)
@@ -785,14 +792,30 @@ func Cmdline(cfg Config) (string, []string, error) {
 			if err := os.MkdirAll(location, 0755); err != nil {
 				return "", nil, err
 			}
-			options := "local"
-			options += fmt.Sprintf(",mount_tag=%s", tag)
-			options += fmt.Sprintf(",path=%s", location)
-			options += fmt.Sprintf(",security_model=%s", *f.NineP.SecurityModel)
-			if !*f.Writable {
-				options += ",readonly"
+
+			switch *y.MountType {
+			case limayaml.NINEP:
+				options := "local"
+				options += fmt.Sprintf(",mount_tag=%s", tag)
+				options += fmt.Sprintf(",path=%s", location)
+				options += fmt.Sprintf(",security_model=%s", *f.NineP.SecurityModel)
+				if !*f.Writable {
+					options += ",readonly"
+				}
+				args = append(args, "-virtfs", options)
+			case limayaml.VIRTIOFS:
+				// Note that read-only mode is not supported on the QEMU/virtiofsd side yet:
+				// https://gitlab.com/virtio-fs/virtiofsd/-/issues/97
+				chardev := fmt.Sprintf("char-virtiofs-%d", i)
+				vhostSock := filepath.Join(cfg.InstanceDir, fmt.Sprintf(filenames.VhostSock, i))
+				args = append(args, "-chardev", fmt.Sprintf("socket,id=%s,path=%s", chardev, vhostSock))
+
+				options := "vhost-user-fs-pci"
+				options += fmt.Sprintf(",queue-size=%d", *f.Virtiofs.QueueSize)
+				options += fmt.Sprintf(",chardev=%s", chardev)
+				options += fmt.Sprintf(",tag=%s", tag)
+				args = append(args, "-device", options)
 			}
-			args = append(args, "-virtfs", options)
 		}
 	}
 
@@ -812,6 +835,99 @@ func Cmdline(cfg Config) (string, []string, error) {
 	return exe, args, nil
 }
 
+func FindVirtiofsd(qemuExe string) (string, error) {
+	type vhostUserBackend struct {
+		BackendType string `json:"type"`
+		Binary      string `json:"binary"`
+	}
+
+	currentUser, err := user.Current()
+	if err != nil {
+		return "", err
+	}
+
+	const relativePath = "share/qemu/vhost-user"
+
+	binDir := filepath.Dir(qemuExe)                              // "/usr/local/bin"
+	usrDir := filepath.Dir(binDir)                               // "/usr/local"
+	userLocalDir := filepath.Join(currentUser.HomeDir, ".local") // "$HOME/.local"
+
+	candidates := []string{
+		filepath.Join(userLocalDir, relativePath),
+		filepath.Join(usrDir, relativePath),
+	}
+
+	if usrDir != "/usr" {
+		candidates = append(candidates, filepath.Join("/usr", relativePath))
+	}
+
+	for _, vhostConfigsDir := range candidates {
+		logrus.Debugf("Checking vhost directory %s", vhostConfigsDir)
+
+		configEntries, err := os.ReadDir(vhostConfigsDir)
+		if err != nil {
+			logrus.Debugf("Failed to list vhost directory: %v", err)
+			continue
+		}
+
+		for _, configEntry := range configEntries {
+			logrus.Debugf("Checking vhost config %s", configEntry.Name())
+			if !strings.HasSuffix(configEntry.Name(), ".json") {
+				continue
+			}
+
+			var config vhostUserBackend
+			contents, err := os.ReadFile(filepath.Join(vhostConfigsDir, configEntry.Name()))
+			if err == nil {
+				err = json.Unmarshal(contents, &config)
+			}
+
+			if err != nil {
+				logrus.Warnf("Failed to load vhost-user config %s: %v", configEntry.Name(), err)
+				continue
+			}
+			logrus.Debugf("%v", config)
+
+			if config.BackendType != "fs" {
+				continue
+			}
+
+			// Only rust virtiofsd supports --version, so use that to make sure this isn't
+			// QEMU's virtiofsd, which requires running as root.
+			cmd := exec.Command(config.Binary, "--version")
+			output, err := cmd.CombinedOutput()
+			if err != nil {
+				logrus.Warnf("Failed to run %s --version (is this QEMU virtiofsd?): %s: %s",
+					config.Binary, err, output)
+				continue
+			}
+
+			return config.Binary, nil
+		}
+	}
+
+	return "", errors.New("Failed to locate virtiofsd")
+}
+
+func VirtiofsdCmdline(cfg Config, mountIndex int) ([]string, error) {
+	mount := cfg.LimaYAML.Mounts[mountIndex]
+	location, err := localpathutil.Expand(mount.Location)
+	if err != nil {
+		return nil, err
+	}
+
+	vhostSock := filepath.Join(cfg.InstanceDir, fmt.Sprintf(filenames.VhostSock, mountIndex))
+	// qemu_driver has to wait for the socket to appear, so make sure any old ones are removed here.
+	if err := os.Remove(vhostSock); err != nil && !errors.Is(err, fs.ErrNotExist) {
+		logrus.Warnf("Failed to remove old vhost socket: %v", err)
+	}
+
+	return []string{
+		"--socket-path", vhostSock,
+		"--shared-dir", location,
+	}, nil
+}
+
 func getExe(arch limayaml.Arch) (string, []string, error) {
 	exeBase := "qemu-system-" + arch
 	var args []string