Support for lima usernet network

Signed-off-by: Balaji Vijayakumar <[email protected]>

Author: balajiv113

cmd/limactl/main.go

@@ -100,6 +100,7 @@ func newApp() *cobra.Command {
 		newEditCommand(),
 		newFactoryResetCommand(),
 		newDiskCommand(),
+		newUsernetCommand(),
 	)
 	return rootCmd
 }

cmd/limactl/usernet.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+
+	"github.com/lima-vm/lima/pkg/networks/usernet"
+	"github.com/spf13/cobra"
+)
+
+func newUsernetCommand() *cobra.Command {
+	var hostagentCommand = &cobra.Command{
+		Use:    "usernet",
+		Short:  "run usernet",
+		Args:   cobra.ExactArgs(0),
+		RunE:   usernetAction,
+		Hidden: true,
+	}
+	hostagentCommand.Flags().StringP("pidfile", "p", "", "write pid to file")
+	hostagentCommand.Flags().StringP("endpoint", "e", "", "exposes usernet api(s) on this endpoint")
+	hostagentCommand.Flags().String("listen-qemu", "", "listen for qemu connections")
+	hostagentCommand.Flags().String("listen-fd", "", "listen for fd connections")
+	hostagentCommand.Flags().Int("mtu", 1500, "mtu")
+	return hostagentCommand
+}
+
+func usernetAction(cmd *cobra.Command, args []string) error {
+
+	pidfile, err := cmd.Flags().GetString("pidfile")
+	if err != nil {
+		return err
+	}
+	if pidfile != "" {
+		if _, err := os.Stat(pidfile); !errors.Is(err, os.ErrNotExist) {
+			return fmt.Errorf("pidfile %q already exists", pidfile)
+		}
+		if err := os.WriteFile(pidfile, []byte(strconv.Itoa(os.Getpid())+"\n"), 0644); err != nil {
+			return err
+		}
+		defer os.RemoveAll(pidfile)
+	}
+	endpoint, err := cmd.Flags().GetString("endpoint")
+	if err != nil {
+		return err
+	}
+	qemuSocket, err := cmd.Flags().GetString("listen-qemu")
+	if err != nil {
+		return err
+	}
+	fdSocket, err := cmd.Flags().GetString("listen-fd")
+	if err != nil {
+		return err
+	}
+
+	mtu, err := cmd.Flags().GetInt("mtu")
+	if err != nil {
+		return err
+	}
+
+	os.RemoveAll(endpoint)
+	os.RemoveAll(qemuSocket)
+	os.RemoveAll(fdSocket)
+
+	return usernet.StartGVisorNetstack(cmd.Context(), &usernet.GVisorNetstackOpts{
+		MTU:        mtu,
+		Endpoint:   endpoint,
+		QemuSocket: qemuSocket,
+		FdSocket:   fdSocket,
+	})
+}

examples/experimental/usernet.yaml

@@ -0,0 +1,15 @@
+# Example to run vz instance with lima usernet enabled
+vmType: "vz"
+images:
+- location: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img"
+  arch: "x86_64"
+- location: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-arm64.img"
+  arch: "aarch64"
+
+mounts:
+- location: "~"
+- location: "/tmp/lima"
+  writable: true
+mountType: "virtiofs"
+networks:
+- lima: default

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/cheggaaa/pb/v3 v3.1.2
 	github.com/containerd/containerd v1.6.19
 	github.com/containerd/continuity v0.3.0
-	github.com/containers/gvisor-tap-vsock v0.5.0
+	github.com/containers/gvisor-tap-vsock v0.5.1-0.20230220110902-2732d3a55dc7
 	github.com/coreos/go-semver v0.3.1
 	github.com/cyphar/filepath-securejoin v0.2.3
 	github.com/digitalocean/go-qemu v0.0.0-20210326154740-ac9e0b687001
@@ -78,10 +78,13 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/kr/fs v0.1.0 // indirect
+	github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-runewidth v0.0.12 // indirect
+	github.com/mdlayher/socket v0.4.0 // indirect
+	github.com/mdlayher/vsock v1.2.0 // indirect
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -95,10 +98,10 @@ require (
 	go.uber.org/multierr v1.7.0 // indirect
 	golang.org/x/crypto v0.4.0 // indirect
 	golang.org/x/mod v0.7.0 // indirect
-	golang.org/x/net v0.4.0 // indirect
+	golang.org/x/net v0.7.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
-	golang.org/x/term v0.3.0 // indirect
-	golang.org/x/text v0.5.0 // indirect
+	golang.org/x/term v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	golang.org/x/tools v0.3.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect

go.sum

@@ -111,8 +111,8 @@ github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDG
 github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
 github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
 github.com/containerd/typeurl v0.0.0-20200205145503-b45ef1f1f737/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg=
-github.com/containers/gvisor-tap-vsock v0.5.0 h1:hoCkrfQ96tjek2BtiW1BHy50zAQCzkqeiAQY96y6NLk=
-github.com/containers/gvisor-tap-vsock v0.5.0/go.mod h1:jrnI5plQtmys5LEKpXcCCrLqZlrHsozQg0V2Jw1UG74=
+github.com/containers/gvisor-tap-vsock v0.5.1-0.20230220110902-2732d3a55dc7 h1:Ej2OTGLHMjcpnGApet9gmEfboe9aTT2eU/civ1Imm6g=
+github.com/containers/gvisor-tap-vsock v0.5.1-0.20230220110902-2732d3a55dc7/go.mod h1:RfDVvmr+DHvp6whJVEwrMn+hfTYIPLWlUPJhWXNUsCM=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -376,6 +376,8 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lima-vm/sshocker v0.3.2 h1:o0WqVzcpt6mzVCuqtS3N3O8kwTx6X4SLr4h7YaRISuE=
 github.com/lima-vm/sshocker v0.3.2/go.mod h1:9SWN6wob210VM6oJkkzvWQOlHSp/rQLB+0fSEc92zig=
+github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2 h1:DZMFueDbfz6PNc1GwDRA8+6lBx1TB9UnxDQliCqR73Y=
+github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2/go.mod h1:SWzULI85WerrFt3u+nIm5F9l7EvxZTKQvd0InF3nmgM=
 github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=
 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -409,6 +411,10 @@ github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZ
 github.com/mdlayher/raw v0.0.0-20190606142536-fef19f00fc18/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg=
 github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065 h1:aFkJ6lx4FPip+S+Uw4aTegFMct9shDvP+79PsSxpm3w=
 github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg=
+github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw=
+github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46NlmWuVoc=
+github.com/mdlayher/vsock v1.2.0 h1:klRY9lndjmg6k/QWbX/ucQ3e2JFRm1M7vfG9hijbQ0A=
+github.com/mdlayher/vsock v1.2.0/go.mod h1:w4kdSTQB9p1l/WwGmAs0V62qQ869qRYoongwgN+Y1HE=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
@@ -444,7 +450,7 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.24.2 h1:J/tulyYK6JwBldPViHJReihxxZ+22FHs0piGjQAvoUE=
+github.com/onsi/gomega v1.27.1 h1:rfztXRbg6nv/5f+Raen9RcGoSecHIFgBBLQK3Wdj754=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
@@ -641,8 +647,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
-golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
-golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
+golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -748,8 +754,8 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
-golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
-golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
+golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -761,8 +767,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
-golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

pkg/cidata/cidata.go

@@ -211,7 +211,11 @@ func GenerateISO9660(instDir, name string, y *limayaml.LimaYAML, udpDNSLocalPort
 
 	slirpMACAddress := limayaml.MACAddress(instDir)
 	args.Networks = append(args.Networks, Network{MACAddress: slirpMACAddress, Interface: networks.SlirpNICName})
-	for _, nw := range y.Networks {
+	firstUsernetIndex := limayaml.GetFirstUsernetIndex(y)
+	for i, nw := range y.Networks {
+		if i == firstUsernetIndex {
+			continue
+		}
 		args.Networks = append(args.Networks, Network{MACAddress: nw.MACAddress, Interface: nw.Interface})
 	}
 

pkg/fd/fd.go

@@ -0,0 +1,109 @@
+//go:build !windows
+// +build !windows
+
+// This file has been adapted from https://github.com/ftrvxmtrx/fd/blob/master/fd.go
+
+// Package fd provides a simple API to pass file descriptors
+// between different OS processes.
+//
+// It can be useful if you want to inherit network connections
+// from another process without closing them.
+//
+// Example scenario:
+//
+//  1. Running server receives a "let's upgrade" message
+//  2. Server opens a Unix domain socket for the "upgrade"
+//  3. Server starts a new copy of itself and passes Unix domain socket name
+//  4. New copy starts reading for the socket
+//  5. Server sends its state over the socket, also sending the number
+//     of network connections to inherit, then it sends those connections
+//     using fd.Put()
+//  6. New copy reads the state and inherits connections using fd.Get(),
+//     checks that everything is OK and sends the "OK" message to the socket
+//  7. Server receives "OK" message and kills itself
+package fd
+
+import (
+	"net"
+	"os"
+	"syscall"
+)
+
+// Get receives single file descriptors from a Unix domain socket.
+//
+// Num specifies the expected number of file descriptors in one message.
+// Internal files' names to be assigned are specified via optional filenames
+// argument.
+//
+// You need to close all files in the returned slice. The slice can be
+// non-empty even if this function returns an error.
+//
+// Use net.FileConn() if you're receiving a network connection.
+func Get(via *net.UnixConn, num int, filenames []string) ([]*os.File, error) {
+	if num < 1 {
+		return nil, nil
+	}
+
+	// get the underlying socket
+	viaf, err := via.File()
+	if err != nil {
+		return nil, err
+	}
+	socket := int(viaf.Fd())
+	defer viaf.Close()
+
+	// recvmsg
+	buf := make([]byte, syscall.CmsgSpace(num*4))
+	_, _, _, _, err = syscall.Recvmsg(socket, nil, buf, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	// parse control msgs
+	var msgs []syscall.SocketControlMessage
+	msgs, err = syscall.ParseSocketControlMessage(buf)
+
+	// convert fds to files
+	res := make([]*os.File, 0, len(msgs))
+	for i := 0; i < len(msgs) && err == nil; i++ {
+		var fds []int
+		fds, err = syscall.ParseUnixRights(&msgs[i])
+
+		for fi, fd := range fds {
+			var filename string
+			if fi < len(filenames) {
+				filename = filenames[fi]
+			}
+
+			res = append(res, os.NewFile(uintptr(fd), filename))
+		}
+	}
+
+	return res, err
+}
+
+// Put sends file descriptors to Unix domain socket.
+//
+// Please note that the number of descriptors in one message is limited
+// and is rather small.
+// Use conn.File() to get a file if you want to put a network connection.
+func Put(via *net.UnixConn, files ...*os.File) error {
+	if len(files) == 0 {
+		return nil
+	}
+
+	viaf, err := via.File()
+	if err != nil {
+		return err
+	}
+	socket := int(viaf.Fd())
+	defer viaf.Close()
+
+	fds := make([]int, len(files))
+	for i := range files {
+		fds[i] = int(files[i].Fd())
+	}
+
+	rights := syscall.UnixRights(fds...)
+	return syscall.Sendmsg(socket, nil, rights, nil, 0)
+}