guestagent: ticker: watch sys_exit_bind with eBPF

cmd/lima-guestagent/daemon_linux.go

@@ -16,6 +16,7 @@ import (
 	"github.com/lima-vm/lima/v2/pkg/guestagent"
 	"github.com/lima-vm/lima/v2/pkg/guestagent/api/server"
 	"github.com/lima-vm/lima/v2/pkg/guestagent/serialport"
+	"github.com/lima-vm/lima/v2/pkg/guestagent/ticker"
 	"github.com/lima-vm/lima/v2/pkg/portfwdserver"
 )
 
@@ -52,17 +53,20 @@ func daemonAction(cmd *cobra.Command, _ []string) error {
 	if os.Geteuid() != 0 {
 		return errors.New("must run as the root user")
 	}
-	logrus.Infof("event tick: %v", tick)
 
-	newTicker := func() (<-chan time.Time, func()) {
-		// TODO: use an equivalent of `bpftrace -e 'tracepoint:syscalls:sys_*_bind { printf("tick\n"); }')`,
-		// without depending on `bpftrace` binary.
-		// The agent binary will need CAP_BPF file cap.
-		ticker := time.NewTicker(tick)
-		return ticker.C, ticker.Stop
+	logrus.Infof("event tick: %v", tick)
+	simpleTicker := ticker.NewSimpleTicker(time.NewTicker(tick))
+	tickerInst := simpleTicker
+	// See /sys/kernel/debug/tracing/available_events for the list of available tracepoints
+	tracepoints := []string{"syscalls:sys_exit_bind"}
+	if ebpfTicker, err := ticker.NewEbpfTicker(tracepoints); err != nil {
+		logrus.WithError(err).Warn("failed to create eBPF ticker, falling back to simple ticker")
+	} else {
+		logrus.Infof("using eBPF ticker with tracepoints: %v", tracepoints)
+		tickerInst = ticker.NewCompoundTicker(simpleTicker, ebpfTicker)
 	}
 
-	agent, err := guestagent.New(ctx, newTicker, tick*20)
+	agent, err := guestagent.New(ctx, tickerInst, tick*20)
 	if err != nil {
 		return err
 	}

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/apparentlymart/go-cidr v1.1.0
 	github.com/balajiv113/fd v0.0.0-20230330094840-143eec500f3e
 	github.com/cheggaaa/pb/v3 v3.1.7 // gomodjail:unconfined
+	github.com/cilium/ebpf v0.19.0 // gomodjail:unconfined
 	github.com/containerd/continuity v0.4.5
 	github.com/containers/gvisor-tap-vsock v0.8.7 // gomodjail:unconfined
 	github.com/coreos/go-semver v0.3.1

go.sum

@@ -34,6 +34,8 @@ github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMU
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/cheggaaa/pb/v3 v3.1.7 h1:2FsIW307kt7A/rz/ZI2lvPO+v3wKazzE4K/0LtTWsOI=
 github.com/cheggaaa/pb/v3 v3.1.7/go.mod h1:/Ji89zfVPeC/u5j8ukD0MBPHt2bzTYp74lQ7KlgFWTQ=
+github.com/cilium/ebpf v0.19.0 h1:Ro/rE64RmFBeA9FGjcTc+KmCeY6jXmryu6FfnzPRIao=
+github.com/cilium/ebpf v0.19.0/go.mod h1:fLCgMo3l8tZmAdM3B2XqdFzXBpwkcSTroaVqN08OWVY=
 github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4=
 github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
@@ -96,6 +98,8 @@ github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF
 github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4=
 github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
 github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
+github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6 h1:teYtXy9B7y5lHTp8V9KPxpYRAVA7dozigQcMiBust1s=
+github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI=
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=

pkg/guestagent/guestagent_linux.go

@@ -21,12 +21,13 @@ import (
 	"github.com/lima-vm/lima/v2/pkg/guestagent/iptables"
 	"github.com/lima-vm/lima/v2/pkg/guestagent/kubernetesservice"
 	"github.com/lima-vm/lima/v2/pkg/guestagent/procnettcp"
+	"github.com/lima-vm/lima/v2/pkg/guestagent/ticker"
 	"github.com/lima-vm/lima/v2/pkg/guestagent/timesync"
 )
 
-func New(ctx context.Context, newTicker func() (<-chan time.Time, func()), iptablesIdle time.Duration) (Agent, error) {
+func New(ctx context.Context, ticker ticker.Ticker, iptablesIdle time.Duration) (Agent, error) {
 	a := &agent{
-		newTicker:                newTicker,
+		ticker:                   ticker,
 		kubernetesServiceWatcher: kubernetesservice.NewServiceWatcher(),
 	}
 
@@ -95,7 +96,7 @@ type agent struct {
 	// Ticker is like time.Ticker.
 	// We can't use inotify for /proc/net/tcp, so we need this ticker to
 	// reload /proc/net/tcp.
-	newTicker func() (<-chan time.Time, func())
+	ticker ticker.Ticker
 
 	worthCheckingIPTables    bool
 	worthCheckingIPTablesMu  sync.RWMutex
@@ -197,8 +198,8 @@ func isEventEmpty(ev *api.Event) bool {
 
 func (a *agent) Events(ctx context.Context, ch chan *api.Event) {
 	defer close(ch)
-	tickerCh, tickerClose := a.newTicker()
-	defer tickerClose()
+	tickerCh := a.ticker.Chan()
+	defer a.ticker.Stop()
 	var st eventState
 	for {
 		var ev *api.Event

pkg/guestagent/ticker/compound.go

@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package ticker
+
+import (
+	"time"
+)
+
+func NewCompoundTicker(t1, t2 Ticker) Ticker {
+	return &compoundTicker{t1, t2}
+}
+
+type compoundTicker struct {
+	t1, t2 Ticker
+}
+
+func (ticker *compoundTicker) Chan() <-chan time.Time {
+	ch := make(chan time.Time)
+	go func() {
+		defer ticker.Stop()
+		for {
+			select {
+			case v, ok := <-ticker.t1.Chan():
+				if !ok {
+					return
+				}
+				ch <- v
+			case v, ok := <-ticker.t2.Chan():
+				if !ok {
+					return
+				}
+				ch <- v
+			}
+		}
+	}()
+	return ch
+}
+
+func (ticker *compoundTicker) Stop() {
+	ticker.t1.Stop()
+	ticker.t2.Stop()
+}

pkg/guestagent/ticker/ebpf_linux.go

@@ -0,0 +1,135 @@
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package ticker
+
+import (
+	"strings"
+	"time"
+
+	"github.com/cilium/ebpf"
+	"github.com/cilium/ebpf/asm"
+	"github.com/cilium/ebpf/link"
+	"github.com/cilium/ebpf/ringbuf"
+	"github.com/sirupsen/logrus"
+)
+
+func NewEbpfTicker(tracepoints []string) (Ticker, error) {
+	var (
+		ticker ebpfTicker
+		err    error
+	)
+	ticker.events, err = ebpf.NewMap(&ebpf.MapSpec{
+		Name:       "lima_ticker_events",
+		Type:       ebpf.RingBuf,
+		MaxEntries: 1 << 20,
+	})
+	if err != nil {
+		ticker.Stop()
+		return nil, err
+	}
+
+	ticker.prog, err = buildEbpfProg(ticker.events)
+	if err != nil {
+		ticker.Stop()
+		return nil, err
+	}
+
+	for _, tp := range tracepoints {
+		tpPair := strings.SplitN(tp, ":", 2)
+		tpLink, err := link.Tracepoint(tpPair[0], tpPair[1], ticker.prog, nil)
+		if err != nil {
+			ticker.Stop()
+			return nil, err
+		}
+		ticker.links = append(ticker.links, tpLink)
+	}
+
+	ticker.reader, err = ringbuf.NewReader(ticker.events)
+	if err != nil {
+		ticker.Stop()
+		return nil, err
+	}
+
+	ticker.ch = make(chan time.Time)
+	go func() {
+		for {
+			_, rdErr := ticker.reader.Read()
+			if rdErr != nil {
+				logrus.WithError(rdErr).Warn("ebpfTicker: failed to read ringbuf")
+				ticker.Stop()
+				return
+			}
+			ticker.ch <- time.Now()
+		}
+	}()
+
+	return &ticker, nil
+}
+
+type ebpfTicker struct {
+	events *ebpf.Map
+	prog   *ebpf.Program
+	links  []link.Link
+	reader *ringbuf.Reader
+	ch     chan time.Time
+}
+
+func (ticker *ebpfTicker) Chan() <-chan time.Time {
+	return ticker.ch
+}
+
+func (ticker *ebpfTicker) Stop() {
+	if ticker.events != nil {
+		_ = ticker.events.Close()
+	}
+	if ticker.prog != nil {
+		_ = ticker.prog.Close()
+	}
+	for _, l := range ticker.links {
+		_ = l.Close()
+	}
+	if ticker.reader != nil {
+		_ = ticker.reader.Close()
+	}
+	if ticker.ch != nil {
+		close(ticker.ch)
+	}
+}
+
+func buildEbpfProg(events *ebpf.Map) (*ebpf.Program, error) {
+	inst := asm.Instructions{
+		// ringbuf = &map
+		asm.LoadMapPtr(asm.R1, events.FD()),
+
+		// data = FP - 8
+		asm.Mov.Reg(asm.R2, asm.R10),
+		asm.Add.Imm(asm.R2, -8),
+
+		// *data = 1
+		asm.StoreImm(asm.R2, 0, 1, asm.Word),
+
+		// size = 1
+		asm.Mov.Imm(asm.R3, 1),
+
+		// flags = 0
+		asm.Mov.Imm(asm.R4, 0),
+
+		// long bpf_ringbuf_output(void *ringbuf, void *data, u64 size, u64 flags)
+		// https://man7.org/linux/man-pages/man7/bpf-helpers.7.html
+		asm.FnRingbufOutput.Call(),
+
+		// return 0
+		asm.Mov.Imm(asm.R0, 0),
+		asm.Return(),
+	}
+
+	spec := &ebpf.ProgramSpec{
+		Name:         "lima_ticker",
+		Type:         ebpf.TracePoint,
+		License:      "Apache-2.0", // No need to be GPL?
+		Instructions: inst,
+	}
+
+	return ebpf.NewProgram(spec)
+}

pkg/guestagent/ticker/simple.go

@@ -0,0 +1,20 @@
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package ticker
+
+import (
+	"time"
+)
+
+func NewSimpleTicker(ticker *time.Ticker) Ticker {
+	return &simpleTicker{Ticker: ticker}
+}
+
+type simpleTicker struct {
+	*time.Ticker
+}
+
+func (ticker *simpleTicker) Chan() <-chan time.Time {
+	return ticker.Ticker.C
+}

pkg/guestagent/ticker/ticker.go

@@ -0,0 +1,13 @@
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package ticker
+
+import (
+	"time"
+)
+
+type Ticker interface {
+	Chan() <-chan time.Time
+	Stop()
+}