diff --git a/cmd/lima-guestagent/daemon_linux.go b/cmd/lima-guestagent/daemon_linux.go index 748196f0c52..87e4a4feaa8 100644 --- a/cmd/lima-guestagent/daemon_linux.go +++ b/cmd/lima-guestagent/daemon_linux.go @@ -16,6 +16,7 @@ import ( "github.com/lima-vm/lima/v2/pkg/guestagent" "github.com/lima-vm/lima/v2/pkg/guestagent/api/server" "github.com/lima-vm/lima/v2/pkg/guestagent/serialport" + "github.com/lima-vm/lima/v2/pkg/guestagent/ticker" "github.com/lima-vm/lima/v2/pkg/portfwdserver" ) @@ -52,17 +53,20 @@ func daemonAction(cmd *cobra.Command, _ []string) error { if os.Geteuid() != 0 { return errors.New("must run as the root user") } - logrus.Infof("event tick: %v", tick) - newTicker := func() (<-chan time.Time, func()) { - // TODO: use an equivalent of `bpftrace -e 'tracepoint:syscalls:sys_*_bind { printf("tick\n"); }')`, - // without depending on `bpftrace` binary. - // The agent binary will need CAP_BPF file cap. - ticker := time.NewTicker(tick) - return ticker.C, ticker.Stop + logrus.Infof("event tick: %v", tick) + simpleTicker := ticker.NewSimpleTicker(time.NewTicker(tick)) + tickerInst := simpleTicker + // See /sys/kernel/debug/tracing/available_events for the list of available tracepoints + tracepoints := []string{"syscalls:sys_exit_bind"} + if ebpfTicker, err := ticker.NewEbpfTicker(tracepoints); err != nil { + logrus.WithError(err).Warn("failed to create eBPF ticker, falling back to simple ticker") + } else { + logrus.Infof("using eBPF ticker with tracepoints: %v", tracepoints) + tickerInst = ticker.NewCompoundTicker(simpleTicker, ebpfTicker) } - agent, err := guestagent.New(ctx, newTicker, tick*20) + agent, err := guestagent.New(ctx, tickerInst, tick*20) if err != nil { return err } diff --git a/go.mod b/go.mod index e7d73ddec33..b29739cf3bf 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/apparentlymart/go-cidr v1.1.0 github.com/balajiv113/fd v0.0.0-20230330094840-143eec500f3e github.com/cheggaaa/pb/v3 v3.1.7 // gomodjail:unconfined + github.com/cilium/ebpf v0.19.0 // gomodjail:unconfined github.com/containerd/continuity v0.4.5 github.com/containers/gvisor-tap-vsock v0.8.7 // gomodjail:unconfined github.com/coreos/go-semver v0.3.1 diff --git a/go.sum b/go.sum index 084213745e5..c14c15beb12 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMU github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cheggaaa/pb/v3 v3.1.7 h1:2FsIW307kt7A/rz/ZI2lvPO+v3wKazzE4K/0LtTWsOI= github.com/cheggaaa/pb/v3 v3.1.7/go.mod h1:/Ji89zfVPeC/u5j8ukD0MBPHt2bzTYp74lQ7KlgFWTQ= +github.com/cilium/ebpf v0.19.0 h1:Ro/rE64RmFBeA9FGjcTc+KmCeY6jXmryu6FfnzPRIao= +github.com/cilium/ebpf v0.19.0/go.mod h1:fLCgMo3l8tZmAdM3B2XqdFzXBpwkcSTroaVqN08OWVY= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= @@ -96,6 +98,8 @@ github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6 h1:teYtXy9B7y5lHTp8V9KPxpYRAVA7dozigQcMiBust1s= +github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= diff --git a/pkg/guestagent/guestagent_linux.go b/pkg/guestagent/guestagent_linux.go index 5b199131f56..b19009982b1 100644 --- a/pkg/guestagent/guestagent_linux.go +++ b/pkg/guestagent/guestagent_linux.go @@ -21,12 +21,13 @@ import ( "github.com/lima-vm/lima/v2/pkg/guestagent/iptables" "github.com/lima-vm/lima/v2/pkg/guestagent/kubernetesservice" "github.com/lima-vm/lima/v2/pkg/guestagent/procnettcp" + "github.com/lima-vm/lima/v2/pkg/guestagent/ticker" "github.com/lima-vm/lima/v2/pkg/guestagent/timesync" ) -func New(ctx context.Context, newTicker func() (<-chan time.Time, func()), iptablesIdle time.Duration) (Agent, error) { +func New(ctx context.Context, ticker ticker.Ticker, iptablesIdle time.Duration) (Agent, error) { a := &agent{ - newTicker: newTicker, + ticker: ticker, kubernetesServiceWatcher: kubernetesservice.NewServiceWatcher(), } @@ -95,7 +96,7 @@ type agent struct { // Ticker is like time.Ticker. // We can't use inotify for /proc/net/tcp, so we need this ticker to // reload /proc/net/tcp. - newTicker func() (<-chan time.Time, func()) + ticker ticker.Ticker worthCheckingIPTables bool worthCheckingIPTablesMu sync.RWMutex @@ -197,8 +198,8 @@ func isEventEmpty(ev *api.Event) bool { func (a *agent) Events(ctx context.Context, ch chan *api.Event) { defer close(ch) - tickerCh, tickerClose := a.newTicker() - defer tickerClose() + tickerCh := a.ticker.Chan() + defer a.ticker.Stop() var st eventState for { var ev *api.Event diff --git a/pkg/guestagent/ticker/compound.go b/pkg/guestagent/ticker/compound.go new file mode 100644 index 00000000000..5373a56216c --- /dev/null +++ b/pkg/guestagent/ticker/compound.go @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright The Lima Authors +// SPDX-License-Identifier: Apache-2.0 + +package ticker + +import ( + "time" +) + +func NewCompoundTicker(t1, t2 Ticker) Ticker { + return &compoundTicker{t1, t2} +} + +type compoundTicker struct { + t1, t2 Ticker +} + +func (ticker *compoundTicker) Chan() <-chan time.Time { + ch := make(chan time.Time) + go func() { + defer ticker.Stop() + for { + select { + case v, ok := <-ticker.t1.Chan(): + if !ok { + return + } + ch <- v + case v, ok := <-ticker.t2.Chan(): + if !ok { + return + } + ch <- v + } + } + }() + return ch +} + +func (ticker *compoundTicker) Stop() { + ticker.t1.Stop() + ticker.t2.Stop() +} diff --git a/pkg/guestagent/ticker/ebpf_linux.go b/pkg/guestagent/ticker/ebpf_linux.go new file mode 100644 index 00000000000..8bd4ddee8da --- /dev/null +++ b/pkg/guestagent/ticker/ebpf_linux.go @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: Copyright The Lima Authors +// SPDX-License-Identifier: Apache-2.0 + +package ticker + +import ( + "strings" + "time" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/asm" + "github.com/cilium/ebpf/link" + "github.com/cilium/ebpf/ringbuf" + "github.com/sirupsen/logrus" +) + +func NewEbpfTicker(tracepoints []string) (Ticker, error) { + var ( + ticker ebpfTicker + err error + ) + ticker.events, err = ebpf.NewMap(&ebpf.MapSpec{ + Name: "lima_ticker_events", + Type: ebpf.RingBuf, + MaxEntries: 1 << 20, + }) + if err != nil { + ticker.Stop() + return nil, err + } + + ticker.prog, err = buildEbpfProg(ticker.events) + if err != nil { + ticker.Stop() + return nil, err + } + + for _, tp := range tracepoints { + tpPair := strings.SplitN(tp, ":", 2) + tpLink, err := link.Tracepoint(tpPair[0], tpPair[1], ticker.prog, nil) + if err != nil { + ticker.Stop() + return nil, err + } + ticker.links = append(ticker.links, tpLink) + } + + ticker.reader, err = ringbuf.NewReader(ticker.events) + if err != nil { + ticker.Stop() + return nil, err + } + + ticker.ch = make(chan time.Time) + go func() { + for { + _, rdErr := ticker.reader.Read() + if rdErr != nil { + logrus.WithError(rdErr).Warn("ebpfTicker: failed to read ringbuf") + ticker.Stop() + return + } + ticker.ch <- time.Now() + } + }() + + return &ticker, nil +} + +type ebpfTicker struct { + events *ebpf.Map + prog *ebpf.Program + links []link.Link + reader *ringbuf.Reader + ch chan time.Time +} + +func (ticker *ebpfTicker) Chan() <-chan time.Time { + return ticker.ch +} + +func (ticker *ebpfTicker) Stop() { + if ticker.events != nil { + _ = ticker.events.Close() + } + if ticker.prog != nil { + _ = ticker.prog.Close() + } + for _, l := range ticker.links { + _ = l.Close() + } + if ticker.reader != nil { + _ = ticker.reader.Close() + } + if ticker.ch != nil { + close(ticker.ch) + } +} + +func buildEbpfProg(events *ebpf.Map) (*ebpf.Program, error) { + inst := asm.Instructions{ + // ringbuf = &map + asm.LoadMapPtr(asm.R1, events.FD()), + + // data = FP - 8 + asm.Mov.Reg(asm.R2, asm.R10), + asm.Add.Imm(asm.R2, -8), + + // *data = 1 + asm.StoreImm(asm.R2, 0, 1, asm.Word), + + // size = 1 + asm.Mov.Imm(asm.R3, 1), + + // flags = 0 + asm.Mov.Imm(asm.R4, 0), + + // long bpf_ringbuf_output(void *ringbuf, void *data, u64 size, u64 flags) + // https://man7.org/linux/man-pages/man7/bpf-helpers.7.html + asm.FnRingbufOutput.Call(), + + // return 0 + asm.Mov.Imm(asm.R0, 0), + asm.Return(), + } + + spec := &ebpf.ProgramSpec{ + Name: "lima_ticker", + Type: ebpf.TracePoint, + License: "Apache-2.0", // No need to be GPL? + Instructions: inst, + } + + return ebpf.NewProgram(spec) +} diff --git a/pkg/guestagent/ticker/simple.go b/pkg/guestagent/ticker/simple.go new file mode 100644 index 00000000000..c9b3446e3f0 --- /dev/null +++ b/pkg/guestagent/ticker/simple.go @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: Copyright The Lima Authors +// SPDX-License-Identifier: Apache-2.0 + +package ticker + +import ( + "time" +) + +func NewSimpleTicker(ticker *time.Ticker) Ticker { + return &simpleTicker{Ticker: ticker} +} + +type simpleTicker struct { + *time.Ticker +} + +func (ticker *simpleTicker) Chan() <-chan time.Time { + return ticker.Ticker.C +} diff --git a/pkg/guestagent/ticker/ticker.go b/pkg/guestagent/ticker/ticker.go new file mode 100644 index 00000000000..edfa5012473 --- /dev/null +++ b/pkg/guestagent/ticker/ticker.go @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright The Lima Authors +// SPDX-License-Identifier: Apache-2.0 + +package ticker + +import ( + "time" +) + +type Ticker interface { + Chan() <-chan time.Time + Stop() +}