diff --git a/agent/topology_probes.go b/agent/topology_probes.go index 978919ff03..a0bc6851fd 100644 --- a/agent/topology_probes.go +++ b/agent/topology_probes.go @@ -140,25 +140,31 @@ func NewTopologyProbeBundle(g *graph.Graph, hostNode *graph.Node) (*probe.Bundle } probeList := []string{"hardware"} - if runtime.GOOS == "linux" { - probeList = append(probeList, "netlink", "netns") - } + /* + * Do not add netlink by default + if runtime.GOOS == "linux" { + probeList = append(probeList, "netlink", "netns") + } + */ probeList = append(probeList, config.GetStringSlice("agent.topology.probes")...) logging.GetLogger().Infof("Topology probes: %v", probeList) if runtime.GOOS == "linux" { - nlHandler, err := NewTopologyProbe("netlink", ctx, bundle) - if err != nil { - return nil, err - } - bundle.AddHandler("netlink", nlHandler) - - nsHandler, err := NewTopologyProbe("netns", ctx, bundle) - if err != nil { - return nil, err - } - bundle.AddHandler("netns", nsHandler) + /* + * Do not add netlink by default + nlHandler, err := NewTopologyProbe("netlink", ctx, bundle) + if err != nil { + return nil, err + } + bundle.AddHandler("netlink", nlHandler) + + nsHandler, err := NewTopologyProbe("netns", ctx, bundle) + if err != nil { + return nil, err + } + bundle.AddHandler("netns", nsHandler) + */ } for _, t := range probeList { diff --git a/analyzer/probes.go b/analyzer/probes.go index e05d9c55cb..0a4841c656 100644 --- a/analyzer/probes.go +++ b/analyzer/probes.go @@ -41,6 +41,9 @@ import ( "github.com/skydive-project/skydive/topology/probes/ovn" "github.com/skydive-project/skydive/topology/probes/ovsdb" "github.com/skydive-project/skydive/topology/probes/peering" + "github.com/skydive-project/skydive/topology/probes/proccon" + "github.com/skydive-project/skydive/topology/probes/procpeering" + "github.com/skydive-project/skydive/topology/probes/snmplldp" ) func registerStaticProbes() { @@ -54,6 +57,7 @@ func registerStaticProbes() { ovsdb.Register() libvirt.Register() ovn.Register() + proccon.Register() } func registerPluginProbes() error { @@ -121,6 +125,12 @@ func NewTopologyProbeBundleFromConfig(g *graph.Graph) (*probe.Bundle, error) { handler, err = istio.NewIstioProbe(g) case "nsm": handler, err = nsm.NewNsmProbe(g) + case "proccon": + handler, err = proccon.NewProbe(g) + case "procpeering": + handler, err = procpeering.NewProbe(g) + case "snmplldp": + handler, err = snmplldp.NewProbe(g) default: logging.GetLogger().Errorf("unknown probe type: %s", t) continue diff --git a/analyzer/server.go b/analyzer/server.go index 8f139d32f0..afa21b5e14 100644 --- a/analyzer/server.go +++ b/analyzer/server.go @@ -332,6 +332,7 @@ func NewServerFromConfig() (*Server, error) { tr.AddTraversalExtension(ge.NewFlowTraversalExtension(tableClient, s.flowStorage)) tr.AddTraversalExtension(ge.NewSocketsTraversalExtension()) tr.AddTraversalExtension(ge.NewDescendantsTraversalExtension()) + tr.AddTraversalExtension(ge.NewAscendantsTraversalExtension()) tr.AddTraversalExtension(ge.NewNextHopTraversalExtension()) tr.AddTraversalExtension(ge.NewGroupTraversalExtension()) diff --git a/etc/skydive.yml.default b/etc/skydive.yml.default index 9e1fb0bbfa..fc18e46cea 100644 --- a/etc/skydive.yml.default +++ b/etc/skydive.yml.default @@ -117,6 +117,8 @@ analyzer: # - istio # - nsm # - ovn + # - procpeering + # - proccon k8s: # kubeconfig resolution order: @@ -175,6 +177,33 @@ analyzer: # * unix:/var/run/ovn/ovnnb_db.sock # address: unix:/var/run/ovn/ovnnb_db.sock + # Probe to listen for outside metrics with network information about procs + proccon: + # Where does this probe listen for new data + listen: 0.0.0.0:4000 + + # Network information received by proccon that does not add new elements to Metadata.TCPConn or TCPListen + # does not produce a flush to the backend. + # To avoid leaving behind the backend, after this number of node revisions, the flush is done. + # The node.Revision field increments one for each modification in TCPConn or TCPListen. + # If the node receive info for both values, node.Revision will increment two times. + # Using revision_flush=1 will keep the backend synced to skydive, but at the expense of many writes. + # Using a big value could lost data in case of a restart. + # This value should be a trade-off between the interval of clients sending data the the garbage_collector.delete_duration + # Eg.: if clients send data each 5', assuming they always send conn+listen info, this will produce 2 updates each 5'. + # If we want to sync each two hours, we will set the value to 12*2/5=48 + revision_flush: 48 + + # Clean old connections (items from Metadata.TCPConn and Metadata.TCPListen) + garbage_collector: + # How often garbage collector will run + # Format: https://golang.org/pkg/time/#ParseDuration + # Eg.: 2h, 20m, 3h45m + interval: 1h + # Network info older than this value will be deleted. + # Same format as "interval" + delete_duration: 3h + replication: # debug: false diff --git a/go.mod b/go.mod index 9a1142b942..94068e255f 100644 --- a/go.mod +++ b/go.mod @@ -29,11 +29,14 @@ require ( github.com/golang/protobuf v1.3.2 github.com/golangci/golangci-lint v1.18.0 github.com/gomatic/renderizer v1.0.1 + github.com/google/go-cmp v0.5.5 // indirect github.com/google/gopacket v1.1.17 github.com/gophercloud/gophercloud v0.13.0 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.4.2 + github.com/gosnmp/gosnmp v1.31.0 // indirect github.com/gosuri/uitable v0.0.0-20160404203958-36ee7e946282 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.11.3 // indirect github.com/hashicorp/golang-lru v0.5.3 github.com/hydrogen18/stoppableListener v0.0.0-20151210151943-dadc9ccc400c github.com/inconshreveable/log15 v0.0.0-20201112154412-8562bdadbbac // indirect @@ -50,7 +53,6 @@ require ( github.com/lunixbochs/struc v0.0.0-20180408203800-02e4c2afbb2a github.com/lxc/lxd v0.0.0-20200330183600-518f06676866 github.com/mailru/easyjson v0.7.6 - github.com/mattn/go-isatty v0.0.12 // indirect github.com/mattn/goveralls v0.0.2 github.com/mitchellh/mapstructure v1.3.3 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 @@ -79,17 +81,21 @@ require ( github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.0 + github.com/stretchr/testify v1.7.0 github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c github.com/tebeka/go2xunit v1.4.10 github.com/tebeka/selenium v0.0.0-20170314201507-657e45ec600f + github.com/tinylib/msgp v1.1.0 // indirect github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect github.com/vishvananda/netlink v1.0.0 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df github.com/voxelbrain/goptions v0.0.0-20180630082107-58cddc247ea2 // indirect github.com/weaveworks/tcptracer-bpf v0.0.0-20170817155301-e080bd747dc6 golang.org/x/net v0.0.0-20201021035429-f5854403a974 + golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f golang.org/x/tools v0.0.0-20210106214847-113979e3529a + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/grpc v1.23.1 gopkg.in/fsnotify/fsnotify.v1 v1.0.0-20180110053347-c2828203cd70 gopkg.in/macaroon-bakery.v2 v2.2.0 // indirect diff --git a/graffiti/api/server/edge.go b/graffiti/api/server/edge.go index 39a113aeb4..7d6ab0e849 100644 --- a/graffiti/api/server/edge.go +++ b/graffiti/api/server/edge.go @@ -90,7 +90,7 @@ func (h *EdgeAPIHandler) Create(resource rest.Resource, createOpts *rest.CreateO graphEdge.UpdatedAt = graphEdge.CreatedAt } if graphEdge.Origin == "" { - graphEdge.Origin = h.g.GetOrigin() + graphEdge.Origin = graph.Origin(h.g.GetHost(), apiOrigin) } if graphEdge.Metadata == nil { graphEdge.Metadata = graph.Metadata{} diff --git a/graffiti/api/types/types.go b/graffiti/api/types/types.go index 92db42d5e6..f3bb3e023d 100644 --- a/graffiti/api/types/types.go +++ b/graffiti/api/types/types.go @@ -88,10 +88,18 @@ func (e *Edge) Validate() error { } // Node object -// easyjson:json // swagger:model type Node graph.Node +// UnmarshalJSON decodes types.Node using the custom graph.Node unmarshal which +// uses MetadataDecoders +func (n *Node) UnmarshalJSON(data []byte) error { + gNode := graph.Node(*n) + err := gNode.UnmarshalJSON(data) + *n = Node(gNode) + return err +} + // GetID returns the node ID func (n *Node) GetID() string { return string(n.ID) diff --git a/graffiti/api/types/types_easyjson.go b/graffiti/api/types/types_easyjson.go index d744dc5ddd..521406fddc 100644 --- a/graffiti/api/types/types_easyjson.go +++ b/graffiti/api/types/types_easyjson.go @@ -436,170 +436,7 @@ func (v *TopologyParams) UnmarshalJSON(data []byte) error { func (v *TopologyParams) UnmarshalEasyJSON(l *jlexer.Lexer) { easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes3(l, v) } -func easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes4(in *jlexer.Lexer, out *Node) { - isTopLevel := in.IsStart() - if in.IsNull() { - if isTopLevel { - in.Consumed() - } - in.Skip() - return - } - in.Delim('{') - for !in.IsDelim('}') { - key := in.UnsafeFieldName(false) - in.WantColon() - if in.IsNull() { - in.Skip() - in.WantComma() - continue - } - switch key { - case "ID": - out.ID = graph.Identifier(in.String()) - case "Metadata": - if in.IsNull() { - in.Skip() - } else { - in.Delim('{') - out.Metadata = make(graph.Metadata) - for !in.IsDelim('}') { - key := string(in.String()) - in.WantColon() - var v7 interface{} - if m, ok := v7.(easyjson.Unmarshaler); ok { - m.UnmarshalEasyJSON(in) - } else if m, ok := v7.(json.Unmarshaler); ok { - _ = m.UnmarshalJSON(in.Raw()) - } else { - v7 = in.Interface() - } - (out.Metadata)[key] = v7 - in.WantComma() - } - in.Delim('}') - } - case "Host": - out.Host = string(in.String()) - case "Origin": - out.Origin = string(in.String()) - case "CreatedAt": - if data := in.Raw(); in.Ok() { - in.AddError((out.CreatedAt).UnmarshalJSON(data)) - } - case "UpdatedAt": - if data := in.Raw(); in.Ok() { - in.AddError((out.UpdatedAt).UnmarshalJSON(data)) - } - case "DeletedAt": - if data := in.Raw(); in.Ok() { - in.AddError((out.DeletedAt).UnmarshalJSON(data)) - } - case "Revision": - out.Revision = int64(in.Int64()) - default: - in.SkipRecursive() - } - in.WantComma() - } - in.Delim('}') - if isTopLevel { - in.Consumed() - } -} -func easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes4(out *jwriter.Writer, in Node) { - out.RawByte('{') - first := true - _ = first - { - const prefix string = ",\"ID\":" - out.RawString(prefix[1:]) - out.String(string(in.ID)) - } - { - const prefix string = ",\"Metadata\":" - out.RawString(prefix) - if in.Metadata == nil && (out.Flags&jwriter.NilMapAsEmpty) == 0 { - out.RawString(`null`) - } else { - out.RawByte('{') - v8First := true - for v8Name, v8Value := range in.Metadata { - if v8First { - v8First = false - } else { - out.RawByte(',') - } - out.String(string(v8Name)) - out.RawByte(':') - if m, ok := v8Value.(easyjson.Marshaler); ok { - m.MarshalEasyJSON(out) - } else if m, ok := v8Value.(json.Marshaler); ok { - out.Raw(m.MarshalJSON()) - } else { - out.Raw(json.Marshal(v8Value)) - } - } - out.RawByte('}') - } - } - { - const prefix string = ",\"Host\":" - out.RawString(prefix) - out.String(string(in.Host)) - } - { - const prefix string = ",\"Origin\":" - out.RawString(prefix) - out.String(string(in.Origin)) - } - { - const prefix string = ",\"CreatedAt\":" - out.RawString(prefix) - out.Raw((in.CreatedAt).MarshalJSON()) - } - { - const prefix string = ",\"UpdatedAt\":" - out.RawString(prefix) - out.Raw((in.UpdatedAt).MarshalJSON()) - } - if true { - const prefix string = ",\"DeletedAt\":" - out.RawString(prefix) - out.Raw((in.DeletedAt).MarshalJSON()) - } - { - const prefix string = ",\"Revision\":" - out.RawString(prefix) - out.Int64(int64(in.Revision)) - } - out.RawByte('}') -} - -// MarshalJSON supports json.Marshaler interface -func (v Node) MarshalJSON() ([]byte, error) { - w := jwriter.Writer{} - easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes4(&w, v) - return w.Buffer.BuildBytes(), w.Error -} - -// MarshalEasyJSON supports easyjson.Marshaler interface -func (v Node) MarshalEasyJSON(w *jwriter.Writer) { - easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes4(w, v) -} - -// UnmarshalJSON supports json.Unmarshaler interface -func (v *Node) UnmarshalJSON(data []byte) error { - r := jlexer.Lexer{Data: data} - easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes4(&r, v) - return r.Error() -} - -// UnmarshalEasyJSON supports easyjson.Unmarshaler interface -func (v *Node) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes4(l, v) -} -func easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes5(in *jlexer.Lexer, out *Edge) { +func easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes4(in *jlexer.Lexer, out *Edge) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -625,27 +462,7 @@ func easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes5(in *j case "ID": out.ID = graph.Identifier(in.String()) case "Metadata": - if in.IsNull() { - in.Skip() - } else { - in.Delim('{') - out.Metadata = make(graph.Metadata) - for !in.IsDelim('}') { - key := string(in.String()) - in.WantColon() - var v9 interface{} - if m, ok := v9.(easyjson.Unmarshaler); ok { - m.UnmarshalEasyJSON(in) - } else if m, ok := v9.(json.Unmarshaler); ok { - _ = m.UnmarshalJSON(in.Raw()) - } else { - v9 = in.Interface() - } - (out.Metadata)[key] = v9 - in.WantComma() - } - in.Delim('}') - } + (out.Metadata).UnmarshalEasyJSON(in) case "Host": out.Host = string(in.String()) case "Origin": @@ -674,7 +491,7 @@ func easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes5(in *j in.Consumed() } } -func easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes5(out *jwriter.Writer, in Edge) { +func easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes4(out *jwriter.Writer, in Edge) { out.RawByte('{') first := true _ = first @@ -696,29 +513,7 @@ func easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes5(out * { const prefix string = ",\"Metadata\":" out.RawString(prefix) - if in.Metadata == nil && (out.Flags&jwriter.NilMapAsEmpty) == 0 { - out.RawString(`null`) - } else { - out.RawByte('{') - v10First := true - for v10Name, v10Value := range in.Metadata { - if v10First { - v10First = false - } else { - out.RawByte(',') - } - out.String(string(v10Name)) - out.RawByte(':') - if m, ok := v10Value.(easyjson.Marshaler); ok { - m.MarshalEasyJSON(out) - } else if m, ok := v10Value.(json.Marshaler); ok { - out.Raw(m.MarshalJSON()) - } else { - out.Raw(json.Marshal(v10Value)) - } - } - out.RawByte('}') - } + (in.Metadata).MarshalEasyJSON(out) } { const prefix string = ",\"Host\":" @@ -756,27 +551,27 @@ func easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes5(out * // MarshalJSON supports json.Marshaler interface func (v Edge) MarshalJSON() ([]byte, error) { w := jwriter.Writer{} - easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes5(&w, v) + easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes4(&w, v) return w.Buffer.BuildBytes(), w.Error } // MarshalEasyJSON supports easyjson.Marshaler interface func (v Edge) MarshalEasyJSON(w *jwriter.Writer) { - easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes5(w, v) + easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes4(w, v) } // UnmarshalJSON supports json.Unmarshaler interface func (v *Edge) UnmarshalJSON(data []byte) error { r := jlexer.Lexer{Data: data} - easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes5(&r, v) + easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes4(&r, v) return r.Error() } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *Edge) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes5(l, v) + easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes4(l, v) } -func easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes6(in *jlexer.Lexer, out *Alert) { +func easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes5(in *jlexer.Lexer, out *Alert) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -821,7 +616,7 @@ func easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes6(in *j in.Consumed() } } -func easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes6(out *jwriter.Writer, in Alert) { +func easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes5(out *jwriter.Writer, in Alert) { out.RawByte('{') first := true _ = first @@ -892,23 +687,23 @@ func easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes6(out * // MarshalJSON supports json.Marshaler interface func (v Alert) MarshalJSON() ([]byte, error) { w := jwriter.Writer{} - easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes6(&w, v) + easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes5(&w, v) return w.Buffer.BuildBytes(), w.Error } // MarshalEasyJSON supports easyjson.Marshaler interface func (v Alert) MarshalEasyJSON(w *jwriter.Writer) { - easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes6(w, v) + easyjson6601e8cdEncodeGithubComSkydiveProjectSkydiveGraffitiApiTypes5(w, v) } // UnmarshalJSON supports json.Unmarshaler interface func (v *Alert) UnmarshalJSON(data []byte) error { r := jlexer.Lexer{Data: data} - easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes6(&r, v) + easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes5(&r, v) return r.Error() } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *Alert) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes6(l, v) + easyjson6601e8cdDecodeGithubComSkydiveProjectSkydiveGraffitiApiTypes5(l, v) } diff --git a/graffiti/graph/elasticsearch.go b/graffiti/graph/elasticsearch.go index 27eebfa7bd..8f243487ca 100644 --- a/graffiti/graph/elasticsearch.go +++ b/graffiti/graph/elasticsearch.go @@ -435,6 +435,16 @@ func (b *ElasticSearchBackend) GetEdges(t Context, m ElementMatcher, e ElementMa edges = dedupEdges(edges) } + for _, e := range edges { + raw, err := edgeToRaw(e) + if err != nil { + b.logger.Errorf("Ignoring edge, failing to marshal: %v", err) + continue + } + + b.prevRevision[e.ID] = raw + } + return edges } @@ -474,6 +484,16 @@ func (b *ElasticSearchBackend) GetNodes(t Context, m ElementMatcher, e ElementMa nodes = dedupNodes(nodes) } + for _, n := range nodes { + raw, err := nodeToRaw(n) + if err != nil { + b.logger.Errorf("Ignoring node, failing to marshal: %v", err) + continue + } + + b.prevRevision[n.ID] = raw + } + return nodes } diff --git a/gremlin/traversal/ascendants.go b/gremlin/traversal/ascendants.go new file mode 100644 index 0000000000..4f9abdd773 --- /dev/null +++ b/gremlin/traversal/ascendants.go @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2018 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy ofthe License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specificlanguage governing permissions and + * limitations under the License. + * + */ + +package traversal + +import ( + "github.com/pkg/errors" + + "github.com/skydive-project/skydive/graffiti/filters" + "github.com/skydive-project/skydive/graffiti/graph" + "github.com/skydive-project/skydive/graffiti/graph/traversal" + "github.com/skydive-project/skydive/topology" +) + +// AscendantsTraversalExtension describes a new extension to enhance the topology +type AscendantsTraversalExtension struct { + AscendantsToken traversal.Token +} + +// AscendantsGremlinTraversalStep rawpackets step +type AscendantsGremlinTraversalStep struct { + context traversal.GremlinTraversalContext + maxDepth int64 + edgeFilter graph.ElementMatcher +} + +// NewAscendantsTraversalExtension returns a new graph traversal extension +func NewAscendantsTraversalExtension() *AscendantsTraversalExtension { + return &AscendantsTraversalExtension{ + AscendantsToken: traversalAscendantsToken, + } +} + +// ScanIdent returns an associated graph token +func (e *AscendantsTraversalExtension) ScanIdent(s string) (traversal.Token, bool) { + switch s { + case "ASCENDANTS": + return e.AscendantsToken, true + } + return traversal.IDENT, false +} + +// ParseStep parses ascendants step +func (e *AscendantsTraversalExtension) ParseStep(t traversal.Token, p traversal.GremlinTraversalContext) (traversal.GremlinTraversalStep, error) { + switch t { + case e.AscendantsToken: + default: + return nil, nil + } + + maxDepth := int64(1) + edgeFilter, _ := topology.OwnershipMetadata().Filter() + + switch len(p.Params) { + case 0: + default: + i := len(p.Params) / 2 * 2 + filter, err := traversal.ParamsToFilter(filters.BoolFilterOp_OR, p.Params[:i]...) + if err != nil { + return nil, errors.Wrap(err, "Ascendants accepts an optional number of key/value tuples and an optional depth") + } + edgeFilter = filter + + if i == len(p.Params) { + break + } + + fallthrough + case 1: + depth, ok := p.Params[len(p.Params)-1].(int64) + if !ok { + return nil, errors.New("Ascendants last argument must be the maximum depth specified as an integer") + } + maxDepth = depth + } + + return &AscendantsGremlinTraversalStep{context: p, maxDepth: maxDepth, edgeFilter: graph.NewElementFilter(edgeFilter)}, nil +} + +// getAscendants given a list of nodes, add them to the ascendants list, get the ascendants of that nodes and call this function with those new nodes +func getAscendants(g *graph.Graph, nodes []*graph.Node, ascendants *[]*graph.Node, currDepth, maxDepth int64, edgeFilter graph.ElementMatcher, visited map[graph.Identifier]bool) { + var ld []*graph.Node + for _, node := range nodes { + if _, ok := visited[node.ID]; !ok { + ld = append(ld, node) + visited[node.ID] = true + } + } + *ascendants = append(*ascendants, ld...) + + if maxDepth == 0 || currDepth < maxDepth { + for _, parent := range nodes { + parents := g.LookupParents(parent, nil, edgeFilter) + getAscendants(g, parents, ascendants, currDepth+1, maxDepth, edgeFilter, visited) + } + } +} + +// Exec Ascendants step +func (d *AscendantsGremlinTraversalStep) Exec(last traversal.GraphTraversalStep) (traversal.GraphTraversalStep, error) { + var ascendants []*graph.Node + + switch tv := last.(type) { + case *traversal.GraphTraversalV: + tv.GraphTraversal.RLock() + getAscendants(tv.GraphTraversal.Graph, tv.GetNodes(), &ascendants, 0, d.maxDepth, d.edgeFilter, make(map[graph.Identifier]bool)) + tv.GraphTraversal.RUnlock() + + return traversal.NewGraphTraversalV(tv.GraphTraversal, ascendants), nil + } + return nil, traversal.ErrExecutionError +} + +// Reduce Ascendants step +func (d *AscendantsGremlinTraversalStep) Reduce(next traversal.GremlinTraversalStep) (traversal.GremlinTraversalStep, error) { + return next, nil +} + +// Context Ascendants step +func (d *AscendantsGremlinTraversalStep) Context() *traversal.GremlinTraversalContext { + return &d.context +} diff --git a/gremlin/traversal/token.go b/gremlin/traversal/token.go index d8177a68ac..e6604e9c89 100644 --- a/gremlin/traversal/token.go +++ b/gremlin/traversal/token.go @@ -33,4 +33,5 @@ const ( traversalNextHopToken traversal.Token = 1011 traversalGroupToken traversal.Token = 1012 traversalMoreThanToken traversal.Token = 1013 + traversalAscendantsToken traversal.Token = 1014 ) diff --git a/tests/api_test.go b/tests/api_test.go index 2a11e3b8aa..d395dce44d 100644 --- a/tests/api_test.go +++ b/tests/api_test.go @@ -40,6 +40,7 @@ func getCrudClient() (c *shttp.CrudClient, err error) { }) return } + func TestAlertAPI(t *testing.T) { client, err := getCrudClient() if err != nil { @@ -623,6 +624,100 @@ func TestAPIPatchNodeTable(t *testing.T) { } } +// TestAPIPatchNodeTypedMetadata tries to patch a node where some of its metadata contains +// typed variables, not just plain dict/lists +func TestAPIPatchNodeTypedMetadata(t *testing.T) { + // Types to test that stored metadata values keep its type after patching + type ProcInfo struct { + CreatedAt int64 + UpdatedAt int64 + Revision int64 + } + type NetworkInfo map[string]ProcInfo + + client, err := getCrudClient() + if err != nil { + t.Fatal(err) + } + + // Create node + originalNodeBytes := []byte(`{ + "ID": "test1", + "Metadata": { + "TID": "test1", + "Name": "name1", + "Type": "type1", + "foo": "bar" + }, + "Host": "host1", + "Origin": "origin1", + "CreatedAt": 0, + "UpdatedAt": 0, + "Revision": 0 + }`) + originalNode := types.Node{} + if err := json.Unmarshal(originalNodeBytes, &originalNode); err != nil { + t.Fatalf("error unmarshal originalNode: %v", err) + } + + // Add typed metadata + ni := NetworkInfo{} + ni["test"] = ProcInfo{ + CreatedAt: 1622466640, + UpdatedAt: 1622466640, + Revision: 1, + } + + originalNode.Metadata.SetField("NetworkInfo", ni) + + if err := client.Create("node", &originalNode, nil); err != nil { + t.Fatalf("Failed to create originalNode: %s", err.Error()) + } + + patchedNode := types.Node{} + + // Apply a patch that will do nothing + patch := []JSONPatch{ + { + Op: "add", + Path: "/Metadata/foo", + Value: "bar", + }, + } + + // Patch node + _, err = client.Update("node", originalNode.GetID(), patch, &patchedNode) + if err != nil { + t.Fatalf("Failed to apply patch: %s", err.Error()) + } + + getPatchedNode := types.Node{} + err = client.Get("node", originalNode.GetID(), &getPatchedNode) + if err != nil { + t.Fatalf("Failed to get node after patching: %s", err.Error()) + } + + // Compare nodes + neighOriginalRaw, err := originalNode.Metadata.GetField("NetworkInfo") + if err != nil { + panic(err) + } + neighPatchesRaw, err := getPatchedNode.Metadata.GetField("NetworkInfo") + if err != nil { + panic(err) + } + + _, ok := neighOriginalRaw.(NetworkInfo) + if !ok { + panic(fmt.Sprintf("Unable to convert original node Metadata.NetworkInfo to type NetworkInfo: %v (%T)", neighOriginalRaw, neighOriginalRaw)) + } + + _, ok = neighPatchesRaw.(NetworkInfo) + if !ok { + panic(fmt.Sprintf("Unable to convert patches node Metadata.NetworkInfo to type NetworkInfo: %v (%T)", neighPatchesRaw, neighPatchesRaw)) + } +} + // Try to patch a non existing node func TestAPIPatchNodeNonExistingNode(t *testing.T) { client, err := getCrudClient() diff --git a/topology/probes/proccon/metadata.go b/topology/probes/proccon/metadata.go new file mode 100644 index 0000000000..6e519e2390 --- /dev/null +++ b/topology/probes/proccon/metadata.go @@ -0,0 +1,53 @@ +//go:generate go run github.com/skydive-project/skydive/graffiti/gendecoder -package github.com/skydive-project/skydive/topology/probes/proccon +//go:generate go run github.com/mailru/easyjson/easyjson $GOFILE + +/* + * Copyright (C) 2019 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy ofthe License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specificlanguage governing permissions and + * limitations under the License. + * + */ + +package proccon + +import ( + "encoding/json" + "fmt" + + "github.com/skydive-project/skydive/graffiti/getter" +) + +// easyjson:json +// gendecoder +type NetworkInfo map[string]ProcInfo + +// ProcInfo store info associated to each TCP connection or listen endpoint. +// It is used to delete old conections and be able to tell which connections +// are seen several times +// easyjson:json +// gendecoder +type ProcInfo struct { + CreatedAt int64 + UpdatedAt int64 + Revision int64 +} + +// MetadataDecoder implements a json message raw decoder +func MetadataDecoder(raw json.RawMessage) (getter.Getter, error) { + var m NetworkInfo + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("unable to unmarshal proccon metadata %s: %s", string(raw), err) + } + + return &m, nil +} diff --git a/topology/probes/proccon/metadata_easyjson.go b/topology/probes/proccon/metadata_easyjson.go new file mode 100644 index 0000000000..354bb79e70 --- /dev/null +++ b/topology/probes/proccon/metadata_easyjson.go @@ -0,0 +1,163 @@ +// Code generated by easyjson for marshaling/unmarshaling. DO NOT EDIT. + +package proccon + +import ( + json "encoding/json" + easyjson "github.com/mailru/easyjson" + jlexer "github.com/mailru/easyjson/jlexer" + jwriter "github.com/mailru/easyjson/jwriter" +) + +// suppress unused package warning +var ( + _ *json.RawMessage + _ *jlexer.Lexer + _ *jwriter.Writer + _ easyjson.Marshaler +) + +func easyjsonBa0ee0e3DecodeGithubComSkydiveProjectSkydiveTopologyProbesProccon(in *jlexer.Lexer, out *ProcInfo) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "CreatedAt": + out.CreatedAt = int64(in.Int64()) + case "UpdatedAt": + out.UpdatedAt = int64(in.Int64()) + case "Revision": + out.Revision = int64(in.Int64()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjsonBa0ee0e3EncodeGithubComSkydiveProjectSkydiveTopologyProbesProccon(out *jwriter.Writer, in ProcInfo) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"CreatedAt\":" + out.RawString(prefix[1:]) + out.Int64(int64(in.CreatedAt)) + } + { + const prefix string = ",\"UpdatedAt\":" + out.RawString(prefix) + out.Int64(int64(in.UpdatedAt)) + } + { + const prefix string = ",\"Revision\":" + out.RawString(prefix) + out.Int64(int64(in.Revision)) + } + out.RawByte('}') +} + +// MarshalJSON supports json.Marshaler interface +func (v ProcInfo) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + easyjsonBa0ee0e3EncodeGithubComSkydiveProjectSkydiveTopologyProbesProccon(&w, v) + return w.Buffer.BuildBytes(), w.Error +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v ProcInfo) MarshalEasyJSON(w *jwriter.Writer) { + easyjsonBa0ee0e3EncodeGithubComSkydiveProjectSkydiveTopologyProbesProccon(w, v) +} + +// UnmarshalJSON supports json.Unmarshaler interface +func (v *ProcInfo) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + easyjsonBa0ee0e3DecodeGithubComSkydiveProjectSkydiveTopologyProbesProccon(&r, v) + return r.Error() +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *ProcInfo) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjsonBa0ee0e3DecodeGithubComSkydiveProjectSkydiveTopologyProbesProccon(l, v) +} +func easyjsonBa0ee0e3DecodeGithubComSkydiveProjectSkydiveTopologyProbesProccon1(in *jlexer.Lexer, out *NetworkInfo) { + isTopLevel := in.IsStart() + if in.IsNull() { + in.Skip() + } else { + in.Delim('{') + *out = make(NetworkInfo) + for !in.IsDelim('}') { + key := string(in.String()) + in.WantColon() + var v1 ProcInfo + (v1).UnmarshalEasyJSON(in) + (*out)[key] = v1 + in.WantComma() + } + in.Delim('}') + } + if isTopLevel { + in.Consumed() + } +} +func easyjsonBa0ee0e3EncodeGithubComSkydiveProjectSkydiveTopologyProbesProccon1(out *jwriter.Writer, in NetworkInfo) { + if in == nil && (out.Flags&jwriter.NilMapAsEmpty) == 0 { + out.RawString(`null`) + } else { + out.RawByte('{') + v2First := true + for v2Name, v2Value := range in { + if v2First { + v2First = false + } else { + out.RawByte(',') + } + out.String(string(v2Name)) + out.RawByte(':') + (v2Value).MarshalEasyJSON(out) + } + out.RawByte('}') + } +} + +// MarshalJSON supports json.Marshaler interface +func (v NetworkInfo) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + easyjsonBa0ee0e3EncodeGithubComSkydiveProjectSkydiveTopologyProbesProccon1(&w, v) + return w.Buffer.BuildBytes(), w.Error +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v NetworkInfo) MarshalEasyJSON(w *jwriter.Writer) { + easyjsonBa0ee0e3EncodeGithubComSkydiveProjectSkydiveTopologyProbesProccon1(w, v) +} + +// UnmarshalJSON supports json.Unmarshaler interface +func (v *NetworkInfo) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + easyjsonBa0ee0e3DecodeGithubComSkydiveProjectSkydiveTopologyProbesProccon1(&r, v) + return r.Error() +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *NetworkInfo) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjsonBa0ee0e3DecodeGithubComSkydiveProjectSkydiveTopologyProbesProccon1(l, v) +} diff --git a/topology/probes/proccon/metadata_gendecoder.go b/topology/probes/proccon/metadata_gendecoder.go new file mode 100644 index 0000000000..2dd1b557ff --- /dev/null +++ b/topology/probes/proccon/metadata_gendecoder.go @@ -0,0 +1,94 @@ +// Code generated - DO NOT EDIT. + +package proccon + +import ( + "github.com/skydive-project/skydive/graffiti/getter" + "strings" +) + +func (obj *ProcInfo) GetFieldBool(key string) (bool, error) { + return false, getter.ErrFieldNotFound +} + +func (obj *ProcInfo) GetFieldInt64(key string) (int64, error) { + switch key { + case "CreatedAt": + return int64(obj.CreatedAt), nil + case "UpdatedAt": + return int64(obj.UpdatedAt), nil + case "Revision": + return int64(obj.Revision), nil + } + return 0, getter.ErrFieldNotFound +} + +func (obj *ProcInfo) GetFieldString(key string) (string, error) { + return "", getter.ErrFieldNotFound +} + +func (obj *ProcInfo) GetFieldKeys() []string { + return []string{ + "CreatedAt", + "UpdatedAt", + "Revision", + } +} + +func (obj *ProcInfo) MatchBool(key string, predicate getter.BoolPredicate) bool { + return false +} + +func (obj *ProcInfo) MatchInt64(key string, predicate getter.Int64Predicate) bool { + if b, err := obj.GetFieldInt64(key); err == nil { + return predicate(b) + } + return false +} + +func (obj *ProcInfo) MatchString(key string, predicate getter.StringPredicate) bool { + return false +} + +func (obj *ProcInfo) GetField(key string) (interface{}, error) { + if i, err := obj.GetFieldInt64(key); err == nil { + return i, nil + } + return nil, getter.ErrFieldNotFound +} + +func (obj *NetworkInfo) GetFieldBool(key string) (bool, error) { + return false, getter.ErrFieldNotFound +} + +func (obj *NetworkInfo) GetFieldInt64(key string) (int64, error) { + return 0, getter.ErrFieldNotFound +} + +func (obj *NetworkInfo) GetFieldString(key string) (string, error) { + return "", getter.ErrFieldNotFound +} + +func (obj *NetworkInfo) GetFieldKeys() []string { + return []string{} +} + +func (obj *NetworkInfo) MatchBool(key string, predicate getter.BoolPredicate) bool { + return false +} + +func (obj *NetworkInfo) MatchInt64(key string, predicate getter.Int64Predicate) bool { + return false +} + +func (obj *NetworkInfo) MatchString(key string, predicate getter.StringPredicate) bool { + return false +} + +func (obj *NetworkInfo) GetField(key string) (interface{}, error) { + return nil, getter.ErrFieldNotFound +} + +func init() { + strings.Index("", ".") +} diff --git a/topology/probes/proccon/metric.go b/topology/probes/proccon/metric.go new file mode 100644 index 0000000000..4a43ee705b --- /dev/null +++ b/topology/probes/proccon/metric.go @@ -0,0 +1,96 @@ +package proccon + +import ( + "encoding/binary" + "time" +) + +// MessagePackTime implements the official timestamp extension type +// https://github.com/msgpack/msgpack/blob/master/spec.md#timestamp-extension-type +// +// tinylib/msgp has been using their own custom extension type and the official extension +// is not available. (https://github.com/tinylib/msgp/issues/214) +type MessagePackTime struct { + time time.Time +} + +// ExtensionType implements the Extension interface +// The extension is registered in Start() +func (*MessagePackTime) ExtensionType() int8 { + return -1 +} + +// Len implements the Extension interface +// The timestamp extension uses variable length encoding depending the input +// +// 32bits: [1970-01-01 00:00:00 UTC, 2106-02-07 06:28:16 UTC) range. If the nanoseconds part is 0 +// 64bits: [1970-01-01 00:00:00.000000000 UTC, 2514-05-30 01:53:04.000000000 UTC) range. +// 96bits: [-584554047284-02-23 16:59:44 UTC, 584554051223-11-09 07:00:16.000000000 UTC) range. +func (t *MessagePackTime) Len() int { + sec := t.time.Unix() + nsec := t.time.Nanosecond() + + if sec < 0 || sec >= (1<<34) { // 96 bits encoding + return 12 + } + if sec >= (1<<32) || nsec != 0 { + return 8 + } + return 4 +} + +// MarshalBinaryTo implements the Extension interface +func (t *MessagePackTime) MarshalBinaryTo(buf []byte) error { + len := t.Len() + + if len == 4 { + sec := t.time.Unix() + binary.BigEndian.PutUint32(buf, uint32(sec)) + } else if len == 8 { + sec := t.time.Unix() + nsec := t.time.Nanosecond() + + data := uint64(nsec)<<34 | (uint64(sec) & 0x03_ffff_ffff) + binary.BigEndian.PutUint64(buf, data) + } else if len == 12 { + sec := t.time.Unix() + nsec := t.time.Nanosecond() + + binary.BigEndian.PutUint32(buf, uint32(nsec)) + binary.BigEndian.PutUint64(buf[4:], uint64(sec)) + } + + return nil +} + +// UnmarshalBinary implements the Extension interface +func (t *MessagePackTime) UnmarshalBinary(buf []byte) error { + len := len(buf) + + if len == 4 { + sec := binary.BigEndian.Uint32(buf) + t.time = time.Unix(int64(sec), 0) + } else if len == 8 { + data := binary.BigEndian.Uint64(buf) + + nsec := (data & 0xfffffffc_00000000) >> 34 + sec := (data & 0x00000003_ffffffff) + + t.time = time.Unix(int64(sec), int64(nsec)) + } else if len == 12 { + nsec := binary.BigEndian.Uint32(buf) + sec := binary.BigEndian.Uint64(buf[4:]) + + t.time = time.Unix(int64(sec), int64(nsec)) + } + + return nil +} + +// Metric represents each of the processes found by the external agent +type Metric struct { + Name string `msg:"name"` + Time MessagePackTime `msg:"time,extension"` + Tags map[string]string `msg:"tags"` + Fields map[string]string `msg:"fields"` +} diff --git a/topology/probes/proccon/proccon.go b/topology/probes/proccon/proccon.go new file mode 100644 index 0000000000..850c70d821 --- /dev/null +++ b/topology/probes/proccon/proccon.go @@ -0,0 +1,714 @@ +/* + * Copyright (C) 2016 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy ofthe License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specificlanguage governing permissions and + * limitations under the License. + * + */ + +// Package proccon provides an additional API to add metrics to Skydive. +// It accept HTTP-JSON POST like this: +// { +// "metrics": [ +// { +// "fields": { +// "conn": "1.2.3.4:80,9.9.9.9:53", +// "listen": "192.168.1.36:8000,192.168.1.22:8000" +// }, +// "name": "procstat_test", +// "tags": { +// "cmdline": "nc -kl 8000", +// "host": "fooBar", +// "process_name": "nc", +// "conn_prefix": "" // optional value +// }, +// "timestamp": 1603890543 +// } +// ] +// } +// +// This will create, if does not already exists, a node Type=Server and other +// node Type=Software linked to the previous one. +// It will find if there is already a software who matches the cmdline. +// If not, it will use one with name "others". +// +// In the software node it will add the network info in the Metadata like: +// "Metadata": { +// "TCPConn": { +// "1.2.3.4:80": { +// "CreatedAt": 161114359940 +// "UpdatedAt": 161114367803 +// "Revision": 2 +// }, +// "9.9.9.9:53": { +// "CreatedAt": 161114359940 +// "UpdatedAt": 161114367803 +// "Revision": 2 +// } +// }, +// "TCPListen": { +// "192.168.1.36:8000": { +// "CreatedAt": 161114359940 +// "UpdatedAt": 161114367803 +// "Revision": 2 +// }, +// "192.168.1.22:8000": { +// "CreatedAt": 161114359940 +// "UpdatedAt": 161114367803 +// "Revision": 2 +// } +// }, +// "Name": "others", +// "Type": "Software" +// } +// +// This plugin it is thought to work with procpeering, which will create edges +// between connected nodes (one node having a TCPConn matching a TCPListen of +// another node). +package proccon + +import ( + "errors" + "fmt" + "io/ioutil" + "net/http" + "strings" + "time" + + uuid "github.com/nu7hatch/gouuid" + "github.com/skydive-project/skydive/config" + "github.com/skydive-project/skydive/graffiti/filters" + "github.com/skydive-project/skydive/graffiti/getter" + "github.com/skydive-project/skydive/graffiti/graph" + "github.com/skydive-project/skydive/graffiti/logging" + "github.com/tinylib/msgp/msgp" +) + +const ( + // ProcconOriginName defines the prefix set in nodes/edges Origin field + ProcconOriginName = "proccon." + // MetadataTypeKey key name to store the type of node in node's metadata + MetadataTypeKey = "Type" + // MetadataNameKey key name to store the name of the server in node's metadata + MetadataNameKey = "Name" + // MetadataTCPConnKey key name to store TCP connections in node's metadata + MetadataTCPConnKey = "TCPConn" + // MetadataListenEndpointKey key name to store TCP listening endpoints in node's metadata + MetadataListenEndpointKey = "TCPListen" + // MetadataRelationTypeKey key name to store the kind of relation in edge's metadata + MetadataRelationTypeKey = "RelationType" + // RelationTypeHasSoftware value of the key RelationType to mark an installed software in a server + RelationTypeHasSoftware = "has_software" + // OthersSoftwareNode name of the node type server where connection info is stored when there is not an specific sofware node + OthersSoftwareNode = "others" + // MetadataTypeServer value of key Type for nodes representing a server + MetadataTypeServer = "Server" + // MetadataTypeSoftware value of key Type for nodes representing a software + MetadataTypeSoftware = "Software" + // MetadataCmdlineKey key name to store the cmdline of known Software in nodes of type software + MetadataCmdlineKey = "Cmdline" + // MetricFieldConn is the field key where connection info is received + MetricFieldConn = "conn" + // MetricFieldListen the field key where listen info is received + MetricFieldListen = "listen" + // MetricTagConnPrefix is an optional tag value to prefix network information with, to be able to + // distinguish between same private IPs in different network partitions + MetricTagConnPrefix = "conn_prefix" +) + +// Probe describes this probe +type Probe struct { + graph *graph.Graph + // GCdone channel to stop garbageCollector ticker + GCdone chan bool + // nodeRevisionForceFlush defines after how many node updates it is synced to the backend. + // A small number will produce a lot of modifications in the backend because of network info updates. + // A big number will leave behind the backend, loosing info in case of a restart of skydive + nodeRevisionForceFlush int64 +} + +// processMetrics get an array of metrics, process each one and return the map with conn info +// which doesn't have a specific Software server. +// Return the number of metrics not proccessed or which returned errors +func (p *Probe) processMetrics(metrics []Metric) int { + numberOfErrors := 0 + + // Group metrics by host + hostMetrics := map[string][]Metric{} + for _, m := range metrics { + host, ok := m.Tags["host"] + if !ok { + logging.GetLogger().Warningf("Metric without host tag: %+v. Ignored", m) + numberOfErrors++ + continue + } + hostMetrics[host] = append(hostMetrics[host], m) + } + + // For each host, process each metrics + for host, metrics := range hostMetrics { + numErr, err := p.processHost(host, metrics) + if err != nil { + logging.GetLogger().Errorf("Processing metrics for host %s: %v", host, err) + } + numberOfErrors += numErr + } + return numberOfErrors +} + +// processHost handle a group of metrics belonging to the same server. +// Get or create a Server node from the host string and add/update metrics to that +// server node. +func (p *Probe) processHost(host string, metrics []Metric) (int, error) { + logging.GetLogger().Debugf("processHost, host=%s, metrics=%+v", host, metrics) + + p.graph.Lock() // Avoid race condition createing twice the same server node + defer p.graph.Unlock() + + // Get server node + hostNode := p.graph.GetNode(getIdentifier(graph.Metadata{ + MetadataTypeKey: MetadataTypeServer, + MetadataNameKey: host, + })) + + var err error + + // Server node does not exists. Create it + if hostNode == nil { + logging.GetLogger().Debugf("Node not found with Metadata.Name '%s', creating it", host) + + logging.GetLogger().Debugf("newNode(Server, %v)", host) + hostNode, err = p.newNode(host, graph.Metadata{ + MetadataNameKey: host, + MetadataTypeKey: MetadataTypeServer, + }) + if err != nil { + return len(metrics), fmt.Errorf("creating %s server node: %v", host, err) + } + } + + numberOfErrors := p.processHostMetrics(hostNode, metrics) + + return numberOfErrors, nil +} + +// processHostMetrics given a Server node and a list of metrics of that node, for each metric +// try to find a Software node with the same cmdline. +// If found, add the network info of the metric to that node and remove it from "others" node. +// If not found, append that network info to "others". +// Return the number of errors seen +func (p *Probe) processHostMetrics(serverNode *graph.Node, metrics []Metric) int { + addToOthers := []Metric{} + removeFromOthers := []Metric{} + numberOfErrors := 0 + + for _, metric := range metrics { + // Get child Software nodes with matching cmdline + childNodes := p.graph.LookupChildren( + serverNode, + graph.Metadata{MetadataTypeKey: MetadataTypeSoftware, MetadataCmdlineKey: metric.Tags["cmdline"]}, + graph.Metadata{MetadataRelationTypeKey: RelationTypeHasSoftware}, + ) + + if len(childNodes) == 0 { + logging.GetLogger().Debugf("Software node not found for Server node '%v' and cmdline '%s', storing in others", nodeName(serverNode), metric.Tags["cmdline"]) + // Accumulate changes to "others" to make only one change to the node + addToOthers = append(addToOthers, metric) + continue + } else if len(childNodes) > 1 { + // This should not happen + logging.GetLogger().Errorf("Found more than one Software node for Server node '%v' and cmdline '%s': %+v. Ignoring", nodeName(serverNode), metric.Tags["cmdline"], childNodes) + numberOfErrors++ + continue + } + + // Software node already exists + swNode := childNodes[0] + + // Append this metric to the list to be deleted from others + removeFromOthers = append(removeFromOthers, metric) + + // Attach that network information to the software node + err := p.addNetworkInfo(swNode, []Metric{metric}) + if err != nil { + logging.GetLogger().Errorf("Not able to add network info to Software '%s' (host %+v)", nodeName(swNode), serverNode) + numberOfErrors++ + } + } + + err := p.handleOthers(serverNode, addToOthers, removeFromOthers) + if err != nil { + logging.GetLogger().Errorf("Not able to handle 'others' Software node: %v", err) + numberOfErrors++ + } + + return numberOfErrors +} + +// ServeHTTP receive HTTP POST requests from Telegraf nodes with processes data +func (p *Probe) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var metrics []Metric + + defer r.Body.Close() + + // Read the Telegraf packet + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Get original client from X-Forwarded-For for error logging purposes + source := "" + if h := r.Header.Get("X-Forwarded-For"); h != "" { + source = strings.Split(h, ",")[0] + } else { + source = strings.Split(r.RemoteAddr, ":")[0] + } + + for { + var metric Metric + leftovervalues, err := metric.UnmarshalMsg(body) + if err != nil { + http.Error(w, err.Error(), http.StatusNotAcceptable) + logging.GetLogger().Warningf("invalid metric: %v (client: %+v)", err, source) + return + } + metrics = append(metrics, metric) + body = leftovervalues + if len(body) == 0 { + break + } + } + + numberOfErrors := p.processMetrics(metrics) + + w.Write([]byte(fmt.Sprintf("total:%d error:%d", len(metrics), numberOfErrors))) +} + +// handleOthers given a Server and two list of metrics, one with the metrics to be +// added to the linked 'others' Software node, and the other list with the metrics to +// be removed from that 'others' node. +// Create the 'others' node if needed. +func (p *Probe) handleOthers( + hostNode *graph.Node, + metricsToBeAdded []Metric, + metricsToBeDeleted []Metric, +) error { + var otherNode *graph.Node + var err error + + logging.GetLogger().Debugf("Handling %d adds and %d removes to other host of Server %s", + len(metricsToBeAdded), + len(metricsToBeDeleted), + nodeName(hostNode), + ) + + othersNodes := p.graph.LookupChildren( + hostNode, + graph.Metadata{MetadataTypeKey: MetadataTypeSoftware, MetadataNameKey: OthersSoftwareNode}, + graph.Metadata{MetadataRelationTypeKey: RelationTypeHasSoftware}, + ) + + if len(othersNodes) == 0 { + if len(metricsToBeAdded) == 0 { + // do nothing if there is no 'others' node and nothing to add + return nil + } + + // create 'others' node + logging.GetLogger().Debugf("newNode(Software, others)") + otherNode, err = p.newNode(hostNode.Host, graph.Metadata{ + MetadataNameKey: OthersSoftwareNode, + MetadataTypeKey: MetadataTypeSoftware, + }) + if err != nil { + return fmt.Errorf("creating 'others' Software node for Server node '%+v': %v", hostNode, err) + } + + // Create edge to link to the host + logging.GetLogger().Debugf("newEdge(%v, %v, has_software)", hostNode, otherNode) + _, err = p.newEdge(hostNode.Host, hostNode, otherNode, graph.Metadata{ + MetadataRelationTypeKey: RelationTypeHasSoftware, + }) + if err != nil { + return fmt.Errorf("linking 'others' Software node to host Server node '%+v': %v", hostNode, err) + } + } else if len(othersNodes) > 1 { + // This should not happen + return fmt.Errorf("Found more than one 'others' Software node for Server node '%+v'. Not removing metrics", hostNode) + } else { + otherNode = othersNodes[0] + } + + err = p.removeFromOthers(otherNode, metricsToBeDeleted) + if err != nil { + return fmt.Errorf("removing connections from the other node of Server %s: %v", nodeName(hostNode), err) + } + + // If removeFromOthers fail, this function will not be executed + err = p.addNetworkInfo(otherNode, metricsToBeAdded) + if err != nil { + return fmt.Errorf("adding connections to the other node of Server %s: %v", nodeName(hostNode), err) + } + + return nil +} + +// removeFromOthers remove connections/listeners from the "others" software node +func (p *Probe) removeFromOthers(otherNode *graph.Node, metricsToBeDeleted []Metric) error { + // Remove from connections/listeners lists in otherNode the values found in "metric" + removeKeysFromList := func(field interface{}, metrics []string) (ret bool) { + infoPtr, ok := field.(*NetworkInfo) + if !ok { + logging.GetLogger().Warningf("Unable to convert %v (%T) to *NetworkInfo", field, field) + return false + } + info := *infoPtr + + for _, v := range metrics { + if _, ok := info[v]; ok { + delete(info, v) + ret = true + } + } + return ret + } + + for _, metric := range metricsToBeDeleted { + removeTCPConn := func(field interface{}) (ret bool) { + metricsTCPConn := strings.Split(metric.Fields[MetricFieldConn], ",") + return removeKeysFromList(field, metricsTCPConn) + } + + err := p.graph.UpdateMetadata(otherNode, MetadataTCPConnKey, removeTCPConn) + if err != nil { + return fmt.Errorf("unable to delete old TCP connections: %v", err) + } + + removeListenEndpoints := func(field interface{}) (ret bool) { + metricsListenersEndpoints := strings.Split(metric.Fields[MetricFieldListen], ",") + return removeKeysFromList(field, metricsListenersEndpoints) + } + + err = p.graph.UpdateMetadata(otherNode, MetadataListenEndpointKey, removeListenEndpoints) + if err != nil { + return fmt.Errorf("unable to delete old listen endpoints: %v", err) + } + } + + return nil +} + +// appendProcInfoData given a list of connetions/endpoints, for each element append it, with +// the appropiate struct, to the netInfo variable. +// Revision is initialized to 1 +func appendProcInfoData(conn []string, metricTimestamp int64, netInfo NetworkInfo) { + for _, c := range conn { + netInfo[c] = ProcInfo{ + CreatedAt: metricTimestamp, + UpdatedAt: metricTimestamp, + Revision: 1, + } + } +} + +// updateNetworkMetadata adds new network info and update current stored one. +// Only return true if there is new data. +// Modifying UpdatedAt and Revision fields are not considered modifications (save flushes to the backed) +// To don't leave the backend too behind, each N node Revisions consider it a modification. +func (p *Probe) updateNetworkMetadata(field interface{}, newData NetworkInfo, nodeRevision int64) (ret bool) { + currentDataPtr, ok := field.(*NetworkInfo) + if !ok { + logging.GetLogger().Warningf("Unable to convert %v (%T) to *NetworkInfo", field, field) + return false + } + currentData := *currentDataPtr + + for k, v := range newData { + netData, ok := currentData[k] + if ok { + // If network info exists, update Revision and UpdatedAt + netData.UpdatedAt = v.UpdatedAt + netData.Revision++ + currentData[k] = netData + } else { + // If the network info did not exists, assign the new values + currentData[k] = v + ret = true + } + } + // If node p.nodeRevisionForceFlush is 100, when nodeRevision is 100, 200, etc, the function + // will return true, forcing an update in the backend. + // Avoid panic if p.nodeRevisionForceFlush is 0 + return ret || (p.nodeRevisionForceFlush != 0 && (nodeRevision%p.nodeRevisionForceFlush == 0)) +} + +// addNetworkInfo append connection and listen endpoints to the metadata of the server +func (p *Probe) addNetworkInfo(node *graph.Node, metrics []Metric) error { + logging.GetLogger().Debugf("addNetworkInfo, node:%s, metrics:%+v", nodeName(node), metrics) + + // Accumulate conn/endpoint of all metrics in this vars + tcpConnStruct := NetworkInfo{} + listenEndpointsStruct := NetworkInfo{} + + for _, metric := range metrics { + // Generate an slice from the comma separated list in the metric received + // Avoid having an array with an empty element if the string received is the empty string + tcpConn := []string{} + if metric.Fields[MetricFieldConn] != "" { + tcpConn = strings.Split(metric.Fields[MetricFieldConn], ",") + // Add ConnPrefix if defined + if metric.Tags[MetricTagConnPrefix] != "" { + for i := 0; i < len(tcpConn); i++ { + tcpConn[i] = metric.Tags[MetricTagConnPrefix] + tcpConn[i] + } + } + } + + // Same for the listen endpoints + listenEndpoints := []string{} + if metric.Fields[MetricFieldListen] != "" { + listenEndpoints = strings.Split(metric.Fields[MetricFieldListen], ",") + // Add ConnPrefix if defined + if metric.Tags[MetricTagConnPrefix] != "" { + for i := 0; i < len(listenEndpoints); i++ { + listenEndpoints[i] = metric.Tags[MetricTagConnPrefix] + listenEndpoints[i] + } + } + } + + // Convert metric timestamp into ms in int64 + timestampMs := metric.Time.time.UnixNano() / int64(time.Millisecond) + + // Network info converted to the data structure stored in the node metadata + appendProcInfoData(tcpConn, timestampMs, tcpConnStruct) + appendProcInfoData(listenEndpoints, timestampMs, listenEndpointsStruct) + } + + // Updates nodes metadata + errTCPConn := p.graph.UpdateMetadata(node, MetadataTCPConnKey, func(field interface{}) bool { + return p.updateNetworkMetadata(field, tcpConnStruct, node.Revision) + }) + + errListenEndpoint := p.graph.UpdateMetadata(node, MetadataListenEndpointKey, func(field interface{}) bool { + return p.updateNetworkMetadata(field, listenEndpointsStruct, node.Revision) + }) + + // UpdateMetadata will fail if the metadata key does not exists + // In that case, use AddMetadata to create that keys + if errTCPConn != nil || errListenEndpoint != nil { + tr := p.graph.StartMetadataTransaction(node) + if errTCPConn != nil { + tr.AddMetadata(MetadataTCPConnKey, &tcpConnStruct) + } + if errListenEndpoint != nil { + tr.AddMetadata(MetadataListenEndpointKey, &listenEndpointsStruct) + } + if err := tr.Commit(); err != nil { + return fmt.Errorf("unable to set metadata in node %s: %v", nodeName(node), err) + } + } + + return nil +} + +// removeOldNetworkInformation delete TCP connections and listen endpoints which update time +// is less than "thresholdTime" +func (p *Probe) removeOldNetworkInformation(node *graph.Node, thresholdTime time.Time) error { + tt := graph.Time(thresholdTime) + + removeOld := func(field interface{}) (ret bool) { + info, ok := field.(*NetworkInfo) + if !ok { + logging.GetLogger().Warningf("Unable to convert %v (%T) to *NetworkInfo", field, field) + return false + } + + for k, v := range *info { + if v.UpdatedAt < tt.UnixMilli() { + delete(*info, k) + ret = true + } + } + return ret + } + + err := p.graph.UpdateMetadata(node, MetadataTCPConnKey, removeOld) + if err != nil { + if errors.Is(err, getter.ErrFieldNotFound) { + logging.GetLogger().Debugf("Unable to delete old network info because Metadata.%v does not exists", MetadataTCPConnKey) + return nil + } + return fmt.Errorf("unable to delete old TCP connections: %v", err) + } + err = p.graph.UpdateMetadata(node, MetadataListenEndpointKey, removeOld) + if err != nil { + if errors.Is(err, getter.ErrFieldNotFound) { + logging.GetLogger().Debugf("Unable to delete old network info because Metadata.%v does not exists", MetadataListenEndpointKey) + return nil + } + return fmt.Errorf("unable to delete old listen endpoints: %v", err) + } + return nil +} + +// cleanSoftwareNodes delete old data from type Software nodes +// "now" is the current time (parametrized to be able to test this function) +// "expiredConnection" is used to determine if connections/listeners are too old +func (p *Probe) cleanSoftwareNodes(expiredConnectionThreshold time.Time) { + // Lock the graph while cleaning. + // Once we get the nodes, we are using that info to update its metadata. + // We have to avoid changes in those hosts while running this cleaner. + p.graph.Lock() + defer p.graph.Unlock() + + softwareNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeSoftware)) + softwareNodes := p.graph.GetNodes(softwareNodeFilter) + + logging.GetLogger().Debugf("GarbageCollector, delete network info older than %v. Processing %v nodes", expiredConnectionThreshold, len(softwareNodes)) + + for _, n := range softwareNodes { + err := p.removeOldNetworkInformation(n, expiredConnectionThreshold) + if err != nil { + logging.GetLogger().Warningf("Deleting old network information on node %+v: %v", n, err) + } + } +} + +// garbageCollector executes periodically functions to clean old data +// interval is how often will be executed the garbageCollector +// expiredConnection is how old have to be a connection to be deleted +func (p *Probe) garbageCollector(interval, expiredConnection time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-p.GCdone: + logging.GetLogger().Debug("Closing garbageCollector") + return + case <-ticker.C: + logging.GetLogger().Debug("Executing garbageCollector") + p.cleanSoftwareNodes(time.Now().Add(-expiredConnection)) + } + } +} + +// nodeName return the Metadata.Name value or ID +// Used in loggers and errors to show a representation of the node +func nodeName(n *graph.Node) string { + name, err := n.Metadata.GetFieldString(MetadataNameKey) + if err == nil { + return name + } + + return string(n.ID) +} + +// newEdge creates and inserts a new edge in the graph, using a random ID, setting the +// Origin field to "proccon"+g.Origin and the Host field to the param "host" +func (p *Probe) newEdge(host string, n *graph.Node, c *graph.Node, m graph.Metadata) (*graph.Edge, error) { + u, _ := uuid.NewV5(uuid.NamespaceOID, []byte(n.ID+c.ID)) + i := graph.Identifier(u.String()) + + e := graph.CreateEdge(i, n, c, m, graph.TimeUTC(), host, ProcconOriginName+p.graph.GetOrigin()) + + if err := p.graph.AddEdge(e); err != nil { + return nil, err + } + return e, nil +} + +// newNode creates and inserts a new node in the graph, using known ID, setting the +// Origin field to "proccon"+g.Origin and the Host field to the param "host" +func (p *Probe) newNode(host string, m graph.Metadata) (*graph.Node, error) { + i := getIdentifier(m) + n := graph.CreateNode(i, m, graph.TimeUTC(), host, ProcconOriginName+p.graph.GetOrigin()) + + if err := p.graph.AddNode(n); err != nil { + return nil, err + } + return n, nil +} + +// getIdentifier generate a node identifier. Server nodes will get always the same identifier, generated from the values of the metadata. The rets of nodes will get a random one. +// Server node identifier is generated with: {Metadata.Type}__{Metadata.Name} +func getIdentifier(metadata graph.Metadata) graph.Identifier { + name, err := metadata.GetField(MetadataNameKey) + if err != nil { + panic("node metadata should always have 'Name'") + } + mType, err := metadata.GetField(MetadataTypeKey) + if err != nil { + panic("node metadata should always have 'Type'") + } + + // Nodes with fixed identifiers + switch mType { + case MetadataTypeServer: + return graph.Identifier(mType.(string) + "__" + name.(string)) + } + + return graph.GenID() +} + +// Start initilizates the proccon probe, starting a web server to receive data and the garbage collector to delete old info +func (p *Probe) Start() error { + // Register msgp MessagePackTime extension + msgp.RegisterExtension(-1, func() msgp.Extension { return new(MessagePackTime) }) + + listenEndpoint := config.GetString("analyzer.topology.proccon.listen") + http.Handle("/", p) + go http.ListenAndServe(listenEndpoint, nil) + logging.GetLogger().Infof("Listening for new network metrics on %v", listenEndpoint) + + p.GCdone = make(chan bool) + + intervalConfig := config.GetString("analyzer.topology.proccon.garbage_collector.interval") + interval, err := time.ParseDuration(intervalConfig) + if err != nil { + logging.GetLogger().Fatalf("Invalid analyzer.topology.proccon.garbage_collector.interval value: %v", err) + } + + deleteDurationConfig := config.GetString("analyzer.topology.proccon.garbage_collector.delete_duration") + deleteDuration, err := time.ParseDuration(deleteDurationConfig) + if err != nil { + logging.GetLogger().Fatalf("Invalid analyzer.topology.proccon.garbage_collector.delete_duration value: %v", err) + } + + p.nodeRevisionForceFlush = int64(config.GetInt("analyzer.topology.proccon.revision_flush")) + + go p.garbageCollector(interval, deleteDuration) + + return nil +} + +// Stop garbageCollector +func (p *Probe) Stop() { + p.GCdone <- true +} + +// NewProbe initialize the probe with the parameters from the config file +func NewProbe(g *graph.Graph) (*Probe, error) { + probe := &Probe{ + graph: g, + } + + return probe, nil +} + +// Register called at initialization to register metadata decoders +func Register() { + graph.NodeMetadataDecoders[MetadataTCPConnKey] = MetadataDecoder + graph.NodeMetadataDecoders[MetadataListenEndpointKey] = MetadataDecoder +} diff --git a/topology/probes/proccon/proccon_gen.go b/topology/probes/proccon/proccon_gen.go new file mode 100644 index 0000000000..ded84ce18a --- /dev/null +++ b/topology/probes/proccon/proccon_gen.go @@ -0,0 +1,413 @@ +package proccon + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +import ( + "fmt" + + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *MessagePackTime) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z MessagePackTime) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 0 + err = en.Append(0x80) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z MessagePackTime) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 0 + o = append(o, 0x80) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *MessagePackTime) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z MessagePackTime) Msgsize() (s int) { + s = 1 + return +} + +// DecodeMsg implements msgp.Decodable +func (z *Metric) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "name": + z.Name, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "time": + err = dc.ReadExtension(&z.Time) + if err != nil { + err = msgp.WrapError(err, "Time") + return + } + case "tags": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Tags") + return + } + if z.Tags == nil { + z.Tags = make(map[string]string, zb0002) + } else if len(z.Tags) > 0 { + for key := range z.Tags { + delete(z.Tags, key) + } + } + for zb0002 > 0 { + zb0002-- + var za0001 string + var za0002 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Tags") + return + } + za0002, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Tags", za0001) + return + } + z.Tags[za0001] = za0002 + } + case "fields": + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Fields") + return + } + if z.Fields == nil { + z.Fields = make(map[string]string, zb0003) + } else if len(z.Fields) > 0 { + for key := range z.Fields { + delete(z.Fields, key) + } + } + for zb0003 > 0 { + zb0003-- + var za0003 string + var za0004 string + za0003, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Fields") + return + } + za0004, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Fields", za0003) + return + } + z.Fields[za0003] = za0004 + } + default: + err = fmt.Errorf("unknown field: %v", msgp.UnsafeString(field)) + return + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *Metric) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "name" + err = en.Append(0x84, 0xa4, 0x6e, 0x61, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Name) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + // write "time" + err = en.Append(0xa4, 0x74, 0x69, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteExtension(&z.Time) + if err != nil { + err = msgp.WrapError(err, "Time") + return + } + // write "tags" + err = en.Append(0xa4, 0x74, 0x61, 0x67, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Tags))) + if err != nil { + err = msgp.WrapError(err, "Tags") + return + } + for za0001, za0002 := range z.Tags { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Tags") + return + } + err = en.WriteString(za0002) + if err != nil { + err = msgp.WrapError(err, "Tags", za0001) + return + } + } + // write "fields" + err = en.Append(0xa6, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Fields))) + if err != nil { + err = msgp.WrapError(err, "Fields") + return + } + for za0003, za0004 := range z.Fields { + err = en.WriteString(za0003) + if err != nil { + err = msgp.WrapError(err, "Fields") + return + } + err = en.WriteIntf(za0004) + if err != nil { + err = msgp.WrapError(err, "Fields", za0003) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *Metric) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "name" + o = append(o, 0x84, 0xa4, 0x6e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.Name) + // string "time" + o = append(o, 0xa4, 0x74, 0x69, 0x6d, 0x65) + o, err = msgp.AppendExtension(o, &z.Time) + if err != nil { + err = msgp.WrapError(err, "Time") + return + } + // string "tags" + o = append(o, 0xa4, 0x74, 0x61, 0x67, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.Tags))) + for za0001, za0002 := range z.Tags { + o = msgp.AppendString(o, za0001) + o = msgp.AppendString(o, za0002) + } + // string "fields" + o = append(o, 0xa6, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.Fields))) + for za0003, za0004 := range z.Fields { + o = msgp.AppendString(o, za0003) + o, err = msgp.AppendIntf(o, za0004) + if err != nil { + err = msgp.WrapError(err, "Fields", za0003) + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Metric) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "name": + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "time": + bts, err = msgp.ReadExtensionBytes(bts, &z.Time) + if err != nil { + err = msgp.WrapError(err, "Time") + return + } + case "tags": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tags") + return + } + if z.Tags == nil { + z.Tags = make(map[string]string, zb0002) + } else if len(z.Tags) > 0 { + for key := range z.Tags { + delete(z.Tags, key) + } + } + for zb0002 > 0 { + var za0001 string + var za0002 string + zb0002-- + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tags") + return + } + za0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tags", za0001) + return + } + z.Tags[za0001] = za0002 + } + case "fields": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Fields") + return + } + if z.Fields == nil { + z.Fields = make(map[string]string, zb0003) + } else if len(z.Fields) > 0 { + for key := range z.Fields { + delete(z.Fields, key) + } + } + for zb0003 > 0 { + var za0003 string + var za0004 string + zb0003-- + za0003, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Fields") + return + } + za0004, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Fields", za0003) + return + } + z.Fields[za0003] = za0004 + } + default: + err = fmt.Errorf("unknown field: %v", msgp.UnsafeString(field)) + return + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *Metric) Msgsize() (s int) { + s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 5 + msgp.ExtensionPrefixSize + z.Time.Len() + 5 + msgp.MapHeaderSize + if z.Tags != nil { + for za0001, za0002 := range z.Tags { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + msgp.StringPrefixSize + len(za0002) + } + } + s += 7 + msgp.MapHeaderSize + if z.Fields != nil { + for za0003, za0004 := range z.Fields { + _ = za0004 + s += msgp.StringPrefixSize + len(za0003) + msgp.GuessSize(za0004) + } + } + return +} diff --git a/topology/probes/proccon/proccon_test.go b/topology/probes/proccon/proccon_test.go new file mode 100644 index 0000000000..accd279164 --- /dev/null +++ b/topology/probes/proccon/proccon_test.go @@ -0,0 +1,1654 @@ +package proccon + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/skydive-project/skydive/graffiti/filters" + "github.com/skydive-project/skydive/graffiti/graph" + "github.com/skydive-project/skydive/graffiti/logging" + "github.com/stretchr/testify/assert" +) + +func TestMain(m *testing.M) { + // Remove comment to change logging level to debug + // logging.InitLogging("id", true, []*logging.LoggerConfig{logging.NewLoggerConfig(logging.NewStdioBackend(os.Stdout), "5", "UTF-8")}) + os.Exit(m.Run()) +} + +func newGraph(t testing.TB) *graph.Graph { + b, err := graph.NewMemoryBackend() + if err != nil { + t.Error(err.Error()) + } + + return graph.NewGraph("testhost", b, "analyzer.testhost") +} + +// sendAgentData simulates the HTTP post of an external agent +func sendAgentData(t *testing.T, p Probe, data []byte, expectedStatus int) { + req, err := http.NewRequest("POST", "/", bytes.NewBuffer(data)) + if err != nil { + t.Fatal(err) + } + + rr := httptest.NewRecorder() + + p.ServeHTTP(rr, req) + + // Check HTTP 200 and empty body + if status := rr.Code; status != expectedStatus { + t.Errorf("handler returned wrong status code: got %v want %v. Error: %v", + status, expectedStatus, rr.Body.String()) + } +} + +// getTCPConn return the list of connections stored in this node Metadata +// Return an empty slice if metadata key does not exists or is empty +func getTCPConn(node *graph.Node) (conn []string, err error) { + data, err := node.Metadata.GetField(MetadataTCPConnKey) + if err != nil { + logging.GetLogger().Warningf("Node %+v does not have metadata key %s", node, MetadataTCPConnKey) + return conn, nil + } + + for c := range *data.(*NetworkInfo) { + conn = append(conn, c) + } + + return conn, err +} + +// getListenEndpoints return the list of listen endpoints stored in this node Metadata +// Return an empty slice if metadata key does not exists or is empty +func getListenEndpoints(node *graph.Node) (listen []string, err error) { + data, err := node.Metadata.GetField(MetadataListenEndpointKey) + if err != nil { + logging.GetLogger().Warningf("Node %+v does not have metadata key %s", node, MetadataListenEndpointKey) + return listen, nil + } + + for l := range *data.(*NetworkInfo) { + listen = append(listen, l) + } + + return listen, err +} + +// TestCreateServerNodeSoftwareOthers receives a metric with an unkown host, it should create a new Server node in the Graph with that name, +// a software node called "others", a edge between them and append the connection info to that software node +func TestCreateServerNodeSoftwareOthers(t *testing.T) { + // GIVEN + p := Probe{} + p.graph = newGraph(t) + + metricServerName := "hostFoo" + metricSoftwareCmdline := "nc -kl 8000" + metricConnections := "1.2.3.4:80,9.9.9.9:53" + metricListen := "192.168.1.36:8000,192.168.1.22:8000" + + metric := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(1603890543, 0), + }, + Tags: map[string]string{ + "cmdline": metricSoftwareCmdline, + "host": metricServerName, + "process_name": "nc", + }, + Fields: map[string]string{ + "conn": metricConnections, + "listen": metricListen, + }, + } + + agentData := []byte{} + agentData, err := metric.MarshalMsg(agentData) + if err != nil { + panic(err) + } + + // WHEN + sendAgentData(t, p, agentData, http.StatusOK) + + // THEN + // Check if Server node has been created correctly + serverNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeServer)) + servers := p.graph.GetNodes(serverNodeFilter) + if len(servers) == 0 { + t.Fatal("Server node not created") + } else if len(servers) > 1 { + t.Error("Too many Server nodes created") + } + + server := servers[0] + serverName, err := server.Metadata.GetFieldString(MetadataNameKey) + if err != nil { + t.Errorf("Server node created but without Metadata.Name") + } + + if serverName != metricServerName { + t.Errorf("Server node created, but with wrong name, %s != %s (expected)", serverName, metricServerName) + } + + // Check if Software node 'others' exists and have the right name and connection info + softwareNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeSoftware)) + softwares := p.graph.GetNodes(softwareNodeFilter) + if len(softwares) == 0 { + t.Fatal("Software node not created") + } else if len(softwares) > 1 { + t.Error("Too many Software nodes created") + } + + software := softwares[0] + softwareName, err := software.Metadata.GetFieldString(MetadataNameKey) + if err != nil { + t.Errorf("Software node created but without Metadata.Name") + } + + if softwareName != OthersSoftwareNode { + t.Errorf("Software node created, but with wrong name: %v", softwareName) + } + + // The software node should have two revisions: + // - creating the node + // - adding TCPConn and TCPListen to that empty node + assert.Equal(t, int64(2), software.Revision) + + softwareTCPConn, err := getTCPConn(software) + if err != nil { + t.Errorf("Not able to get TCP connection info") + } + assert.ElementsMatch(t, softwareTCPConn, strings.Split(metricConnections, ",")) + + softwareListenEndpoints, err := getListenEndpoints(software) + if err != nil { + t.Errorf("Not able to get TCP listen endpoints") + } + assert.ElementsMatch(t, softwareListenEndpoints, strings.Split(metricListen, ",")) + + // Check the edge between these two nodes + hasSoftwareEdgesFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataRelationTypeKey, RelationTypeHasSoftware)) + edges := p.graph.GetEdges(hasSoftwareEdgesFilter) + if len(edges) == 0 { + t.Fatal("has_software edge not created") + } else if len(edges) > 1 { + t.Error("Too many has_software edges created") + } + + edge := edges[0] + if edge.Parent != server.ID { + t.Errorf("Edge parent is not Server node") + } + if edge.Child != software.ID { + t.Errorf("Edge child is not Software node") + } +} + +// TestPresentServerCreateSoftwareToOthers receives a metric of a server node already in the graph, but without a "others" software node. It should +// create that "others" software node and append the connection info. +func TestPresentServerCreateSoftwareToOthers(t *testing.T) { + // GIVEN + p := Probe{} + + p.graph = newGraph(t) + + givenServerName := "hostFoo" + + serverNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: givenServerName, + MetadataTypeKey: MetadataTypeServer, + }) + if err != nil { + t.Errorf("Unable to create server %s", givenServerName) + } + + // WHEN + metricServerName := givenServerName + metricSoftwareCmdline := "nc -kl 8000" + metricConnections := "1.2.3.4:80,9.9.9.9:53" + metricListen := "192.168.1.36:8000,192.168.1.22:8000" + metric := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(1603890543, 0), + }, + Tags: map[string]string{ + "cmdline": metricSoftwareCmdline, + "host": metricServerName, + "process_name": "nc", + }, + Fields: map[string]string{ + "conn": metricConnections, + "listen": metricListen, + }, + } + + agentData := []byte{} + agentData, err = metric.MarshalMsg(agentData) + if err != nil { + panic(err) + } + + sendAgentData(t, p, agentData, http.StatusOK) + + // THEN + // Check if Software node 'others' exists and have the right name and connection info + softwareNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeSoftware)) + softwares := p.graph.GetNodes(softwareNodeFilter) + if len(softwares) == 0 { + t.Fatal("Software node not created") + } else if len(softwares) > 1 { + t.Error("Too many Software nodes created") + } + + software := softwares[0] + softwareName, err := software.Metadata.GetFieldString(MetadataNameKey) + if err != nil { + t.Errorf("Software node created but without Metadata.Name") + } + + if softwareName != OthersSoftwareNode { + t.Errorf("Software node created, but with wrong name: %v", softwareName) + } + + softwareTCPConn, err := getTCPConn(software) + if err != nil { + t.Errorf("Not able to get TCP connection info") + } + assert.ElementsMatch(t, softwareTCPConn, strings.Split(metricConnections, ",")) + + softwareListenEndpoints, err := getListenEndpoints(software) + if err != nil { + t.Errorf("Not able to get TCP listen endpoints") + } + assert.ElementsMatch(t, softwareListenEndpoints, strings.Split(metricListen, ",")) + + // Check there ir an edge connecting software to server node + e := p.graph.GetNodeEdges(software, graph.NewElementFilter(filters.NewTermStringFilter(MetadataRelationTypeKey, RelationTypeHasSoftware))) + assert.Len(t, e, 1) + assert.Equal(t, e[0].Parent, serverNode.ID) +} + +// TestFillOthersSoftwareNode given a present server and empty 'others' software node, add the connection info to that node +func TestFillOthersSoftwareNode(t *testing.T) { + // GIVEN + p := Probe{} + + p.graph = newGraph(t) + + givenServerName := "hostFoo" + + givenNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: givenServerName, + MetadataTypeKey: MetadataTypeServer, + }) + if err != nil { + t.Errorf("Unable to create server %s", givenServerName) + } + + givenOtherNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: OthersSoftwareNode, + MetadataTypeKey: MetadataTypeSoftware, + }) + if err != nil { + t.Error("Unable to create software others") + } + + _, err = p.graph.NewEdge("", givenNode, givenOtherNode, graph.Metadata{ + MetadataRelationTypeKey: RelationTypeHasSoftware, + }) + if err != nil { + t.Errorf("Unable to create edge between server %s and software others", givenServerName) + } + + // WHEN + metricServerName := "hostFoo" + metricSoftwareCmdline := "nc -kl 8000" + metricConnections := "1.2.3.4:80,9.9.9.9:53" + metricListen := "192.168.1.36:8000,192.168.1.22:8000" + + metric := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(1603890543, 0), + }, + Tags: map[string]string{ + "cmdline": metricSoftwareCmdline, + "host": metricServerName, + "process_name": "nc", + }, + Fields: map[string]string{ + "conn": metricConnections, + "listen": metricListen, + }, + } + + agentData := []byte{} + agentData, err = metric.MarshalMsg(agentData) + if err != nil { + panic(err) + } + + sendAgentData(t, p, agentData, http.StatusOK) + + // THEN + // Check if Software node 'others' exists and have the right name and connection info + softwareNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeSoftware)) + softwares := p.graph.GetNodes(softwareNodeFilter) + if len(softwares) == 0 { + t.Fatal("Software node not created") + } else if len(softwares) > 1 { + t.Error("Too many Software nodes created") + } + + software := softwares[0] + softwareName, err := software.Metadata.GetFieldString(MetadataNameKey) + if err != nil { + t.Errorf("Software node created but without Metadata.Name") + } + + if softwareName != OthersSoftwareNode { + t.Errorf("Software node created, but with wrong name: %v", softwareName) + } + + softwareTCPConn, err := getTCPConn(software) + if err != nil { + t.Errorf("Not able to get TCP connection info") + } + assert.ElementsMatch(t, softwareTCPConn, strings.Split(metricConnections, ",")) + + softwareListenEndpoints, err := getListenEndpoints(software) + if err != nil { + t.Errorf("Not able to get TCP listen endpoints") + } + assert.ElementsMatch(t, softwareListenEndpoints, strings.Split(metricListen, ",")) +} + +// TestSendingInvalidFormatDataJSON sends a JSON body, instead of msgpack, to the server, it should +// reject the the message +func TestSendingInvalidFormatDataJSON(t *testing.T) { + // GIVEN + p := Probe{} + + p.graph = newGraph(t) + + // WHEN + agentData := []byte(` +{ + "metrics": [ + { + "fields": { + "conn": "1.1.1.1:90", + "listen": "192.168.1.1:980" + }, + "name": "procstat_test", + "tags": { + "cmdline": "nc -kl 980", + "host": "foobar", + "process_name": "nc" + }, + "timestamp": 123456789 + } + ] +}`) + + sendAgentData(t, p, agentData, http.StatusNotAcceptable) + + // THEN + softwareNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeSoftware)) + assert.Len(t, p.graph.GetNodes(softwareNodeFilter), 0) +} + +// TestMetricDateIsUsed checks that CreatedAt and UpdatedAt fields are set to the metric timestamp for new metrics +func TestMetricDateIsUsed(t *testing.T) { + // GIVEN + p := Probe{} + + p.graph = newGraph(t) + + givenServerName := "hostFoo" + + givenNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: givenServerName, + MetadataTypeKey: MetadataTypeServer, + }) + if err != nil { + t.Errorf("Unable to create server %s", givenServerName) + } + + givenOtherNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: OthersSoftwareNode, + MetadataTypeKey: MetadataTypeSoftware, + }) + if err != nil { + t.Error("Unable to create software others") + } + + _, err = p.graph.NewEdge("", givenNode, givenOtherNode, graph.Metadata{ + MetadataRelationTypeKey: RelationTypeHasSoftware, + }) + if err != nil { + t.Errorf("Unable to create edge between server %s and software others", givenServerName) + } + + // WHEN + metricServerName := "hostFoo" + metricSoftwareCmdline := "nc -kl 8000" + metricConnections := "1.2.3.4:80" + metricListen := "192.168.1.36:8000" + var metricTimestamp int64 = 1555555555 + + metric := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(metricTimestamp, 0), + }, + Tags: map[string]string{ + "cmdline": metricSoftwareCmdline, + "host": metricServerName, + "process_name": "nc", + }, + Fields: map[string]string{ + "conn": metricConnections, + "listen": metricListen, + }, + } + + agentData := []byte{} + agentData, err = metric.MarshalMsg(agentData) + if err != nil { + panic(err) + } + + sendAgentData(t, p, agentData, http.StatusOK) + + // THEN + softwareNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeSoftware)) + software := p.graph.GetNodes(softwareNodeFilter)[0] + + metadataTCPConnRaw, _ := software.Metadata.GetField(MetadataTCPConnKey) + connMetadata := (*metadataTCPConnRaw.(*NetworkInfo))[metricConnections] + + metadataListenEndpointsRaw, _ := software.Metadata.GetField(MetadataListenEndpointKey) + listenMetadata := (*metadataListenEndpointsRaw.(*NetworkInfo))[metricListen] + + expectedMetadata := ProcInfo{ + CreatedAt: metricTimestamp * 1000, // CreatedAt is in milliseconds + UpdatedAt: metricTimestamp * 1000, // UpdatedAt is in milliseconds + Revision: 1, + } + + assert.Equal(t, connMetadata, expectedMetadata) + assert.Equal(t, listenMetadata, expectedMetadata) +} + +// TestMultipleMetricsToOtherOnlyOneRevision if several metrics are received in the same packet and are going to the "others" software, +// the node.Revision field should only increase by one. Network info from both connetions should be added to "others" +func TestMultipleMetricsToOtherOnlyOneRevision(t *testing.T) { + // GIVEN + p := Probe{} + + p.graph = newGraph(t) + + givenServerName := "hostFoo" + + givenNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: givenServerName, + MetadataTypeKey: MetadataTypeServer, + }) + if err != nil { + t.Errorf("Unable to create server %s", givenServerName) + } + + givenOtherNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: OthersSoftwareNode, + MetadataTypeKey: MetadataTypeSoftware, + }) + if err != nil { + t.Error("Unable to create software others") + } + + _, err = p.graph.NewEdge("", givenNode, givenOtherNode, graph.Metadata{ + MetadataRelationTypeKey: RelationTypeHasSoftware, + }) + if err != nil { + t.Errorf("Unable to create edge between server %s and software others", givenServerName) + } + + // WHEN + metricServerName := "hostFoo" + metricSoftwareCmdline := "nc -kl 8000" + metricConnections := "1.2.3.4:80" + metricListen := "192.168.1.36:8000" + + metricSoftwareCmdline2 := "nc -kl 8888" + metricConnections2 := "1.2.3.4:88" + metricListen2 := "192.168.1.36:8888" + + metricSoftwareCmdline3 := "nc -kl 9999" + metricConnections3 := "1.2.3.4:99" + metricListen3 := "192.168.1.36:9999" + + var metricTimestamp int64 = 1555555555 + + metric1 := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(metricTimestamp, 0), + }, + Tags: map[string]string{ + "cmdline": metricSoftwareCmdline, + "host": metricServerName, + "process_name": "nc", + }, + Fields: map[string]string{ + "conn": metricConnections, + "listen": metricListen, + }, + } + + metric2 := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(metricTimestamp, 0), + }, + Tags: map[string]string{ + "cmdline": metricSoftwareCmdline2, + "host": metricServerName, + "process_name": "nc", + }, + Fields: map[string]string{ + "conn": metricConnections2, + "listen": metricListen2, + }, + } + + metric3 := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(metricTimestamp, 0), + }, + Tags: map[string]string{ + "cmdline": metricSoftwareCmdline3, + "host": metricServerName, + "process_name": "nc", + }, + Fields: map[string]string{ + "conn": metricConnections3, + "listen": metricListen3, + }, + } + + agentData := []byte{} + agentData, err = metric1.MarshalMsg(agentData) + if err != nil { + panic(err) + } + agentData, err = metric2.MarshalMsg(agentData) + if err != nil { + panic(err) + } + agentData, err = metric3.MarshalMsg(agentData) + if err != nil { + panic(err) + } + + // First send + sendAgentData(t, p, agentData, http.StatusOK) + + // THEN + softwareNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeSoftware)) + software := p.graph.GetNodes(softwareNodeFilter)[0] + + // The software node should have two revisions: + // - creating the node + // - adding TCPConn and TCPListen to that empty node + assert.Equal(t, int64(2), software.Revision) + + metadataTCPConnRaw, _ := software.Metadata.GetField(MetadataTCPConnKey) + connMetadata := (*metadataTCPConnRaw.(*NetworkInfo)) + assert.Len(t, connMetadata, 3) + + metadataListenEndpointsRaw, _ := software.Metadata.GetField(MetadataListenEndpointKey) + listenMetadata := (*metadataListenEndpointsRaw.(*NetworkInfo)) + assert.Len(t, listenMetadata, 3) +} + +// TestTwoMetricsTwoOthersNode given a metric message with two metrics from different servers (tag.host), it should create +// two different Server node and two different "others" Software node +func TestTwoMetricsTwoOthersNode(t *testing.T) { + // GIVEN + p := Probe{} + p.graph = newGraph(t) + + metricServerName := "hostFoo" + metricSoftwareCmdline := "nc -kl 8000" + metricConnections := "1.2.3.4:80" + metricListen := "192.168.1.36:8000" + + metricServerName2 := "hostBar" + metricSoftwareCmdline2 := "nc -kl 8888" + metricConnections2 := "1.2.3.4:88" + metricListen2 := "192.168.1.36:8888" + + var metricTimestamp int64 = 1555555555 + + metric1 := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(metricTimestamp, 0), + }, + Tags: map[string]string{ + "cmdline": metricSoftwareCmdline, + "host": metricServerName, + "process_name": "nc", + }, + Fields: map[string]string{ + "conn": metricConnections, + "listen": metricListen, + }, + } + + metric2 := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(metricTimestamp, 0), + }, + Tags: map[string]string{ + "cmdline": metricSoftwareCmdline2, + "host": metricServerName2, + "process_name": "nc", + }, + Fields: map[string]string{ + "conn": metricConnections2, + "listen": metricListen2, + }, + } + + var err error + agentData := []byte{} + agentData, err = metric1.MarshalMsg(agentData) + if err != nil { + panic(err) + } + agentData, err = metric2.MarshalMsg(agentData) + if err != nil { + panic(err) + } + + // WHEN + sendAgentData(t, p, []byte(agentData), http.StatusOK) + + // THEN + serverNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeServer)) + servers := p.graph.GetNodes(serverNodeFilter) + assert.Len(t, servers, 2) + + softwareNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeSoftware)) + softwares := p.graph.GetNodes(softwareNodeFilter) + assert.Len(t, softwares, 2) +} + +// TestMetricDateIsUsedWhenUpdating consecutives updates for the same metric should increase Revision and use the metric timestamp in the UpdatedAt field +func TestMetricDateIsUsedWhenUpdating(t *testing.T) { + // GIVEN + p := Probe{} + + p.graph = newGraph(t) + + givenServerName := "hostFoo" + + givenNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: givenServerName, + MetadataTypeKey: MetadataTypeServer, + }) + if err != nil { + t.Errorf("Unable to create server %s", givenServerName) + } + + givenOtherNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: OthersSoftwareNode, + MetadataTypeKey: MetadataTypeSoftware, + }) + if err != nil { + t.Error("Unable to create software others") + } + + _, err = p.graph.NewEdge("", givenNode, givenOtherNode, graph.Metadata{ + MetadataRelationTypeKey: RelationTypeHasSoftware, + }) + if err != nil { + t.Errorf("Unable to create edge between server %s and software others", givenServerName) + } + + // WHEN + metricServerName := "hostFoo" + metricSoftwareCmdline := "nc -kl 8000" + metricConnections := "1.2.3.4:80" + metricListen := "192.168.1.36:8000" + var metricTimestamp int64 = 1555555555 + + metric := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(metricTimestamp, 0), + }, + Tags: map[string]string{ + "cmdline": metricSoftwareCmdline, + "host": metricServerName, + "process_name": "nc", + }, + Fields: map[string]string{ + "conn": metricConnections, + "listen": metricListen, + }, + } + + agentData := []byte{} + agentData, err = metric.MarshalMsg(agentData) + if err != nil { + panic(err) + } + + // First send + sendAgentData(t, p, agentData, http.StatusOK) + + // Second send + var secondMetricTimestamp int64 = 1666666666 + metric.Time = MessagePackTime{time: time.Unix(secondMetricTimestamp, 0)} + agentData = []byte{} + agentData, err = metric.MarshalMsg(agentData) + if err != nil { + panic(err) + } + + sendAgentData(t, p, agentData, http.StatusOK) + + // THEN + softwareNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeSoftware)) + software := p.graph.GetNodes(softwareNodeFilter)[0] + + metadataTCPConnRaw, _ := software.Metadata.GetField(MetadataTCPConnKey) + connMetadata := (*metadataTCPConnRaw.(*NetworkInfo))[metricConnections] + + metadataListenEndpointsRaw, _ := software.Metadata.GetField(MetadataListenEndpointKey) + listenMetadata := (*metadataListenEndpointsRaw.(*NetworkInfo))[metricListen] + + expectedMetadata := ProcInfo{ + CreatedAt: metricTimestamp * 1000, // CreatedAt is in milliseconds + UpdatedAt: secondMetricTimestamp * 1000, // UpdatedAt is in milliseconds + Revision: 2, + } + + assert.Equal(t, connMetadata, expectedMetadata) + assert.Equal(t, listenMetadata, expectedMetadata) +} + +// TestFillOthersSoftwareNodeWithConnPrefix if the received metrics has the tag connPrefix, IPs stored in Skydive should prefix that value +func TestFillOthersSoftwareNodeWithConnPrefix(t *testing.T) { + // GIVEN + p := Probe{} + + p.graph = newGraph(t) + + givenServerName := "hostFoo" + + givenNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: givenServerName, + MetadataTypeKey: MetadataTypeServer, + }) + if err != nil { + t.Errorf("Unable to create server %s", givenServerName) + } + + givenOtherNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: OthersSoftwareNode, + MetadataTypeKey: MetadataTypeSoftware, + }) + if err != nil { + t.Error("Unable to create software others") + } + + _, err = p.graph.NewEdge("", givenNode, givenOtherNode, graph.Metadata{ + MetadataRelationTypeKey: RelationTypeHasSoftware, + }) + if err != nil { + t.Errorf("Unable to create edge between server %s and software others", givenServerName) + } + + // WHEN + metricServerName := "hostFoo" + metricSoftwareCmdline := "nc -kl 8000" + metricConnections := "1.2.3.4:80,9.9.9.9:53" + metricListen := "192.168.1.36:8000,192.168.1.22:8000" + connPrefix := "foobar-" + + metric := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(1603890543, 0), + }, + Tags: map[string]string{ + "cmdline": metricSoftwareCmdline, + "host": metricServerName, + "process_name": "nc", + "conn_prefix": connPrefix, + }, + Fields: map[string]string{ + "conn": metricConnections, + "listen": metricListen, + }, + } + + agentData := []byte{} + agentData, err = metric.MarshalMsg(agentData) + if err != nil { + panic(err) + } + + sendAgentData(t, p, agentData, http.StatusOK) + + // THEN + // Check if Software node 'others' exists and have the right name and connection info + softwareNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeSoftware)) + softwares := p.graph.GetNodes(softwareNodeFilter) + if len(softwares) == 0 { + t.Fatal("Software node not created") + } else if len(softwares) > 1 { + t.Error("Too many Software nodes created") + } + + software := softwares[0] + softwareTCPConn, _ := getTCPConn(software) + assert.ElementsMatch(t, softwareTCPConn, []string{"foobar-1.2.3.4:80", "foobar-9.9.9.9:53"}) + + softwareListenEndpoints, _ := getListenEndpoints(software) + assert.ElementsMatch(t, softwareListenEndpoints, []string{"foobar-192.168.1.36:8000", "foobar-192.168.1.22:8000"}) +} + +// TestNewMetricUpdateNetworkMetadata given a present 'others' software node with some data, if a new metric is received, it should update the network metadata +func TestNewMetricUpdateNetworkMetadata(t *testing.T) { + // GIVEN + p := Probe{} + + p.graph = newGraph(t) + + givenServerName := "hostFoo" + givenOthersSoftwareTCPConnections := []string{"1.2.3.4:80"} + givenOthersSoftwareListenEndpoints := []string{"192.168.0.1:22"} + + givenNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: givenServerName, + MetadataTypeKey: MetadataTypeServer, + }) + if err != nil { + t.Errorf("Unable to create server %s", givenServerName) + } + + givenOtherNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: OthersSoftwareNode, + MetadataTypeKey: MetadataTypeSoftware, + }) + if err != nil { + t.Error("Unable to create software others") + } + + _, err = p.graph.NewEdge("", givenNode, givenOtherNode, graph.Metadata{ + MetadataRelationTypeKey: RelationTypeHasSoftware, + }) + if err != nil { + t.Errorf("Unable to create edge between server %s and software others", givenServerName) + } + + // This function handles its own lock + m := Metric{ + Fields: map[string]string{ + MetricFieldConn: strings.Join(givenOthersSoftwareTCPConnections, ","), + MetricFieldListen: strings.Join(givenOthersSoftwareListenEndpoints, ","), + }, + } + err = p.addNetworkInfo(givenOtherNode, []Metric{m}) + if err != nil { + t.Error("Adding network connections to others Software node") + } + + // This created other nod should have 2 revisions + // - creating the node + // - adding TCPConn and TCPListen to that empty node + assert.Equal(t, int64(2), givenOtherNode.Revision) + + // WHEN + metricSoftwareCmdline := "nc -kl 8000" + metric := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(1603890543, 0), + }, + Tags: map[string]string{ + "cmdline": metricSoftwareCmdline, + "host": givenServerName, + "process_name": "nc", + }, + Fields: map[string]string{ + "conn": strings.Join(givenOthersSoftwareTCPConnections, ","), + "listen": strings.Join(givenOthersSoftwareListenEndpoints, ","), + }, + } + + agentData := []byte{} + agentData, err = metric.MarshalMsg(agentData) + if err != nil { + panic(err) + } + + time.Sleep(time.Millisecond) // To be able to see a difference between UpdatedAt and CreatedAt + sendAgentData(t, p, agentData, http.StatusOK) + + // THEN + softwareNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeSoftware)) + software := p.graph.GetNodes(softwareNodeFilter)[0] + + // After handling the metrics, node should have increased its revision by two + // - adding TCPConn to existant Metadata + // - adding TCPListen to existant Metadata + assert.Equal(t, int64(4), givenOtherNode.Revision) + + softwareTCPConn, err := software.Metadata.GetField(MetadataTCPConnKey) + if err != nil { + t.Fatalf("Software 'others' must have the %v key in metadata", MetadataTCPConnKey) + } + + procInfoTCPConnPtr, ok := softwareTCPConn.(*NetworkInfo) + assert.True(t, ok) + procInfoTCPConn := *procInfoTCPConnPtr + + // It should have just one connection + assert.Len(t, procInfoTCPConn, 1) + + conn := procInfoTCPConn[givenOthersSoftwareTCPConnections[0]] + + // This connection should have the revision metadata field set to 1, as it have received a new metric after creation + assert.Equal(t, int64(2), conn.Revision) + // It should shown an update time newer than creating time + assert.Greater(t, conn.UpdatedAt, conn.CreatedAt) + + softwareListenEndpoint, err := software.Metadata.GetField(MetadataListenEndpointKey) + if err != nil { + t.Fatalf("Software 'others' must have the %v key in metadata", MetadataListenEndpointKey) + } + + procInfoListenEndpointPtr, ok := softwareListenEndpoint.(*NetworkInfo) + assert.True(t, ok) + procInfoListenEndpoint := *procInfoListenEndpointPtr + + // It should have just one listener + assert.Len(t, procInfoListenEndpoint, 1) + + listen := procInfoListenEndpoint[givenOthersSoftwareListenEndpoints[0]] + + // This listener should have the revision metadata field set to 1, as it have received a new metric after creation + assert.Equal(t, int64(2), listen.Revision) + // It should shown an update time newer than creating time + assert.Greater(t, listen.UpdatedAt, listen.CreatedAt) +} + +// TestAppendConnectionInfoToOthersSoftwareNode given a present 'others' software node with some data, check that new data is appended and old data +// is kept +func TestAppendConnectionInfoToOthersSoftwareNode(t *testing.T) { + // GIVEN + p := Probe{} + + p.graph = newGraph(t) + + givenServerName := "hostFoo" + givenOthersSoftwareTCPConnections := []string{"1.2.3.4:80", "8.8.8.8:443"} + givenOthersSoftwareListenEndpoints := []string{"192.168.0.1:22", "10.0.1.1:22"} + + p.graph.Lock() + givenNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: givenServerName, + MetadataTypeKey: MetadataTypeServer, + }) + if err != nil { + t.Errorf("Unable to create server %s", givenServerName) + } + + givenOtherNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: OthersSoftwareNode, + MetadataTypeKey: MetadataTypeSoftware, + }) + if err != nil { + t.Error("Unable to create software others") + } + + _, err = p.graph.NewEdge("", givenNode, givenOtherNode, graph.Metadata{ + MetadataRelationTypeKey: RelationTypeHasSoftware, + }) + if err != nil { + t.Errorf("Unable to create edge between server %s and software others", givenServerName) + } + p.graph.Unlock() + + // This function handles its own lock + m := Metric{ + Fields: map[string]string{ + MetricFieldConn: strings.Join(givenOthersSoftwareTCPConnections, ","), + MetricFieldListen: strings.Join(givenOthersSoftwareListenEndpoints, ","), + }, + } + err = p.addNetworkInfo(givenOtherNode, []Metric{m}) + if err != nil { + t.Error("Adding network connections to others Software node") + } + + // WHEN + metricSoftwareCmdline := "nc -kl 8000" + metricConnections := []string{} + metricListen := []string{"192.168.1.36:8000", "192.168.1.22:8000", "10.0.1.1:22"} + + metric := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(1603890543, 0), + }, + Tags: map[string]string{ + "cmdline": metricSoftwareCmdline, + "host": givenServerName, + "process_name": "nc", + }, + Fields: map[string]string{ + "conn": strings.Join(metricConnections, ","), + "listen": strings.Join(metricListen, ","), + }, + } + + agentData := []byte{} + agentData, err = metric.MarshalMsg(agentData) + if err != nil { + panic(err) + } + + sendAgentData(t, p, agentData, http.StatusOK) + + // THEN + softwareNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeSoftware)) + software := p.graph.GetNodes(softwareNodeFilter)[0] + + softwareTCPConn, err := getTCPConn(software) + if err != nil { + t.Errorf("Not able to get TCP connection info") + } + // TCP connections should be the union of the present ones with those in the metric + expectedTCPConn := append(givenOthersSoftwareTCPConnections, metricConnections...) + assert.ElementsMatch(t, softwareTCPConn, expectedTCPConn) + + softwareListenEndpoints, err := getListenEndpoints(software) + if err != nil { + t.Errorf("Not able to get TCP listen endpoints") + } + // Listen endpoints should be the union of the present ones with those in the metric + expectedListenEndpoints := []string{"192.168.0.1:22", "10.0.1.1:22", "192.168.1.36:8000", "192.168.1.22:8000"} + assert.ElementsMatch(t, softwareListenEndpoints, expectedListenEndpoints) +} + +// +// TestFillKnownSoftwareNode given a Server node and a liked Software node, if the metric received matches the cmdline of the software node, the +// connection info should be appended to that software node +func TestFillKnownSoftwareNode(t *testing.T) { + // GIVEN + p := Probe{} + + p.graph = newGraph(t) + + givenSoftwareName := "PostgreSQL" + cmdline := "/usr/bin/postgres -D /var/lib/postgres/data" + givenServerName := "hostFoo" + givenSoftwareTCPConnections := []string{} + givenSoftwareListenEndpoints := []string{"192.168.0.1:5432", "10.0.1.1:5432"} + + p.graph.Lock() + givenNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: givenServerName, + MetadataTypeKey: MetadataTypeServer, + }) + if err != nil { + t.Errorf("Unable to create server %s", givenServerName) + } + + givenSWNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: givenSoftwareName, + MetadataTypeKey: MetadataTypeSoftware, + MetadataCmdlineKey: cmdline, + }) + if err != nil { + t.Error("Unable to create software others") + } + + _, err = p.graph.NewEdge("", givenNode, givenSWNode, graph.Metadata{ + MetadataRelationTypeKey: RelationTypeHasSoftware, + }) + if err != nil { + t.Errorf("Unable to create edge between server %s and software others", givenServerName) + } + p.graph.Unlock() + + // This function handles its own lock + m := Metric{ + Fields: map[string]string{ + MetricFieldConn: strings.Join(givenSoftwareTCPConnections, ","), + MetricFieldListen: strings.Join(givenSoftwareListenEndpoints, ","), + }, + } + err = p.addNetworkInfo(givenSWNode, []Metric{m}) + if err != nil { + t.Error("Adding network connections to others Software node") + } + + // WHEN + metricConnections := []string{} + metricListen := []string{"192.168.1.36:8000", "192.168.1.22:8000", "10.0.1.1:22"} + + metric := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(1603890543, 0), + }, + Tags: map[string]string{ + "cmdline": cmdline, + "host": givenServerName, + "process_name": "nc", + }, + Fields: map[string]string{ + "conn": strings.Join(metricConnections, ","), + "listen": strings.Join(metricListen, ","), + }, + } + + agentData := []byte{} + agentData, err = metric.MarshalMsg(agentData) + if err != nil { + panic(err) + } + + sendAgentData(t, p, agentData, http.StatusOK) + + // THEN + softwareNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeSoftware)) + software := p.graph.GetNodes(softwareNodeFilter)[0] + + softwareName, _ := software.Metadata.GetFieldString(MetadataNameKey) + assert.Equal(t, givenSoftwareName, softwareName) + + softwareTCPConn, err := getTCPConn(software) + if err != nil { + t.Errorf("Not able to get TCP connection info") + } + // TCP connections should be the union of the present ones with those in the metric + expectedTCPConn := metricConnections + assert.ElementsMatch(t, softwareTCPConn, expectedTCPConn) + + softwareListenEndpoints, err := getListenEndpoints(software) + if err != nil { + t.Errorf("Not able to get TCP listen endpoints") + } + // Listen endpoints should be the union of the present ones with those in the metric + expectedListenEndpoints := []string{"192.168.0.1:5432", "10.0.1.1:5432", "192.168.1.36:8000", "192.168.1.22:8000", "10.0.1.1:22"} + assert.ElementsMatch(t, softwareListenEndpoints, expectedListenEndpoints) +} + +// TestClearOldConnections check if old connections stored in the metadata are deleted correctly by the appropiate function +func TestClearOldConnections(t *testing.T) { + // GIVEN + p := Probe{} + p.graph = newGraph(t) + + metricServerName := "hostFoo" + metricSoftwareCmdline := "nc -kl 8000" + metricConnections := "1.2.3.4:80,9.9.9.9:53" + metricListen := "192.168.1.36:8000,192.168.1.22:8000" + metric := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(1603890543, 0), + }, + Tags: map[string]string{ + "cmdline": metricSoftwareCmdline, + "host": metricServerName, + "process_name": "nc", + }, + Fields: map[string]string{ + "conn": metricConnections, + "listen": metricListen, + }, + } + + var err error + agentData := []byte{} + agentData, err = metric.MarshalMsg(agentData) + if err != nil { + panic(err) + } + + sendAgentData(t, p, agentData, http.StatusOK) + swNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeSoftware)) + software := p.graph.GetNodes(swNodeFilter)[0] + + // WHEN + // This should delete all connections, as the threshold time is in the future + p.removeOldNetworkInformation(software, time.Now().Add(time.Hour)) + + // THEN + softwareTCPConn, err := getTCPConn(software) + if err != nil { + t.Errorf("Not able to get TCP connection info") + } + assert.Empty(t, softwareTCPConn) + + softwareListenEndpoints, err := getListenEndpoints(software) + if err != nil { + t.Errorf("Not able to get TCP listen endpoints") + } + assert.Empty(t, softwareListenEndpoints) +} + +// TestClearOldConnectionsKeepNewerConnections for a given node with old and new connections, check that clearing old connections +// do not delete new connections +func TestClearOldConnectionsKeepNewerConnections(t *testing.T) { + // GIVEN + p := Probe{} + p.graph = newGraph(t) + + software, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: OthersSoftwareNode, + MetadataTypeKey: MetadataTypeSoftware, + MetadataTCPConnKey: &NetworkInfo{ + "1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + "1.2.3.4:80": { + CreatedAt: graph.TimeNow().UnixMilli(), + UpdatedAt: graph.TimeNow().UnixMilli(), + Revision: 1, + }, + }, + MetadataListenEndpointKey: &NetworkInfo{ + "1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + "1.2.3.4:80": { + CreatedAt: graph.TimeNow().UnixMilli(), + UpdatedAt: graph.TimeNow().UnixMilli(), + Revision: 1, + }, + }, + }) + if err != nil { + t.Error("Unable to create software others") + } + + // WHEN + // Should remove old connections but no the new ones + p.removeOldNetworkInformation(software, time.Now().Add(-time.Hour)) + + // THEN + softwareTCPConn, err := getTCPConn(software) + if err != nil { + t.Errorf("Not able to get TCP connection info") + } + assert.Len(t, softwareTCPConn, 1) + + softwareListenEndpoints, err := getListenEndpoints(software) + if err != nil { + t.Errorf("Not able to get TCP listen endpoints") + } + assert.Len(t, softwareListenEndpoints, 1) +} + +// TestCleanTCPListenIfTCPConnIsInvalid check that TCPListen metadata is cleaned even when TCPConn +// has an invalid data type +func TestCleanTCPListenIfTCPConnIsInvalid(t *testing.T) { + // GIVEN + p := Probe{} + p.graph = newGraph(t) + + software, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: OthersSoftwareNode, + MetadataTypeKey: MetadataTypeSoftware, + MetadataTCPConnKey: "", + MetadataListenEndpointKey: &NetworkInfo{ + "1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + "1.2.3.4:80": { + CreatedAt: graph.TimeNow().UnixMilli(), + UpdatedAt: graph.TimeNow().UnixMilli(), + Revision: 1, + }, + }, + }) + if err != nil { + t.Error("Unable to create software others") + } + + // WHEN + // Should remove old connections but no the new ones + p.removeOldNetworkInformation(software, time.Now().Add(-time.Hour)) + + // THEN + softwareListenEndpoints, err := getListenEndpoints(software) + if err != nil { + t.Errorf("Not able to get TCP listen endpoints") + } + assert.Len(t, softwareListenEndpoints, 1) +} + +// TestDoNotPanicIfInvalidTCPConnOrTCPListenDataType checks that parsing invalid data do not panic +func TestDoNotPanicIfInvalidTCPConnOrTCPListenDataType(t *testing.T) { + // GIVEN + p := Probe{} + p.graph = newGraph(t) + + software, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: OthersSoftwareNode, + MetadataTypeKey: MetadataTypeSoftware, + MetadataTCPConnKey: "", + MetadataListenEndpointKey: "", + }) + if err != nil { + t.Error("Unable to create software others") + } + + // WHEN + // Should remove old connections but no the new ones + p.removeOldNetworkInformation(software, time.Now().Add(-time.Hour)) +} + +// TestDoNotReturnErrorIfTCPConnOrTCPListenKeysDoesNotExists missing keys in Software node does not +// should return an error, only a debug log trace +func TestDoNotReturnErrorIfTCPConnOrTCPListenKeysDoesNotExists(t *testing.T) { + // GIVEN + p := Probe{} + p.graph = newGraph(t) + + softwareNoTCPConn, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: OthersSoftwareNode, + MetadataTypeKey: MetadataTypeSoftware, + MetadataListenEndpointKey: "", + }) + if err != nil { + t.Error("Unable to create software others") + } + + softwareNoTCPListen, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: OthersSoftwareNode, + MetadataTypeKey: MetadataTypeSoftware, + MetadataTCPConnKey: "", + }) + if err != nil { + t.Error("Unable to create software others") + } + + // WHEN + // Should remove old connections but no the new ones + assert.NoError(t, p.removeOldNetworkInformation(softwareNoTCPConn, time.Now().Add(-time.Hour))) + assert.NoError(t, p.removeOldNetworkInformation(softwareNoTCPListen, time.Now().Add(-time.Hour))) +} + +// TestCleanSoftwareNodes check if garbage collector function delete correctly old connections in all software nodes +func TestClearSoftwareNodes(t *testing.T) { + // GIVEN + p := Probe{} + p.graph = newGraph(t) + + metricServerName := "hostFoo" + metricSoftwareCmdline := "nc -kl 8000" + metricConnections := "1.2.3.4:80,9.9.9.9:53" + metricListen := "192.168.1.36:8000,192.168.1.22:8000" + metric := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(1603890543, 0), + }, + Tags: map[string]string{ + "cmdline": metricSoftwareCmdline, + "host": metricServerName, + "process_name": "nc", + }, + Fields: map[string]string{ + "conn": metricConnections, + "listen": metricListen, + }, + } + + var err error + agentData := []byte{} + agentData, err = metric.MarshalMsg(agentData) + if err != nil { + panic(err) + } + + // hostFoo server + sendAgentData(t, p, agentData, http.StatusOK) + // hostBar server + metric.Tags["host"] = "hostBar" + agentData = []byte{} + agentData, err = metric.MarshalMsg(agentData) + if err != nil { + panic(err) + } + sendAgentData(t, p, agentData, http.StatusOK) + + // WHEN + // This should delete all connections in all nodes, as the threshold time is in the future + p.cleanSoftwareNodes(time.Now().Add(time.Hour)) + + // THEN + swNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeSoftware)) + for _, software := range p.graph.GetNodes(swNodeFilter) { + softwareTCPConn, err := getTCPConn(software) + if err != nil { + t.Errorf("Not able to get TCP connection info") + } + assert.Empty(t, softwareTCPConn) + + softwareListenEndpoints, err := getListenEndpoints(software) + if err != nil { + t.Errorf("Not able to get TCP listen endpoints") + } + assert.Empty(t, softwareListenEndpoints) + } +} + +// +// TestMigrateConnInfoFromOthersToKnownSoftwareNode check if connection info previously sent to 'others' is moved to a known software node if it +// is created after +func TestMigrateConnInfoFromOthersToKnownSoftwareNode(t *testing.T) { + // GIVEN + p := Probe{} + p.graph = newGraph(t) + + metricServerName := "hostFoo" + metricSoftwareCmdline := "nc -kl 8000" + metricConnections := "1.2.3.4:80,9.9.9.9:53" + metricListen := "192.168.1.36:8000,192.168.1.22:8000" + metric := Metric{ + Name: "procstat_test", + Time: MessagePackTime{ + time: time.Unix(1603890543, 0), + }, + Tags: map[string]string{ + "cmdline": metricSoftwareCmdline, + "host": metricServerName, + "process_name": "nc", + }, + Fields: map[string]string{ + "conn": metricConnections, + "listen": metricListen, + }, + } + + var err error + agentData := []byte{} + agentData, err = metric.MarshalMsg(agentData) + if err != nil { + panic(err) + } + + // This should create server "hostFoo" and software "others" + sendAgentData(t, p, agentData, http.StatusOK) + + // WHEN + // Create a software node for netcat linked to "hostFoo" server + netcatSoftwareName := "Netcat" + netcatNode, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: netcatSoftwareName, + MetadataTypeKey: MetadataTypeSoftware, + MetadataCmdlineKey: metricSoftwareCmdline, + }) + if err != nil { + t.Errorf("Unable to create server %s", netcatSoftwareName) + } + + serverNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataTypeKey, MetadataTypeServer)) + server := p.graph.GetNodes(serverNodeFilter)[0] + _, err = p.graph.NewEdge("", server, netcatNode, graph.Metadata{ + MetadataRelationTypeKey: RelationTypeHasSoftware, + }) + if err != nil { + t.Errorf("Unable to create edge between server %s and software %s", server, netcatSoftwareName) + } + + // Send again the process metrics + sendAgentData(t, p, []byte(agentData), http.StatusOK) + + // THEN + // others software should have not connections, neither listeners + swNodeFilter := graph.NewElementFilter(filters.NewTermStringFilter(MetadataNameKey, OthersSoftwareNode)) + others := p.graph.GetNodes(swNodeFilter)[0] + + othersTCPConn, err := getTCPConn(others) + if err != nil { + t.Errorf("Not able to get TCP connection info for others node") + } + assert.Empty(t, othersTCPConn) + + othersListenEndpoints, err := getListenEndpoints(others) + if err != nil { + t.Errorf("Not able to get TCP listen endpoints for others node") + } + assert.Empty(t, othersListenEndpoints) + + // netcat software should have two connections and two listener + swNodeFilter = graph.NewElementFilter(filters.NewTermStringFilter(MetadataNameKey, netcatSoftwareName)) + netcat := p.graph.GetNodes(swNodeFilter)[0] + + netcatTCPConn, err := getTCPConn(netcat) + if err != nil { + t.Errorf("Not able to get TCP connection info for netcat node") + } + assert.Len(t, netcatTCPConn, len(strings.Split(metricConnections, ","))) + + netcatListenEndpoints, err := getListenEndpoints(netcat) + if err != nil { + t.Errorf("Not able to get TCP listen endpoints for netcat node") + } + assert.Len(t, netcatListenEndpoints, len(strings.Split(metricListen, ","))) +} + +// TestNotSignalUpdateForKnownNetworkUpdates checks that updateNetworkMetadata does not return true (modificated) if +// the only modification is updating the fields UpdatedAt and Revision +func TestNotSignalUpdateForKnownNetworkUpdates(t *testing.T) { + newNetworkInfo := generateProcInfoData([]string{"1.1.1.1:80"}, 1e9) + nodeNetworkInfo := NetworkInfo{} + var nodeRevisionForceFlush int64 = 100 + p := Probe{ + nodeRevisionForceFlush: nodeRevisionForceFlush, + } + + // First time updateNetworkMetadata is called, there is no info in nodeNetworkInfo, so it should return true + assert.Truef(t, p.updateNetworkMetadata(&nodeNetworkInfo, newNetworkInfo, 1), "new network info") + + // This time, its the same network info, just with a new timestamp, the function should not mark is as an update + newNetworkInfo = generateProcInfoData([]string{"1.1.1.1:80"}, 1e9+1) + assert.Falsef(t, p.updateNetworkMetadata(&nodeNetworkInfo, newNetworkInfo, 2), "update without new network info") + + // If a new connection is added, it should return true (modification) + newNetworkInfo = generateProcInfoData([]string{"2.2.2.2:8000"}, 1e9+2) + assert.Truef(t, p.updateNetworkMetadata(&nodeNetworkInfo, newNetworkInfo, 3), "new network info") + + // After several node modifications the function should return "true" even if it has only + // updated the UpdatedAt and Revision values. + // This is to avoid leaving the backend behind too much + assert.True(t, p.updateNetworkMetadata(&nodeNetworkInfo, newNetworkInfo, nodeRevisionForceFlush), "forced flush iteration %v", nodeRevisionForceFlush) + assert.True(t, p.updateNetworkMetadata(&nodeNetworkInfo, newNetworkInfo, nodeRevisionForceFlush*2), "forced flush iteration %v", nodeRevisionForceFlush*2) +} + +func generateProcInfoData(conn []string, metricTimestamp int64) NetworkInfo { + ret := NetworkInfo{} + for _, c := range conn { + ret[c] = ProcInfo{ + CreatedAt: metricTimestamp, + UpdatedAt: metricTimestamp, + Revision: 1, + } + } + + return ret +} + +// BenchmarkProcessMetricsSameNode replicate how this probe will receive the data. +// Usually each POST will contain only metrics of the same host. +func BenchmarkProcessMetricsSameNode(b *testing.B) { + p := Probe{} + p.graph = newGraph(b) + + metrics := []Metric{} + + // Create nodes in the backend + for i := 1; i < 10000; i++ { + _, err := p.newNode("host", graph.Metadata{ + MetadataNameKey: fmt.Sprintf("foo-%d", i), + MetadataTypeKey: MetadataTypeServer, + }) + if err != nil { + panic(err) + } + } + + // Create an array of 1000 metrics of the same node (same tags.host) + for i := 0; i < 1000; i++ { + metrics = append(metrics, Metric{ + Name: "tcp", + Time: MessagePackTime{ + time: time.Now(), + }, + Tags: map[string]string{ + "host": "foo", + "cmdline": fmt.Sprintf("foo-%d", i), + }, + Fields: map[string]string{ + "f1": "f1", + "f2": "f2", + "f3": "f3", + }, + }) + } + + for i := 0; i < b.N; i++ { + p.processMetrics(metrics) + } +} diff --git a/topology/probes/procpeering/procpeering.go b/topology/probes/procpeering/procpeering.go new file mode 100644 index 0000000000..4bd19f9013 --- /dev/null +++ b/topology/probes/procpeering/procpeering.go @@ -0,0 +1,236 @@ +/* + * Copyright (C) 2016 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy ofthe License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specificlanguage governing permissions and + * limitations under the License. + * + */ + +// Package procpeering generate edges (type tcp_conn) between nodes having a TCP connection between them. +// Nodes should have the fields Metadata.TCPConn / Metadata.TCPListen +// The match will be produced when a node have a connection endpoint (the destination) already seen +// in another node (in its TCPListen). +// It means that the first node is connecting to the IP:port of the second node. +// +// This is thought to work with "proccon", which will be the one in charge to add the network info to nodes +// +// To be able to create matchings, when a node with listeners is created or modified, an index is updated. +// When another node with connections is created/modified, it tries to match the listeners. +// +// If we start Skydive with data already present in the backend, those listeners will not be available in the +// index until the nodes are modified. +package procpeering + +import ( + "fmt" + + uuid "github.com/nu7hatch/gouuid" + "github.com/skydive-project/skydive/graffiti/graph" + "github.com/skydive-project/skydive/graffiti/logging" + "github.com/skydive-project/skydive/topology/probes/proccon" +) + +const ( + // ProcpeeringOriginName defines the prefix set in nodes/edges Origin field + ProcpeeringOriginName = "procpeering." + // RelationTypeConnection value for edges connecting two software nodes connected by TCP + RelationTypeConnection = "tcp_conn" +) + +// Probe describes graph peering based on TCP connections +type Probe struct { + graph.DefaultGraphListener + graph *graph.Graph + listenIndexer *graph.Indexer + connIndexer *graph.Indexer + linker *graph.ResourceLinker +} + +// Start the TCP peering resolver probe +func (p *Probe) Start() error { + logging.GetLogger().Debug("TCP connections peering") + + err := p.listenIndexer.Start() + if err != nil { + return fmt.Errorf("starting listenIndexer: %v", err) + } + + err = p.connIndexer.Start() + if err != nil { + return fmt.Errorf("starting connIndexer: %v", err) + } + + err = p.linker.Start() + if err != nil { + return fmt.Errorf("starting linker: %v", err) + } + + return nil +} + +// Stop the probe +func (p *Probe) Stop() { + logging.GetLogger().Debug("TCP connections peering") + p.listenIndexer.Stop() + p.connIndexer.Stop() + p.linker.Stop() +} + +// OnError implements the LinkerEventListener interface +func (p *Probe) OnError(err error) { + logging.GetLogger().Error(err) +} + +type simpleLinker struct { + probe *Probe +} + +// GetABLinks get nodes with new connections and have to return edges to listeners +func (l *simpleLinker) GetABLinks(nodeConnection *graph.Node) (edges []*graph.Edge) { + tcpConnIface, err := nodeConnection.Metadata.GetField(proccon.MetadataTCPConnKey) + if err != nil { + logging.GetLogger().Debugf("incorrect node '%v' metadata, expecting TCPConn key: %v", nodeConnection, err) + } + + tcpConnPtr, ok := tcpConnIface.(*proccon.NetworkInfo) + if !ok { + return []*graph.Edge{} + } + + tcpConn := *tcpConnPtr // TODO quitar esto y poner en el for *tcpConnPtr + + // Iterate over node connections and try to find a listener match + for outgoingConn := range tcpConn { // cambiado de string a interface + // Only find using the hash with ip and port, ignore process as it will be different in the other side of the connection + nodesListeners, _ := l.probe.listenIndexer.Get(outgoingConn) + + // Create link from our node to the listener + for _, nodeListener := range nodesListeners { + logging.GetLogger().Debugf("Match %s -> %s (%s)", nodeName(nodeConnection), outgoingConn, nodeName(nodeListener)) + + // Create edge, but do not insert into the graph, GetABLinks caller will do it + u, _ := uuid.NewV5(uuid.NamespaceOID, []byte(nodeConnection.ID+nodeListener.ID)) + i := graph.Identifier(u.String()) + + edges = append(edges, graph.CreateEdge( + i, + nodeConnection, + nodeListener, + graph.Metadata{ + "RelationType": RelationTypeConnection, + "Destination": outgoingConn, + // We do not have the origin connection info, it is not stored in the node metadata, we only store the destination endpoint + }, + graph.TimeUTC(), + nodeConnection.Host, // Host to the same value of the connection originator node + ProcpeeringOriginName+l.probe.graph.GetOrigin(), // Procpeering creator of the edge + )) + } + + // Show an error if we find more than one listener + if len(nodesListeners) > 1 { + logging.GetLogger().Warningf("node '%+v' connection %v has found more than one listener endpoint: %v", nodeName(nodeConnection), outgoingConn, nodesListeners) + } + } + + return edges +} + +// GetBALinks not used, we only one listener handler for new connections +func (l *simpleLinker) GetBALinks(n *graph.Node) (edges []*graph.Edge) { + return nil +} + +// connectionsEndpointHasher creates an index of outgoing connections +func connectionsEndpointHasher(n *graph.Node) map[string]interface{} { + tcpConnIface, err := n.Metadata.GetField(proccon.MetadataTCPConnKey) + if err != nil { + return nil + } + tcpConn, ok := tcpConnIface.(*proccon.NetworkInfo) + if !ok { + // Ignore invalid values for TCPConn metadata + return map[string]interface{}{} + } + + kv := make(map[string]interface{}, len(*tcpConn)) + for k := range *tcpConn { + // Only create the hash with ip and port, ignore process as it will be different in the other side of the connection + // We need to hash the value to be able to query this indexer later (Get func) + kv[graph.Hash(k)] = k + } + + logging.GetLogger().Debugf("Connection index for node %s: %v", nodeName(n), kv) + + return kv +} + +// listenEndpointHasher creates an index of listeners +func listenEndpointHasher(n *graph.Node) map[string]interface{} { + tcpListenIface, err := n.Metadata.GetField(proccon.MetadataListenEndpointKey) + if err != nil { + return nil + } + + tcpListen, ok := tcpListenIface.(*proccon.NetworkInfo) + if !ok { + // Ignore invalid values for TCPListen metadata + return map[string]interface{}{} + } + + kv := make(map[string]interface{}, len(*tcpListen)) + for k := range *tcpListen { + // Only create the hash with ip and port, ignore process as it will be different in the other side of the connection + // We need to hash the value to be able to query this indexer later (Get func) + kv[graph.Hash(k)] = nil + } + + logging.GetLogger().Debugf("Listen index for node %s: %v", nodeName(n), kv) + return kv +} + +// nodeName return the Metadata.Name value or ID +// Used in loggers and errors to show a representation of the node +func nodeName(n *graph.Node) string { + name, err := n.Metadata.GetFieldString(proccon.MetadataNameKey) + if err == nil { + return name + } + + return string(n.ID) +} + +// NewProbe creates a new graph node peering probe +func NewProbe(g *graph.Graph) (*Probe, error) { + probe := &Probe{ + graph: g, + listenIndexer: graph.NewIndexer(g, g, listenEndpointHasher, false), + connIndexer: graph.NewIndexer(g, g, connectionsEndpointHasher, false), + } + + // Only generate events with connections, not with listeners + probe.linker = graph.NewResourceLinker( + g, + []graph.ListenerHandler{probe.connIndexer}, + nil, + &simpleLinker{probe: probe}, + nil, + ) + + // Notify errors using OnError + probe.linker.AddEventListener(probe) + + // Subscribirnos para obtener eventos de node updated + g.AddEventListener(probe) + + return probe, nil +} diff --git a/topology/probes/procpeering/procpeering_test.go b/topology/probes/procpeering/procpeering_test.go new file mode 100644 index 0000000000..c4a4ffac07 --- /dev/null +++ b/topology/probes/procpeering/procpeering_test.go @@ -0,0 +1,426 @@ +package procpeering + +import ( + "os" + "testing" + + "github.com/skydive-project/skydive/graffiti/filters" + "github.com/skydive-project/skydive/graffiti/graph" + "github.com/skydive-project/skydive/topology/probes/proccon" + "github.com/stretchr/testify/assert" +) + +func TestMain(m *testing.M) { + // Remove comment to change logging level to debug + // logging.InitLogging("id", true, []*logging.LoggerConfig{logging.NewLoggerConfig(logging.NewStdioBackend(os.Stdout), "5", "UTF-8")}) + os.Exit(m.Run()) +} + +func newGraph(t *testing.T) *graph.Graph { + b, err := graph.NewMemoryBackend() + if err != nil { + t.Error(err.Error()) + } + + return graph.NewGraph("testhost", b, "analyzer.testhost") +} + +// TestMatchConnectionListener check if given a Software with a determined listener, if a new Server appears with a connection matching the listener, an edge should be created +func TestMatchConnectionListener(t *testing.T) { + // GIVEN + g := newGraph(t) + p, _ := NewProbe(g) + p.Start() + + softwareServer, err := p.graph.NewNode(graph.GenID(), graph.Metadata{ + proccon.MetadataNameKey: "swServer", + proccon.MetadataTypeKey: proccon.MetadataTypeSoftware, + proccon.MetadataTCPConnKey: &proccon.NetworkInfo{}, + proccon.MetadataListenEndpointKey: &proccon.NetworkInfo{ + "1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + }, + }) + if err != nil { + t.Errorf("Unable to create software server: %v", err) + } + + // WHEN + softwareClient, err := p.graph.NewNode(graph.GenID(), graph.Metadata{ + proccon.MetadataNameKey: "swClient", + proccon.MetadataTypeKey: proccon.MetadataTypeSoftware, + proccon.MetadataTCPConnKey: &proccon.NetworkInfo{ + "1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + }, + }) + if err != nil { + t.Errorf("Unable to create software client: %v", err) + } + + // THEN + hasSoftwareEdgeFilter := graph.NewElementFilter(filters.NewTermStringFilter(proccon.MetadataRelationTypeKey, RelationTypeConnection)) + hasSoftwareEdges := g.GetEdges(hasSoftwareEdgeFilter) + if len(hasSoftwareEdges) == 0 { + t.Fatalf("Edge %s not created", proccon.RelationTypeHasSoftware) + } else if len(hasSoftwareEdges) > 1 { + t.Errorf("Too many edges %s created", proccon.RelationTypeHasSoftware) + } + + hasSoftwareEdge := hasSoftwareEdges[0] + + assert.Equal(t, hasSoftwareEdge.Parent, softwareClient.ID) + assert.Equal(t, hasSoftwareEdge.Child, softwareServer.ID) +} + +// TestMatchConnectionListenerWithPrefixedIPs check edge creation when IPs have a prefix to differentiate private subnets +func TestMatchConnectionListenerWithPrefixedIPs(t *testing.T) { + // GIVEN + g := newGraph(t) + p, _ := NewProbe(g) + p.Start() + + softwareServer, err := p.graph.NewNode(graph.GenID(), graph.Metadata{ + proccon.MetadataNameKey: "swServer", + proccon.MetadataTypeKey: proccon.MetadataTypeSoftware, + proccon.MetadataTCPConnKey: &proccon.NetworkInfo{}, + proccon.MetadataListenEndpointKey: &proccon.NetworkInfo{ + "foobar-1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + }, + }) + if err != nil { + t.Errorf("Unable to create software server: %v", err) + } + + // WHEN + softwareClient, err := p.graph.NewNode(graph.GenID(), graph.Metadata{ + proccon.MetadataNameKey: "swClient", + proccon.MetadataTypeKey: proccon.MetadataTypeSoftware, + proccon.MetadataTCPConnKey: &proccon.NetworkInfo{ + "foobar-1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + }, + }) + if err != nil { + t.Errorf("Unable to create software client: %v", err) + } + + // THEN + hasSoftwareEdgeFilter := graph.NewElementFilter(filters.NewTermStringFilter(proccon.MetadataRelationTypeKey, RelationTypeConnection)) + hasSoftwareEdges := g.GetEdges(hasSoftwareEdgeFilter) + if len(hasSoftwareEdges) == 0 { + t.Fatalf("Edge %s not created", proccon.RelationTypeHasSoftware) + } else if len(hasSoftwareEdges) > 1 { + t.Errorf("Too many edges %s created", proccon.RelationTypeHasSoftware) + } + + hasSoftwareEdge := hasSoftwareEdges[0] + + assert.Equal(t, hasSoftwareEdge.Parent, softwareClient.ID) + assert.Equal(t, hasSoftwareEdge.Child, softwareServer.ID) +} + +// TestNoMatchConnectionListenerWithDifferentPrefixedIPs check edge creation when IPs have a prefix to differentiate private subnets +func TestNoMatchConnectionListenerWithDifferentPrefixedIPs(t *testing.T) { + // GIVEN + g := newGraph(t) + p, _ := NewProbe(g) + p.Start() + + _, err := p.graph.NewNode(graph.GenID(), graph.Metadata{ + proccon.MetadataNameKey: "swServer", + proccon.MetadataTypeKey: proccon.MetadataTypeSoftware, + proccon.MetadataTCPConnKey: &proccon.NetworkInfo{}, + proccon.MetadataListenEndpointKey: &proccon.NetworkInfo{ + "foo-1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + }, + }) + if err != nil { + t.Errorf("Unable to create software server: %v", err) + } + + // WHEN + _, err = p.graph.NewNode(graph.GenID(), graph.Metadata{ + proccon.MetadataNameKey: "swClient", + proccon.MetadataTypeKey: proccon.MetadataTypeSoftware, + proccon.MetadataTCPConnKey: &proccon.NetworkInfo{ + "bar-1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + }, + }) + if err != nil { + t.Errorf("Unable to create software client: %v", err) + } + + // THEN + hasSoftwareEdgeFilter := graph.NewElementFilter(filters.NewTermStringFilter(proccon.MetadataRelationTypeKey, RelationTypeConnection)) + hasSoftwareEdges := g.GetEdges(hasSoftwareEdgeFilter) + assert.Empty(t, hasSoftwareEdges) +} + +// TestMatchConnectionListenerUpdatedNode given a Software node without network info, it is updated to add a determined listener, if a new Server appears with a connection matching the listener, an edge should be created +func TestMatchConnectionListenerUpdatedNode(t *testing.T) { + // GIVEN + g := newGraph(t) + p, _ := NewProbe(g) + p.Start() + + softwareServer, err := p.graph.NewNode(graph.GenID(), graph.Metadata{ + proccon.MetadataNameKey: "swServer", + proccon.MetadataTypeKey: proccon.MetadataTypeSoftware, + }) + if err != nil { + t.Errorf("Unable to create software server: %v", err) + } + + p.graph.AddMetadata(softwareServer, proccon.MetadataListenEndpointKey, &proccon.NetworkInfo{ + "1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + }) + + // WHEN + softwareClient, err := p.graph.NewNode(graph.GenID(), graph.Metadata{ + proccon.MetadataNameKey: "swClient", + proccon.MetadataTypeKey: proccon.MetadataTypeSoftware, + proccon.MetadataTCPConnKey: &proccon.NetworkInfo{ + "1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + }, + }) + if err != nil { + t.Errorf("Unable to create software client: %v", err) + } + + // THEN + hasSoftwareEdgeFilter := graph.NewElementFilter(filters.NewTermStringFilter(proccon.MetadataRelationTypeKey, RelationTypeConnection)) + hasSoftwareEdges := g.GetEdges(hasSoftwareEdgeFilter) + if len(hasSoftwareEdges) == 0 { + t.Fatalf("Edge %s not created", proccon.RelationTypeHasSoftware) + } else if len(hasSoftwareEdges) > 1 { + t.Errorf("Too many edges %s created", proccon.RelationTypeHasSoftware) + } + + hasSoftwareEdge := hasSoftwareEdges[0] + + // Conex from parent (client) to child (server) + assert.Equal(t, hasSoftwareEdge.Parent, softwareClient.ID) + assert.Equal(t, hasSoftwareEdge.Child, softwareServer.ID) +} + +// TestUpdatingNetworkingMetadataDoesNotCreateNewEdges check that updating network metadata does not duplicate edges +func TestUpdatingNetworkingMetadataDoesNotCreateNewEdges(t *testing.T) { + // GIVEN + g := newGraph(t) + p, _ := NewProbe(g) + p.Start() + + softwareServer, err := p.graph.NewNode(graph.GenID(), graph.Metadata{ + proccon.MetadataNameKey: "swServer", + proccon.MetadataTypeKey: proccon.MetadataTypeSoftware, + proccon.MetadataTCPConnKey: &proccon.NetworkInfo{}, + proccon.MetadataListenEndpointKey: &proccon.NetworkInfo{ + "1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + }, + }) + if err != nil { + t.Errorf("Unable to create software server: %v", err) + } + + softwareClient, err := p.graph.NewNode(graph.GenID(), graph.Metadata{ + proccon.MetadataNameKey: "swClient", + proccon.MetadataTypeKey: proccon.MetadataTypeSoftware, + proccon.MetadataTCPConnKey: &proccon.NetworkInfo{ + "1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + }, + }) + if err != nil { + t.Errorf("Unable to create software client: %v", err) + } + + // WHEN + // Once the edge tcp_conn is connected, we modify the network metadata for server and client + p.graph.AddMetadata(softwareServer, proccon.MetadataListenEndpointKey, &proccon.NetworkInfo{ + "1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 1, + Revision: 2, + }, + }) + + p.graph.AddMetadata(softwareClient, proccon.MetadataTCPConnKey, &proccon.NetworkInfo{ + "1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 1, + Revision: 2, + }, + }) + + // THEN + hasSoftwareEdgeFilter := graph.NewElementFilter(filters.NewTermStringFilter(proccon.MetadataRelationTypeKey, RelationTypeConnection)) + hasSoftwareEdges := g.GetEdges(hasSoftwareEdgeFilter) + if len(hasSoftwareEdges) == 0 { + t.Fatalf("Edge %s not created", proccon.RelationTypeHasSoftware) + } else if len(hasSoftwareEdges) > 1 { + t.Errorf("Too many edges %s created", proccon.RelationTypeHasSoftware) + } + + hasSoftwareEdge := hasSoftwareEdges[0] + + assert.Equal(t, hasSoftwareEdge.Parent, softwareClient.ID) + assert.Equal(t, hasSoftwareEdge.Child, softwareServer.ID) +} + +// TestRemovedConnectionFromMetadataDeleteConnectionEdge if we have two nodes connected with a tcp_conn edge, if the connection from the client node is removed, edge should be deleted +func TestRemovedConnectionFromMetadataDeleteConnectionEdge(t *testing.T) { + // GIVEN + g := newGraph(t) + p, _ := NewProbe(g) + p.Start() + + _, err := p.graph.NewNode(graph.GenID(), graph.Metadata{ + proccon.MetadataNameKey: "swServer", + proccon.MetadataTypeKey: proccon.MetadataTypeSoftware, + proccon.MetadataTCPConnKey: &proccon.NetworkInfo{}, + proccon.MetadataListenEndpointKey: &proccon.NetworkInfo{ + "1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + }, + }) + if err != nil { + t.Errorf("Unable to create software server: %v", err) + } + + softwareClient, err := p.graph.NewNode(graph.GenID(), graph.Metadata{ + proccon.MetadataNameKey: "swClient", + proccon.MetadataTypeKey: proccon.MetadataTypeSoftware, + proccon.MetadataTCPConnKey: &proccon.NetworkInfo{ + "1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + }, + }) + if err != nil { + t.Errorf("Unable to create software client: %v", err) + } + + // Check that we have the tcp_conn edge + hasSoftwareEdgeFilter := graph.NewElementFilter(filters.NewTermStringFilter(proccon.MetadataRelationTypeKey, RelationTypeConnection)) + assert.Len(t, g.GetEdges(hasSoftwareEdgeFilter), 1) + + // WHEN + // Once the edge tcp_conn is connected, we remove the connection from the client + p.graph.AddMetadata(softwareClient, proccon.MetadataTCPConnKey, &proccon.NetworkInfo{}) + + // THEN + assert.Empty(t, g.GetEdges(hasSoftwareEdgeFilter)) +} + +// TestRemovedListenerFromMetadataDeleteEdgesFromClients if a server node updates its metadata to delete a listener, all edges from clients should be removed in the next update of clients +func TestRemovedListenerFromMetadataDeleteEdgesFromClients(t *testing.T) { + // GIVEN + g := newGraph(t) + p, _ := NewProbe(g) + p.Start() + + softwareServer, err := p.graph.NewNode(graph.GenID(), graph.Metadata{ + proccon.MetadataNameKey: "swServer", + proccon.MetadataTypeKey: proccon.MetadataTypeSoftware, + proccon.MetadataTCPConnKey: &proccon.NetworkInfo{}, + proccon.MetadataListenEndpointKey: &proccon.NetworkInfo{ + "1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + }, + }) + if err != nil { + t.Errorf("Unable to create software server: %v", err) + } + + // Create numberOfClients clients connected to the server + numberOfClients := 3 + clients := make([]*graph.Node, 3) + + for i := 0; i < numberOfClients; i++ { + c, err := p.graph.NewNode(graph.GenID(), graph.Metadata{ + proccon.MetadataNameKey: "swClient", + proccon.MetadataTypeKey: proccon.MetadataTypeSoftware, + proccon.MetadataTCPConnKey: &proccon.NetworkInfo{ + "1.1.1.1:80": { + CreatedAt: 0, + UpdatedAt: 0, + Revision: 1, + }, + }, + }) + if err != nil { + t.Errorf("Unable to create software client: %v", err) + } + clients[i] = c + } + // Check that we have the three tcp_conn edges + connectionEdgeFilter := graph.NewElementFilter(filters.NewTermStringFilter(proccon.MetadataRelationTypeKey, RelationTypeConnection)) + assert.Len(t, g.GetEdges(connectionEdgeFilter), numberOfClients) + + // WHEN + // Once we have all clients connected, delete the listener + p.graph.AddMetadata(softwareServer, proccon.MetadataListenEndpointKey, &proccon.NetworkInfo{}) + + // Edges will be still there, for simplicity they are only handled when modifications take place in clients + assert.Len(t, g.GetEdges(connectionEdgeFilter), numberOfClients) + + // Simulate a new reception of network data in client + for _, c := range clients { + p.graph.UpdateMetadata(c, proccon.MetadataTCPConnKey, func(field interface{}) bool { + info := *field.(*proccon.NetworkInfo) + for _, v := range info { + v.UpdatedAt = graph.TimeNow().UnixMilli() + v.Revision++ + } + return true + }) + } + + // THEN + assert.Empty(t, g.GetEdges(connectionEdgeFilter)) +} diff --git a/topology/probes/snmplldp/snmp.go b/topology/probes/snmplldp/snmp.go new file mode 100644 index 0000000000..626a8602ae --- /dev/null +++ b/topology/probes/snmplldp/snmp.go @@ -0,0 +1,135 @@ +package snmplldp + +import ( + "fmt" + "strings" + "time" + + snmp "github.com/gosnmp/gosnmp" +) + +type lldpInfo struct { + name string + desc string + chassisID string + remoteTable map[string]*lldpRemote +} + +type lldpRemote struct { + name string + desc string + chassisID string +} + +type lldpObjName int + +const ( + localName lldpObjName = iota + localDesc + localChassisID + remoteName + remoteDesc + remoteChassisID +) + +const noObjMsg = "A required object (or instance) doesn't exist." + +func getLLDPInfo(host string, port uint16, community string) (*lldpInfo, error) { + snmpClient := &snmp.GoSNMP{ + Version: snmp.Version2c, + Timeout: time.Duration(10) * time.Second, + + Target: host, + Port: port, + Community: community, + } + + info := &lldpInfo{} + + err := snmpClient.Connect() + if err != nil { + return info, err + } + defer snmpClient.Conn.Close() + + // Object/OID table + localObjects := map[lldpObjName]string{ + localName: "1.0.8802.1.1.2.1.3.3.0", + localDesc: "1.0.8802.1.1.2.1.3.4.0", + localChassisID: "1.0.8802.1.1.2.1.3.2.0", + } + + // Retrieve local system info + for object, oid := range localObjects { + result, err := snmpClient.Get([]string{oid}) + if err != nil { + return info, err + } + + e := result.Variables[0] + if e.Type == snmp.NoSuchObject || e.Type == snmp.NoSuchInstance { + err = fmt.Errorf("%v OID: %v", noObjMsg, oid) + return info, err + } + + switch object { + case localName: + info.name = string(e.Value.([]byte)) + case localDesc: + info.desc = string(e.Value.([]byte)) + case localChassisID: + hexrep := fmt.Sprintf("% x", e.Value) + info.chassisID = strings.ReplaceAll(hexrep, " ", ":") + } + + } + + info.remoteTable, err = getRemoteTable(snmpClient) + return info, err +} + +func getRemoteTable(client *snmp.GoSNMP) (map[string]*lldpRemote, error) { + // Column (object)/OID table + columns := map[lldpObjName]string{ + remoteName: "1.0.8802.1.1.2.1.4.1.1.9", + remoteDesc: "1.0.8802.1.1.2.1.4.1.1.10", + remoteChassisID: "1.0.8802.1.1.2.1.4.1.1.5", + } + + table := make(map[string]*lldpRemote) + var results []snmp.SnmpPDU + var err error = nil + + for column, oid := range columns { + results, err = client.WalkAll(oid) + if err != nil { + return table, err + } + + // Retrieve the rows (instances) of the column (object) + for _, e := range results { + if e.Type == snmp.NoSuchObject || e.Type == snmp.NoSuchInstance { + err = fmt.Errorf("%v OID: %v", noObjMsg, oid) + return table, err + } + + row := strings.ReplaceAll(e.Name, oid, "") + if table[row] == nil { + table[row] = &lldpRemote{} + } + + switch column { + case remoteName: + table[row].name = string(e.Value.([]byte)) + case remoteDesc: + table[row].desc = string(e.Value.([]byte)) + case remoteChassisID: + hexrep := fmt.Sprintf("% x", e.Value) + table[row].chassisID = strings.ReplaceAll(hexrep, " ", ":") + } + + } + } + + return table, err +} diff --git a/topology/probes/snmplldp/snmplldp.go b/topology/probes/snmplldp/snmplldp.go new file mode 100644 index 0000000000..6770308f0b --- /dev/null +++ b/topology/probes/snmplldp/snmplldp.go @@ -0,0 +1,171 @@ +package snmplldp + +import ( + "strconv" + "strings" + "time" + + "github.com/skydive-project/skydive/config" + "github.com/skydive-project/skydive/graffiti/filters" + "github.com/skydive-project/skydive/graffiti/graph" + "github.com/skydive-project/skydive/graffiti/logging" +) + +type Device struct { + host string + port uint16 + community string +} + +type Probe struct { + graph *graph.Graph + log logging.Logger + quit chan bool + devices []Device +} + +func (p *Probe) buildGraph() { + for _, dev := range p.devices { + lldp, err := getLLDPInfo(dev.host, dev.port, dev.community) + if err != nil { + p.log.Errorf("Obtaining LLDP info from %+v: %v", dev, err) + continue + } + + // Create local device node + localNode, err := p.newNode(lldp.name, lldp.chassisID, lldp.desc) + if err != nil { + p.log.Errorf("Creating a node: %v", err) + continue + } + + // Create remote devices nodes and connect each one to the local + // device with an edge + for _, remote := range lldp.remoteTable { + remoteNode, err := p.newNode(remote.name, remote.chassisID, remote.desc) + if err != nil { + p.log.Errorf("Creating a node: %v", err) + continue + } + + err = p.newEdge(localNode, remoteNode) + if err != nil { + p.log.Errorf("Creating an edge: %v", err) + continue + } + } + } +} + +func (p *Probe) newNode(name string, chassisID string, desc string) (*graph.Node, error) { + p.graph.Lock() + defer p.graph.Unlock() + + filter := graph.NewElementFilter(filters.NewTermStringFilter("ChassisID", chassisID)) + node := p.graph.LookupFirstNode(filter) + var err error = nil + + if node == nil { + m := graph.Metadata{ + "Name": name, + "ChassisID": chassisID, + "Description": desc, + "Type": "host", + } + orig := "snmplldp." + p.graph.GetOrigin() + nodeID := graph.GenID() + + node = graph.CreateNode(nodeID, m, graph.TimeUTC(), name, orig) + + err = p.graph.AddNode(node) + if err == nil { + p.log.Debugf("New Node: %s (%s)", name, chassisID) + } + } + + return node, err +} + +func (p *Probe) newEdge(localNode *graph.Node, remoteNode *graph.Node) error { + var err error = nil + + p.graph.Lock() + defer p.graph.Unlock() + + if !p.graph.AreLinked(localNode, remoteNode, nil) { + m := graph.Metadata{"RelationType": "node"} + orig := "snmplldp." + p.graph.GetOrigin() + edgeID := graph.GenID() + + edge := graph.CreateEdge(edgeID, localNode, remoteNode, m, graph.TimeUTC(), "", orig) + + err = p.graph.AddEdge(edge) + if err == nil { + p.log.Debugf("New Edge: %s - %s", localNode.Host, remoteNode.Host) + } + } + + return err +} + +// Start the probe +func (p *Probe) Start() error { + intervalConfig := config.GetString("analyzer.topology.snmplldp.interval") + interval, err := time.ParseDuration(intervalConfig) + if err != nil { + p.log.Fatalf("Invalid analyzer.topology.snmplldp.interval value: %v", err) + } + + devices := config.GetStringSlice("analyzer.topology.snmplldp.devices") + for _, dev := range devices { + connTriplet := strings.Split(dev, ":") + if len(connTriplet) != 3 { + p.log.Fatalf("Invalid analyzer.topology.snmplldp.devices value") + } + + port, err := strconv.ParseUint(connTriplet[1], 10, 16) + if err != nil { + p.log.Fatalf("Invalid analyzer.topology.snmplldp.devices value: %v", err) + } + + p.devices = append(p.devices, + Device{ + host: connTriplet[0], + port: uint16(port), + community: connTriplet[2], + }) + } + + // First execution + p.buildGraph() + + // Start recurrent execution every interval + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-p.quit: + return + case <-ticker.C: + p.buildGraph() + } + } + }() + + return nil +} + +// Stop the probe +func (p *Probe) Stop() { + p.quit <- true +} + +func NewProbe(g *graph.Graph) (*Probe, error) { + return &Probe{ + graph: g, + log: logging.GetLogger(), + quit: make(chan bool), + }, nil +} diff --git a/validator/validator.go b/validator/validator.go index f922708a1d..f721939b64 100644 --- a/validator/validator.go +++ b/validator/validator.go @@ -176,6 +176,7 @@ func isGremlinExpr(v interface{}, param string) error { tr.AddTraversalExtension(ge.NewSocketsTraversalExtension()) tr.AddTraversalExtension(ge.NewRawPacketsTraversalExtension()) tr.AddTraversalExtension(ge.NewDescendantsTraversalExtension()) + tr.AddTraversalExtension(ge.NewAscendantsTraversalExtension()) tr.AddTraversalExtension(ge.NewNextHopTraversalExtension()) tr.AddTraversalExtension(ge.NewGroupTraversalExtension())