Develop (#112)

* DEV-39965: Allow the user to specify a parent group during installation

* DEV-39965: Allow the user to specify a parent group during installation

* DEV-40505: Sync the k8s resource to santaba when argus launches

* DEV-40434: The alert status of Pods and Services are disabled

* DEV-40505: Sync the k8s resource to santaba when argus launches

* DEV-40505: Sync the k8s resource to santaba when argus launches

* DEV-40779: Enable the alert on service group when it is created by argus

* DEV-41432: Use IP as hostname for argus related devices

* DEV-41432: Use IP as hostname for argus related devices

* DEV-41459: Add a new device type for k8s devices

* DEV-41432: Use IP as hostname for argus related devices

* DEV-40217: Support to set log levels and improve logs in argus related projects

* DEV-40217: Support to set log levels and improve logs in argus related projects

* DEV-41682: Improve the argus code for the CI failure in GitHub

* Dev-40217 support to set log levels and improve

Dev-40217 support to set log levels and improve

* DEV-41947: Improve the initsync logic to prevent lost data after k8s device is updated

* DEV-41947: Improve the initsync logic to prevent lost data after k8s device is updated

* DEV-42060: Don't create collector device for Kubernetes Clusters

* DEV-47062 Add distinguishing property to host network pods

* DEV-49046 Fix the internal IP cannot found bug in argus

* DEV-49046 Fix the internal IP cannot found bug in argus

* Update node_test.go

* DEV-48974 Upgrade Argus from golang 1.9 to 1.11

* DEV-48974 Upgrade Argus from golang 1.9 to 1.11

* DEV-49251 improve panic log process

* DEV-50734 Support all service type, and add the logs

* DEV-41301 Upgrade the go sdk to v2

* DEV-41301 Upgrade the go sdk to v2

* DEV-41301 Upgrade the go sdk to v2

* DEV-41301 Upgrade the go sdk to v2

* DEV-50734 Support all service type, modify the UpdateFunc

* DEV-41301 Upgrade the go sdk to v2

* DEV-50929 Add k8s deployments monitoring by Argus

* DEV-49251 Add the defer to initSync and improve log output

* DEV-41301 Upgrade the go sdk to v2

* DEV-50929 Improve the code with comments

* DEV-50734 Improve the log message

* DEV-49251 Improve the log message

* DEV-49251 Improve the comments

* DEV-51405 Fix the argus log format

* DEV-51567 Fixed the label value of k8s monitored

* DEV-49675 Synchronous develop branch, add the recover panic logic for deployments

* Modify the format by Travis CI check

* DEV-51677 Improve the code according to github comments

* Improve the variable naming

* DEV-51932 fix uptime widget for k8s resources

* DEV-49675 Restore the dockerfile

* DEV-52508 Fixed the CI check

* DEV-52508 Argus deployment rbac privilege improve

* DEV-52508 Argus deployment rbac privilege improve

* DEV-52508 Argus deployment rbac privilege compatibility improvement

* DEV-52508 Argus deployment rbac privilege compatibility improvement

* DEV-52508 Argus deployment rbac privilege compatibility improvement

* DEV-52036 Allow for duplicate pod names in one LM account

* DEV-52508 Improve by the comments

* DEV-52508 Fix the comments

* DEV-52508 Fix the comments

* remove logo link

* DEV-52036 Allow for duplicate pod names in one LM account

* DEV-52036 Allow for duplicate pod names in one LM account

* DEV-52036 Allow for duplicate pod names in one LM account

* DEV-52036 Fix the comments

* DEV-52036 Fix the comments

* DEV-52991 Add UT

Author: DzXiaoLMCD

pkg/constants/constants.go

@@ -84,6 +84,8 @@ const (
 const (
 	// K8sClusterNamePropertyKey is the key of the unique auto property kubernetes cluster name
 	K8sClusterNamePropertyKey = "auto.clustername"
+	// K8sResourceNamePropertyKey is the key of the custom property used to record resource name
+	K8sResourceNamePropertyKey = "auto.resourcename"
 	// K8sResourceCreatedOnPropertyKey is the key of the custom property used to record resource create timestamp
 	K8sResourceCreatedOnPropertyKey = "kubernetes.resourceCreatedOn"
 	// K8sDeviceType is the type value of the k8s device

pkg/device/device.go

@@ -3,6 +3,7 @@ package device
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/logicmonitor/k8s-argus/pkg/config"
 	"github.com/logicmonitor/k8s-argus/pkg/constants"
@@ -55,40 +56,92 @@ func buildDevice(c *config.Config, client api.CollectorSetControllerClient, opti
 
 // checkAndUpdateExistingDevice tries to find and update the devices which needs to be changed
 func (m *Manager) checkAndUpdateExistingDevice(device *models.Device) (*models.Device, error) {
-	oldDevice, err := m.FindByDisplayName(*device.DisplayName)
+	displayNameWithClusterName := fmt.Sprintf("%s-%s", *device.DisplayName, m.Config().ClusterName)
+	existingDevices, err := m.FindByDisplayNames(*device.DisplayName, displayNameWithClusterName)
 	if err != nil {
 		return nil, err
 	}
-	if oldDevice == nil {
-		return nil, fmt.Errorf("can not find the device: %s", *device.DisplayName)
+	if len(existingDevices) == 0 {
+		return nil, fmt.Errorf("cannot find devices with name: %s", *device.DisplayName)
 	}
-
-	// the device which is not changed will be ignored
-	if *device.Name == *oldDevice.Name {
-		log.Infof("No changes to device (%s). Ignoring update", *device.DisplayName)
-		return device, nil
-	}
-
-	// the device of the other cluster will be ignored
-	oldClusterName := ""
-	if oldDevice.CustomProperties != nil && len(oldDevice.CustomProperties) > 0 {
-		for _, cp := range oldDevice.CustomProperties {
-			if *cp.Name == constants.K8sClusterNamePropertyKey {
-				oldClusterName = *cp.Value
+	for _, existingDevice := range existingDevices {
+		clusterName := m.GetPropertyValue(existingDevice, constants.K8sClusterNamePropertyKey)
+		if clusterName == m.Config().ClusterName {
+			// the device which is not changed will be ignored
+			if *existingDevice.Name == *device.Name {
+				log.Infof("No changes to device (%s). Ignoring update", *device.DisplayName)
+				return device, nil
 			}
+			// the clusterName is the same and hostName is not the same, need update
+			*device.DisplayName = *existingDevice.DisplayName
+			newDevice, err2 := m.updateAndReplace(existingDevice.ID, device)
+			if err2 != nil {
+				return nil, err2
+			}
+			log.Infof("Updating existing device (%s)", *newDevice.DisplayName)
+			return newDevice, nil
 		}
 	}
-	if oldClusterName != m.Config().ClusterName {
-		log.Infof("Device (%s) belongs to a different cluster (%s). Ignoring update", *device.DisplayName, oldClusterName)
-		return device, nil
+	// duplicate device exists. update displayName and re-add
+	renamedDevice, err := m.renameAndAddDevice(device)
+	if err != nil {
+		log.Errorf("rename device failed: %v", err)
+		return nil, fmt.Errorf("rename device failed")
 	}
+	return renamedDevice, nil
+}
 
-	newDevice, err := m.updateAndReplace(oldDevice.ID, device)
+// renameAndAddDevice rename display name and then add the device
+func (m *Manager) renameAndAddDevice(device *models.Device) (*models.Device, error) {
+	resourceName := m.GetPropertyValue(device, constants.K8sResourceNamePropertyKey)
+	if resourceName == "" {
+		resourceName = *device.DisplayName
+	}
+	renameResourceName := fmt.Sprintf("%s-%s", resourceName, m.Config().ClusterName)
+	existingDevice, err := m.FindByDisplayName(renameResourceName)
+	if err != nil {
+		log.Warnf("Get device(%s) failed, err: %v", resourceName, err)
+	}
+	if existingDevice != nil {
+		if m.Config().ClusterName == m.GetPropertyValue(existingDevice, constants.K8sClusterNamePropertyKey) {
+			device.DisplayName = existingDevice.DisplayName
+			return m.updateAndReplace(existingDevice.ID, device)
+		}
+		return nil, fmt.Errorf("exist displayName: %s", renameResourceName)
+	}
+	log.Infof("Rename device: %s -> %s", *device.DisplayName, renameResourceName)
+	device.DisplayName = &renameResourceName
+	params := lm.NewAddDeviceParams()
+	addFromWizard := false
+	params.SetAddFromWizard(&addFromWizard)
+	params.SetBody(device)
+	restResponse, err := m.LMClient.LM.AddDevice(params)
 	if err != nil {
 		return nil, err
 	}
-	log.Infof("Finished updating the device: %s", *newDevice.DisplayName)
-	return newDevice, nil
+	return restResponse.Payload, nil
+}
+
+// GetPropertyValue get device property value by property name
+func (m *Manager) GetPropertyValue(device *models.Device, propertyName string) string {
+	if device == nil {
+		return ""
+	}
+	if len(device.CustomProperties) > 0 {
+		for _, cp := range device.CustomProperties {
+			if *cp.Name == propertyName {
+				return *cp.Value
+			}
+		}
+	}
+	if len(device.SystemProperties) > 0 {
+		for _, cp := range device.SystemProperties {
+			if *cp.Name == propertyName {
+				return *cp.Value
+			}
+		}
+	}
+	return ""
 }
 
 func (m *Manager) updateAndReplace(id int32, device *models.Device) (*models.Device, error) {
@@ -124,6 +177,37 @@ func (m *Manager) FindByDisplayName(name string) (*models.Device, error) {
 	return nil, nil
 }
 
+// FindByDisplayNames implements types.DeviceManager.
+func (m *Manager) FindByDisplayNames(displayNames ...string) ([]*models.Device, error) {
+	if len(displayNames) == 0 {
+		return []*models.Device{}, nil
+	}
+	filter := fmt.Sprintf("displayName:\"%s\"", strings.Join(displayNames, "\"|\""))
+	params := lm.NewGetDeviceListParams()
+	params.SetFilter(&filter)
+	restResponse, err := m.LMClient.LM.GetDeviceList(params)
+	if err != nil {
+		return nil, err
+	}
+	log.Debugf("%#v", restResponse)
+	return restResponse.Payload.Items, nil
+}
+
+// FindByDisplayNameAndClusterName implements types.DeviceManager.
+func (m *Manager) FindByDisplayNameAndClusterName(displayName string) (*models.Device, error) {
+	displayNameWithClusterName := fmt.Sprintf("%s-%s", displayName, m.Config().ClusterName)
+	devices, err := m.FindByDisplayNames(displayName, displayNameWithClusterName)
+	if err != nil {
+		return nil, err
+	}
+	for _, device := range devices {
+		if m.Config().ClusterName == m.GetPropertyValue(device, constants.K8sClusterNamePropertyKey) {
+			return device, nil
+		}
+	}
+	return nil, nil
+}
+
 // Add implements types.DeviceManager.
 func (m *Manager) Add(options ...types.DeviceOption) (*models.Device, error) {
 	device := buildDevice(m.Config(), m.ControllerClient, options...)
@@ -152,7 +236,6 @@ func (m *Manager) Add(options ...types.DeviceOption) (*models.Device, error) {
 		return nil, err
 	}
 	log.Debugf("%#v", restResponse)
-
 	return restResponse.Payload, nil
 }
 
@@ -166,7 +249,7 @@ func (m *Manager) UpdateAndReplaceByID(id int32, options ...types.DeviceOption)
 
 // UpdateAndReplaceByDisplayName implements types.DeviceManager.
 func (m *Manager) UpdateAndReplaceByDisplayName(name string, options ...types.DeviceOption) (*models.Device, error) {
-	d, err := m.FindByDisplayName(name)
+	d, err := m.FindByDisplayNameAndClusterName(name)
 	if err != nil {
 		return nil, err
 	}
@@ -175,7 +258,7 @@ func (m *Manager) UpdateAndReplaceByDisplayName(name string, options ...types.De
 		log.Warnf("Could not find device %q", name)
 		return nil, nil
 	}
-
+	options = append(options, m.DisplayName(*d.DisplayName))
 	// Update the device.
 	device, err := m.UpdateAndReplaceByID(d.ID, options...)
 	if err != nil {
@@ -210,7 +293,7 @@ func (m *Manager) UpdateAndReplaceFieldByID(id int32, field string, options ...t
 
 // UpdateAndReplaceFieldByDisplayName implements types.DeviceManager.
 func (m *Manager) UpdateAndReplaceFieldByDisplayName(name string, field string, options ...types.DeviceOption) (*models.Device, error) {
-	d, err := m.FindByDisplayName(name)
+	d, err := m.FindByDisplayNameAndClusterName(name)
 	if err != nil {
 		return nil, err
 	}
@@ -219,7 +302,7 @@ func (m *Manager) UpdateAndReplaceFieldByDisplayName(name string, field string,
 		log.Infof("Could not find device %q", name)
 		return nil, nil
 	}
-
+	options = append(options, m.DisplayName(*d.DisplayName))
 	// Update the device.
 	device, err := m.UpdateAndReplaceFieldByID(d.ID, field, options...)
 	if err != nil {
@@ -239,7 +322,7 @@ func (m *Manager) DeleteByID(id int32) error {
 
 // DeleteByDisplayName implements types.DeviceManager.
 func (m *Manager) DeleteByDisplayName(name string) error {
-	d, err := m.FindByDisplayName(name)
+	d, err := m.FindByDisplayNameAndClusterName(name)
 	if err != nil {
 		return err
 	}

pkg/device/device_test.go

@@ -0,0 +1,52 @@
+package device
+
+import (
+	"testing"
+
+	"github.com/logicmonitor/lm-sdk-go/models"
+)
+
+func TestGetPropertyValue(t *testing.T) {
+	deviceName := "test-device"
+
+	customPropertiesName1 := "name1"
+	customPropertiesValue1 := "value1"
+	customPropertiesName2 := "name2"
+	customPropertiesValue2 := "value2"
+
+	systemPropertiesName1 := "system-name1"
+	systemPropertiesValue1 := "system-value1"
+	systemPropertiesName2 := "system-name2"
+	systemPropertiesValue2 := "system-value2"
+
+	device := &models.Device{
+		Name:        &deviceName,
+		DisplayName: &deviceName,
+		CustomProperties: []*models.NameAndValue{
+			{
+				Name:  &customPropertiesName1,
+				Value: &customPropertiesValue1,
+			}, {
+				Name:  &customPropertiesName2,
+				Value: &customPropertiesValue2,
+			},
+		},
+		SystemProperties: []*models.NameAndValue{
+			{
+				Name:  &systemPropertiesName1,
+				Value: &systemPropertiesValue1,
+			}, {
+				Name:  &systemPropertiesName2,
+				Value: &systemPropertiesValue2,
+			},
+		},
+	}
+
+	manage := Manager{}
+	value := manage.GetPropertyValue(device, customPropertiesName1)
+	t.Logf("name=%s, value=%s", customPropertiesName1, value)
+	value = manage.GetPropertyValue(device, systemPropertiesName2)
+	t.Logf("name=%s, value=%s", systemPropertiesName2, value)
+	value = manage.GetPropertyValue(device, "non-exist-name")
+	t.Logf("name=%s, value=%s", "non-exist-name", value)
+}

pkg/sync/initsyncer.go

@@ -158,21 +158,19 @@ func (i *InitSyncer) syncDevices(resourceType string, resourcesMap map[string]st
 	for _, device := range devices {
 		// the "auto.clustername" property checking is used to prevent unexpected deletion of the normal non-k8s device
 		// which may be assigned to the cluster group
-		cps := device.CustomProperties
-		autoClusterName := ""
-		for _, cp := range cps {
-			if *cp.Name == constants.K8sClusterNamePropertyKey {
-				autoClusterName = *cp.Value
-				break
-			}
-		}
+		autoClusterName := i.DeviceManager.GetPropertyValue(device, constants.K8sClusterNamePropertyKey)
 		if autoClusterName != i.DeviceManager.Config().ClusterName {
 			log.Infof("Ignore the device (%v) which does not have property %v:%v",
 				*device.DisplayName, constants.K8sClusterNamePropertyKey, i.DeviceManager.Config().ClusterName)
 			continue
 		}
-
-		_, exist := resourcesMap[*device.DisplayName]
+		// the displayName may be renamed, we should use the constants.K8sResourceNamePropertyKey property value
+		resourceName := i.DeviceManager.GetPropertyValue(device, constants.K8sResourceNamePropertyKey)
+		// for compatibility, if resourceName is empty, use display name
+		if resourceName == "" {
+			resourceName = *device.DisplayName
+		}
+		_, exist := resourcesMap[resourceName]
 		if !exist {
 			log.Infof("Delete the non-exist %v device: %v", resourceType, *device.DisplayName)
 			err := i.DeviceManager.DeleteByID(device.ID)

pkg/types/types.go

@@ -41,6 +41,10 @@ type DeviceMapper interface {
 	// FindByDisplayName searches for a device by it's display name. It will return a device if and only if
 	// one device was found, and return nil otherwise.
 	FindByDisplayName(string) (*models.Device, error)
+	// FindByDisplayNames searches for devices by the specified string by its display name. It will return the device list.
+	FindByDisplayNames(...string) ([]*models.Device, error)
+	// FindByDisplayNameAndClusterName searches for device by the specified string by its display name and clusterName. It will return a device if and only if
+	FindByDisplayNameAndClusterName(string) (*models.Device, error)
 	// Add adds a device to a LogicMonitor account.
 	Add(...DeviceOption) (*models.Device, error)
 	// UpdateAndReplaceByID updates a device using the 'replace' OpType.

pkg/watch/deployment/deployment.go

@@ -127,6 +127,7 @@ func (w *Watcher) args(deployment *v1beta2.Deployment, category string) []types.
 		w.Auto("selflink", deployment.SelfLink),
 		w.Auto("uid", string(deployment.UID)),
 		w.Custom(constants.K8sResourceCreatedOnPropertyKey, strconv.FormatInt(deployment.CreationTimestamp.Unix(), 10)),
+		w.Custom(constants.K8sResourceNamePropertyKey, fmtDeploymentDisplayName(deployment)),
 	}
 }
 

pkg/watch/node/node.go

@@ -157,6 +157,7 @@ func (w *Watcher) args(node *v1.Node, category string) []types.DeviceOption {
 		w.Auto("selflink", node.SelfLink),
 		w.Auto("uid", string(node.UID)),
 		w.Custom(constants.K8sResourceCreatedOnPropertyKey, strconv.FormatInt(node.CreationTimestamp.Unix(), 10)),
+		w.Custom(constants.K8sResourceNamePropertyKey, node.Name),
 	}
 }
 

pkg/watch/pod/pod.go

@@ -157,6 +157,7 @@ func (w *Watcher) args(pod *v1.Pod, category string) []types.DeviceOption {
 		w.Auto("uid", string(pod.UID)),
 		w.System("ips", pod.Status.PodIP),
 		w.Custom(constants.K8sResourceCreatedOnPropertyKey, strconv.FormatInt(pod.CreationTimestamp.Unix(), 10)),
+		w.Custom(constants.K8sResourceNamePropertyKey, pod.Name),
 	}
 	if pod.Spec.HostNetwork {
 		options = append(options, w.Custom("kubernetes.pod.hostNetwork", "true"))

pkg/watch/service/service.go

@@ -143,6 +143,7 @@ func (w *Watcher) args(service *v1.Service, category string) []types.DeviceOptio
 		w.Auto("selflink", service.SelfLink),
 		w.Auto("uid", string(service.UID)),
 		w.Custom(constants.K8sResourceCreatedOnPropertyKey, strconv.FormatInt(service.CreationTimestamp.Unix(), 10)),
+		w.Custom(constants.K8sResourceNamePropertyKey, fmtServiceDisplayName(service)),
 	}
 }