diff --git a/pkg/cloudprovider/providers/oci/instances.go b/pkg/cloudprovider/providers/oci/instances.go index 106a8f26cf..07bdfc5fb0 100644 --- a/pkg/cloudprovider/providers/oci/instances.go +++ b/pkg/cloudprovider/providers/oci/instances.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "net" + "strings" "github.com/oracle/oci-go-sdk/v65/core" "k8s.io/apimachinery/pkg/labels" @@ -29,6 +30,12 @@ import ( cloudprovider "k8s.io/cloud-provider" ) +const ( + OpenShiftTagNamesapcePrefix = "openshift-" + OpenShiftBootVolumeType = "boot-volume-type" + OpenShiftBootVolumeISCSI = "ISCSI" +) + var _ cloudprovider.Instances = &CloudProvider{} // mapNodeNameToInstanceName maps a kube NodeName to a OCI instance display @@ -97,6 +104,38 @@ func (cp *CloudProvider) extractNodeAddresses(ctx context.Context, instanceID st addresses = append(addresses, api.NodeAddress{Type: api.NodeExternalIP, Address: ip.String()}) } + OpenShiftTagNamesapce := cp.getOpenShiftTagNamespaceByInstance(ctx, instanceID) + + if OpenShiftTagNamesapce != "" { + secondaryVnics, err := cp.client.Compute().GetSecondaryVNICsForInstance(ctx, compartmentID, instanceID) + if err != nil { + return nil, err + } + + if secondaryVnics == nil || len(secondaryVnics) == 0 { + return addresses, nil + } + for _, secondaryVnic := range secondaryVnics { + if cp.checkOpenShiftISCSIBootVolumeTagByVnic(ctx, secondaryVnic, OpenShiftTagNamesapce) { + if (secondaryVnic.IsPrimary == nil || !*secondaryVnic.IsPrimary) && secondaryVnic.PrivateIp != nil && *secondaryVnic.PrivateIp != "" { + ip := net.ParseIP(*secondaryVnic.PrivateIp) + if ip == nil { + return nil, fmt.Errorf("instance has invalid private address: %q", *secondaryVnic.PrivateIp) + } + addresses = append(addresses, api.NodeAddress{Type: api.NodeInternalIP, Address: ip.String()}) + } + + if (secondaryVnic.IsPrimary == nil || !*secondaryVnic.IsPrimary) && secondaryVnic.PublicIp != nil && *secondaryVnic.PublicIp != "" { + ip := net.ParseIP(*secondaryVnic.PublicIp) + if ip == nil { + return nil, errors.Errorf("instance has invalid public address: %q", *secondaryVnic.PublicIp) + } + addresses = append(addresses, api.NodeAddress{Type: api.NodeExternalIP, Address: ip.String()}) + } + } + } + } + // Changing this can have wide reaching impact. // // if vnic.HostnameLabel != nil && *vnic.HostnameLabel != "" { @@ -312,3 +351,35 @@ func (cp *CloudProvider) getCompartmentIDByNodeName(nodeName string) (string, er cp.logger.Debug("CompartmentID annotation is not present") return "", errors.New("compartmentID annotation missing in the node. Would retry") } + +func (cp *CloudProvider) getOpenShiftTagNamespaceByInstance(ctx context.Context, instanceID string) string { + instance, err := cp.client.Compute().GetInstance(ctx, instanceID) + if err != nil { + return "" + } + + if instance.DefinedTags == nil { + return "" + } + + for namespace := range instance.DefinedTags { + if strings.HasPrefix(namespace, OpenShiftTagNamesapcePrefix) { + return namespace + } + } + return "" +} + +func (cp *CloudProvider) checkOpenShiftISCSIBootVolumeTagByVnic(ctx context.Context, vnic *core.Vnic, namespace string) bool { + if vnic.DefinedTags == nil { + return false + } + + if tags, namespaceExists := vnic.DefinedTags[namespace]; namespaceExists { + // Check if the boot volume type key exists and its value is ISCSI + if bootVolume, keyExists := tags[OpenShiftBootVolumeType]; keyExists && bootVolume == OpenShiftBootVolumeISCSI { + return true + } + } + return false +} diff --git a/pkg/cloudprovider/providers/oci/instances_test.go b/pkg/cloudprovider/providers/oci/instances_test.go index 2e5f62fefa..5bb7e73569 100644 --- a/pkg/cloudprovider/providers/oci/instances_test.go +++ b/pkg/cloudprovider/providers/oci/instances_test.go @@ -344,6 +344,10 @@ func (MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compartm return instanceVnics[instanceID], nil } +func (c *MockComputeClient) GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error) { + return []*core.Vnic{instanceVnics[instanceID]}, nil +} + func (MockComputeClient) FindVolumeAttachment(ctx context.Context, compartmentID, volumeID string) (core.VolumeAttachment, error) { return nil, nil } diff --git a/pkg/csi/driver/bv_controller_test.go b/pkg/csi/driver/bv_controller_test.go index a5b9011d9c..4ade3b9e0e 100644 --- a/pkg/csi/driver/bv_controller_test.go +++ b/pkg/csi/driver/bv_controller_test.go @@ -568,6 +568,10 @@ func (c *MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compa return nil, nil } +func (c *MockComputeClient) GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error) { + return nil, nil +} + func (c *MockComputeClient) FindVolumeAttachment(ctx context.Context, compartmentID, volumeID string) (core.VolumeAttachment, error) { var page *string var requestMetadata common.RequestMetadata diff --git a/pkg/oci/client/compute.go b/pkg/oci/client/compute.go index e5f5b96c63..49bc7afc70 100644 --- a/pkg/oci/client/compute.go +++ b/pkg/oci/client/compute.go @@ -16,6 +16,7 @@ package client import ( "context" + "log" "strings" "github.com/oracle/oci-go-sdk/v65/core" @@ -34,6 +35,8 @@ type ComputeInterface interface { GetPrimaryVNICForInstance(ctx context.Context, compartmentID, instanceID string) (*core.Vnic, error) + GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error) + VolumeAttachmentInterface } @@ -182,6 +185,55 @@ func (c *client) GetPrimaryVNICForInstance(ctx context.Context, compartmentID, i return nil, errors.WithStack(errNotFound) } +func (c *client) GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error) { + logger := c.logger.With("instanceID", instanceID, "compartmentID", compartmentID) + secondaryVnics := []*core.Vnic{} + var page *string + for { + resp, err := c.listVNICAttachments(ctx, core.ListVnicAttachmentsRequest{ + InstanceId: &instanceID, + CompartmentId: &compartmentID, + Page: page, + RequestMetadata: c.requestMetadata, + }) + + if err != nil { + return nil, err + } + + for _, attachment := range resp.Items { + if attachment.LifecycleState != core.VnicAttachmentLifecycleStateAttached { + logger.With("vnicAttachmentID", *attachment.Id).Info("VNIC attachment is not in attached state") + log.Println(logger) + continue + } + + if attachment.VnicId == nil { + // Should never happen but lets be extra cautious as field is non-mandatory in OCI API. + logger.With("vnicAttachmentID", *attachment.Id).Error("VNIC attachment is attached but has no VNIC ID") + log.Println(logger) + continue + } + + // TODO(apryde): Cache map[instanceID]primaryVNICID. + vnic, err := c.GetVNIC(ctx, *attachment.VnicId) + if err != nil { + return nil, err + } + + if !*vnic.IsPrimary { + secondaryVnics = append(secondaryVnics, vnic) + } + } + + if page = resp.OpcNextPage; resp.OpcNextPage == nil { + break + } + } + + return secondaryVnics, nil +} + func (c *client) GetInstanceByNodeName(ctx context.Context, compartmentID, vcnID, nodeName string) (*core.Instance, error) { // First try lookup by display name. instance, err := c.getInstanceByDisplayName(ctx, compartmentID, nodeName) diff --git a/pkg/volume/provisioner/block/block_test.go b/pkg/volume/provisioner/block/block_test.go index 6dff0d4995..54901cc8ba 100644 --- a/pkg/volume/provisioner/block/block_test.go +++ b/pkg/volume/provisioner/block/block_test.go @@ -246,6 +246,10 @@ func (c *MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compa return nil, nil } +func (c *MockComputeClient) GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error) { + return nil, nil +} + func (c *MockComputeClient) FindVolumeAttachment(ctx context.Context, compartmentID, volumeID string) (core.VolumeAttachment, error) { return nil, nil } diff --git a/pkg/volume/provisioner/fss/fss_test.go b/pkg/volume/provisioner/fss/fss_test.go index 23eb97bf67..eacb08b718 100644 --- a/pkg/volume/provisioner/fss/fss_test.go +++ b/pkg/volume/provisioner/fss/fss_test.go @@ -244,6 +244,10 @@ func (c *MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compa return nil, nil } +func (c *MockComputeClient) GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error) { + return nil, nil +} + func (c *MockComputeClient) FindVolumeAttachment(ctx context.Context, compartmentID, volumeID string) (core.VolumeAttachment, error) { return nil, nil }