Skip to content

Commit f8d8a9c

Browse files
Pearl1594nvazquezrohityadavcloud
authored
NSX Integration fixes (#8906)
* Prevent addition of duplicate PF rules on scale up and no rules left behind on scale down (#32) * fix missing dependency injection * NSX: Fix concurrency issues on port forwarding rules deletion (#37) * Fix concurrency issues on port forwarding rules deletion * Refactor objectExists * Fix unit test * Fix test * Small fixes * CKS: Externalize control and worker node setup wait time and installation attempts (#38) * NSX: Add shared network support (#41) * NSX: Fix number of physical networks for Guest traffic checks and leftover rules on CKS cluster deletion (#45) * Fix pf rules removal on CKS cluster deletion * Fix check for number of physical networks for guest traffic * Fix unit test * fix logger * NSX: Handle CheckHealthCommand to avoid host disconnection and errors on APIs * NSX: Handle CheckHealthCommand to avoid host disconnection and errors on APIs * Remove unused string * fix logger * Update UDP active monitor to ICMP * Fix NPE on restarting VPC with additional public IPs * NSX / VPC: Reuse Source NAT IP from systemVM range on restarts * CKS: Public IP not found for VPC networks * Externalize retries and inverval for NSX segment deletion (#67) * remove unused import * remove duplicate imports * remove unused import * revert externalizing cks settings * fix test * Refactor log messages * Address comments * Fix issue caused due to forward merge: 90fe1d --------- Co-authored-by: Nicolas Vazquez <[email protected]> Co-authored-by: Rohit Yadav <[email protected]>
1 parent f156c4e commit f8d8a9c

File tree

24 files changed

+338
-140
lines changed

24 files changed

+338
-140
lines changed

‎api/src/main/java/com/cloud/network/nsx/NsxService.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,19 @@
1818

1919
import com.cloud.network.IpAddress;
2020
import com.cloud.network.vpc.Vpc;
21+
import org.apache.cloudstack.framework.config.ConfigKey;
2122

2223
public interface NsxService {
2324

25+
ConfigKey<Integer> NSX_API_FAILURE_RETRIES = new ConfigKey<>("Advanced", Integer.class,
26+
"nsx.api.failure.retries", "30",
27+
"Number of retries for NSX API operations in case of failures",
28+
true, ConfigKey.Scope.Zone);
29+
ConfigKey<Integer> NSX_API_FAILURE_INTERVAL = new ConfigKey<>("Advanced", Integer.class,
30+
"nsx.api.failure.interval", "60",
31+
"Waiting time (in seconds) before retrying an NSX API operation in case of failure",
32+
true, ConfigKey.Scope.Zone);
33+
2434
boolean createVpcNetwork(Long zoneId, long accountId, long domainId, Long vpcId, String vpcName, boolean sourceNatEnabled);
2535
boolean updateVpcSourceNatIp(Vpc vpc, IpAddress address);
2636
}

‎engine/components-api/src/main/java/com/cloud/network/vpc/VpcManager.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,15 +115,16 @@ public interface VpcManager {
115115
throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException;
116116

117117
/**
118-
* Assigns source nat public IP address to VPC
118+
* Assigns source nat public IP address to VPC.
119+
* In case of NSX backed VPCs: CloudStack deploys VRs with Public NIC IP different to the VPC source NAT IP, the source NAT IP is on the NSX Public range
119120
*
120121
* @param owner
121122
* @param vpc
122123
* @return public IP address object
123124
* @throws InsufficientAddressCapacityException
124125
* @throws ConcurrentOperationException
125126
*/
126-
PublicIp assignSourceNatIpAddressToVpc(Account owner, Vpc vpc) throws InsufficientAddressCapacityException, ConcurrentOperationException;
127+
PublicIp assignSourceNatIpAddressToVpc(Account owner, Vpc vpc, Long podId) throws InsufficientAddressCapacityException, ConcurrentOperationException;
127128

128129
/**
129130
* Validates network offering to find if it can be used for network creation in VPC

‎engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1645,7 +1645,7 @@ public void implementNetworkElementsAndResources(final DeployDestination dest, f
16451645
if (ips.isEmpty()) {
16461646
final Vpc vpc = _vpcMgr.getActiveVpc(network.getVpcId());
16471647
logger.debug("Creating a source nat ip for vpc {}", vpc);
1648-
_vpcMgr.assignSourceNatIpAddressToVpc(owner, vpc);
1648+
_vpcMgr.assignSourceNatIpAddressToVpc(owner, vpc, null);
16491649
}
16501650
} else {
16511651
ips = _ipAddressDao.listByAssociatedNetwork(network.getId(), true);

‎engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/FirewallRuleDetailVO.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,4 +79,8 @@ public long getResourceId() {
7979
public boolean isDisplay() {
8080
return display;
8181
}
82+
83+
public void setValue(String value) {
84+
this.value = value;
85+
}
8286
}

‎plugins/affinity-group-processors/host-affinity/src/main/java/org/apache/cloudstack/affinity/HostAffinityProcessor.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,9 @@ public void process(VirtualMachineProfile vmProfile, DeploymentPlan plan, Exclud
6363
Transaction.execute(new TransactionCallbackNoReturn() {
6464
@Override
6565
public void doInTransactionWithoutResult(TransactionStatus status) {
66-
_affinityGroupDao.listByIds(affinityGroupIdList, true);
66+
if (!affinityGroupIdList.isEmpty()) {
67+
_affinityGroupDao.listByIds(affinityGroupIdList, true);
68+
}
6769
for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
6870
processAffinityGroup(vmGroupMapping, plan, vm, vmList);
6971
}
@@ -149,7 +151,9 @@ public boolean check(VirtualMachineProfile vmProfile, DeployDestination plannedD
149151
return Transaction.execute(new TransactionCallback<Boolean>() {
150152
@Override
151153
public Boolean doInTransaction(TransactionStatus status) {
152-
_affinityGroupDao.listByIds(affinityGroupIds, true);
154+
if (!affinityGroupIds.isEmpty()) {
155+
_affinityGroupDao.listByIds(affinityGroupIds, true);
156+
}
153157
for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
154158
if (!checkAffinityGroup(vmGroupMapping, vm, plannedHostId)) {
155159
return false;

‎plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,9 @@ public void process(VirtualMachineProfile vmProfile, DeploymentPlan plan, Exclud
7878
Transaction.execute(new TransactionCallbackNoReturn() {
7979
@Override
8080
public void doInTransactionWithoutResult(TransactionStatus status) {
81-
_affinityGroupDao.listByIds(affinityGroupIds, true);
81+
if (!affinityGroupIds.isEmpty()) {
82+
_affinityGroupDao.listByIds(affinityGroupIds, true);
83+
}
8284
for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
8385
processAffinityGroup(vmGroupMapping, avoid, vm);
8486
}
@@ -165,7 +167,9 @@ public boolean check(VirtualMachineProfile vmProfile, DeployDestination plannedD
165167
return Transaction.execute(new TransactionCallback<Boolean>() {
166168
@Override
167169
public Boolean doInTransaction(TransactionStatus status) {
168-
_affinityGroupDao.listByIds(affinityGroupIds, true);
170+
if (!affinityGroupIds.isEmpty()) {
171+
_affinityGroupDao.listByIds(affinityGroupIds, true);
172+
}
169173
for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
170174
// if more than 1 VM's are present in the group then check for
171175
// conflict due to parallel deployment

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -360,7 +360,7 @@ protected IpAddress getVpcTierKubernetesPublicIp(Network network) {
360360
return null;
361361
}
362362
IpAddress address = ipAddressDao.findByUuid(detailsVO.getValue());
363-
if (address == null || network.getVpcId() != address.getVpcId()) {
363+
if (address == null || !Objects.equals(network.getVpcId(), address.getVpcId())) {
364364
logger.warn(String.format("Public IP with ID: %s linked to the Kubernetes cluster: %s is not usable", detailsVO.getValue(), kubernetesCluster.getName()));
365365
return null;
366366
}

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java‎

Lines changed: 38 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,31 @@
1717

1818
package com.cloud.kubernetes.cluster.actionworkers;
1919

20+
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
21+
22+
import java.io.File;
23+
import java.io.IOException;
24+
import java.util.ArrayList;
25+
import java.util.HashMap;
26+
import java.util.List;
27+
import java.util.Map;
28+
import java.util.concurrent.ConcurrentHashMap;
29+
import java.util.stream.Collectors;
30+
31+
import javax.inject.Inject;
32+
33+
import com.cloud.network.rules.FirewallManager;
34+
import com.cloud.offering.NetworkOffering;
35+
import com.cloud.offerings.dao.NetworkOfferingDao;
36+
import org.apache.cloudstack.api.ApiConstants;
37+
import org.apache.cloudstack.api.BaseCmd;
38+
import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd;
39+
import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd;
40+
import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd;
41+
import org.apache.commons.codec.binary.Base64;
42+
import org.apache.commons.collections.CollectionUtils;
43+
import org.apache.commons.lang3.StringUtils;
44+
2045
import com.cloud.capacity.CapacityManager;
2146
import com.cloud.dc.ClusterDetailsDao;
2247
import com.cloud.dc.ClusterDetailsVO;
@@ -61,9 +86,7 @@
6186
import com.cloud.network.vpc.NetworkACLItemDao;
6287
import com.cloud.network.vpc.NetworkACLItemVO;
6388
import com.cloud.network.vpc.NetworkACLService;
64-
import com.cloud.offering.NetworkOffering;
6589
import com.cloud.offering.ServiceOffering;
66-
import com.cloud.offerings.dao.NetworkOfferingDao;
6790
import com.cloud.resource.ResourceManager;
6891
import com.cloud.storage.Volume;
6992
import com.cloud.storage.VolumeApiService;
@@ -88,29 +111,9 @@
88111
import com.cloud.vm.VmDetailConstants;
89112
import com.cloud.vm.dao.VMInstanceDao;
90113
import org.apache.cloudstack.api.ApiCommandResourceType;
91-
import org.apache.cloudstack.api.ApiConstants;
92-
import org.apache.cloudstack.api.BaseCmd;
93-
import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd;
94-
import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd;
95-
import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd;
96114
import org.apache.cloudstack.context.CallContext;
97-
import org.apache.commons.codec.binary.Base64;
98-
import org.apache.commons.collections.CollectionUtils;
99-
import org.apache.commons.lang3.StringUtils;
100115
import org.apache.logging.log4j.Level;
101116

102-
import javax.inject.Inject;
103-
import java.io.File;
104-
import java.io.IOException;
105-
import java.util.ArrayList;
106-
import java.util.HashMap;
107-
import java.util.List;
108-
import java.util.Map;
109-
import java.util.concurrent.ConcurrentHashMap;
110-
import java.util.stream.Collectors;
111-
112-
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
113-
114117
public class KubernetesClusterResourceModifierActionWorker extends KubernetesClusterActionWorker {
115118

116119
@Inject
@@ -134,6 +137,8 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
134137
@Inject
135138
protected RulesService rulesService;
136139
@Inject
140+
protected FirewallManager firewallManager;
141+
@Inject
137142
protected PortForwardingRulesDao portForwardingRulesDao;
138143
@Inject
139144
protected ResourceManager resourceManager;
@@ -169,6 +174,7 @@ private String getKubernetesNodeConfig(final String joinIp, final boolean ejectI
169174
final String joinIpKey = "{{ k8s_control_node.join_ip }}";
170175
final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}";
171176
final String ejectIsoKey = "{{ k8s.eject.iso }}";
177+
172178
String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\"";
173179
String sshKeyPair = kubernetesCluster.getKeyPair();
174180
if (StringUtils.isNotEmpty(sshKeyPair)) {
@@ -181,7 +187,6 @@ private String getKubernetesNodeConfig(final String joinIp, final boolean ejectI
181187
k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp);
182188
k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster));
183189
k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso));
184-
185190
k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig);
186191

187192
return k8sNodeConfig;
@@ -522,17 +527,22 @@ protected FirewallRule removeSshFirewallRule(final IpAddress publicIp) {
522527

523528
protected void removePortForwardingRules(final IpAddress publicIp, final Network network, final Account account, final List<Long> removedVMIds) throws ResourceUnavailableException {
524529
if (!CollectionUtils.isEmpty(removedVMIds)) {
530+
List<PortForwardingRuleVO> pfRules = new ArrayList<>();
531+
List<PortForwardingRuleVO> revokedRules = new ArrayList<>();
525532
for (Long vmId : removedVMIds) {
526-
List<PortForwardingRuleVO> pfRules = portForwardingRulesDao.listByNetwork(network.getId());
533+
pfRules.addAll(portForwardingRulesDao.listByNetwork(network.getId()));
527534
for (PortForwardingRuleVO pfRule : pfRules) {
528535
if (pfRule.getVirtualMachineId() == vmId) {
529536
portForwardingRulesDao.remove(pfRule.getId());
537+
logger.trace("Marking PF rule {} with Revoke state", pfRule);
538+
pfRule.setState(FirewallRule.State.Revoke);
539+
revokedRules.add(pfRule);
530540
logger.debug("The Port forwarding rule [%s] with the id [%s] was removed.", pfRule.getName(), pfRule.getId());
531541
break;
532542
}
533543
}
534544
}
535-
rulesService.applyPortForwardingRules(publicIp.getId(), account);
545+
firewallManager.applyRules(revokedRules, false, true);
536546
}
537547
}
538548

@@ -542,10 +552,11 @@ protected void removePortForwardingRules(final IpAddress publicIp, final Network
542552
for (PortForwardingRuleVO pfRule : pfRules) {
543553
if (startPort <= pfRule.getSourcePortStart() && pfRule.getSourcePortStart() <= endPort) {
544554
portForwardingRulesDao.remove(pfRule.getId());
545-
logger.debug("The Port forwarding rule [{}] with the id [{}] was removed.", pfRule.getName(), pfRule.getId());
555+
logger.debug("The Port forwarding rule [{}] with the id [{}] was mark as revoked.", pfRule.getName(), pfRule.getId());
556+
pfRule.setState(FirewallRule.State.Revoke);
546557
}
547558
}
548-
rulesService.applyPortForwardingRules(publicIp.getId(), account);
559+
firewallManager.applyRules(pfRules, false, true);
549560
}
550561

551562
protected void removeLoadBalancingRule(final IpAddress publicIp, final Network network,

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ private String getKubernetesControlNodeConfig(final String controlNodeIp, final
139139
final String clusterToken = "{{ k8s_control_node.cluster.token }}";
140140
final String clusterInitArgsKey = "{{ k8s_control_node.cluster.initargs }}";
141141
final String ejectIsoKey = "{{ k8s.eject.iso }}";
142+
142143
final List<String> addresses = new ArrayList<>();
143144
addresses.add(controlNodeIp);
144145
if (!serverIp.equals(controlNodeIp)) {
@@ -243,6 +244,7 @@ private String getKubernetesAdditionalControlNodeConfig(final String joinIp, fin
243244
final String sshPubKey = "{{ k8s.ssh.pub.key }}";
244245
final String clusterHACertificateKey = "{{ k8s_control_node.cluster.ha.certificate.key }}";
245246
final String ejectIsoKey = "{{ k8s.eject.iso }}";
247+
246248
String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\"";
247249
String sshKeyPair = kubernetesCluster.getKeyPair();
248250
if (StringUtils.isNotEmpty(sshKeyPair)) {

‎plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/NsxAnswer.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@
2020
import com.cloud.agent.api.Command;
2121

2222
public class NsxAnswer extends Answer {
23+
24+
private boolean objectExists;
25+
2326
public NsxAnswer(final Command command, final boolean success, final String details) {
2427
super(command, success, details);
2528
}
@@ -28,4 +31,11 @@ public NsxAnswer(final Command command, final Exception e) {
2831
super(command, e);
2932
}
3033

34+
public boolean isObjectExistent() {
35+
return objectExists;
36+
}
37+
38+
public void setObjectExists(boolean objectExisted) {
39+
this.objectExists = objectExisted;
40+
}
3141
}

0 commit comments

Comments
 (0)