Cloudstack 4.11.3.0 shapeblue1 (#15)

* Fixed: IP address of Shared Network VR is not persistent when VR is destroyed/recreated

* Clean up DHCP/DNS config on VM expunge and on detaching a VM

* increase DHCP lease to ~infinite

* To handle deletion w/ expunge of VM with interfaces in Isolated and shared networks

* client: fix for jetty session timeout

* ui: fix migrate host form no host popup

* removed the cloud-plugin-network-vsp dependency

* Refactored code

* Removed unused imports

* Refactored the code

* Usage event to store zone id while uploading template

* refactored code

Author: Pearl1594

client/pom.xml

@@ -207,11 +207,6 @@
       <artifactId>cloud-plugin-network-vcs</artifactId>
       <version>${project.version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.apache.cloudstack</groupId>
-      <artifactId>cloud-plugin-network-vsp</artifactId>
-      <version>${project.version}</version>
-    </dependency>
     <dependency>
       <groupId>org.apache.cloudstack</groupId>
       <artifactId>cloud-plugin-hypervisor-xenserver</artifactId>

client/src/org/apache/cloudstack/ServerDaemon.java

@@ -40,13 +40,15 @@
 import org.eclipse.jetty.server.handler.MovedContextHandler;
 import org.eclipse.jetty.server.handler.RequestLogHandler;
 import org.eclipse.jetty.server.handler.gzip.GzipHandler;
+import org.eclipse.jetty.server.session.SessionHandler;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
 import org.eclipse.jetty.webapp.WebAppContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.cloud.utils.Pair;
 import com.cloud.utils.PropertiesUtil;
 import com.google.common.base.Strings;
 
@@ -170,7 +172,8 @@ public void start() throws Exception {
         server.addConnector(httpConnector);
 
         // Setup handlers
-        server.setHandler(createHandlers());
+        Pair<SessionHandler,HandlerCollection> pair = createHandlers();
+        server.setHandler(pair.second());
 
         // Extra config options
         server.setStopAtShutdown(true);
@@ -198,6 +201,8 @@ public void start() throws Exception {
         }
 
         server.start();
+        // Must set the session timeout after the server has started
+        pair.first().setMaxInactiveInterval(sessionTimeout * 60);
         server.join();
     }
 
@@ -215,11 +220,10 @@ public void destroy() {
     /////////////// Private methods ///////////////////
     ///////////////////////////////////////////////////
 
-    private HandlerCollection createHandlers() {
+    private Pair<SessionHandler,HandlerCollection> createHandlers() {
         final WebAppContext webApp = new WebAppContext();
         webApp.setContextPath(contextPath);
         webApp.setInitParameter("org.eclipse.jetty.servlet.Default.dirAllowed", "false");
-        webApp.getSessionHandler().setMaxInactiveInterval(sessionTimeout * 60);
 
         // GZIP handler
         final GzipHandler gzipHandler = new GzipHandler();
@@ -238,14 +242,14 @@ private HandlerCollection createHandlers() {
         final RequestLogHandler log = new RequestLogHandler();
         log.setRequestLog(createRequestLog());
 
-        // Redirect root context handler
+        // Redirect root context handler_war
         MovedContextHandler rootRedirect = new MovedContextHandler();
         rootRedirect.setContextPath("/");
         rootRedirect.setNewContextURL(contextPath);
         rootRedirect.setPermanent(true);
 
         // Put rootRedirect at the end!
-        return new HandlerCollection(log, gzipHandler, rootRedirect);
+        return new Pair<>(webApp.getSessionHandler(), new HandlerCollection(log, gzipHandler, rootRedirect));
     }
 
     private RequestLog createRequestLog() {

engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -1951,9 +1951,18 @@ protected void removeNic(final VirtualMachineProfile vm, final NicVO nic) {
             }
         }
 
+        final NetworkVO network = _networksDao.findById(nic.getNetworkId());
+        if (network != null && network.getTrafficType() == TrafficType.Guest) {
+            final String nicIp = Strings.isNullOrEmpty(nic.getIPv4Address()) ? nic.getIPv6Address() : nic.getIPv4Address();
+            if (!Strings.isNullOrEmpty(nicIp)) {
+                NicProfile nicProfile = new NicProfile(nic.getIPv4Address(), nic.getIPv6Address(), nic.getMacAddress());
+                nicProfile.setId(nic.getId());
+                cleanupNicDhcpDnsEntry(network, vm, nicProfile);
+            }
+        }
+
         nic.setState(Nic.State.Deallocating);
         _nicDao.update(nic.getId(), nic);
-        final NetworkVO network = _networksDao.findById(nic.getNetworkId());
         final NicProfile profile = new NicProfile(nic, network, null, null, null, _networkModel.isSecurityGroupSupportedInNetwork(network), _networkModel.getNetworkTag(
                 vm.getHypervisorType(), network));
 

plugins/pom.xml

@@ -77,7 +77,6 @@
     <module>network-elements/palo-alto</module>
     <module>network-elements/netscaler</module>
     <module>network-elements/nicira-nvp</module>
-    <module>network-elements/nuage-vsp</module>
     <module>network-elements/bigswitch</module>
     <module>network-elements/brocade-vcs</module>
     <module>network-elements/stratosphere-ssp</module>

server/src/com/cloud/network/NetworkModelImpl.java

@@ -2302,6 +2302,9 @@ public NicVO getPlaceholderNicForRouter(Network network, Long podId) {
                         ipv6 = _ipv6Dao.findByNetworkIdAndIp(network.getId(), nic.getIPv6Address());
                     }
                     //return nic only when its ip address belong to the pod range (for the Basic zone case)
+                    if (vlans.isEmpty()) {
+                        return nic;
+                    }
                     for (Vlan vlan : vlans) {
                         if (ip != null && ip.getVlanId() == vlan.getId()) {
                             return nic;

server/src/com/cloud/network/element/VirtualRouterElement.java

@@ -895,7 +895,6 @@ public VirtualRouterProvider getCreatedElement(final long id) {
     @Override
     public boolean release(final Network network, final NicProfile nic, final VirtualMachineProfile vm, final ReservationContext context) throws ConcurrentOperationException,
     ResourceUnavailableException {
-        removeDhcpEntry(network, nic, vm);
         return true;
     }
 

server/src/com/cloud/network/router/CommandSetupHelper.java

@@ -1062,7 +1062,7 @@ private NicVO findDefaultDnsIp(final long userVmId) {
         final NicVO defaultNic = _nicDao.findDefaultNicForVM(userVmId);
 
         // check if DNS provider is the domR
-        if (!_networkModel.isProviderSupportServiceInNetwork(defaultNic.getNetworkId(), Service.Dns, Provider.VirtualRouter)) {
+        if (defaultNic == null || !_networkModel.isProviderSupportServiceInNetwork(defaultNic.getNetworkId(), Service.Dns, Provider.VirtualRouter)) {
             return null;
         }
 

server/src/com/cloud/storage/ImageStoreUploadMonitorImpl.java

@@ -25,6 +25,8 @@
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.api.query.dao.TemplateJoinDao;
+import com.cloud.api.query.vo.TemplateJoinVO;
 import com.cloud.configuration.Resource;
 import com.cloud.event.EventTypes;
 import com.cloud.event.UsageEventUtils;
@@ -102,6 +104,8 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
     private AlertManager _alertMgr;
     @Inject
     private VMTemplateZoneDao _vmTemplateZoneDao;
+    @Inject
+    private TemplateJoinDao _templateJoinDao;
 
     private long _nodeId;
     private ScheduledExecutorService _executor = null;
@@ -313,7 +317,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
 
                             // publish usage events
                             UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_UPLOAD, tmpVolume.getAccountId(),
-                                    tmpVolumeDataStore.getDataStoreId(), tmpVolume.getId(), tmpVolume.getName(),
+                                    tmpVolume.getDataCenterId(), tmpVolume.getId(), tmpVolume.getName(),
                                     null, null, tmpVolumeDataStore.getPhysicalSize(), tmpVolumeDataStore.getSize(),
                                     Volume.class.getName(), tmpVolume.getUuid());
 
@@ -402,9 +406,10 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
                             if (tmpTemplate.getFormat() == Storage.ImageFormat.ISO) {
                                 etype = EventTypes.EVENT_ISO_CREATE;
                             }
-                            UsageEventUtils.publishUsageEvent(etype, tmpTemplate.getAccountId(), tmpTemplateDataStore.getDataStoreId(), tmpTemplate.getId(), tmpTemplate.getName(), null, null,
-                                    tmpTemplateDataStore.getPhysicalSize(), tmpTemplateDataStore.getSize(), VirtualMachineTemplate.class.getName(), tmpTemplate.getUuid());
-
+                            TemplateJoinVO vo = _templateJoinDao.findById(tmpTemplate.getId());
+                            assert (vo != null) : "Couldn't find the template view for given template ID";
+                            UsageEventUtils.publishUsageEvent(etype, tmpTemplate.getAccountId(), vo.getDataCenterId(), tmpTemplate.getId(), tmpTemplate.getName(), null, null,
+                                        tmpTemplateDataStore.getPhysicalSize(), tmpTemplateDataStore.getSize(), VirtualMachineTemplate.class.getName(), tmpTemplate.getUuid());
                             if (s_logger.isDebugEnabled()) {
                                 s_logger.debug("Template " + tmpTemplate.getUuid() + " uploaded successfully");
                             }

server/src/com/cloud/vm/UserVmManagerImpl.java

@@ -302,7 +302,6 @@
 import com.cloud.vm.snapshot.VMSnapshotManager;
 import com.cloud.vm.snapshot.VMSnapshotVO;
 import com.cloud.vm.snapshot.dao.VMSnapshotDao;
-import com.google.common.base.Strings;
 
 
 public class UserVmManagerImpl extends ManagerBase implements UserVmManager, VirtualMachineGuru, UserVmService, Configurable {
@@ -4382,12 +4381,6 @@ public void finalizeStop(VirtualMachineProfile profile, Answer answer) {
         for (final NicVO nic : nics) {
             final NetworkVO network = _networkDao.findById(nic.getNetworkId());
             if (network != null && network.getTrafficType() == TrafficType.Guest) {
-                final String nicIp = Strings.isNullOrEmpty(nic.getIPv4Address()) ? nic.getIPv6Address() : nic.getIPv4Address();
-                if (!Strings.isNullOrEmpty(nicIp)) {
-                    NicProfile nicProfile = new NicProfile(nic.getIPv4Address(), nic.getIPv6Address(), nic.getMacAddress());
-                    nicProfile.setId(nic.getId());
-                    _networkMgr.cleanupNicDhcpDnsEntry(network, profile, nicProfile);
-                }
                 if (nic.getBroadcastUri() != null && nic.getBroadcastUri().getScheme().equals("pvlan")) {
                     NicProfile nicProfile = new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), 0, false, "pvlan-nic");
                     setupVmForPvlan(false, vm.getHostId(), nicProfile);

systemvm/debian/opt/cloud/bin/cs/CsDhcp.py

@@ -159,9 +159,15 @@ def write_hosts(self):
 
     def add(self, entry):
         self.add_host(entry['ipv4_address'], entry['host_name'])
-        # lease time boils down to once a month
-        # with a splay of 60 hours to prevent storms
-        lease = randint(700, 760)
+        # Lease time set to effectively infinite (36000+ days) since we properly control all DHCP/DNS config via CloudStack.
+        # Infinite time helps avoid some edge cases which could cause DHCPNAK being sent to VMs since
+        # (RHEL) system lose routes when they receive DHCPNAK.
+        # When VM is expunged, its active lease and DHCP/DNS config is properly removed from related files in VR,
+        # so the infinite duration of lease does not cause any issues or garbage.
+        # There will be soon a PR which also regenerates the /var/lib/misc/dnsmasq.leases (active lease DB file)
+        # in the new VR (when restarting network with cleanup), which will help around RHEL edge cases (described above)
+        # for the VMs who are already running in productions systems with 30d lease time.
+        lease = randint(870000, 870010)
 
         if entry['default_entry']:
             self.cloud.add("%s,%s,%s,%sh" % (entry['mac_address'],