CLOUDSTACK-8609: [VMware] VM is not accessible after migration across clusters (#2091)

[VMware] VM is not accessible after migration across clusters.

Once a VM is successfully started, don't delete the files associated with the unregistered VM, if the files are in a storage that is being used by the new VM.
Attempt to unregister a VM in another DC, only if there is a host associated with a VM.

This closes #556

Author: sureshanaparti

plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java

@@ -1668,6 +1668,7 @@ protected StartAnswer execute(StartCommand cmd) {
         String existingVmName = null;
         VirtualMachineFileInfo existingVmFileInfo = null;
         VirtualMachineFileLayoutEx existingVmFileLayout = null;
+        List<DatastoreMO> existingDatastores = new ArrayList<DatastoreMO>();
 
         Pair<String, String> names = composeVmNames(vmSpec);
         String vmInternalCSName = names.first();
@@ -1790,6 +1791,7 @@ protected StartAnswer execute(StartCommand cmd) {
                         existingVmName = existingVmInDc.getName();
                         existingVmFileInfo = existingVmInDc.getFileInfo();
                         existingVmFileLayout = existingVmInDc.getFileLayout();
+                        existingDatastores = existingVmInDc.getAllDatastores();
                         existingVmInDc.unregisterVm();
                     }
                     Pair<ManagedObjectReference, DatastoreMO> rootDiskDataStoreDetails = null;
@@ -2256,7 +2258,18 @@ protected StartAnswer execute(StartCommand cmd) {
 
             // Since VM was successfully powered-on, if there was an existing VM in a different cluster that was unregistered, delete all the files associated with it.
             if (existingVmName != null && existingVmFileLayout != null) {
-                deleteUnregisteredVmFiles(existingVmFileLayout, dcMo, true);
+                List<String> vmDatastoreNames = new ArrayList<String>();
+                for (DatastoreMO vmDatastore : vmMo.getAllDatastores()) {
+                    vmDatastoreNames.add(vmDatastore.getName());
+                }
+                // Don't delete files that are in a datastore that is being used by the new VM as well (zone-wide datastore).
+                List<String> skipDatastores = new ArrayList<String>();
+                for (DatastoreMO existingDatastore : existingDatastores) {
+                    if (vmDatastoreNames.contains(existingDatastore.getName())) {
+                        skipDatastores.add(existingDatastore.getName());
+                    }
+                }
+                deleteUnregisteredVmFiles(existingVmFileLayout, dcMo, true, skipDatastores);
             }
 
             return startAnswer;
@@ -2944,7 +2957,14 @@ private void postDiskConfigBeforeStart(VirtualMachineMO vmMo, VirtualMachineTO v
         }
     }
 
-    private void deleteUnregisteredVmFiles(VirtualMachineFileLayoutEx vmFileLayout, DatacenterMO dcMo, boolean deleteDisks) throws Exception {
+    private void checkAndDeleteDatastoreFile(String filePath, List<String> skipDatastores, DatastoreMO dsMo, DatacenterMO dcMo) throws Exception {
+        if (dsMo != null && dcMo != null && (skipDatastores == null || !skipDatastores.contains(dsMo.getName()))) {
+            s_logger.debug("Deleting file: " + filePath);
+            dsMo.deleteFile(filePath, dcMo.getMor(), true);
+        }
+    }
+
+    private void deleteUnregisteredVmFiles(VirtualMachineFileLayoutEx vmFileLayout, DatacenterMO dcMo, boolean deleteDisks, List<String> skipDatastores) throws Exception {
         s_logger.debug("Deleting files associated with an existing VM that was unregistered");
         DatastoreFile vmFolder = null;
         try {
@@ -2957,23 +2977,20 @@ private void deleteUnregisteredVmFiles(VirtualMachineFileLayoutEx vmFileLayout,
                 else if (file.getType().equals("config"))
                     vmFolder = new DatastoreFile(fileInDatastore.getDatastoreName(), fileInDatastore.getDir());
                 DatastoreMO dsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(fileInDatastore.getDatastoreName()));
-                s_logger.debug("Deleting file: " + file.getName());
-                dsMo.deleteFile(file.getName(), dcMo.getMor(), true, VmwareManager.s_vmwareSearchExcludeFolder.value());
+                checkAndDeleteDatastoreFile(file.getName(), skipDatastores, dsMo, dcMo);
             }
             // Delete files that are present in the VM folder - this will take care of the VM disks as well.
             DatastoreMO vmFolderDsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(vmFolder.getDatastoreName()));
             String[] files = vmFolderDsMo.listDirContent(vmFolder.getPath());
             if (deleteDisks) {
                 for (String file : files) {
                     String vmDiskFileFullPath = String.format("%s/%s", vmFolder.getPath(), file);
-                    s_logger.debug("Deleting file: " + vmDiskFileFullPath);
-                    vmFolderDsMo.deleteFile(vmDiskFileFullPath, dcMo.getMor(), true, VmwareManager.s_vmwareSearchExcludeFolder.value());
+                    checkAndDeleteDatastoreFile(vmDiskFileFullPath, skipDatastores, vmFolderDsMo, dcMo);
                 }
             }
             // Delete VM folder
             if (deleteDisks || files.length == 0) {
-                s_logger.debug("Deleting folder: " + vmFolder.getPath());
-                vmFolderDsMo.deleteFolder(vmFolder.getPath(), dcMo.getMor());
+                checkAndDeleteDatastoreFile(vmFolder.getPath(), skipDatastores, vmFolderDsMo, dcMo);
             }
         } catch (Exception e) {
             String message = "Failed to delete files associated with an existing VM that was unregistered due to " + VmwareHelper.getExceptionMessage(e);
@@ -4908,7 +4925,7 @@ protected Answer execute(UnregisterVMCommand cmd) {
                     VirtualMachineFileLayoutEx vmFileLayout = vmMo.getFileLayout();
                     context.getService().unregisterVM(vmMo.getMor());
                     if (cmd.getCleanupVmFiles()) {
-                        deleteUnregisteredVmFiles(vmFileLayout, dataCenterMo, false);
+                        deleteUnregisteredVmFiles(vmFileLayout, dataCenterMo, false, null);
                     }
                     return new Answer(cmd, true, "unregister succeeded");
                 } catch (Exception e) {

vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java

@@ -34,6 +34,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.log4j.Logger;
 
 import com.google.gson.Gson;
@@ -932,6 +933,38 @@ else if (prop.getName().startsWith("value[")) {
         return networks;
     }
 
+    public List<DatastoreMO> getAllDatastores() throws Exception {
+        PropertySpec pSpec = new PropertySpec();
+        pSpec.setType("Datastore");
+        pSpec.getPathSet().add("name");
+
+        TraversalSpec vmDatastoreTraversal = new TraversalSpec();
+        vmDatastoreTraversal.setType("VirtualMachine");
+        vmDatastoreTraversal.setPath("datastore");
+        vmDatastoreTraversal.setName("vmDatastoreTraversal");
+
+        ObjectSpec oSpec = new ObjectSpec();
+        oSpec.setObj(_mor);
+        oSpec.setSkip(Boolean.TRUE);
+        oSpec.getSelectSet().add(vmDatastoreTraversal);
+
+        PropertyFilterSpec pfSpec = new PropertyFilterSpec();
+        pfSpec.getPropSet().add(pSpec);
+        pfSpec.getObjectSet().add(oSpec);
+        List<PropertyFilterSpec> pfSpecArr = new ArrayList<PropertyFilterSpec>();
+        pfSpecArr.add(pfSpec);
+
+        List<ObjectContent> ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr);
+
+        List<DatastoreMO> datastores = new ArrayList<DatastoreMO>();
+        if (CollectionUtils.isNotEmpty(ocs)) {
+            for (ObjectContent oc : ocs) {
+                datastores.add(new DatastoreMO(_context, oc.getObj()));
+            }
+        }
+        return datastores;
+    }
+
     /**
      * Retrieve path info to access VM files via vSphere web interface
      * @return [0] vm-name, [1] data-center-name, [2] datastore-name