kvm: live storage migration intra cluster from NFS source and destination (#2983)

Feature Specification: https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=95653548

Live storage migration on KVM under these conditions:

From source and destination hosts within the same cluster
From NFS primary storage to NFS cluster-wide primary storage
Source NFS and destination NFS storage mounted on hosts
In order to enable this functionality, database should be updated in order to enable live storage capacibilty for KVM, if previous conditions are met. This is due to existing conflicts between qemu and libvirt versions. This has been tested on CentOS 6 hosts.

Additional notes:

To use this feature set the storage_motion_supported=1 in the hypervisor_capability table for KVM. This is done by default as the feature may not work in some environments, read below.
This feature of online storage+VM migration for KVM will only work with CentOS6 and possible Ubuntu as KVM hosts but not with CentOS7 due to:
https://bugs.centos.org/view.php?id=14026
https://bugzilla.redhat.com/show_bug.cgi?id=1219541
On CentOS7 the error we see is: " error: unable to execute QEMU command 'migrate': this feature or command is not currently supported" (reference https://ask.openstack.org/en/question/94186/live-migration-unable-to-execute-qemu-command-migrate/). Reading through various lists looks like the migrate feature with qemu may be available with paid versions of RHEL-EV but not centos7 however this works with CentOS6.
Fix for CentOS 7:

Create repo file on /etc/yum.repos.d/:
[qemu-kvm-rhev]
name=oVirt rebuilds of qemu-kvm-rhev
baseurl=http://resources.ovirt.org/pub/ovirt-3.5/rpm/el7Server/
mirrorlist=http://resources.ovirt.org/pub/yum-repo/mirrorlist-ovirt-3.5-el7Server
enabled=1
skip_if_unavailable=1
gpgcheck=0
yum install qemu-kvm-common-ev-2.3.0-29.1.el7.x86_64 qemu-kvm-ev-2.3.0-29.1.el7.x86_64 qemu-img-ev-2.3.0-29.1.el7.x86_64
Reboot host

Signed-off-by: Rohit Yadav <[email protected]>

Author: nvazquez

api/src/main/java/com/cloud/storage/MigrationOptions.java

@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.storage;
+
+import java.io.Serializable;
+
+public class MigrationOptions implements Serializable {
+
+    private String srcPoolUuid;
+    private Storage.StoragePoolType srcPoolType;
+    private Type type;
+    private String srcBackingFilePath;
+    private boolean copySrcTemplate;
+    private String srcVolumeUuid;
+    private int timeout;
+
+    public enum Type {
+        LinkedClone, FullClone
+    }
+
+    public MigrationOptions() {
+    }
+
+    public MigrationOptions(String srcPoolUuid, Storage.StoragePoolType srcPoolType, String srcBackingFilePath, boolean copySrcTemplate) {
+        this.srcPoolUuid = srcPoolUuid;
+        this.srcPoolType = srcPoolType;
+        this.type = Type.LinkedClone;
+        this.srcBackingFilePath = srcBackingFilePath;
+        this.copySrcTemplate = copySrcTemplate;
+    }
+
+    public MigrationOptions(String srcPoolUuid, Storage.StoragePoolType srcPoolType, String srcVolumeUuid) {
+        this.srcPoolUuid = srcPoolUuid;
+        this.srcPoolType = srcPoolType;
+        this.type = Type.FullClone;
+        this.srcVolumeUuid = srcVolumeUuid;
+    }
+
+    public String getSrcPoolUuid() {
+        return srcPoolUuid;
+    }
+
+    public Storage.StoragePoolType getSrcPoolType() {
+        return srcPoolType;
+    }
+
+    public String getSrcBackingFilePath() {
+        return srcBackingFilePath;
+    }
+
+    public boolean isCopySrcTemplate() {
+        return copySrcTemplate;
+    }
+
+    public String getSrcVolumeUuid() {
+        return srcVolumeUuid;
+    }
+
+    public Type getType() {
+        return type;
+    }
+
+    public int getTimeout() {
+        return timeout;
+    }
+
+    public void setTimeout(int timeout) {
+        this.timeout = timeout;
+    }
+}

core/src/main/java/com/cloud/agent/api/CreateVMSnapshotCommand.java

@@ -32,6 +32,10 @@ public CreateVMSnapshotCommand(String vmName, String vmUuid, VMSnapshotTO snapsh
         this.vmUuid = vmUuid;
     }
 
+    public CreateVMSnapshotCommand(String vmName, VMSnapshotTO snapshot) {
+        super(vmName, snapshot, null, null);
+    }
+
     public String getVmUuid() {
         return vmUuid;
     }

core/src/main/java/com/cloud/agent/api/DeleteVMSnapshotCommand.java

@@ -27,4 +27,8 @@ public class DeleteVMSnapshotCommand extends VMSnapshotBaseCommand {
     public DeleteVMSnapshotCommand(String vmName, VMSnapshotTO snapshot, List<VolumeObjectTO> volumeTOs, String guestOSType) {
         super(vmName, snapshot, volumeTOs, guestOSType);
     }
+
+    public DeleteVMSnapshotCommand(String vmName, VMSnapshotTO snapshot) {
+        super(vmName, snapshot, null, null);
+    }
 }

core/src/main/java/com/cloud/agent/api/MigrateCommand.java

@@ -30,6 +30,7 @@ public class MigrateCommand extends Command {
     private String vmName;
     private String destIp;
     private Map<String, MigrateDiskInfo> migrateStorage;
+    private boolean migrateStorageManaged;
     private boolean autoConvergence;
     private String hostGuid;
     private boolean isWindows;
@@ -56,6 +57,14 @@ public Map<String, MigrateDiskInfo> getMigrateStorage() {
         return migrateStorage != null ? new HashMap<>(migrateStorage) : new HashMap<String, MigrateDiskInfo>();
     }
 
+    public boolean isMigrateStorageManaged() {
+        return migrateStorageManaged;
+    }
+
+    public void setMigrateStorageManaged(boolean migrateStorageManaged) {
+        this.migrateStorageManaged = migrateStorageManaged;
+    }
+
     public void setAutoConvergence(boolean autoConvergence) {
         this.autoConvergence = autoConvergence;
     }

core/src/main/java/com/cloud/agent/api/storage/CheckStorageAvailabilityCommand.java

@@ -0,0 +1,43 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api.storage;
+
+import com.cloud.agent.api.Command;
+import com.cloud.storage.Storage;
+
+import java.util.Map;
+
+public class CheckStorageAvailabilityCommand extends Command {
+
+    private Map<String, Storage.StoragePoolType> poolsMap;
+
+    public CheckStorageAvailabilityCommand(Map<String, Storage.StoragePoolType> poolsMap) {
+        this.poolsMap = poolsMap;
+    }
+
+    public Map<String, Storage.StoragePoolType> getPoolsMap() {
+        return poolsMap;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+}

core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java

@@ -19,6 +19,7 @@
 
 package org.apache.cloudstack.storage.to;
 
+import com.cloud.storage.MigrationOptions;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 
 import com.cloud.agent.api.to.DataObjectType;
@@ -59,6 +60,7 @@ public class VolumeObjectTO implements DataTO {
     private Long iopsWriteRateMaxLength;
     private DiskCacheMode cacheMode;
     private Hypervisor.HypervisorType hypervisorType;
+    private MigrationOptions migrationOptions;
 
     public VolumeObjectTO() {
 
@@ -97,6 +99,7 @@ public VolumeObjectTO(VolumeInfo volume) {
         cacheMode = volume.getCacheMode();
         hypervisorType = volume.getHypervisorType();
         setDeviceId(volume.getDeviceId());
+        this.migrationOptions = volume.getMigrationOptions();
     }
 
     public String getUuid() {
@@ -300,4 +303,8 @@ public void setCacheMode(DiskCacheMode cacheMode) {
     public DiskCacheMode getCacheMode() {
         return cacheMode;
     }
+
+    public MigrationOptions getMigrationOptions() {
+        return migrationOptions;
+    }
 }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java

@@ -21,6 +21,7 @@
 import com.cloud.agent.api.Answer;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.offering.DiskOffering.DiskCacheMode;
+import com.cloud.storage.MigrationOptions;
 import com.cloud.storage.Volume;
 import com.cloud.vm.VirtualMachine;
 
@@ -71,4 +72,11 @@ public interface VolumeInfo extends DataObject, Volume {
     Long getIopsWriteRateMaxLength();
 
     DiskCacheMode getCacheMode();
+
+    /**
+     * Currently available for KVM volumes
+     */
+    MigrationOptions getMigrationOptions();
+
+    void setMigrationOptions(MigrationOptions migrationOptions);
 }

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -87,11 +87,19 @@ public interface StorageManager extends StorageService {
     ConfigKey<Integer> KvmStorageOnlineMigrationWait = new ConfigKey<>(Integer.class,
             "kvm.storage.online.migration.wait",
             "Storage",
-            "10800",
+            "86400",
             "Timeout in seconds for online (live) storage migration to complete on KVM (migrateVirtualMachineWithVolume)",
             true,
             ConfigKey.Scope.Global,
             null);
+    ConfigKey<Boolean> KvmAutoConvergence = new ConfigKey<>(Boolean.class,
+            "kvm.auto.convergence",
+            "Storage",
+            "false",
+            "Setting this to 'true' allows KVM to use auto convergence to complete VM migration (libvirt version 1.2.3+ and QEMU version 1.6+)",
+            true,
+            ConfigKey.Scope.Global,
+            null);
     ConfigKey<Integer> MaxNumberOfManagedClusteredFileSystems = new ConfigKey<>(Integer.class,
             "max.number.managed.clustered.file.systems",
             "Storage",

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -120,7 +120,6 @@
 import com.cloud.agent.manager.allocator.HostAllocator;
 import com.cloud.alert.AlertManager;
 import com.cloud.capacity.CapacityManager;
-import com.cloud.configuration.Config;
 import com.cloud.dc.ClusterDetailsDao;
 import com.cloud.dc.ClusterDetailsVO;
 import com.cloud.dc.DataCenter;
@@ -2367,11 +2366,8 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
             final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
             final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType()));
 
-            String autoConvergence = _configDao.getValue(Config.KvmAutoConvergence.toString());
-            boolean kvmAutoConvergence = Boolean.parseBoolean(autoConvergence);
-
+            boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value();
             mc.setAutoConvergence(kvmAutoConvergence);
-
             mc.setHostGuid(dest.getHost().getGuid());
 
             try {
@@ -3897,11 +3893,8 @@ private void orchestrateMigrateForScale(final String vmUuid, final long srcHostI
             final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
             final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType()));
 
-            String autoConvergence = _configDao.getValue(Config.KvmAutoConvergence.toString());
-            boolean kvmAutoConvergence = Boolean.parseBoolean(autoConvergence);
-
+            boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value();
             mc.setAutoConvergence(kvmAutoConvergence);
-
             mc.setHostGuid(dest.getHost().getGuid());
 
             try {

engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java

@@ -24,6 +24,8 @@
 
 import javax.inject.Inject;
 
+import com.cloud.storage.ScopeType;
+import com.cloud.storage.Storage;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
@@ -54,6 +56,7 @@
 import com.cloud.storage.dao.VMTemplatePoolDao;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.VirtualMachineManager;
+import org.apache.commons.collections.MapUtils;
 
 /**
  * Extends {@link StorageSystemDataMotionStrategy}, allowing KVM hosts to migrate VMs with the ROOT volume on a non managed local storage pool.
@@ -77,21 +80,69 @@ public class KvmNonManagedStorageDataMotionStrategy extends StorageSystemDataMot
      * Note that the super implementation (override) is called by {@link #canHandle(Map, Host, Host)} which ensures that {@link #internalCanHandle(Map)} will be executed only if the source host is KVM.
      */
     @Override
-    protected StrategyPriority internalCanHandle(Map<VolumeInfo, DataStore> volumeMap) {
-        if (super.internalCanHandle(volumeMap) == StrategyPriority.CANT_HANDLE) {
-            Set<VolumeInfo> volumeInfoSet = volumeMap.keySet();
+    protected StrategyPriority internalCanHandle(Map<VolumeInfo, DataStore> volumeMap, Host srcHost, Host destHost) {
+        if (super.internalCanHandle(volumeMap, srcHost, destHost) == StrategyPriority.CANT_HANDLE) {
+            if (canHandleKVMNonManagedLiveNFSStorageMigration(volumeMap, srcHost, destHost) == StrategyPriority.CANT_HANDLE) {
+                Set<VolumeInfo> volumeInfoSet = volumeMap.keySet();
 
-            for (VolumeInfo volumeInfo : volumeInfoSet) {
-                StoragePoolVO storagePoolVO = _storagePoolDao.findById(volumeInfo.getPoolId());
-                if (storagePoolVO.getPoolType() != StoragePoolType.Filesystem && storagePoolVO.getPoolType() != StoragePoolType.NetworkFilesystem) {
-                    return StrategyPriority.CANT_HANDLE;
+                for (VolumeInfo volumeInfo : volumeInfoSet) {
+                    StoragePoolVO storagePoolVO = _storagePoolDao.findById(volumeInfo.getPoolId());
+                    if (storagePoolVO.getPoolType() != StoragePoolType.Filesystem && storagePoolVO.getPoolType() != StoragePoolType.NetworkFilesystem) {
+                        return StrategyPriority.CANT_HANDLE;
+                    }
                 }
             }
             return StrategyPriority.HYPERVISOR;
         }
         return StrategyPriority.CANT_HANDLE;
     }
 
+    /**
+     * Allow KVM live storage migration for non managed storage when:
+     * - Source host and destination host are different, and are on the same cluster
+     * - Source and destination storage are NFS
+     * - Destination storage is cluster-wide
+     */
+    protected StrategyPriority canHandleKVMNonManagedLiveNFSStorageMigration(Map<VolumeInfo, DataStore> volumeMap,
+                                                                             Host srcHost, Host destHost) {
+        if (srcHost.getId() != destHost.getId() &&
+                srcHost.getClusterId().equals(destHost.getClusterId()) &&
+                isSourceNfsPrimaryStorage(volumeMap) &&
+                isDestinationNfsPrimaryStorageClusterWide(volumeMap)) {
+            return StrategyPriority.HYPERVISOR;
+        }
+        return StrategyPriority.CANT_HANDLE;
+    }
+
+    /**
+     * True if volumes source storage are NFS
+     */
+    protected boolean isSourceNfsPrimaryStorage(Map<VolumeInfo, DataStore> volumeMap) {
+        if (MapUtils.isNotEmpty(volumeMap)) {
+            for (VolumeInfo volumeInfo : volumeMap.keySet()) {
+                StoragePoolVO storagePoolVO = _storagePoolDao.findById(volumeInfo.getPoolId());
+                return storagePoolVO != null &&
+                        storagePoolVO.getPoolType() == Storage.StoragePoolType.NetworkFilesystem;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * True if destination storage is cluster-wide NFS
+     */
+    protected boolean isDestinationNfsPrimaryStorageClusterWide(Map<VolumeInfo, DataStore> volumeMap) {
+        if (MapUtils.isNotEmpty(volumeMap)) {
+            for (DataStore dataStore : volumeMap.values()) {
+                StoragePoolVO storagePoolVO = _storagePoolDao.findById(dataStore.getId());
+                return storagePoolVO != null &&
+                        storagePoolVO.getPoolType() == Storage.StoragePoolType.NetworkFilesystem &&
+                        storagePoolVO.getScope() == ScopeType.CLUSTER;
+            }
+        }
+        return false;
+    }
+
     /**
      * Configures a {@link MigrateDiskInfo} object configured for migrating a File System volume and calls rootImageProvisioning.
      */
@@ -135,7 +186,7 @@ protected void setVolumePath(VolumeVO volume) {
      */
     @Override
     protected boolean shouldMigrateVolume(StoragePoolVO sourceStoragePool, Host destHost, StoragePoolVO destStoragePool) {
-        return sourceStoragePool.getPoolType() == StoragePoolType.Filesystem;
+        return sourceStoragePool.getPoolType() == StoragePoolType.Filesystem || sourceStoragePool.getPoolType() == StoragePoolType.NetworkFilesystem;
     }
 
     /**