CLOUDSTACK-9620: Enhancements for managed storage

Allowed zone-wide primary storage based on a custom plug-in to be added via the GUI in a KVM-only environment (previously this only worked for XenServer and VMware)

Added support for root disks on managed storage with KVM

Added support for volume snapshots with managed storage on KVM

Enable creating a template directly from a volume (i.e. without having to go through a volume snapshot) on KVM with managed storage

Only allow the resizing of a volume for managed storage on KVM if the volume in question is either not attached to a VM or is attached to a VM in the Stopped state.

Included support for Reinstall VM on KVM with managed storage

Enabled offline migration on KVM from non-managed storage to managed storage and vice versa

Included support for online storage migration on KVM with managed storage (NFS and Ceph to managed storage)

Added support to download (extract) a managed-storage volume to a QCOW2 file

When uploading a file from outside of CloudStack to CloudStack, set the min and max IOPS, if applicable.

Included support for the KVM auto-convergence feature

The compression flag was actually added in version 1.0.3 (1000003) as opposed to version 1.3.0 (1003000) (changed this to reflect the correct version)

On KVM when using iSCSI-based managed storage, if the user shuts a VM down from the guest OS (as opposed to doing so from CloudStack), we need to pass to the KVM agent a list of applicable iSCSI volumes that need to be disconnected.

Added a new Global Setting: kvm.storage.live.migration.wait

For XenServer, added a check to enforce that only volumes from zone-wide managed storage can be storage motioned from a host in one cluster to a host in another cluster (cannot do so at the time being with volumes from cluster-scoped managed storage)

Don’t allow Storage XenMotion on a VM that has any managed-storage volume with one or more snapshots.

Enabled for managed storage with VMware: Template caching, create snapshot, delete snapshot, create volume from snapshot, and create template from snapshot

Added an SIOC API plug-in to support VMware SIOC

When starting a VM that uses managed storage in a cluster other than the one it last was running in, we need to remove the reference to the iSCSI volume from the original cluster.

Added the ability to revert a volume to a snapshot

Enabled cluster-scoped managed storage

Added support for VMware dynamic discovery

Author: Mike Tutkowski

api/src/com/cloud/agent/api/to/DiskTO.java

@@ -27,6 +27,7 @@ public class DiskTO {
     public static final String CHAP_INITIATOR_SECRET = "chapInitiatorSecret";
     public static final String CHAP_TARGET_USERNAME = "chapTargetUsername";
     public static final String CHAP_TARGET_SECRET = "chapTargetSecret";
+    public static final String SCSI_NAA_DEVICE_ID = "scsiNaaDeviceId";
     public static final String MANAGED = "managed";
     public static final String IQN = "iqn";
     public static final String STORAGE_HOST = "storageHost";
@@ -36,6 +37,9 @@ public class DiskTO {
     public static final String PROTOCOL_TYPE = "protocoltype";
     public static final String PATH = "path";
     public static final String UUID = "uuid";
+    public static final String VMDK = "vmdk";
+    public static final String EXPAND_DATASTORE = "expandDatastore";
+    public static final String TEMPLATE_RESIGN = "templateResign";
 
     private DataTO data;
     private Long diskSeq;

api/src/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java

@@ -78,6 +78,14 @@ public class ResizeVolumeCmd extends BaseAsyncCmd {
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
 
+    public ResizeVolumeCmd() {}
+
+    public ResizeVolumeCmd(Long id, Long minIops, Long maxIops) {
+        this.id = id;
+        this.minIops = minIops;
+        this.maxIops = maxIops;
+    }
+
     //TODO use the method getId() instead of this one.
     public Long getEntityId() {
         return id;

client/pom.xml

@@ -1147,6 +1147,21 @@
         </dependency>
       </dependencies>
     </profile>
+    <profile>
+        <id>sfsioc</id>
+        <activation>
+            <property>
+                <name>noredist</name>
+            </property>
+        </activation>
+        <dependencies>
+            <dependency>
+                <groupId>org.apache.cloudstack</groupId>
+                <artifactId>cloud-plugin-api-solidfire-sioc</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+        </dependencies>
+    </profile>
     <profile>
       <id>quickcloud</id>
       <activation>

core/src/com/cloud/agent/api/MigrateCommand.java

@@ -19,15 +19,20 @@
 
 package com.cloud.agent.api;
 
+import java.util.HashMap;
+import java.util.Map;
+
 import com.cloud.agent.api.to.VirtualMachineTO;
 
 public class MigrateCommand extends Command {
-    String vmName;
-    String destIp;
-    String hostGuid;
-    boolean isWindows;
-    VirtualMachineTO vmTO;
-    boolean executeInSequence = false;
+    private String vmName;
+    private String destIp;
+    private Map<String, MigrateDiskInfo> migrateStorage;
+    private boolean autoConvergence;
+    private String hostGuid;
+    private boolean isWindows;
+    private VirtualMachineTO vmTO;
+    private boolean executeInSequence = false;
 
     protected MigrateCommand() {
     }
@@ -40,6 +45,26 @@ public MigrateCommand(String vmName, String destIp, boolean isWindows, VirtualMa
         this.executeInSequence = executeInSequence;
     }
 
+    public void setMigrateStorage(Map<String, MigrateDiskInfo> migrateStorage) {
+        this.migrateStorage = migrateStorage;
+    }
+
+    public Map<String, MigrateDiskInfo> getMigrateStorage() {
+        return migrateStorage != null ? new HashMap<>(migrateStorage) : new HashMap<String, MigrateDiskInfo>();
+    }
+
+    public boolean isMigrateStorage() {
+        return migrateStorage != null && !migrateStorage.isEmpty();
+    }
+
+    public void setAutoConvergence(boolean autoConvergence) {
+        this.autoConvergence = autoConvergence;
+    }
+
+    public boolean isAutoConvergence() {
+        return autoConvergence;
+    }
+
     public boolean isWindows() {
         return isWindows;
     }
@@ -68,4 +93,79 @@ public String getHostGuid() {
     public boolean executeInSequence() {
         return executeInSequence;
     }
+
+    public static class MigrateDiskInfo {
+        public enum DiskType {
+            FILE, BLOCK;
+
+            @Override
+            public String toString() {
+                switch(this) {
+                    case FILE: return "file";
+                    case BLOCK: return "block";
+                    default: throw new IllegalArgumentException();
+                }
+            }
+        }
+
+        public enum DriverType {
+            QCOW2, RAW;
+
+            @Override
+            public String toString() {
+                switch(this) {
+                    case QCOW2: return "qcow2";
+                    case RAW: return "raw";
+                    default: throw new IllegalArgumentException();
+                }
+            }
+        }
+
+        public enum Source {
+            FILE, DEV;
+
+            @Override
+            public String toString() {
+                switch(this) {
+                    case FILE: return "file";
+                    case DEV: return "dev";
+                    default: throw new IllegalArgumentException();
+                }
+            }
+        }
+
+        private final String serialNumber;
+        private final DiskType diskType;
+        private final DriverType driverType;
+        private final Source source;
+        private final String sourceText;
+
+        public MigrateDiskInfo(final String serialNumber, final DiskType diskType, final DriverType driverType, final Source source, final String sourceText) {
+            this.serialNumber = serialNumber;
+            this.diskType = diskType;
+            this.driverType = driverType;
+            this.source = source;
+            this.sourceText = sourceText;
+        }
+
+        public String getSerialNumber() {
+            return serialNumber;
+        }
+
+        public DiskType getDiskType() {
+            return diskType;
+        }
+
+        public DriverType getDriverType() {
+            return driverType;
+        }
+
+        public Source getSource() {
+            return source;
+        }
+
+        public String getSourceText() {
+            return sourceText;
+        }
+    }
 }

core/src/com/cloud/agent/api/ModifyTargetsAnswer.java

@@ -19,5 +19,16 @@
 
 package com.cloud.agent.api;
 
+import java.util.List;
+
 public class ModifyTargetsAnswer extends Answer {
+    private List<String> connectedPaths;
+
+    public void setConnectedPaths(List<String> connectedPaths) {
+        this.connectedPaths = connectedPaths;
+    }
+
+    public List<String> getConnectedPaths() {
+        return connectedPaths;
+    }
 }

core/src/com/cloud/agent/api/ModifyTargetsCommand.java

@@ -23,7 +23,11 @@
 import java.util.Map;
 
 public class ModifyTargetsCommand extends Command {
+    public enum TargetTypeToRemove { BOTH, NEITHER, STATIC, DYNAMIC }
+
     public static final String IQN = "iqn";
+    public static final String STORAGE_TYPE = "storageType";
+    public static final String STORAGE_UUID = "storageUuid";
     public static final String STORAGE_HOST = "storageHost";
     public static final String STORAGE_PORT = "storagePort";
     public static final String CHAP_NAME = "chapName";
@@ -32,6 +36,9 @@ public class ModifyTargetsCommand extends Command {
     public static final String MUTUAL_CHAP_SECRET = "mutualChapSecret";
 
     private boolean add;
+    private boolean applyToAllHostsInCluster;
+    private TargetTypeToRemove targetTypeToRemove = TargetTypeToRemove.BOTH;
+    private boolean removeAsync;
     private List<Map<String, String>> targets;
 
     public void setAdd(boolean add) {
@@ -42,6 +49,30 @@ public boolean getAdd() {
         return add;
     }
 
+    public void setApplyToAllHostsInCluster(boolean applyToAllHostsInCluster) {
+        this.applyToAllHostsInCluster = applyToAllHostsInCluster;
+    }
+
+    public boolean getApplyToAllHostsInCluster() {
+        return applyToAllHostsInCluster;
+    }
+
+    public void setTargetTypeToRemove(TargetTypeToRemove targetTypeToRemove) {
+        this.targetTypeToRemove = targetTypeToRemove;
+    }
+
+    public TargetTypeToRemove getTargetTypeToRemove() {
+        return targetTypeToRemove;
+    }
+
+    public void setRemoveAsync(boolean removeAsync) {
+        this.removeAsync = removeAsync;
+    }
+
+    public boolean isRemoveAsync() {
+        return removeAsync;
+    }
+
     public void setTargets(List<Map<String, String>> targets) {
         this.targets = targets;
     }

core/src/com/cloud/agent/api/PrepareForMigrationCommand.java

@@ -22,7 +22,8 @@
 import com.cloud.agent.api.to.VirtualMachineTO;
 
 public class PrepareForMigrationCommand extends Command {
-    VirtualMachineTO vm;
+    private VirtualMachineTO vm;
+    private boolean rollback;
 
     protected PrepareForMigrationCommand() {
     }
@@ -35,6 +36,14 @@ public VirtualMachineTO getVirtualMachine() {
         return vm;
     }
 
+    public void setRollback(boolean rollback) {
+        this.rollback = rollback;
+    }
+
+    public boolean isRollback() {
+        return rollback;
+    }
+
     @Override
     public boolean executeInSequence() {
         return true;

core/src/com/cloud/agent/api/StartAnswer.java

@@ -24,9 +24,12 @@
 import com.cloud.agent.api.to.VirtualMachineTO;
 
 public class StartAnswer extends Answer {
+    public static final String PATH = "path";
+    public static final String IMAGE_FORMAT = "imageFormat";
+
     VirtualMachineTO vm;
     String hostGuid;
-    Map<String, String> _iqnToPath;
+    private Map<String, Map<String, String>> _iqnToData;
 
     protected StartAnswer() {
     }
@@ -61,11 +64,11 @@ public String getHost_guid() {
         return hostGuid;
     }
 
-    public void setIqnToPath(Map<String, String> iqnToPath) {
-        _iqnToPath = iqnToPath;
+    public void setIqnToData(Map<String, Map<String, String>> iqnToData) {
+        _iqnToData = iqnToData;
     }
 
-    public Map<String, String> getIqnToPath() {
-        return _iqnToPath;
+    public Map<String, Map<String, String>> getIqnToData() {
+        return _iqnToData;
     }
 }

core/src/com/cloud/agent/api/StopCommand.java

@@ -22,6 +22,10 @@
 import com.cloud.agent.api.to.GPUDeviceTO;
 import com.cloud.vm.VirtualMachine;
 
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.List;
+
 public class StopCommand extends RebootCommand {
     private boolean isProxy = false;
     private String urlPort = null;
@@ -30,6 +34,11 @@ public class StopCommand extends RebootCommand {
     boolean checkBeforeCleanup = false;
     String controlIp = null;
     boolean forceStop = false;
+    /**
+     * On KVM when using iSCSI-based managed storage, if the user shuts a VM down from the guest OS (as opposed to doing so from CloudStack),
+     * we need to pass to the KVM agent a list of applicable iSCSI volumes that need to be disconnected.
+     */
+    private List<Map<String, String>> volumesToDisconnect = new ArrayList<>();
 
     protected StopCommand() {
     }
@@ -102,4 +111,12 @@ public void setControlIp(String controlIp){
     public boolean isForceStop() {
         return forceStop;
     }
+
+    public void setVolumesToDisconnect(List<Map<String, String>> volumesToDisconnect) {
+        this.volumesToDisconnect = volumesToDisconnect;
+    }
+
+    public List<Map<String, String>> getVolumesToDisconnect() {
+        return volumesToDisconnect;
+    }
 }