FIX5: new qemu-guest-agent based patching for KVM (#72)

This introduces a new patching script for patching systemvms on KVM
using qemu-guest-agent that runs inside the systemvm on startup. This
also removes the vport device which was previously used by the legacy
patching script and instead uses the modern and new uniform guest
agent vport for host-guest communication.

Also updates the sytemvmtemplate build config to use the latest Debian
9.8.0 iso.

Signed-off-by: Rohit Yadav <[email protected]>

Author: rohityadavcloud

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -47,7 +47,6 @@
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 
-import com.cloud.resource.RequestWrapper;
 import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
@@ -146,6 +145,7 @@
 import com.cloud.network.Networks.BroadcastDomainType;
 import com.cloud.network.Networks.RouterPrivateIpStrategy;
 import com.cloud.network.Networks.TrafficType;
+import com.cloud.resource.RequestWrapper;
 import com.cloud.resource.ServerResource;
 import com.cloud.resource.ServerResourceBase;
 import com.cloud.storage.JavaStorageLayer;
@@ -199,7 +199,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
 
     private String _modifyVlanPath;
     private String _versionstringpath;
-    private String _patchViaSocketPath;
+    private String _patchScriptPath;
     private String _createvmPath;
     private String _manageSnapshotPath;
     private String _resizeVolumePath;
@@ -682,9 +682,9 @@ public boolean configure(final String name, final Map<String, Object> params) th
             throw new ConfigurationException("Unable to find versions.sh");
         }
 
-        _patchViaSocketPath = Script.findScript(kvmScriptsDir + "/patch/", "patchviasocket.py");
-        if (_patchViaSocketPath == null) {
-            throw new ConfigurationException("Unable to find patchviasocket.py");
+        _patchScriptPath = Script.findScript(kvmScriptsDir, "patch.sh");
+        if (_patchScriptPath == null) {
+            throw new ConfigurationException("Unable to find patch.sh");
         }
 
         _heartBeatPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh");
@@ -1362,13 +1362,13 @@ private boolean checkOvsNetwork(final String networkName) {
     }
 
     public boolean passCmdLine(final String vmName, final String cmdLine) throws InternalErrorException {
-        final Script command = new Script(_patchViaSocketPath, 5 * 1000, s_logger);
+        final Script command = new Script(_patchScriptPath, 30 * 1000, s_logger);
         String result;
         command.add("-n", vmName);
-        command.add("-p", cmdLine.replaceAll(" ", "%"));
+        command.add("-c", cmdLine);
         result = command.execute();
         if (result != null) {
-            s_logger.error("passcmd failed:" + result);
+            s_logger.error("Passing cmdline failed:" + result);
             return false;
         }
         return true;
@@ -2141,12 +2141,6 @@ So if getMinSpeed() returns null we fall back to getSpeed().
         final SerialDef serial = new SerialDef("pty", null, (short)0);
         devices.addDevice(serial);
 
-        /* Add a VirtIO channel for SystemVMs for communication and provisioning */
-        if (vmTO.getType() != VirtualMachine.Type.User) {
-            devices.addDevice(new ChannelDef(vmTO.getName() + ".vport", ChannelDef.ChannelType.UNIX,
-                              new File(_qemuSocketsPath + "/" + vmTO.getName() + ".agent")));
-        }
-
         if (_rngEnable) {
             final RngDef rngDevice = new RngDef(_rngPath, _rngBackendModel, _rngRateBytes, _rngRatePeriod);
             devices.addDevice(rngDevice);

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java

@@ -115,17 +115,18 @@ public Answer execute(final StartCommand command, final LibvirtComputingResource
 
             // pass cmdline info to system vms
             if (vmSpec.getType() != VirtualMachine.Type.User) {
-                //wait and try passCmdLine for 5 minutes at most for CLOUDSTACK-2823
                 String controlIp = null;
                 for (final NicTO nic : nics) {
                     if (nic.getType() == TrafficType.Control) {
                         controlIp = nic.getIp();
                         break;
                     }
                 }
-                for (int count = 0; count < 30; count++) {
+                // try to patch and SSH into the systemvm for up to 5 minutes
+                for (int count = 0; count < 10; count++) {
+                    // wait and try passCmdLine for 30 seconds at most for CLOUDSTACK-2823
                     libvirtComputingResource.passCmdLine(vmName, vmSpec.getBootArgs());
-                    //check router is up?
+                    // check router is up?
                     final VirtualRoutingResource virtRouterResource = libvirtComputingResource.getVirtRouterResource();
                     final boolean result = virtRouterResource.connect(controlIp, 1, 5000);
                     if (result) {

plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParserTest.java

@@ -22,14 +22,14 @@
 import java.io.File;
 import java.util.List;
 
-import junit.framework.TestCase;
-
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ChannelDef;
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef;
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef;
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.RngDef;
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.WatchDogDef;
 
+import junit.framework.TestCase;
+
 public class LibvirtDomainXMLParserTest extends TestCase {
 
     public void testDomainXMLParser() {
@@ -45,9 +45,6 @@ public void testDomainXMLParser() {
         InterfaceDef.GuestNetType ifType = InterfaceDef.GuestNetType.BRIDGE;
 
         ChannelDef.ChannelType channelType = ChannelDef.ChannelType.UNIX;
-        ChannelDef.ChannelState channelState = ChannelDef.ChannelState.DISCONNECTED;
-        String ssvmAgentPath =  "/var/lib/libvirt/qemu/s-2970-VM.agent";
-        String ssvmAgentName = "s-2970-VM.vport";
         String guestAgentPath = "/var/lib/libvirt/qemu/guest-agent.org.qemu.guest_agent.0";
         String guestAgentName = "org.qemu.guest_agent.0";
 
@@ -155,12 +152,6 @@ public void testDomainXMLParser() {
                      "<target type='serial' port='0'/>" +
                      "<alias name='serial0'/>" +
                      "</console>" +
-                     "<channel type='unix'>" +
-                     "<source mode='bind' path='/var/lib/libvirt/qemu/s-2970-VM.agent'/>" +
-                     "<target type='virtio' name='s-2970-VM.vport' state='disconnected'/>" +
-                     "<alias name='channel0'/>" +
-                     "<address type='virtio-serial' controller='0' bus='0' port='1'/>" +
-                     "</channel>" +
                      "<input type='tablet' bus='usb'>" +
                      "<alias name='input0'/>" +
                      "</input>" +
@@ -215,14 +206,9 @@ public void testDomainXMLParser() {
             assertEquals(channelType, channels.get(i).getChannelType());
         }
 
-        /* SSVM provisioning port/channel */
-        assertEquals(channelState, channels.get(0).getChannelState());
-        assertEquals(new File(ssvmAgentPath), channels.get(0).getPath());
-        assertEquals(ssvmAgentName, channels.get(0).getName());
-
         /* Qemu Guest Agent port/channel */
-        assertEquals(new File(guestAgentPath), channels.get(1).getPath());
-        assertEquals(guestAgentName, channels.get(1).getName());
+        assertEquals(new File(guestAgentPath), channels.get(0).getPath());
+        assertEquals(guestAgentName, channels.get(0).getName());
 
         List<InterfaceDef> ifs = parser.getInterfaces();
         for (int i = 0; i < ifs.size(); i++) {

plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java

@@ -21,13 +21,13 @@
 
 import java.io.File;
 
-import junit.framework.TestCase;
-
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ChannelDef;
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef;
 import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.SCSIDef;
 import com.cloud.utils.Pair;
 
+import junit.framework.TestCase;
+
 public class LibvirtVMDefTest extends TestCase {
 
     public void testInterfaceEtehrnet() {
@@ -180,7 +180,7 @@ public void testRngDef() {
     public void testChannelDef() {
         ChannelDef.ChannelType type = ChannelDef.ChannelType.UNIX;
         ChannelDef.ChannelState state = ChannelDef.ChannelState.CONNECTED;
-        String name = "v-136-VM.vport";
+        String name = "v-136-VM.org.qemu.guest_agent.0";
         File path = new File("/var/lib/libvirt/qemu/" + name);
 
         ChannelDef channelDef = new ChannelDef(name, type, state, path);

scripts/installer/windows/client.wxs

@@ -1046,7 +1046,7 @@
                                             <File Id="fil47212822DDAFFCD71C79615CF4583BAF" KeyPath="yes" Source="!(wix.SourceClient)\WEB-INF\classes\scripts\vm\hypervisor\kvm\kvmheartbeat.sh" />
                                         </Component>
                                         <Component Id="cmpF2BBDD336FEC0B34B3C744ACF1E4B959" Guid="{56D8ECF7-49F8-4B26-A8F4-662252C0A647}">
-                                            <File Id="filD809C7F728AC5D1BD36E7DB403BFA141" KeyPath="yes" Source="!(wix.SourceClient)\WEB-INF\classes\scripts\vm\hypervisor\kvm\patchviasocket.py" />
+                                            <File Id="filD809C7F728AC5D1BD36E7DB403BFA141" KeyPath="yes" Source="!(wix.SourceClient)\WEB-INF\classes\scripts\vm\hypervisor\kvm\patch.sh" />
                                         </Component>
                                         <Component Id="cmp2F4D4D81563D153E86B0A652A83D363A" Guid="{7BFC7637-E33D-4BC4-8B25-3CDEA601110C}">
                                             <File Id="fil349420D6088A01C9F63E27634623F5BE" KeyPath="yes" Source="!(wix.SourceClient)\WEB-INF\classes\scripts\vm\hypervisor\kvm\setup_agent.sh" />

scripts/vm/hypervisor/kvm/patch.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+# Get the VM name and cmdline
+while getopts "n:c:h" opt; do
+  case ${opt} in
+    n )
+      name=$OPTARG
+      ;;
+    c )
+      cmdline=$(echo $OPTARG | base64 -w 0)
+      ;;
+    h )
+      echo "Usage: $0 -n [VM name] -c [command line]"
+      exit 0
+      ;;
+  esac
+done
+
+SSHKEY_FILE="/root/.ssh/id_rsa.pub.cloud"
+if [ ! -e $SSHKEY_FILE ]; then
+    echo "SSH public key file $SSHKEY_FILE not found!"
+    exit 1
+fi
+
+if ! which virsh > /dev/null; then
+    echo "Libvirt CLI 'virsh' not found"
+    exit 1
+fi
+
+# Read the SSH public key
+sshkey=$(cat $SSHKEY_FILE | base64 -w 0)
+
+# Method to send and write payload inside the VM
+send_file() {
+    local name=${1}
+    local path=${2}
+    local content=${@:3}
+    local fd=$(virsh qemu-agent-command $name "{\"execute\":\"guest-file-open\", \"arguments\":{\"path\":\"$path\",\"mode\":\"w+\"}}" | sed 's/[^:]*:\([^}]*\).*/\1/')
+    virsh qemu-agent-command $name "{\"execute\":\"guest-file-write\", \"arguments\":{\"handle\":$fd,\"buf-b64\":\"$content\"}}" > /dev/null
+    virsh qemu-agent-command $name "{\"execute\":\"guest-file-close\", \"arguments\":{\"handle\":$fd}}" > /dev/null
+}
+
+# Wait for the guest agent to come online
+while ! virsh qemu-agent-command $name '{"execute":"guest-ping"}' >/dev/null 2>&1
+do
+    sleep 0.1
+done
+
+# Test guest agent sanity
+while [ $(virsh qemu-agent-command $name '{"execute":"guest-sync","arguments":{"id":1234567890}}' 2>/dev/null) != '{"return":1234567890}' ]
+do
+    sleep 0.1
+done
+
+# Write ssh public key
+send_file $name "/root/.ssh/authorized_keys" $sshkey
+
+# Fix ssh public key permission
+virsh qemu-agent-command $name '{"execute":"guest-exec","arguments":{"path":"chmod","arg":["go-rwx","/root/.ssh/authorized_keys"]}}' > /dev/null
+
+# Write cmdline payload
+send_file $name "/var/cache/cloud/cmdline" $cmdline