vmm_tests: Adds a regression test to verify that duplicate aer requests are not sent after servicing with keepalive. (#2135)

Add a regression test to verify that the driver does not issue
additional AERs after restore if something is already pending. Also some
minor restructuring of tests.

Author: gurasinghMS

vmm_tests/vmm_tests/tests/tests/multiarch/openhcl_servicing.rs

@@ -24,7 +24,6 @@ use petri::PetriVmBuilder;
 use petri::PetriVmmBackend;
 use petri::ResolvedArtifact;
 use petri::openvmm::OpenVmmPetriBackend;
-use petri::pipette::PipetteClient;
 use petri::pipette::cmd;
 use petri::vtl2_settings::ControllerType;
 use petri::vtl2_settings::Vtl2LunBuilder;
@@ -40,6 +39,7 @@ use petri_artifacts_vmm_test::artifacts::openhcl_igvm::LATEST_STANDARD_X64;
 use petri_artifacts_vmm_test::artifacts::openhcl_igvm::RELEASE_25_05_LINUX_DIRECT_X64;
 #[allow(unused_imports)]
 use petri_artifacts_vmm_test::artifacts::openhcl_igvm::RELEASE_25_05_STANDARD_AARCH64;
+use pipette_client::PipetteClient;
 use scsidisk_resources::SimpleScsiDiskHandle;
 use storvsp_resources::ScsiControllerHandle;
 use storvsp_resources::ScsiDeviceAndPath;
@@ -281,29 +281,38 @@ async fn servicing_keepalive_with_nvme_fault(
             ),
         );
 
-    let (mut vm, agent) = create_keepalive_test_config(config, fault_configuration).await?;
+    apply_fault_with_keepalive(
+        config,
+        fault_configuration,
+        fault_start_updater,
+        (igvm_file,),
+    )
+    .await
+}
 
-    agent.ping().await?;
-    let sh = agent.unix_shell();
+/// Verifies that the driver awaits an existing AER instead of issuing a new one after servicing.
+#[openvmm_test(openhcl_linux_direct_x64 [LATEST_LINUX_DIRECT_TEST_X64])]
+async fn servicing_keepalive_verify_no_duplicate_aers(
+    config: PetriVmBuilder<OpenVmmPetriBackend>,
+    (igvm_file,): (ResolvedArtifact<impl petri_artifacts_common::tags::IsOpenhclIgvm>,),
+) -> Result<(), anyhow::Error> {
+    let mut fault_start_updater = CellUpdater::new(false);
 
-    // Make sure the disk showed up.
-    cmd!(sh, "ls /dev/sda").run().await?;
+    let fault_configuration = FaultConfiguration::new(fault_start_updater.cell())
+        .with_admin_queue_fault(
+            AdminQueueFaultConfig::new().with_submission_queue_fault(
+                CommandMatchBuilder::new().match_cdw0_opcode(nvme_spec::AdminOpcode::ASYNCHRONOUS_EVENT_REQUEST.0).build(),
+                QueueFaultBehavior::Panic("Received a duplicate ASYNCHRONOUS_EVENT_REQUEST command during servicing with keepalive enabled. THERE IS A BUG SOMEWHERE.".to_string()),
+            ),
+        );
 
-    // CREATE_IO_COMPLETION_QUEUE is blocked. This will panic out without keepalive enabled.
-    fault_start_updater.set(true).await;
-    vm.restart_openhcl(
-        igvm_file.clone(),
-        OpenHclServicingFlags {
-            enable_nvme_keepalive: true,
-            ..Default::default()
-        },
+    apply_fault_with_keepalive(
+        config,
+        fault_configuration,
+        fault_start_updater,
+        (igvm_file,),
     )
-    .await?;
-
-    fault_start_updater.set(false).await;
-    agent.ping().await?;
-
-    Ok(())
+    .await
 }
 
 /// Test servicing an OpenHCL VM from the current version to itself with NVMe keepalive support
@@ -321,7 +330,6 @@ async fn servicing_keepalive_with_nvme_identify_fault(
     // Reduce the reported size of the namespace to 256 blocks instead of the original 512.
     let mut buf: u64 = 256;
     let buf = buf.as_mut_bytes();
-
     let fault_configuration = FaultConfiguration::new(fault_start_updater.cell())
         .with_admin_queue_fault(
             AdminQueueFaultConfig::new().with_completion_queue_fault(
@@ -338,6 +346,21 @@ async fn servicing_keepalive_with_nvme_identify_fault(
             ),
         );
 
+    apply_fault_with_keepalive(
+        config,
+        fault_configuration,
+        fault_start_updater,
+        (igvm_file,),
+    )
+    .await
+}
+
+async fn apply_fault_with_keepalive(
+    config: PetriVmBuilder<OpenVmmPetriBackend>,
+    fault_configuration: FaultConfiguration,
+    mut fault_start_updater: CellUpdater<bool>,
+    (igvm_file,): (ResolvedArtifact<impl petri_artifacts_common::tags::IsOpenhclIgvm>,),
+) -> Result<(), anyhow::Error> {
     let (mut vm, agent) = create_keepalive_test_config(config, fault_configuration).await?;
 
     agent.ping().await?;
@@ -346,7 +369,6 @@ async fn servicing_keepalive_with_nvme_identify_fault(
     // Make sure the disk showed up.
     cmd!(sh, "ls /dev/sda").run().await?;
 
-    // IDENTIFY:NAMESPACE is faulty. It will report a changed namespace size. The driver is still expected to make progress.
     fault_start_updater.set(true).await;
     vm.restart_openhcl(
         igvm_file.clone(),