vmm_test: MANA VF reset tests

Cargo.lock

@@ -2876,6 +2876,22 @@ dependencies = [
  "vm_resource",
 ]
 
+[[package]]
+name = "gdma_test"
+version = "0.0.0"
+dependencies = [
+ "async-trait",
+ "futures",
+ "gdma",
+ "gdma_defs",
+ "gdma_resources",
+ "pal_async",
+ "pci_resources",
+ "test_with_tracing",
+ "vm_resource",
+ "zerocopy",
+]
+
 [[package]]
 name = "generation_id"
 version = "0.0.0"
@@ -6037,6 +6053,7 @@ dependencies = [
  "disklayer_vhdx",
  "firmware_uefi",
  "gdma",
+ "gdma_test",
  "guest_crash_device",
  "guest_emulation_device",
  "guest_emulation_log",

Cargo.toml

@@ -280,6 +280,7 @@ missing_dev_resources = { path = "vm/devices/missing_dev_resources" }
 gdma = { path = "vm/devices/net/gdma" }
 gdma_defs = { path = "vm/devices/net/gdma_defs" }
 gdma_resources = { path = "vm/devices/net/gdma_resources" }
+gdma_test = { path = "vm/devices/net/gdma_test" }
 linux_net_bindings = { path = "vm/devices/net/linux_net_bindings" }
 mana_driver = { path = "vm/devices/net/mana_driver" }
 vfio_sys = { path = "vm/devices/user_driver/vfio_sys" }

openvmm/openvmm_resources/Cargo.toml

@@ -62,6 +62,7 @@ vmgs_broker.workspace = true
 # PCI devices
 cxl_spec.workspace = true
 gdma.workspace = true
+gdma_test.workspace = true
 nvme.workspace = true
 nvme_test.workspace = true
 

openvmm/openvmm_resources/src/lib.rs

@@ -88,6 +88,7 @@ vm_resource::register_static_resolvers! {
     // PCI devices
     cxl_spec::test::resolver::CxlTestDeviceResolver,
     gdma::resolver::GdmaDeviceResolver,
+    gdma_test::resolver::GdmaTestDeviceResolver,
     nvme::resolver::NvmeControllerResolver,
     nvme_test::resolver::NvmeFaultControllerResolver,
     virtio::resolver::VirtioPciResolver,

petri/src/vm/openvmm/mod.rs

@@ -19,6 +19,7 @@ mod start;
 pub use hugetlb::HUGETLB_2MB_PAGE_SIZE;
 #[cfg(target_os = "linux")]
 pub use hugetlb::ensure_2mb_hugetlb_pages;
+pub use modify::ManaTestControl;
 pub use runtime::OpenVmmFramebufferAccess;
 pub use runtime::OpenVmmInspector;
 pub use runtime::PetriVmOpenVmm;

petri/src/vm/openvmm/modify.rs

@@ -15,9 +15,12 @@ use chipset_resources::battery::HostBatteryUpdate;
 use disk_backend_resources::LayeredDiskHandle;
 use disk_backend_resources::layer::RamDiskLayerHandle;
 use gdma_resources::GdmaDeviceHandle;
+use gdma_resources::GdmaTestDeviceHandle;
+use gdma_resources::GdmaTestRequest;
 use gdma_resources::VportDefinition;
 use get_resources::ged::IgvmAttestTestConfig;
 use guid::Guid;
+use mesh::rpc::RpcSend;
 use net_backend_resources::mac_address::MacAddress;
 use nvme_resources::NamespaceDefinition;
 use nvme_resources::NvmeControllerHandle;
@@ -35,6 +38,8 @@ use openvmm_defs::config::Vtl2BaseAddressType;
 use vm_resource::IntoResource;
 use vmotherboard::ChipsetDeviceHandle;
 
+const MANA_VTL0_INSTANCE: Guid = guid::guid!("f8615163-d915-4743-a7d8-efa75db7b85a");
+
 impl PetriVmConfigOpenVmm {
     /// Enable the VTL0 alias map.
     // TODO: Remove once #912 is fixed.
@@ -154,6 +159,64 @@ impl PetriVmConfigOpenVmm {
         self
     }
 
+    /// Add a test-controllable VTL2 PCIe NIC and its paired VTL0 VF.
+    pub fn with_nic_test_control(mut self, test_config: ManaTestConfig) -> Self {
+        let vtl2_endpoint = net_backend_resources::consomme::ConsommeHandle {
+            cidr: None,
+            ports: Vec::new(),
+            recv: None,
+        }
+        .into_resource();
+        let vtl0_endpoint = net_backend_resources::consomme::ConsommeHandle {
+            cidr: None,
+            ports: Vec::new(),
+            recv: None,
+        }
+        .into_resource();
+
+        if let Some(vtl2_settings) = self.runtime_config.vtl2_settings.as_mut() {
+            self.config.vpci_devices.extend([
+                VpciDeviceConfig {
+                    vtl: DeviceVtl::Vtl2,
+                    instance_id: MANA_INSTANCE,
+                    resource: GdmaTestDeviceHandle {
+                        vports: vec![VportDefinition {
+                            mac_address: NIC_MAC_ADDRESS,
+                            endpoint: vtl2_endpoint,
+                        }],
+                        request_recv: test_config.request_recv,
+                    }
+                    .into_resource(),
+                    vnode: None,
+                },
+                VpciDeviceConfig {
+                    vtl: DeviceVtl::Vtl0,
+                    instance_id: MANA_VTL0_INSTANCE,
+                    resource: GdmaDeviceHandle {
+                        vports: vec![VportDefinition {
+                            mac_address: NIC_MAC_ADDRESS,
+                            endpoint: vtl0_endpoint,
+                        }],
+                    }
+                    .into_resource(),
+                    vnode: None,
+                },
+            ]);
+
+            vtl2_settings.dynamic.as_mut().unwrap().nic_devices.push(
+                vtl2_settings_proto::NicDeviceLegacy {
+                    instance_id: MANA_INSTANCE.to_string(),
+                    subordinate_instance_id: Some(MANA_VTL0_INSTANCE.to_string()),
+                    max_sub_channels: None,
+                },
+            );
+        } else {
+            panic!("with_nic_test_control requires VTL2 settings");
+        }
+
+        self
+    }
+
     /// Add a PCIe NVMe device to the VM using the NVMe emulator.
     pub fn with_pcie_nvme(mut self, port_name: &str, subsystem_id: Guid) -> Self {
         self.config.pcie_devices.push(PcieDeviceConfig {
@@ -589,3 +652,45 @@ impl PetriVmConfigOpenVmm {
         self
     }
 }
+
+/// Control interface for issuing test requests to the MANA emulator.
+///
+/// Created with [`ManaTestControl::new`] and connected with
+/// [`PetriVmConfigOpenVmm::with_nic_test_control`].
+pub struct ManaTestControl {
+    request_send: mesh::Sender<mesh::rpc::Rpc<GdmaTestRequest, ()>>,
+}
+
+/// Configuration token for a test-controllable MANA emulator.
+pub struct ManaTestConfig {
+    request_recv: mesh::Receiver<mesh::rpc::Rpc<GdmaTestRequest, ()>>,
+}
+
+impl ManaTestControl {
+    /// Creates a test control and its corresponding device configuration.
+    pub fn new() -> (Self, ManaTestConfig) {
+        let (request_send, request_recv) = mesh::channel();
+        (Self { request_send }, ManaTestConfig { request_recv })
+    }
+
+    /// Requests VF reconfiguration through the emulated hardware channel.
+    ///
+    /// Completion means the reset EQE has been posted to the HWC EQ.
+    /// It may take time for the EQE to be processed.
+    /// Then, VF Reconfiguration will be completed asynchronously.
+    ///
+    /// `revoke_vtl0_vf`: when `true` the guest VTL0 VF is revoked as part of
+    /// the reconfiguration.
+    pub async fn inject_vf_reset(&self, revoke_vtl0_vf: bool) -> Result<(), mesh::rpc::RpcError> {
+        self.request_send
+            .call(|rpc| rpc, GdmaTestRequest::VfReset { revoke_vtl0_vf })
+            .await
+    }
+
+    /// Shuts down test control after all preceding requests have completed.
+    pub async fn shutdown(self) -> Result<(), mesh::rpc::RpcError> {
+        self.request_send
+            .call(|rpc| rpc, GdmaTestRequest::Shutdown)
+            .await
+    }
+}

vm/devices/get/guest_emulation_device/src/lib.rs

@@ -645,6 +645,7 @@ impl<T: RingMem + Unpin> GedChannel<T> {
             HostRequests::CREATE_RAM_GPA_RANGE => self.handle_create_ram_gpa_range(message_buf)?,
             HostRequests::RESET_RAM_GPA_RANGE => self.handle_reset_ram_gpa_range(message_buf)?,
             HostRequests::LOAD_FIRMWARE => self.handle_load_firmware(message_buf)?,
+            HostRequests::VPCI_DEVICE_CONTROL => self.handle_vpci_device_control(message_buf)?,
             _ => {
                 tracing::error!(message_id = ?header.message_id(), "unexpected message");
                 return Err(Error::InvalidSequence);
@@ -1084,6 +1085,24 @@ impl<T: RingMem + Unpin> GedChannel<T> {
         Ok(())
     }
 
+    fn handle_vpci_device_control(&mut self, message_buf: &[u8]) -> Result<(), Error> {
+        let request = get_protocol::VpciDeviceControlRequest::read_from_prefix(message_buf)
+            .map_err(|_| Error::MessageTooSmall)?
+            .0;
+        let status = match request.code {
+            get_protocol::VpciDeviceControlCode::OFFER
+            | get_protocol::VpciDeviceControlCode::REVOKE
+            | get_protocol::VpciDeviceControlCode::RESET => {
+                get_protocol::VpciDeviceControlStatus::SUCCESS
+            }
+            _ => get_protocol::VpciDeviceControlStatus::INVALID_REQUEST,
+        };
+        let response = get_protocol::VpciDeviceControlResponse::new(status);
+        self.channel
+            .try_send(response.as_bytes())
+            .map_err(Error::Vmbus)
+    }
+
     fn handle_host_notification(
         &mut self,
         header: get_protocol::HeaderHostNotification,

vm/devices/net/gdma/Cargo.toml

@@ -6,6 +6,9 @@ name = "gdma"
 edition.workspace = true
 rust-version.workspace = true
 
+[features]
+test_helpers = []
+
 [dependencies]
 gdma_defs.workspace = true
 gdma_resources.workspace = true

vm/devices/net/gdma/src/bnic.rs

@@ -35,6 +35,7 @@ use gdma_defs::Wqe;
 use gdma_defs::access::WqeAccess;
 use gdma_defs::bnic as bnic_defs;
 use gdma_defs::bnic::ManaDestroyWqobjReq;
+use gdma_defs::bnic::ManaMoveFilterVTL2PrivilegedReq;
 use gdma_defs::bnic::ManaTxShortOob;
 use gdma_defs::bnic::Tristate;
 use guestmem::GuestMemory;
@@ -190,6 +191,7 @@ struct Vport {
     task: TaskControl<TxRxState, TxRxTask>,
     queue_cfg: QueueCfg,
     serial_no: u32,
+    direction_to_vtl0: bool,
 }
 
 impl InspectMut for Vport {
@@ -201,6 +203,7 @@ impl InspectMut for Vport {
             .field("tx_cq", self.queue_cfg.tx.map(|(_wq, cq)| cq))
             .field("rx_wq", self.queue_cfg.rx.map(|(wq, _cq)| wq))
             .field("rx_cq", self.queue_cfg.rx.map(|(_wq, cq)| cq))
+            .field("direction_to_vtl0", self.direction_to_vtl0)
             .merge(&mut self.task);
     }
 }
@@ -228,6 +231,7 @@ impl BasicNic {
                         task: TaskControl::new(TxRxState),
                         queue_cfg: QueueCfg { tx: None, rx: None },
                         serial_no: 0,
+                        direction_to_vtl0: false,
                     }
                 },
             )
@@ -424,19 +428,30 @@ impl BasicNic {
                 }
             }
             ManaCommandCode::MANA_VTL2_MOVE_FILTER => {
-                anyhow::bail!("unsupported command MANA_VTL2_MOVE_FILTER");
+                let req: ManaMoveFilterVTL2PrivilegedReq = read
+                    .read_plain()
+                    .context("reading move vport filter request")?;
+                let vport = self
+                    .vports
+                    .get_mut(req.vport as usize)
+                    .context("invalid vport")?;
+                vport.direction_to_vtl0 = match req.direction_to_vtl0 {
+                    0 => false,
+                    1 => true,
+                    value => anyhow::bail!("invalid filter direction {value}"),
+                };
             }
             ManaCommandCode::MANA_VTL2_QUERY_FILTER_STATE => {
                 let req: gdma_defs::bnic::ManaQueryFilterStateReq = read
                     .read_plain()
                     .context("reading query vport filter state request")?;
-                let _ = self
+                let vport = self
                     .vports
-                    .get_mut(req.vport as usize)
+                    .get(req.vport as usize)
                     .context("invalid vport")?;
 
                 let resp = gdma_defs::bnic::ManaQueryFilterStateResponse {
-                    direction_to_vtl0: 0,
+                    direction_to_vtl0: vport.direction_to_vtl0.into(),
                     reserved: [0; 7],
                 };
 

vm/devices/net/gdma/src/lib.rs

@@ -121,6 +121,35 @@ enum SmcError {
 
 pub use bnic::BnicConfig;
 
+/// Helpers for cross-crate testing.
+#[cfg(feature = "test_helpers")]
+pub mod test_helpers {
+    use super::GdmaDevice;
+    use super::VportConfig;
+    use super::queues;
+    use gdma_resources::VportDefinition;
+    use std::sync::Arc;
+    use vm_resource::ResourceResolver;
+
+    /// Resolves vport definitions for a test GDMA device.
+    pub async fn resolve_vports(
+        resolver: &ResourceResolver,
+        vports: Vec<VportDefinition>,
+    ) -> Result<Vec<VportConfig>, super::resolver::Error> {
+        super::resolver::resolve_vports(resolver, vports).await
+    }
+
+    /// Returns a function that injects EQEs into the hardware channel EQ.
+    pub fn hwc_eq_injector(device: &GdmaDevice) -> impl Fn(u8, &[u8]) + Send + Sync + 'static {
+        let queues = Arc::downgrade(&device.queues);
+        move |ty, data| {
+            if let Some(queues) = queues.upgrade() {
+                queues.post_eq(queues::ID_OFFSET as u32, ty, data)
+            }
+        }
+    }
+}
+
 pub struct VportConfig {
     pub mac_address: MacAddress,
     pub endpoint: Box<dyn Endpoint>,

vm/devices/net/gdma/src/queues.rs

@@ -37,7 +37,7 @@ use zerocopy::IntoBytes;
 use zerocopy::KnownLayout;
 
 // Offset the queue IDs seen by the guest.
-const ID_OFFSET: usize = 24;
+pub(crate) const ID_OFFSET: usize = 24;
 
 struct CqEq<T> {
     region: DmaRegion,