feat(vmm): add netd-managed macvtap networking

docs/macvtap-networking.md

@@ -0,0 +1,55 @@
+# Macvtap networking
+
+Macvtap mode gives each CVM a layer-2 identity on an existing host network
+without adding the parent interface to a Linux bridge. The VMM delegates the
+privileged interface lifecycle to `dstack-vmm netd`; manifests never contain
+the unstable `/dev/tapN` device path.
+
+## Configuration
+
+Configure a NIC through the manifest or VMM RPC:
+
+```json
+{
+  "mode": "macvtap",
+  "parent": "eth0",
+  "macvtap_mode": "private"
+}
+```
+
+`parent` must name an existing host interface. `macvtap_mode` may be
+`private`, `bridge`, `vepa`, or `passthru`; an empty value selects `private`.
+The configured netd socket and caller allowlist apply in the same way as for
+libvirt-filtered bridge networking.
+
+## Lifecycle
+
+For every macvtap NIC, the VMM sends netd the VM identity, NIC index, parent,
+and the same deterministic MAC address passed to QEMU. Netd then:
+
+1. derives the stable `dt<hash>` interface name;
+2. replaces any stale interface with that name;
+3. creates and activates the macvtap interface;
+4. reads its kernel-assigned ifindex and waits for `/dev/tap<ifindex>`; and
+5. returns that runtime device path to the VMM.
+
+The per-VM launcher opens the character device, places it at the fd referenced
+by QEMU's `-netdev tap,fd=...` argument, and then execs QEMU. This keeps device
+paths out of persistent VM
+configuration, works with both Supervisor and systemd process managers, and
+does not pass network fds through `sudo`.
+
+VM shutdown removes the interface by its deterministic identity. The device
+node disappears with the interface; its numeric path is never reused as an
+identity or cleanup key.
+
+## Limitations
+
+- The host and a macvtap guest do not communicate directly through the parent
+  interface by default. Add a host macvlan/macvtap endpoint if that path is
+  required.
+- Libvirt nwfilter bindings apply only to bridge mode. Macvtap deployments
+  must enforce network policy in the physical network or with another host
+  mechanism.
+- Real-host testing requires `CAP_NET_ADMIN`, a working udev setup for
+  `/dev/tapN`, and an upstream network that accepts multiple MAC addresses.

docs/tutorials/vmm-configuration.md

@@ -113,7 +113,6 @@ cid_pool_size = 1000
 max_allocable_vcpu = 124                        # Adjust: total cores - 4
 max_allocable_memory_in_mb = 990616             # Adjust: total MB - 16384
 qmp_socket = false
-user = ""
 use_mrconfigid = true
 qemu_pci_hole64_size = 0
 qemu_hotplug_off = false

dstack/vmm/Cargo.toml

@@ -25,7 +25,7 @@ sha2.workspace = true
 hex.workspace = true
 fs-err.workspace = true
 getrandom = { workspace = true, features = ["std"] }
-nix = { workspace = true, features = ["user"] }
+nix = { workspace = true, features = ["fs", "process", "signal", "user"] }
 dirs.workspace = true
 which.workspace = true
 clap = { workspace = true, features = ["derive", "string"] }

dstack/vmm/rpc/proto/vmm_rpc.proto

@@ -126,10 +126,14 @@ message VmConfiguration {
 
 // Per-VM networking configuration.
 message NetworkingConfig {
-  // Networking mode: "bridge", "user"
+  // Networking mode: "bridge", "user", "macvtap"
   string mode = 1;
   // Per-VM bridge interface name. Empty = node default bridge.
   string bridge_name = 2;
+  // Parent host interface for macvtap mode.
+  string parent = 3;
+  // macvtap forwarding mode. Empty selects "private".
+  string macvtap_mode = 4;
 }
 
 // Requested GPU layout for a CVM.

dstack/vmm/src/app.rs

@@ -5,7 +5,10 @@
 use crate::{
     config::{Config, NetworkFilterMode, Networking, NetworkingMode, ProcessAnnotation, Protocol},
     logrotate,
-    netd::{self, InterfaceIdentity, PrepareBridgeRequest, Request as NetdRequest},
+    netd::{
+        self, InterfaceIdentity, PrepareBridgeRequest, PrepareMacvtapRequest,
+        Request as NetdRequest,
+    },
 };
 
 use anyhow::{bail, Context, Result};
@@ -21,7 +24,7 @@ use dstack_vmm_rpc::{
 use fs_err as fs;
 use guest_api::client::DefaultClient as GuestClient;
 use id_pool::IdPool;
-use nix::unistd::{Uid, User};
+use nix::unistd::Uid;
 use or_panic::ResultOrPanic;
 use ra_rpc::client::RaClient;
 use serde::{Deserialize, Serialize};
@@ -447,7 +450,7 @@ impl App {
                 append_boot_separator(&path);
             }
 
-            let runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm);
+            let mut runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm);
             let devices = self.try_allocate_gpus(&vm_config.manifest)?;
             let gpu_host_config = self.config.cvm.gpu.clone();
             let devices_to_sanitize = devices.clone();
@@ -456,15 +459,33 @@ impl App {
             })
             .await
             .context("GPU sanitization task failed")??;
-            let processes = vm_config.config_qemu(&work_dir, &self.config.cvm, &devices)?;
-            work_dir.set_runtime_networks(&runtime_networks)?;
             if let Err(error) = self
-                .prepare_filtered_networks(&vm_config, &runtime_networks)
+                .prepare_filtered_networks(&vm_config, &mut runtime_networks)
                 .await
             {
                 let _ = work_dir.clear_runtime_networks();
                 return Err(error);
             }
+            let processes = match vm_config.config_qemu(
+                &work_dir,
+                &self.config.cvm,
+                &devices,
+                &runtime_networks,
+            ) {
+                Ok(processes) => processes,
+                Err(error) => {
+                    let _ = self
+                        .remove_filtered_networks(&vm_config.manifest.id, &runtime_networks)
+                        .await;
+                    return Err(error);
+                }
+            };
+            if let Err(error) = work_dir.set_runtime_networks(&runtime_networks) {
+                let _ = self
+                    .remove_filtered_networks(&vm_config.manifest.id, &runtime_networks)
+                    .await;
+                return Err(error);
+            }
             {
                 let mut state = self.lock();
                 let vm_state = state.get_mut(id).context("VM not found")?;
@@ -520,70 +541,84 @@ impl App {
     async fn prepare_filtered_networks(
         &self,
         vm: &VmConfig,
-        networks: &[Networking],
+        networks: &mut [Networking],
     ) -> Result<()> {
-        if self.config.cvm.network_filter.mode == NetworkFilterMode::None {
+        if self.config.cvm.network_filter.mode == NetworkFilterMode::None
+            && !networks
+                .iter()
+                .any(|network| network.mode == NetworkingMode::Macvtap)
+        {
             return Ok(());
         }
-        let qemu_uid = if self.config.cvm.user.is_empty() {
-            Uid::effective().as_raw()
-        } else {
-            User::from_name(&self.config.cvm.user)
-                .context("failed to resolve QEMU user")?
-                .with_context(|| format!("QEMU user {} does not exist", self.config.cvm.user))?
-                .uid
-                .as_raw()
-        };
+        let qemu_uid = Uid::effective().as_raw();
         let mut prepared = Vec::new();
-        for (nic_index, network) in networks.iter().enumerate() {
-            if network.mode != NetworkingMode::Bridge {
+        for (nic_index, network) in networks.iter_mut().enumerate() {
+            if network.mode == NetworkingMode::Bridge
+                && self.config.cvm.network_filter.mode == NetworkFilterMode::None
+            {
                 continue;
             }
             let identity = InterfaceIdentity {
                 instance_id: self.config.cvm.instance_id.clone(),
                 vm_id: vm.manifest.id.clone(),
                 nic_index,
             };
-            let request = PrepareBridgeRequest {
-                identity: identity.clone(),
-                bridge: network.bridge.clone(),
-                mac: network::mac_address_for_vm_index(
-                    &vm.manifest.id,
-                    &network.mac_prefix_bytes(),
-                    nic_index,
-                ),
-                qemu_uid,
-                filter: self.config.cvm.network_filter.filter.clone(),
-                parameters: self.config.cvm.network_filter.parameters.clone(),
+            let mac = network::mac_address_for_vm_index(
+                &vm.manifest.id,
+                &network.mac_prefix_bytes(),
+                nic_index,
+            );
+            let request = match network.mode {
+                NetworkingMode::Bridge => NetdRequest::PrepareBridge(PrepareBridgeRequest {
+                    identity: identity.clone(),
+                    bridge: network.bridge.clone(),
+                    mac,
+                    qemu_uid,
+                    filter: self.config.cvm.network_filter.filter.clone(),
+                    parameters: self.config.cvm.network_filter.parameters.clone(),
+                }),
+                NetworkingMode::Macvtap => NetdRequest::PrepareMacvtap(PrepareMacvtapRequest {
+                    identity: identity.clone(),
+                    parent: network.parent.clone(),
+                    mac,
+                    qemu_uid,
+                    mode: network.macvtap_mode.clone(),
+                }),
+                NetworkingMode::User | NetworkingMode::Custom => continue,
             };
-            if let Err(error) = netd::request(
-                &self.config.netd.socket,
-                &NetdRequest::PrepareBridge(request),
-            )
-            .await
-            {
-                // The client may have timed out while netd was still finishing
-                // this Prepare. Remove the in-flight identity first; netd's
-                // serialized accept loop processes it after Prepare completes.
-                if let Err(cleanup_error) = netd::request(
-                    &self.config.netd.socket,
-                    &NetdRequest::Remove {
-                        identity: identity.clone(),
-                    },
-                )
-                .await
-                {
-                    warn!(%cleanup_error, "failed to roll back in-flight filtered network");
-                }
-                for identity in prepared.into_iter().rev() {
-                    if let Err(cleanup_error) =
-                        netd::request(&self.config.netd.socket, &NetdRequest::Remove { identity })
-                            .await
+            let response = match netd::request(&self.config.netd.socket, &request).await {
+                Ok(response) => response,
+                Err(error) => {
+                    // The client may have timed out while netd was still finishing
+                    // this Prepare. Remove the in-flight identity first; netd's
+                    // serialized accept loop processes it after Prepare completes.
+                    if let Err(cleanup_error) = netd::request(
+                        &self.config.netd.socket,
+                        &NetdRequest::Remove {
+                            identity: identity.clone(),
+                        },
+                    )
+                    .await
                     {
-                        warn!(%cleanup_error, "failed to roll back prepared filtered network");
+                        warn!(%cleanup_error, "failed to roll back in-flight filtered network");
                     }
+                    for identity in prepared.into_iter().rev() {
+                        if let Err(cleanup_error) = netd::request(
+                            &self.config.netd.socket,
+                            &NetdRequest::Remove { identity },
+                        )
+                        .await
+                        {
+                            warn!(%cleanup_error, "failed to roll back prepared filtered network");
+                        }
+                    }
+                    return Err(error).context("failed to prepare libvirt-filtered networking");
                 }
-                return Err(error).context("failed to prepare libvirt-filtered networking");
+            };
+            if network.mode == NetworkingMode::Macvtap {
+                network.device = response
+                    .device
+                    .context("netd response omitted macvtap device")?;
             }
             prepared.push(identity);
         }
@@ -595,12 +630,24 @@ impl App {
         vm_id: &str,
         networks: &[Networking],
     ) -> Result<()> {
-        if self.config.cvm.network_filter.mode == NetworkFilterMode::None {
+        if self.config.cvm.network_filter.mode == NetworkFilterMode::None
+            && !networks
+                .iter()
+                .any(|network| network.mode == NetworkingMode::Macvtap)
+        {
             return Ok(());
         }
         let mut first_error = None;
         for (nic_index, network) in networks.iter().enumerate().rev() {
-            if network.mode != NetworkingMode::Bridge {
+            if network.mode == NetworkingMode::Bridge
+                && self.config.cvm.network_filter.mode == NetworkFilterMode::None
+            {
+                continue;
+            }
+            if !matches!(
+                network.mode,
+                NetworkingMode::Bridge | NetworkingMode::Macvtap
+            ) {
                 continue;
             }
             let identity = InterfaceIdentity {
@@ -2166,6 +2213,9 @@ mod tests {
         manifest.networks = vec![Networking {
             mode: NetworkingMode::Bridge,
             bridge: "dstack-br0".to_string(),
+            parent: String::new(),
+            macvtap_mode: String::new(),
+            device: String::new(),
             mac_prefix: String::new(),
             net: String::new(),
             dhcp_start: String::new(),
@@ -2428,6 +2478,9 @@ mod tests {
         bridge_manifest.networks = vec![Networking {
             mode: NetworkingMode::Bridge,
             bridge: "dstack-br0".to_string(),
+            parent: String::new(),
+            macvtap_mode: String::new(),
+            device: String::new(),
             mac_prefix: "02:aa:bb".to_string(),
             net: String::new(),
             dhcp_start: String::new(),