From 0cbefa49ac453db2c03e66c8eb7958f93d7c19dc Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 16 Aug 2026 22:34:14 -0700 Subject: [PATCH 01/11] feat(vmm): add netd-managed macvtap networking --- docs/macvtap-networking.md | 55 ++++++++ dstack/vmm/rpc/proto/vmm_rpc.proto | 6 +- dstack/vmm/src/app.rs | 108 +++++++++++---- dstack/vmm/src/app/qemu.rs | 72 +++++++++- dstack/vmm/src/app/vm_info.rs | 4 + dstack/vmm/src/config.rs | 25 ++++ dstack/vmm/src/main_service.rs | 30 ++++ dstack/vmm/src/netd.rs | 122 +++++++++++++++-- dstack/vmm/src/vm_launcher.rs | 213 ++++++++++++++++++++++++----- 9 files changed, 559 insertions(+), 76 deletions(-) create mode 100644 docs/macvtap-networking.md diff --git a/docs/macvtap-networking.md b/docs/macvtap-networking.md new file mode 100644 index 000000000..1bd30e9ea --- /dev/null +++ b/docs/macvtap-networking.md @@ -0,0 +1,55 @@ +# Macvtap networking + +Macvtap mode gives each CVM a layer-2 identity on an existing host network +without adding the parent interface to a Linux bridge. The VMM delegates the +privileged interface lifecycle to `dstack-vmm netd`; manifests never contain +the unstable `/dev/tapN` device path. + +## Configuration + +Configure a NIC through the manifest or VMM RPC: + +```json +{ + "mode": "macvtap", + "parent": "eth0", + "macvtap_mode": "private" +} +``` + +`parent` must name an existing host interface. `macvtap_mode` may be +`private`, `bridge`, `vepa`, or `passthru`; an empty value selects `private`. +The configured netd socket and caller allowlist apply in the same way as for +libvirt-filtered bridge networking. + +## Lifecycle + +For every macvtap NIC, the VMM sends netd the VM identity, NIC index, parent, +and the same deterministic MAC address passed to QEMU. Netd then: + +1. derives the stable `dt` interface name; +2. replaces any stale interface with that name; +3. creates and activates the macvtap interface; +4. reads its kernel-assigned ifindex and waits for `/dev/tap`; and +5. returns that runtime device path to the VMM. + +The per-VM launcher opens the root-owned character device before dropping to +`cvm.user`, places it at the fd referenced by QEMU's `-netdev tap,fd=...` +argument, and then execs QEMU. This keeps device paths out of persistent VM +configuration, works with both Supervisor and systemd process managers, and +does not pass network fds through `sudo`. + +VM shutdown removes the interface by its deterministic identity. The device +node disappears with the interface; its numeric path is never reused as an +identity or cleanup key. + +## Limitations + +- The host and a macvtap guest do not communicate directly through the parent + interface by default. Add a host macvlan/macvtap endpoint if that path is + required. +- Libvirt nwfilter bindings apply only to bridge mode. Macvtap deployments + must enforce network policy in the physical network or with another host + mechanism. +- Real-host testing requires `CAP_NET_ADMIN`, a working udev setup for + `/dev/tapN`, and an upstream network that accepts multiple MAC addresses. diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 734086e2e..ef4786af9 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -126,10 +126,14 @@ message VmConfiguration { // Per-VM networking configuration. message NetworkingConfig { - // Networking mode: "bridge", "user" + // Networking mode: "bridge", "user", "macvtap" string mode = 1; // Per-VM bridge interface name. Empty = node default bridge. string bridge_name = 2; + // Parent host interface for macvtap mode. + string parent = 3; + // macvtap forwarding mode. Empty selects "private". + string macvtap_mode = 4; } // Requested GPU layout for a CVM. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index ec31b3e5b..84a041daf 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -447,7 +447,7 @@ impl App { append_boot_separator(&path); } - let runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm); + let mut runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm); let devices = self.try_allocate_gpus(&vm_config.manifest)?; let gpu_host_config = self.config.cvm.gpu.clone(); let devices_to_sanitize = devices.clone(); @@ -456,15 +456,28 @@ impl App { }) .await .context("GPU sanitization task failed")??; - let processes = vm_config.config_qemu(&work_dir, &self.config.cvm, &devices)?; - work_dir.set_runtime_networks(&runtime_networks)?; if let Err(error) = self - .prepare_filtered_networks(&vm_config, &runtime_networks) + .prepare_filtered_networks(&vm_config, &mut runtime_networks) .await { let _ = work_dir.clear_runtime_networks(); return Err(error); } + let processes = match vm_config.config_qemu(&work_dir, &self.config.cvm, &devices) { + Ok(processes) => processes, + Err(error) => { + let _ = self + .remove_filtered_networks(&vm_config.manifest.id, &runtime_networks) + .await; + return Err(error); + } + }; + if let Err(error) = work_dir.set_runtime_networks(&runtime_networks) { + let _ = self + .remove_filtered_networks(&vm_config.manifest.id, &runtime_networks) + .await; + return Err(error); + } { let mut state = self.lock(); let vm_state = state.get_mut(id).context("VM not found")?; @@ -520,9 +533,13 @@ impl App { async fn prepare_filtered_networks( &self, vm: &VmConfig, - networks: &[Networking], + networks: &mut [Networking], ) -> Result<()> { - if self.config.cvm.network_filter.mode == NetworkFilterMode::None { + if self.config.cvm.network_filter.mode == NetworkFilterMode::None + && !networks + .iter() + .any(|network| network.mode == NetworkingMode::Macvtap) + { return Ok(()); } let qemu_uid = if self.config.cvm.user.is_empty() { @@ -535,8 +552,16 @@ impl App { .as_raw() }; let mut prepared = Vec::new(); - for (nic_index, network) in networks.iter().enumerate() { - if network.mode != NetworkingMode::Bridge { + for (nic_index, network) in networks.iter_mut().enumerate() { + if network.mode == NetworkingMode::Bridge + && self.config.cvm.network_filter.mode == NetworkFilterMode::None + { + continue; + } + if !matches!( + network.mode, + NetworkingMode::Bridge | NetworkingMode::Macvtap + ) { continue; } let identity = InterfaceIdentity { @@ -544,24 +569,30 @@ impl App { vm_id: vm.manifest.id.clone(), nic_index, }; - let request = PrepareBridgeRequest { - identity: identity.clone(), - bridge: network.bridge.clone(), - mac: network::mac_address_for_vm_index( - &vm.manifest.id, - &network.mac_prefix_bytes(), - nic_index, - ), - qemu_uid, - filter: self.config.cvm.network_filter.filter.clone(), - parameters: self.config.cvm.network_filter.parameters.clone(), + let mac = network::mac_address_for_vm_index( + &vm.manifest.id, + &network.mac_prefix_bytes(), + nic_index, + ); + let request = match network.mode { + NetworkingMode::Bridge => NetdRequest::PrepareBridge(PrepareBridgeRequest { + identity: identity.clone(), + bridge: network.bridge.clone(), + mac, + qemu_uid, + filter: self.config.cvm.network_filter.filter.clone(), + parameters: self.config.cvm.network_filter.parameters.clone(), + }), + NetworkingMode::Macvtap => NetdRequest::PrepareMacvtap { + identity: identity.clone(), + parent: network.parent.clone(), + mac, + mode: network.macvtap_mode.clone(), + }, + _ => unreachable!(), }; - if let Err(error) = netd::request( - &self.config.netd.socket, - &NetdRequest::PrepareBridge(request), - ) - .await - { + let response = netd::request(&self.config.netd.socket, &request).await; + if let Err(error) = response { // The client may have timed out while netd was still finishing // this Prepare. Remove the in-flight identity first; netd's // serialized accept loop processes it after Prepare completes. @@ -585,6 +616,11 @@ impl App { } return Err(error).context("failed to prepare libvirt-filtered networking"); } + if network.mode == NetworkingMode::Macvtap { + network.device = response? + .device + .context("netd response omitted macvtap device")?; + } prepared.push(identity); } Ok(()) @@ -595,12 +631,24 @@ impl App { vm_id: &str, networks: &[Networking], ) -> Result<()> { - if self.config.cvm.network_filter.mode == NetworkFilterMode::None { + if self.config.cvm.network_filter.mode == NetworkFilterMode::None + && !networks + .iter() + .any(|network| network.mode == NetworkingMode::Macvtap) + { return Ok(()); } let mut first_error = None; for (nic_index, network) in networks.iter().enumerate().rev() { - if network.mode != NetworkingMode::Bridge { + if network.mode == NetworkingMode::Bridge + && self.config.cvm.network_filter.mode == NetworkFilterMode::None + { + continue; + } + if !matches!( + network.mode, + NetworkingMode::Bridge | NetworkingMode::Macvtap + ) { continue; } let identity = InterfaceIdentity { @@ -2166,6 +2214,9 @@ mod tests { manifest.networks = vec![Networking { mode: NetworkingMode::Bridge, bridge: "dstack-br0".to_string(), + parent: String::new(), + macvtap_mode: String::new(), + device: String::new(), mac_prefix: String::new(), net: String::new(), dhcp_start: String::new(), @@ -2428,6 +2479,9 @@ mod tests { bridge_manifest.networks = vec![Networking { mode: NetworkingMode::Bridge, bridge: "dstack-br0".to_string(), + parent: String::new(), + macvtap_mode: String::new(), + device: String::new(), mac_prefix: "02:aa:bb".to_string(), net: String::new(), dhcp_start: String::new(), diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 67115c0fe..78e821330 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -18,7 +18,7 @@ use crate::{ CvmConfig, CvmPlatform, NetworkFilterMode, Networking, NetworkingMode, ProcessAnnotation, }, netd::{tap_name, InterfaceIdentity}, - vm_launcher::{ChildCommand, LaunchSpec}, + vm_launcher::{ChildCommand, LaunchSpec, OpenFile}, }; use anyhow::{bail, Context, Result}; use bon::Builder; @@ -351,7 +351,14 @@ impl VmConfig { prepared: &prepared, } .build()?; + let has_macvtap = prepared + .networks + .iter() + .any(|network| network.mode == NetworkingMode::Macvtap); let Some(socket) = prepared.swtpm_socket.as_deref() else { + if has_macvtap { + return self.wrap_launcher(cfg, &prepared, process, None, None); + } return Ok(vec![process]); }; let swtpm_path = prepared @@ -380,16 +387,63 @@ impl VmConfig { "--flags".into(), "not-need-init,startup-clear".into(), ]; + self.wrap_launcher( + cfg, + &prepared, + process, + Some(ChildCommand { + command: swtpm_path.to_string_lossy().into_owned(), + args: swtpm_args, + }), + Some(socket.to_path_buf()), + ) + } + + fn wrap_launcher( + &self, + cfg: &CvmConfig, + prepared: &PreparedQemuLaunch, + mut process: ProcessConfig, + swtpm: Option, + swtpm_socket: Option, + ) -> Result> { + let identity = if cfg.user.is_empty() { + None + } else { + let user = User::from_name(&cfg.user) + .context("failed to resolve QEMU user")? + .with_context(|| format!("QEMU user {} does not exist", cfg.user))?; + if process.command != "sudo" || process.args.first().map(String::as_str) != Some("-u") { + bail!("unexpected QEMU privilege wrapper"); + } + process.command = process + .args + .get(2) + .context("sudo command is missing")? + .clone(); + process.args.drain(0..3); + Some((user.uid.as_raw(), user.gid.as_raw())) + }; + let open_files = prepared + .networks + .iter() + .enumerate() + .filter(|(_, network)| network.mode == NetworkingMode::Macvtap) + .map(|(index, network)| OpenFile { + fd: (3 + index) as i32, + path: network.device.clone().into(), + }) + .collect(); let spec = LaunchSpec { qemu: ChildCommand { command: process.command, args: process.args, }, - swtpm: ChildCommand { - command: swtpm_path.to_string_lossy().into_owned(), - args: swtpm_args, - }, - swtpm_socket: socket.to_path_buf(), + swtpm, + swtpm_socket, + open_files, + uid: identity.map(|value| value.0), + gid: identity.map(|value| value.1), startup_timeout_ms: 5_000, shutdown_timeout_ms: 10_000, }; @@ -635,6 +689,12 @@ impl QemuCommandBuilder<'_> { } networking.netdev.clone() } + NetworkingMode::Macvtap => { + if networking.device.is_empty() { + bail!("macvtap interface {index} has not been prepared by netd"); + } + format!("tap,id={net_id},fd={},vhost=off", 3 + index) + } }; command.arg("-netdev").arg(netdev); command.arg("-device").arg(net_device); diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index 79d1a78b9..eab200244 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -38,6 +38,7 @@ fn networking_mode_name(mode: NetworkingMode) -> &'static str { NetworkingMode::Bridge => "bridge", NetworkingMode::User => "user", NetworkingMode::Custom => "custom", + NetworkingMode::Macvtap => "macvtap", } } @@ -46,6 +47,7 @@ fn networking_backend_name(mode: NetworkingMode) -> &'static str { NetworkingMode::Bridge => "tap_bridge", NetworkingMode::User => "slirp", NetworkingMode::Custom => "custom", + NetworkingMode::Macvtap => "macvtap", } } @@ -57,6 +59,8 @@ fn networking_to_proto(networking: &Networking) -> pb::NetworkingConfig { } else { String::new() }, + parent: networking.parent.clone(), + macvtap_mode: networking.macvtap_mode.clone(), } } diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index e13464af8..57ce67cb0 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -786,6 +786,19 @@ fn validate_networking(networking: &Networking) -> Result<()> { !networking.netdev.trim().is_empty(), "cvm.networking.netdev must not be empty in custom mode" ), + NetworkingMode::Macvtap => { + anyhow::ensure!( + !networking.parent.trim().is_empty(), + "cvm.networking.parent must not be empty in macvtap mode" + ); + anyhow::ensure!( + matches!( + networking.macvtap_mode.as_str(), + "" | "private" | "bridge" | "vepa" | "passthru" + ), + "cvm.networking.macvtap_mode must be private, bridge, vepa, or passthru" + ); + } NetworkingMode::User => {} } Ok(()) @@ -797,6 +810,7 @@ pub enum NetworkingMode { User, Bridge, Custom, + Macvtap, } /// Flat networking configuration. The `mode` field selects which backend is @@ -811,6 +825,17 @@ pub struct Networking { #[serde(default)] pub bridge: String, + // ── Macvtap fields ──────────────────────────────────────────── + /// Parent host interface for macvtap (e.g., "eth0"). + #[serde(default)] + pub parent: String, + /// macvtap forwarding mode. Empty selects "private". + #[serde(default)] + pub macvtap_mode: String, + /// Runtime-only character device returned by netd. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub device: String, + // ── MAC prefix ───────────────────────────────────────────────── /// Fixed MAC address prefix (0-3 colon-separated hex bytes, e.g. "02:ab:cd"). /// Remaining bytes are derived from the VM ID hash. diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 839fb990a..84066bf1d 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -354,6 +354,7 @@ fn networking_from_proto(proto: &rpc::NetworkingConfig) -> Result NetworkingMode::Bridge, "user" => NetworkingMode::User, + "macvtap" => NetworkingMode::Macvtap, "" if bridge.is_empty() => return Ok(None), "" => bail!("networking mode is required when bridge is set"), "custom" => bail!("custom networking mode is manifest-only"), @@ -362,9 +363,17 @@ fn networking_from_proto(proto: &rpc::NetworkingConfig) -> Result "user".to_string(), NetworkingMode::Bridge => "bridge".to_string(), NetworkingMode::Custom => String::new(), + NetworkingMode::Macvtap => "macvtap".to_string(), }, default_bridge: default_networking.bridge.clone(), }), @@ -1221,6 +1232,7 @@ mod tests { request.networks = vec![rpc::NetworkingConfig { mode: "user".to_string(), bridge_name: String::new(), + ..Default::default() }]; let manifest = create_manifest_from_vm_config(request, &test_cvm_config()).unwrap(); @@ -1230,11 +1242,27 @@ mod tests { assert!(!manifest.networks[0].net.is_empty()); } + #[test] + fn macvtap_networking_preserves_parent_and_mode() { + let networks = networks_from_proto(&[rpc::NetworkingConfig { + mode: "macvtap".to_string(), + parent: "eth0".to_string(), + macvtap_mode: "private".to_string(), + ..Default::default() + }]) + .unwrap(); + + assert_eq!(networks[0].mode, NetworkingMode::Macvtap); + assert_eq!(networks[0].parent, "eth0"); + assert_eq!(networks[0].macvtap_mode, "private"); + } + #[test] fn bridge_name_is_rejected_for_user_mode() { let err = networks_from_proto(&[rpc::NetworkingConfig { mode: "user".to_string(), bridge_name: "dstack-br0".to_string(), + ..Default::default() }]) .unwrap_err(); @@ -1246,6 +1274,7 @@ mod tests { let err = networks_from_proto(&[rpc::NetworkingConfig { mode: String::new(), bridge_name: String::new(), + ..Default::default() }]) .unwrap_err(); @@ -1257,6 +1286,7 @@ mod tests { let err = networks_from_proto(&[rpc::NetworkingConfig { mode: "custom".to_string(), bridge_name: String::new(), + ..Default::default() }]) .unwrap_err(); diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index e8ac8c820..cbb0973a8 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -10,7 +10,10 @@ use std::{ io::Write as _, os::{ fd::AsRawFd, - unix::{fs::PermissionsExt, net::UnixStream as StdUnixStream}, + unix::{ + fs::{FileTypeExt, PermissionsExt}, + net::UnixStream as StdUnixStream, + }, }, path::Path, process::{Command, Stdio}, @@ -61,6 +64,14 @@ pub struct PrepareBridgeRequest { #[serde(tag = "operation", rename_all = "snake_case")] pub enum Request { PrepareBridge(PrepareBridgeRequest), + PrepareMacvtap { + #[serde(flatten)] + identity: InterfaceIdentity, + parent: String, + mac: String, + #[serde(default)] + mode: String, + }, Remove { #[serde(flatten)] identity: InterfaceIdentity, @@ -79,6 +90,8 @@ struct Response { #[serde(default, skip_serializing_if = "Option::is_none")] tap: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + device: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] error: Option, } @@ -99,9 +112,14 @@ pub fn instance_id(configured: &str, run_path: &Path) -> String { format!("path-{}", hex::encode(&digest[..8])) } -pub async fn request(socket: &Path, request: &Request) -> Result { +pub struct PreparedInterface { + pub device: Option, +} + +pub async fn request(socket: &Path, request: &Request) -> Result { let operation = match request { Request::PrepareBridge(_) => "prepare_bridge", + Request::PrepareMacvtap { .. } => "prepare_macvtap", Request::Remove { .. } => "remove", Request::Check { .. } => "check", }; @@ -131,7 +149,12 @@ pub async fn request(socket: &Path, request: &Request) -> Result { response.error.as_deref().unwrap_or("unknown error") ); } - response.tap.context("netd response omitted TAP name") + Ok(PreparedInterface { + device: { + response.tap.context("netd response omitted TAP name")?; + response.device + }, + }) }; timeout(Duration::from_secs(30), exchange) .await @@ -176,9 +199,10 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul .await .and_then(|request| handle_request(&config.libvirt_uri, request)) { - Ok(tap) => Response { + Ok((tap, device)) => Response { ok: true, tap: Some(tap), + device, error: None, }, Err(error) => { @@ -186,6 +210,7 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul Response { ok: false, tap: None, + device: None, error: Some(format!("{error:#}")), } } @@ -195,6 +220,7 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul Response { ok: false, tap: None, + device: None, error: Some("caller UID is not authorized".into()), } }; @@ -216,15 +242,24 @@ async fn read_request(stream: &mut UnixStream) -> Result { serde_json::from_slice(&message).context("invalid netd request") } -fn handle_request(libvirt_uri: &str, request: Request) -> Result { +fn handle_request(libvirt_uri: &str, request: Request) -> Result<(String, Option)> { let _lock = OperationLock::acquire()?; match request { - Request::PrepareBridge(request) => prepare_bridge(libvirt_uri, &request), + Request::PrepareBridge(request) => { + prepare_bridge(libvirt_uri, &request).map(|tap| (tap, None)) + } + Request::PrepareMacvtap { + identity, + parent, + mac, + mode, + } => prepare_macvtap(libvirt_uri, &identity, &parent, &mac, &mode) + .map(|(tap, device)| (tap, Some(device))), Request::Remove { identity } => { validate_identity(&identity)?; let tap = tap_name(&identity); remove_interface(libvirt_uri, &tap)?; - Ok(tap) + Ok((tap, None)) } Request::Check { identity } => { validate_identity(&identity)?; @@ -232,8 +267,65 @@ fn handle_request(libvirt_uri: &str, request: Request) -> Result { if !Path::new("/sys/class/net").join(&tap).exists() { bail!("TAP {tap} does not exist"); } - virsh(libvirt_uri, &["nwfilter-binding-dumpxml", &tap], None)?; - Ok(tap) + if !is_macvtap(&tap) { + virsh(libvirt_uri, &["nwfilter-binding-dumpxml", &tap], None)?; + } + Ok((tap, None)) + } + } +} + +fn prepare_macvtap( + libvirt_uri: &str, + identity: &InterfaceIdentity, + parent: &str, + mac: &str, + mode: &str, +) -> Result<(String, String)> { + validate_identity(identity)?; + validate_name("parent", parent, 15, "_.-")?; + if !Path::new("/sys/class/net").join(parent).exists() { + bail!("parent interface {parent} does not exist"); + } + validate_mac(mac)?; + let mode = if mode.is_empty() { "private" } else { mode }; + if !matches!(mode, "private" | "bridge" | "vepa" | "passthru") { + bail!("invalid macvtap mode"); + } + let tap = tap_name(identity); + remove_interface(libvirt_uri, &tap)?; + ip(&[ + "link", "add", "link", parent, "name", &tap, "address", mac, "type", "macvtap", "mode", + mode, + ])?; + let result = (|| { + let ifindex = + std::fs::read_to_string(Path::new("/sys/class/net").join(&tap).join("ifindex")) + .context("failed to read macvtap ifindex")?; + let device = format!("/dev/tap{}", ifindex.trim()); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !Path::new(&device).exists() { + if std::time::Instant::now() >= deadline { + bail!("timed out waiting for macvtap device {device}"); + } + std::thread::sleep(Duration::from_millis(25)); + } + let metadata = std::fs::symlink_metadata(&device) + .with_context(|| format!("failed to inspect macvtap device {device}"))?; + if !metadata.file_type().is_char_device() { + bail!("macvtap device {device} is not a character device"); + } + ip(&["link", "set", "dev", &tap, "up"])?; + Ok(device) + })(); + match result { + Ok(device) => { + info!(%tap, %parent, %mode, %device, "prepared macvtap"); + Ok((tap, device)) + } + Err(error) => { + let _ = remove_interface(libvirt_uri, &tap); + Err(error) } } } @@ -295,10 +387,13 @@ fn prepare_bridge(libvirt_uri: &str, request: &PrepareBridgeRequest) -> Result Result<()> { + let macvtap = is_macvtap(tap); if Path::new("/sys/class/net").join(tap).exists() { let _ = ip(&["link", "set", "dev", tap, "down"]); } - delete_binding(libvirt_uri, tap)?; + if !macvtap { + delete_binding(libvirt_uri, tap)?; + } if Path::new("/sys/class/net").join(tap).exists() { ip(&["link", "delete", "dev", tap])?; info!(%tap, "removed filtered TAP"); @@ -306,6 +401,13 @@ fn remove_interface(libvirt_uri: &str, tap: &str) -> Result<()> { Ok(()) } +fn is_macvtap(interface: &str) -> bool { + Path::new("/sys/class/net") + .join(interface) + .join("macvtap") + .exists() +} + fn delete_binding(uri: &str, tap: &str) -> Result<()> { let output = Command::new(VIRSH_PATH) .args(["--connect", uri, "nwfilter-binding-delete", tap]) diff --git a/dstack/vmm/src/vm_launcher.rs b/dstack/vmm/src/vm_launcher.rs index 7f9cff7b9..0fb405cf4 100644 --- a/dstack/vmm/src/vm_launcher.rs +++ b/dstack/vmm/src/vm_launcher.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; use std::os::unix::process::CommandExt; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -23,14 +24,28 @@ pub struct ChildCommand { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct LaunchSpec { pub qemu: ChildCommand, - pub swtpm: ChildCommand, - pub swtpm_socket: PathBuf, + #[serde(default)] + pub swtpm: Option, + #[serde(default)] + pub swtpm_socket: Option, + #[serde(default)] + pub open_files: Vec, + #[serde(default)] + pub uid: Option, + #[serde(default)] + pub gid: Option, #[serde(default = "default_startup_timeout_ms")] pub startup_timeout_ms: u64, #[serde(default = "default_shutdown_timeout_ms")] pub shutdown_timeout_ms: u64, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OpenFile { + pub fd: i32, + pub path: PathBuf, +} + fn default_startup_timeout_ms() -> u64 { 5_000 } @@ -51,10 +66,44 @@ impl Drop for SocketCleanup { } } -fn spawn_child(spec: &ChildCommand) -> Result { +fn spawn_child( + spec: &ChildCommand, + open_files: &[OpenFile], + identity: Option<(u32, u32)>, +) -> Result { let parent = unsafe { libc::getpid() }; let mut command = Command::new(&spec.command); command.args(&spec.args); + let opened = open_files + .iter() + .map(|file| { + fs_err::OpenOptions::new() + .read(true) + .write(true) + .open(&file.path) + .with_context(|| format!("failed to open {}", file.path.display())) + }) + .collect::>>()?; + let max_target = open_files.iter().map(|file| file.fd).max().unwrap_or(2); + let inherited = opened + .iter() + .map(|file| { + let fd = + unsafe { libc::fcntl(file.as_raw_fd(), libc::F_DUPFD_CLOEXEC, max_target + 1) }; + if fd < 0 { + Err(std::io::Error::last_os_error()) + .context("failed to reserve inherited file descriptor") + } else { + // SAFETY: fcntl returned a new descriptor owned by this process. + Ok(unsafe { OwnedFd::from_raw_fd(fd) }) + } + }) + .collect::>>()?; + let fds = open_files + .iter() + .zip(&inherited) + .map(|(file, opened)| (file.fd, opened.as_raw_fd())) + .collect::>(); // SAFETY: pre_exec only invokes async-signal-safe libc operations. Checking // the parent after PR_SET_PDEATHSIG closes the fork/parent-exit race. unsafe { @@ -68,6 +117,36 @@ fn spawn_child(spec: &ChildCommand) -> Result { if libc::getppid() != parent { libc::raise(libc::SIGKILL); } + for &(target, source) in &fds { + if target < 3 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "open file target fd must be at least 3", + )); + } + if source == target { + if libc::fcntl(target, libc::F_SETFD, 0) < 0 { + return Err(std::io::Error::last_os_error()); + } + } else if libc::dup2(source, target) < 0 { + return Err(std::io::Error::last_os_error()); + } + } + if let Some((uid, gid)) = identity { + if libc::geteuid() == 0 { + if libc::setgroups(0, std::ptr::null()) != 0 + || libc::setgid(gid) != 0 + || libc::setuid(uid) != 0 + { + return Err(std::io::Error::last_os_error()); + } + } else if libc::geteuid() != uid || libc::getegid() != gid { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "launcher cannot switch to the configured QEMU user", + )); + } + } Ok(()) }); } @@ -126,24 +205,30 @@ pub async fn run(spec_path: &Path) -> Result<()> { let raw = fs_err::read(spec_path) .with_context(|| format!("failed to read launch spec {}", spec_path.display()))?; let spec: LaunchSpec = serde_json::from_slice(&raw).context("failed to parse launch spec")?; - let _socket_cleanup = SocketCleanup(spec.swtpm_socket.clone()); - if spec.swtpm_socket.exists() { - fs_err::remove_file(&spec.swtpm_socket).context("failed to remove stale swtpm socket")?; + let _socket_cleanup = spec.swtpm_socket.clone().map(SocketCleanup); + if let Some(socket) = &spec.swtpm_socket { + if socket.exists() { + fs_err::remove_file(socket).context("failed to remove stale swtpm socket")?; + } } let grace = Duration::from_millis(spec.shutdown_timeout_ms); let mut terminate = signal(SignalKind::terminate()).context("failed to watch SIGTERM")?; let mut interrupt = signal(SignalKind::interrupt()).context("failed to watch SIGINT")?; - let mut swtpm = spawn_child(&spec.swtpm)?; + let mut swtpm = if let Some(command) = &spec.swtpm { + Some(spawn_child(command, &[], None)?) + } else { + None + }; enum StartupExit { Ready, Error(anyhow::Error), Signal, } - let startup_exit = { + let startup_exit = if let (Some(swtpm), Some(socket)) = (&mut swtpm, &spec.swtpm_socket) { let startup = wait_for_swtpm( - &mut swtpm, - &spec.swtpm_socket, + swtpm, + socket, Instant::now() + Duration::from_millis(spec.startup_timeout_ms), ); tokio::pin!(startup); @@ -155,29 +240,37 @@ pub async fn run(spec_path: &Path) -> Result<()> { _ = terminate.recv() => StartupExit::Signal, _ = interrupt.recv() => StartupExit::Signal, } + } else { + StartupExit::Ready }; match startup_exit { StartupExit::Ready => {} StartupExit::Error(error) => { - stop_child(&mut swtpm, "swtpm", grace).await; + if let Some(child) = &mut swtpm { + stop_child(child, "swtpm", grace).await; + } return Err(error); } StartupExit::Signal => { - stop_child(&mut swtpm, "swtpm", grace).await; + if let Some(child) = &mut swtpm { + stop_child(child, "swtpm", grace).await; + } return Ok(()); } } - let mut qemu = match spawn_child(&spec.qemu) { + let mut qemu = match spawn_child(&spec.qemu, &spec.open_files, spec.uid.zip(spec.gid)) { Ok(child) => child, Err(error) => { - stop_child(&mut swtpm, "swtpm", grace).await; + if let Some(child) = &mut swtpm { + stop_child(child, "swtpm", grace).await; + } return Err(error); } }; info!( qemu_pid = qemu.id(), - swtpm_pid = swtpm.id(), + swtpm_pid = swtpm.as_ref().and_then(|child| child.id()), "VM processes started" ); @@ -190,19 +283,26 @@ pub async fn run(spec_path: &Path) -> Result<()> { _ = terminate.recv() => Exit::Signal, _ = interrupt.recv() => Exit::Signal, status = qemu.wait() => Exit::Qemu(status.context("failed to wait for QEMU")?), - status = swtpm.wait() => Exit::Swtpm(status.context("failed to wait for swtpm")?), + status = async { + match &mut swtpm { + Some(child) => child.wait().await, + None => std::future::pending().await, + } + } => Exit::Swtpm(status.context("failed to wait for swtpm")?), }; match exit { Exit::Signal => { - tokio::join!( - stop_child(&mut qemu, "qemu", grace), - stop_child(&mut swtpm, "swtpm", grace) - ); + stop_child(&mut qemu, "qemu", grace).await; + if let Some(child) = &mut swtpm { + stop_child(child, "swtpm", grace).await; + } Ok(()) } Exit::Qemu(status) => { - stop_child(&mut swtpm, "swtpm", grace).await; + if let Some(child) = &mut swtpm { + stop_child(child, "swtpm", grace).await; + } if status.success() { Ok(()) } else { @@ -241,6 +341,31 @@ mod tests { sleep(Duration::from_secs(5)).await; } + #[tokio::test] + async fn passes_open_file_to_qemu_without_swtpm() -> Result<()> { + let dir = tempfile::tempdir()?; + let input = dir.path().join("input"); + let output = dir.path().join("output"); + fs_err::write(&input, b"macvtap-fd")?; + let spec_path = dir.path().join("spec.json"); + let spec = LaunchSpec { + qemu: shell(format!("cat <&3 > {}", output.display())), + swtpm: None, + swtpm_socket: None, + open_files: vec![OpenFile { fd: 3, path: input }], + uid: None, + gid: None, + startup_timeout_ms: 1_000, + shutdown_timeout_ms: 1_000, + }; + fs_err::write(&spec_path, serde_json::to_vec(&spec)?)?; + + run(&spec_path).await?; + + assert_eq!(fs_err::read(output)?, b"macvtap-fd"); + Ok(()) + } + #[tokio::test] async fn qemu_failure_stops_and_reaps_swtpm() { let dir = tempfile::tempdir().unwrap(); @@ -248,8 +373,14 @@ mod tests { let swtpm_pid = dir.path().join("swtpm.pid"); let spec = LaunchSpec { qemu: shell("exit 7".into()), - swtpm: shell(format!("echo $$ > {}; sleep 30", swtpm_pid.display())), - swtpm_socket: socket.clone(), + swtpm: Some(shell(format!( + "echo $$ > {}; sleep 30", + swtpm_pid.display() + ))), + swtpm_socket: Some(socket.clone()), + open_files: vec![], + uid: None, + gid: None, startup_timeout_ms: 2_000, shutdown_timeout_ms: 500, }; @@ -273,8 +404,11 @@ mod tests { let qemu_pid = dir.path().join("qemu.pid"); let spec = LaunchSpec { qemu: shell(format!("echo $$ > {}; sleep 30", qemu_pid.display())), - swtpm: shell("sleep 0.2; exit 9".into()), - swtpm_socket: socket.clone(), + swtpm: Some(shell("sleep 0.2; exit 9".into())), + swtpm_socket: Some(socket.clone()), + open_files: vec![], + uid: None, + gid: None, startup_timeout_ms: 2_000, shutdown_timeout_ms: 500, }; @@ -298,8 +432,14 @@ mod tests { let swtpm_pid = dir.path().join("swtpm.pid"); let spec = LaunchSpec { qemu: shell("exit 0".into()), - swtpm: shell(format!("echo $$ > {}; sleep 30", swtpm_pid.display())), - swtpm_socket: socket.clone(), + swtpm: Some(shell(format!( + "echo $$ > {}; sleep 30", + swtpm_pid.display() + ))), + swtpm_socket: Some(socket.clone()), + open_files: vec![], + uid: None, + gid: None, startup_timeout_ms: 100, shutdown_timeout_ms: 500, }; @@ -323,8 +463,14 @@ mod tests { let swtpm_pid = dir.path().join("swtpm.pid"); let spec = LaunchSpec { qemu: shell("sleep 0.1; exit 0".into()), - swtpm: shell(format!("echo $$ > {}; sleep 30", swtpm_pid.display())), - swtpm_socket: socket.clone(), + swtpm: Some(shell(format!( + "echo $$ > {}; sleep 30", + swtpm_pid.display() + ))), + swtpm_socket: Some(socket.clone()), + open_files: vec![], + uid: None, + gid: None, startup_timeout_ms: 2_000, shutdown_timeout_ms: 500, }; @@ -350,11 +496,14 @@ mod tests { let qemu_marker = dir.path().join("qemu-started"); let spec = LaunchSpec { qemu: shell(format!("touch {}", qemu_marker.display())), - swtpm: ChildCommand { + swtpm: Some(ChildCommand { command: dir.path().join("missing-swtpm").display().to_string(), args: vec![], - }, - swtpm_socket: socket.clone(), + }), + swtpm_socket: Some(socket.clone()), + open_files: vec![], + uid: None, + gid: None, startup_timeout_ms: 100, shutdown_timeout_ms: 100, }; From 2e0f44a41f1889f4ca95c01960d4e7f33f9a27ce Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 16 Aug 2026 23:01:14 -0700 Subject: [PATCH 02/11] refactor(vmm): remove unused QEMU user setting --- docs/macvtap-networking.md | 6 ++-- docs/tutorials/vmm-configuration.md | 1 - dstack/vmm/src/app.rs | 12 ++----- dstack/vmm/src/app/qemu.rs | 50 +++-------------------------- dstack/vmm/src/config.rs | 3 -- dstack/vmm/src/vm_launcher.rs | 41 ++--------------------- dstack/vmm/vmm.toml | 1 - 7 files changed, 13 insertions(+), 101 deletions(-) diff --git a/docs/macvtap-networking.md b/docs/macvtap-networking.md index 1bd30e9ea..af6a2d0e6 100644 --- a/docs/macvtap-networking.md +++ b/docs/macvtap-networking.md @@ -33,9 +33,9 @@ and the same deterministic MAC address passed to QEMU. Netd then: 4. reads its kernel-assigned ifindex and waits for `/dev/tap`; and 5. returns that runtime device path to the VMM. -The per-VM launcher opens the root-owned character device before dropping to -`cvm.user`, places it at the fd referenced by QEMU's `-netdev tap,fd=...` -argument, and then execs QEMU. This keeps device paths out of persistent VM +The per-VM launcher opens the character device, places it at the fd referenced +by QEMU's `-netdev tap,fd=...` argument, and then execs QEMU. This keeps device +paths out of persistent VM configuration, works with both Supervisor and systemd process managers, and does not pass network fds through `sudo`. diff --git a/docs/tutorials/vmm-configuration.md b/docs/tutorials/vmm-configuration.md index 332217c0c..53068e6a2 100644 --- a/docs/tutorials/vmm-configuration.md +++ b/docs/tutorials/vmm-configuration.md @@ -113,7 +113,6 @@ cid_pool_size = 1000 max_allocable_vcpu = 124 # Adjust: total cores - 4 max_allocable_memory_in_mb = 990616 # Adjust: total MB - 16384 qmp_socket = false -user = "" use_mrconfigid = true qemu_pci_hole64_size = 0 qemu_hotplug_off = false diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 84a041daf..795b7755a 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -21,7 +21,7 @@ use dstack_vmm_rpc::{ use fs_err as fs; use guest_api::client::DefaultClient as GuestClient; use id_pool::IdPool; -use nix::unistd::{Uid, User}; +use nix::unistd::Uid; use or_panic::ResultOrPanic; use ra_rpc::client::RaClient; use serde::{Deserialize, Serialize}; @@ -542,15 +542,7 @@ impl App { { return Ok(()); } - let qemu_uid = if self.config.cvm.user.is_empty() { - Uid::effective().as_raw() - } else { - User::from_name(&self.config.cvm.user) - .context("failed to resolve QEMU user")? - .with_context(|| format!("QEMU user {} does not exist", self.config.cvm.user))? - .uid - .as_raw() - }; + let qemu_uid = Uid::effective().as_raw(); let mut prepared = Vec::new(); for (nic_index, network) in networks.iter_mut().enumerate() { if network.mode == NetworkingMode::Bridge diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 78e821330..415c2b89b 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -24,12 +24,9 @@ use anyhow::{bail, Context, Result}; use bon::Builder; use dstack_types::shared_filenames::HOST_SHARED_DISK_LABEL; use fs_err as fs; -use nix::unistd::User; use serde::Serialize; use std::collections::HashMap; -use std::os::unix::fs::PermissionsExt; use std::{ - fs::Permissions, io::Write, path::{Path, PathBuf}, process::{Command, Stdio}, @@ -203,7 +200,7 @@ impl PreparedQemuLaunch { gpus: &GpuConfig, ) -> Result { let workdir = VmWorkDir::new(workdir); - prepare_data_disk(vm, &workdir, cfg)?; + prepare_data_disk(vm, &workdir)?; prepare_shared_dir(&workdir)?; let app_compose = workdir.app_compose().context("failed to get app compose")?; let platform = cfg.resolved_platform(); @@ -293,7 +290,7 @@ impl PreparedQemuLaunch { } } -fn prepare_data_disk(vm: &VmConfig, workdir: &VmWorkDir, cfg: &CvmConfig) -> Result<()> { +fn prepare_data_disk(vm: &VmConfig, workdir: &VmWorkDir) -> Result<()> { let hda_path = workdir.hda_path(); if !hda_path.exists() { create_hd( @@ -302,9 +299,6 @@ fn prepare_data_disk(vm: &VmConfig, workdir: &VmWorkDir, cfg: &CvmConfig) -> Res &format!("{}G", vm.manifest.disk_size), )?; } - if !cfg.user.is_empty() { - fs::set_permissions(&hda_path, Permissions::from_mode(0o660))?; - } Ok(()) } @@ -357,7 +351,7 @@ impl VmConfig { .any(|network| network.mode == NetworkingMode::Macvtap); let Some(socket) = prepared.swtpm_socket.as_deref() else { if has_macvtap { - return self.wrap_launcher(cfg, &prepared, process, None, None); + return self.wrap_launcher(&prepared, process, None, None); } return Ok(vec![process]); }; @@ -365,14 +359,7 @@ impl VmConfig { .swtpm_path .as_ref() .context("missing swtpm executable for configured socket")?; - let (socket_uid, socket_gid) = if cfg.user.is_empty() { - (unsafe { libc::geteuid() }, unsafe { libc::getegid() }) - } else { - let user = User::from_name(&cfg.user) - .context("failed to resolve QEMU user")? - .with_context(|| format!("QEMU user {} does not exist", cfg.user))?; - (user.uid.as_raw(), user.gid.as_raw()) - }; + let (socket_uid, socket_gid) = (unsafe { libc::geteuid() }, unsafe { libc::getegid() }); let swtpm_args = vec![ "socket".into(), @@ -388,7 +375,6 @@ impl VmConfig { "not-need-init,startup-clear".into(), ]; self.wrap_launcher( - cfg, &prepared, process, Some(ChildCommand { @@ -401,29 +387,11 @@ impl VmConfig { fn wrap_launcher( &self, - cfg: &CvmConfig, prepared: &PreparedQemuLaunch, - mut process: ProcessConfig, + process: ProcessConfig, swtpm: Option, swtpm_socket: Option, ) -> Result> { - let identity = if cfg.user.is_empty() { - None - } else { - let user = User::from_name(&cfg.user) - .context("failed to resolve QEMU user")? - .with_context(|| format!("QEMU user {} does not exist", cfg.user))?; - if process.command != "sudo" || process.args.first().map(String::as_str) != Some("-u") { - bail!("unexpected QEMU privilege wrapper"); - } - process.command = process - .args - .get(2) - .context("sudo command is missing")? - .clone(); - process.args.drain(0..3); - Some((user.uid.as_raw(), user.gid.as_raw())) - }; let open_files = prepared .networks .iter() @@ -442,8 +410,6 @@ impl VmConfig { swtpm, swtpm_socket, open_files, - uid: identity.map(|value| value.0), - gid: identity.map(|value| value.1), startup_timeout_ms: 5_000, shutdown_timeout_ms: 10_000, }; @@ -850,12 +816,6 @@ impl QemuCommandBuilder<'_> { if let Some(cpus) = &self.prepared.numa_cpus { arguments.splice(0..0, ["taskset", "-c", cpus].into_iter().map(String::from)); } - if !self.cfg.user.is_empty() { - arguments.splice( - 0..0, - ["sudo", "-u", &self.cfg.user].into_iter().map(String::from), - ); - } let command = arguments.remove(0); let note = serde_json::to_string(&ProcessAnnotation { diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 57ce67cb0..47ffc3672 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -334,9 +334,6 @@ pub struct CvmConfig { pub qmp_socket: bool, /// GPU configuration pub gpu: GpuConfig, - /// Use sudo to run the VM - pub user: String, - /// Auto restart configuration pub auto_restart: AutoRestartConfig, diff --git a/dstack/vmm/src/vm_launcher.rs b/dstack/vmm/src/vm_launcher.rs index 0fb405cf4..c7dcee734 100644 --- a/dstack/vmm/src/vm_launcher.rs +++ b/dstack/vmm/src/vm_launcher.rs @@ -30,10 +30,6 @@ pub struct LaunchSpec { pub swtpm_socket: Option, #[serde(default)] pub open_files: Vec, - #[serde(default)] - pub uid: Option, - #[serde(default)] - pub gid: Option, #[serde(default = "default_startup_timeout_ms")] pub startup_timeout_ms: u64, #[serde(default = "default_shutdown_timeout_ms")] @@ -66,11 +62,7 @@ impl Drop for SocketCleanup { } } -fn spawn_child( - spec: &ChildCommand, - open_files: &[OpenFile], - identity: Option<(u32, u32)>, -) -> Result { +fn spawn_child(spec: &ChildCommand, open_files: &[OpenFile]) -> Result { let parent = unsafe { libc::getpid() }; let mut command = Command::new(&spec.command); command.args(&spec.args); @@ -132,21 +124,6 @@ fn spawn_child( return Err(std::io::Error::last_os_error()); } } - if let Some((uid, gid)) = identity { - if libc::geteuid() == 0 { - if libc::setgroups(0, std::ptr::null()) != 0 - || libc::setgid(gid) != 0 - || libc::setuid(uid) != 0 - { - return Err(std::io::Error::last_os_error()); - } - } else if libc::geteuid() != uid || libc::getegid() != gid { - return Err(std::io::Error::new( - std::io::ErrorKind::PermissionDenied, - "launcher cannot switch to the configured QEMU user", - )); - } - } Ok(()) }); } @@ -216,7 +193,7 @@ pub async fn run(spec_path: &Path) -> Result<()> { let mut terminate = signal(SignalKind::terminate()).context("failed to watch SIGTERM")?; let mut interrupt = signal(SignalKind::interrupt()).context("failed to watch SIGINT")?; let mut swtpm = if let Some(command) = &spec.swtpm { - Some(spawn_child(command, &[], None)?) + Some(spawn_child(command, &[])?) } else { None }; @@ -259,7 +236,7 @@ pub async fn run(spec_path: &Path) -> Result<()> { } } - let mut qemu = match spawn_child(&spec.qemu, &spec.open_files, spec.uid.zip(spec.gid)) { + let mut qemu = match spawn_child(&spec.qemu, &spec.open_files) { Ok(child) => child, Err(error) => { if let Some(child) = &mut swtpm { @@ -353,8 +330,6 @@ mod tests { swtpm: None, swtpm_socket: None, open_files: vec![OpenFile { fd: 3, path: input }], - uid: None, - gid: None, startup_timeout_ms: 1_000, shutdown_timeout_ms: 1_000, }; @@ -379,8 +354,6 @@ mod tests { ))), swtpm_socket: Some(socket.clone()), open_files: vec![], - uid: None, - gid: None, startup_timeout_ms: 2_000, shutdown_timeout_ms: 500, }; @@ -407,8 +380,6 @@ mod tests { swtpm: Some(shell("sleep 0.2; exit 9".into())), swtpm_socket: Some(socket.clone()), open_files: vec![], - uid: None, - gid: None, startup_timeout_ms: 2_000, shutdown_timeout_ms: 500, }; @@ -438,8 +409,6 @@ mod tests { ))), swtpm_socket: Some(socket.clone()), open_files: vec![], - uid: None, - gid: None, startup_timeout_ms: 100, shutdown_timeout_ms: 500, }; @@ -469,8 +438,6 @@ mod tests { ))), swtpm_socket: Some(socket.clone()), open_files: vec![], - uid: None, - gid: None, startup_timeout_ms: 2_000, shutdown_timeout_ms: 500, }; @@ -502,8 +469,6 @@ mod tests { }), swtpm_socket: Some(socket.clone()), open_files: vec![], - uid: None, - gid: None, startup_timeout_ms: 100, shutdown_timeout_ms: 100, }; diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index 599c3e5c6..fd27137d6 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -43,7 +43,6 @@ max_allocable_memory_in_mb = 100_000 # MB # Enable QMP socket qmp_socket = false # The user to run the VM as. If empty, the VM will be run as the current user. -user = "" # Unique namespace when multiple dstack-vmm instances share one host. When # empty, a stable value is derived from run_path. instance_id = "" From 012d8958e560ca9edae6835ecb8f4ce18e822190 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 16 Aug 2026 23:35:37 -0700 Subject: [PATCH 03/11] refactor(vmm): separate macvtap prepare RPC --- dstack/vmm/src/app.rs | 9 +++++--- dstack/vmm/src/netd.rs | 51 +++++++++++++++++++++++++++++------------- 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 795b7755a..2d99ddc6d 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -5,7 +5,10 @@ use crate::{ config::{Config, NetworkFilterMode, Networking, NetworkingMode, ProcessAnnotation, Protocol}, logrotate, - netd::{self, InterfaceIdentity, PrepareBridgeRequest, Request as NetdRequest}, + netd::{ + self, InterfaceIdentity, PrepareBridgeRequest, PrepareMacvtapRequest, + Request as NetdRequest, + }, }; use anyhow::{bail, Context, Result}; @@ -575,12 +578,12 @@ impl App { filter: self.config.cvm.network_filter.filter.clone(), parameters: self.config.cvm.network_filter.parameters.clone(), }), - NetworkingMode::Macvtap => NetdRequest::PrepareMacvtap { + NetworkingMode::Macvtap => NetdRequest::PrepareMacvtap(PrepareMacvtapRequest { identity: identity.clone(), parent: network.parent.clone(), mac, mode: network.macvtap_mode.clone(), - }, + }), _ => unreachable!(), }; let response = netd::request(&self.config.netd.socket, &request).await; diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index cbb0973a8..cbd8e489d 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -60,18 +60,21 @@ pub struct PrepareBridgeRequest { pub parameters: BTreeMap, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PrepareMacvtapRequest { + #[serde(flatten)] + pub identity: InterfaceIdentity, + pub parent: String, + pub mac: String, + #[serde(default)] + pub mode: String, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "operation", rename_all = "snake_case")] pub enum Request { PrepareBridge(PrepareBridgeRequest), - PrepareMacvtap { - #[serde(flatten)] - identity: InterfaceIdentity, - parent: String, - mac: String, - #[serde(default)] - mode: String, - }, + PrepareMacvtap(PrepareMacvtapRequest), Remove { #[serde(flatten)] identity: InterfaceIdentity, @@ -119,7 +122,7 @@ pub struct PreparedInterface { pub async fn request(socket: &Path, request: &Request) -> Result { let operation = match request { Request::PrepareBridge(_) => "prepare_bridge", - Request::PrepareMacvtap { .. } => "prepare_macvtap", + Request::PrepareMacvtap(_) => "prepare_macvtap", Request::Remove { .. } => "remove", Request::Check { .. } => "check", }; @@ -248,13 +251,14 @@ fn handle_request(libvirt_uri: &str, request: Request) -> Result<(String, Option Request::PrepareBridge(request) => { prepare_bridge(libvirt_uri, &request).map(|tap| (tap, None)) } - Request::PrepareMacvtap { - identity, - parent, - mac, - mode, - } => prepare_macvtap(libvirt_uri, &identity, &parent, &mac, &mode) - .map(|(tap, device)| (tap, Some(device))), + Request::PrepareMacvtap(request) => prepare_macvtap( + libvirt_uri, + &request.identity, + &request.parent, + &request.mac, + &request.mode, + ) + .map(|(tap, device)| (tap, Some(device))), Request::Remove { identity } => { validate_identity(&identity)?; let tap = tap_name(&identity); @@ -717,6 +721,21 @@ mod tests { assert!(value.get("identity").is_none()); } + #[test] + fn macvtap_prepare_has_a_dedicated_operation() { + let request = Request::PrepareMacvtap(PrepareMacvtapRequest { + identity: identity("instance", "vm", 1), + parent: "eth0".into(), + mac: "02:00:00:00:00:01".into(), + mode: "private".into(), + }); + let value = serde_json::to_value(request).unwrap(); + assert_eq!(value["operation"], "prepare_macvtap"); + assert_eq!(value["instance_id"], "instance"); + assert_eq!(value["parent"], "eth0"); + assert!(value.get("identity").is_none()); + } + #[tokio::test] async fn disconnected_client_is_confined_to_one_connection() { let (mut server, client) = UnixStream::pair().unwrap(); From 7d34414f9a56c463a9f42d5fef1f44e8ce9b4842 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 16 Aug 2026 23:53:50 -0700 Subject: [PATCH 04/11] feat(vmm): exec single-process launches in place --- dstack/vmm/src/vm_launcher.rs | 132 +++++++++++++++++++++++----------- 1 file changed, 89 insertions(+), 43 deletions(-) diff --git a/dstack/vmm/src/vm_launcher.rs b/dstack/vmm/src/vm_launcher.rs index c7dcee734..afa9e9746 100644 --- a/dstack/vmm/src/vm_launcher.rs +++ b/dstack/vmm/src/vm_launcher.rs @@ -62,10 +62,19 @@ impl Drop for SocketCleanup { } } -fn spawn_child(spec: &ChildCommand, open_files: &[OpenFile]) -> Result { - let parent = unsafe { libc::getpid() }; - let mut command = Command::new(&spec.command); - command.args(&spec.args); +impl LaunchSpec { + fn is_single_process(&self) -> bool { + self.swtpm.is_none() + } +} + +struct PreparedOpenFiles { + _opened: Vec, + _inherited: Vec, + mappings: Vec<(i32, i32)>, +} + +fn prepare_open_files(open_files: &[OpenFile]) -> Result { let opened = open_files .iter() .map(|file| { @@ -91,47 +100,74 @@ fn spawn_child(spec: &ChildCommand, open_files: &[OpenFile]) -> Result { } }) .collect::>>()?; - let fds = open_files + let mappings = open_files .iter() .zip(&inherited) .map(|(file, opened)| (file.fd, opened.as_raw_fd())) .collect::>(); + Ok(PreparedOpenFiles { + _opened: opened, + _inherited: inherited, + mappings, + }) +} + +fn prepare_exec(expected_parent: libc::pid_t, mappings: &[(i32, i32)]) -> std::io::Result<()> { + if unsafe { libc::setpgid(0, 0) } != 0 { + return Err(std::io::Error::last_os_error()); + } + if unsafe { libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) } != 0 { + return Err(std::io::Error::last_os_error()); + } + if unsafe { libc::getppid() } != expected_parent { + unsafe { libc::raise(libc::SIGKILL) }; + } + for &(target, source) in mappings { + if target < 3 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "open file target fd must be at least 3", + )); + } + if source == target { + if unsafe { libc::fcntl(target, libc::F_SETFD, 0) } < 0 { + return Err(std::io::Error::last_os_error()); + } + } else if unsafe { libc::dup2(source, target) } < 0 { + return Err(std::io::Error::last_os_error()); + } + } + Ok(()) +} + +fn spawn_child(spec: &ChildCommand, open_files: &[OpenFile]) -> Result { + let parent = unsafe { libc::getpid() }; + let mut command = Command::new(&spec.command); + command.args(&spec.args); + let prepared = prepare_open_files(open_files)?; + let mappings = prepared.mappings.clone(); // SAFETY: pre_exec only invokes async-signal-safe libc operations. Checking // the parent after PR_SET_PDEATHSIG closes the fork/parent-exit race. unsafe { - command.as_std_mut().pre_exec(move || { - if libc::setpgid(0, 0) != 0 { - return Err(std::io::Error::last_os_error()); - } - if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) != 0 { - return Err(std::io::Error::last_os_error()); - } - if libc::getppid() != parent { - libc::raise(libc::SIGKILL); - } - for &(target, source) in &fds { - if target < 3 { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "open file target fd must be at least 3", - )); - } - if source == target { - if libc::fcntl(target, libc::F_SETFD, 0) < 0 { - return Err(std::io::Error::last_os_error()); - } - } else if libc::dup2(source, target) < 0 { - return Err(std::io::Error::last_os_error()); - } - } - Ok(()) - }); + command + .as_std_mut() + .pre_exec(move || prepare_exec(parent, &mappings)); } command .spawn() .with_context(|| format!("failed to start {}", spec.command)) } +fn exec_in_place(spec: &ChildCommand, open_files: &[OpenFile]) -> Result<()> { + let prepared = prepare_open_files(open_files)?; + let parent = unsafe { libc::getppid() }; + prepare_exec(parent, &prepared.mappings).context("failed to prepare in-place QEMU exec")?; + let mut command = std::process::Command::new(&spec.command); + command.args(&spec.args); + let error = command.exec(); + Err(error).with_context(|| format!("failed to exec {}", spec.command)) +} + async fn stop_child(child: &mut Child, name: &str, grace: Duration) { let Some(pid) = child.id() else { return; @@ -182,6 +218,9 @@ pub async fn run(spec_path: &Path) -> Result<()> { let raw = fs_err::read(spec_path) .with_context(|| format!("failed to read launch spec {}", spec_path.display()))?; let spec: LaunchSpec = serde_json::from_slice(&raw).context("failed to parse launch spec")?; + if spec.is_single_process() { + return exec_in_place(&spec.qemu, &spec.open_files); + } let _socket_cleanup = spec.swtpm_socket.clone().map(SocketCleanup); if let Some(socket) = &spec.swtpm_socket { if socket.exists() { @@ -319,26 +358,33 @@ mod tests { } #[tokio::test] - async fn passes_open_file_to_qemu_without_swtpm() -> Result<()> { + async fn passes_open_file_to_child() -> Result<()> { let dir = tempfile::tempdir()?; let input = dir.path().join("input"); let output = dir.path().join("output"); fs_err::write(&input, b"macvtap-fd")?; - let spec_path = dir.path().join("spec.json"); - let spec = LaunchSpec { - qemu: shell(format!("cat <&3 > {}", output.display())), + let command = shell(format!("cat <&3 > {}", output.display())); + let open_files = vec![OpenFile { fd: 3, path: input }]; + let mut child = spawn_child(&command, &open_files)?; + assert!(child.wait().await?.success()); + + assert_eq!(fs_err::read(output)?, b"macvtap-fd"); + Ok(()) + } + + #[test] + fn single_process_requires_no_swtpm() { + let mut spec = LaunchSpec { + qemu: shell("exit 0".into()), swtpm: None, swtpm_socket: None, - open_files: vec![OpenFile { fd: 3, path: input }], + open_files: vec![], startup_timeout_ms: 1_000, shutdown_timeout_ms: 1_000, }; - fs_err::write(&spec_path, serde_json::to_vec(&spec)?)?; - - run(&spec_path).await?; - - assert_eq!(fs_err::read(output)?, b"macvtap-fd"); - Ok(()) + assert!(spec.is_single_process()); + spec.swtpm = Some(shell("exit 0".into())); + assert!(!spec.is_single_process()); } #[tokio::test] From a0272eb1f070d2e4f6eb91b73230db8c742bed8e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 17 Aug 2026 00:13:21 -0700 Subject: [PATCH 05/11] refactor(vmm): avoid implicit launcher panic paths --- dstack/vmm/src/app.rs | 8 +------- dstack/vmm/src/vm_launcher.rs | 4 ++++ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 2d99ddc6d..1954f5d63 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -553,12 +553,6 @@ impl App { { continue; } - if !matches!( - network.mode, - NetworkingMode::Bridge | NetworkingMode::Macvtap - ) { - continue; - } let identity = InterfaceIdentity { instance_id: self.config.cvm.instance_id.clone(), vm_id: vm.manifest.id.clone(), @@ -584,7 +578,7 @@ impl App { mac, mode: network.macvtap_mode.clone(), }), - _ => unreachable!(), + NetworkingMode::User | NetworkingMode::Custom => continue, }; let response = netd::request(&self.config.netd.socket, &request).await; if let Err(error) = response { diff --git a/dstack/vmm/src/vm_launcher.rs b/dstack/vmm/src/vm_launcher.rs index afa9e9746..af5a72574 100644 --- a/dstack/vmm/src/vm_launcher.rs +++ b/dstack/vmm/src/vm_launcher.rs @@ -69,7 +69,11 @@ impl LaunchSpec { } struct PreparedOpenFiles { + // Retain the original device handles through the fork or exec handoff; + // they close automatically when this prepared set leaves scope. _opened: Vec, + // Own the collision-free source fds used by dup2. Dropping these before + // fork or exec would close the source fds while `mappings` still refers to them. _inherited: Vec, mappings: Vec<(i32, i32)>, } From 97b9216ec348c45df3dacd4781ae2a3aeb3b1baa Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 17 Aug 2026 00:16:21 -0700 Subject: [PATCH 06/11] refactor(vmm): use safe effective ID wrappers --- dstack/vmm/src/app/qemu.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 415c2b89b..6de1455a3 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -24,6 +24,7 @@ use anyhow::{bail, Context, Result}; use bon::Builder; use dstack_types::shared_filenames::HOST_SHARED_DISK_LABEL; use fs_err as fs; +use nix::unistd::{Gid, Uid}; use serde::Serialize; use std::collections::HashMap; use std::{ @@ -359,7 +360,7 @@ impl VmConfig { .swtpm_path .as_ref() .context("missing swtpm executable for configured socket")?; - let (socket_uid, socket_gid) = (unsafe { libc::geteuid() }, unsafe { libc::getegid() }); + let (socket_uid, socket_gid) = (Uid::effective().as_raw(), Gid::effective().as_raw()); let swtpm_args = vec![ "socket".into(), From abc4e539ff206d9b6cb81983ef1c7f216fcfdece Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 17 Aug 2026 00:20:32 -0700 Subject: [PATCH 07/11] refactor(vmm): minimize launcher unsafe calls --- dstack/vmm/Cargo.toml | 2 +- dstack/vmm/src/vm_launcher.rs | 61 ++++++++++++++++------------------- 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/dstack/vmm/Cargo.toml b/dstack/vmm/Cargo.toml index 941802b05..0dfbf2fa4 100644 --- a/dstack/vmm/Cargo.toml +++ b/dstack/vmm/Cargo.toml @@ -25,7 +25,7 @@ sha2.workspace = true hex.workspace = true fs-err.workspace = true getrandom = { workspace = true, features = ["std"] } -nix = { workspace = true, features = ["user"] } +nix = { workspace = true, features = ["fs", "process", "signal", "user"] } dirs.workspace = true which.workspace = true clap = { workspace = true, features = ["derive", "string"] } diff --git a/dstack/vmm/src/vm_launcher.rs b/dstack/vmm/src/vm_launcher.rs index af5a72574..4ce1bebba 100644 --- a/dstack/vmm/src/vm_launcher.rs +++ b/dstack/vmm/src/vm_launcher.rs @@ -8,6 +8,15 @@ use std::path::{Path, PathBuf}; use std::time::Duration; use anyhow::{bail, Context, Result}; +use nix::{ + errno::Errno, + fcntl::{fcntl, FcntlArg, FdFlag}, + sys::{ + prctl, + signal::{kill, raise, Signal}, + }, + unistd::{dup2, getpid, getppid, setpgid, Pid}, +}; use serde::{Deserialize, Serialize}; use tokio::process::{Child, Command}; use tokio::signal::unix::{signal, SignalKind}; @@ -93,15 +102,10 @@ fn prepare_open_files(open_files: &[OpenFile]) -> Result { let inherited = opened .iter() .map(|file| { - let fd = - unsafe { libc::fcntl(file.as_raw_fd(), libc::F_DUPFD_CLOEXEC, max_target + 1) }; - if fd < 0 { - Err(std::io::Error::last_os_error()) - .context("failed to reserve inherited file descriptor") - } else { - // SAFETY: fcntl returned a new descriptor owned by this process. - Ok(unsafe { OwnedFd::from_raw_fd(fd) }) - } + let fd = fcntl(file.as_raw_fd(), FcntlArg::F_DUPFD_CLOEXEC(max_target + 1)) + .context("failed to reserve inherited file descriptor")?; + // SAFETY: F_DUPFD_CLOEXEC returned a new descriptor owned by this process. + Ok(unsafe { OwnedFd::from_raw_fd(fd) }) }) .collect::>>()?; let mappings = open_files @@ -117,14 +121,10 @@ fn prepare_open_files(open_files: &[OpenFile]) -> Result { } fn prepare_exec(expected_parent: libc::pid_t, mappings: &[(i32, i32)]) -> std::io::Result<()> { - if unsafe { libc::setpgid(0, 0) } != 0 { - return Err(std::io::Error::last_os_error()); - } - if unsafe { libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) } != 0 { - return Err(std::io::Error::last_os_error()); - } - if unsafe { libc::getppid() } != expected_parent { - unsafe { libc::raise(libc::SIGKILL) }; + setpgid(Pid::from_raw(0), Pid::from_raw(0))?; + prctl::set_pdeathsig(Signal::SIGKILL)?; + if getppid().as_raw() != expected_parent { + raise(Signal::SIGKILL)?; } for &(target, source) in mappings { if target < 3 { @@ -134,18 +134,16 @@ fn prepare_exec(expected_parent: libc::pid_t, mappings: &[(i32, i32)]) -> std::i )); } if source == target { - if unsafe { libc::fcntl(target, libc::F_SETFD, 0) } < 0 { - return Err(std::io::Error::last_os_error()); - } - } else if unsafe { libc::dup2(source, target) } < 0 { - return Err(std::io::Error::last_os_error()); + fcntl(target, FcntlArg::F_SETFD(FdFlag::empty()))?; + } else { + dup2(source, target)?; } } Ok(()) } fn spawn_child(spec: &ChildCommand, open_files: &[OpenFile]) -> Result { - let parent = unsafe { libc::getpid() }; + let parent = getpid().as_raw(); let mut command = Command::new(&spec.command); command.args(&spec.args); let prepared = prepare_open_files(open_files)?; @@ -164,7 +162,7 @@ fn spawn_child(spec: &ChildCommand, open_files: &[OpenFile]) -> Result { fn exec_in_place(spec: &ChildCommand, open_files: &[OpenFile]) -> Result<()> { let prepared = prepare_open_files(open_files)?; - let parent = unsafe { libc::getppid() }; + let parent = getppid().as_raw(); prepare_exec(parent, &prepared.mappings).context("failed to prepare in-place QEMU exec")?; let mut command = std::process::Command::new(&spec.command); command.args(&spec.args); @@ -176,10 +174,8 @@ async fn stop_child(child: &mut Child, name: &str, grace: Duration) { let Some(pid) = child.id() else { return; }; - // SAFETY: pid comes from the live Child handle and SIGTERM has no pointer arguments. - if unsafe { libc::kill(-(pid as libc::pid_t), libc::SIGTERM) } != 0 { - let error = std::io::Error::last_os_error(); - if error.raw_os_error() != Some(libc::ESRCH) { + if let Err(error) = kill(Pid::from_raw(-(pid as libc::pid_t)), Signal::SIGTERM) { + if error != Errno::ESRCH { warn!(%pid, %name, %error, "failed to terminate child"); } } @@ -188,9 +184,8 @@ async fn stop_child(child: &mut Child, name: &str, grace: Duration) { Ok(Err(error)) => warn!(%pid, %name, %error, "failed to wait for child"), Err(_) => { warn!(%pid, %name, "child did not stop gracefully; killing"); - if unsafe { libc::kill(-(pid as libc::pid_t), libc::SIGKILL) } != 0 { - let error = std::io::Error::last_os_error(); - if error.raw_os_error() != Some(libc::ESRCH) { + if let Err(error) = kill(Pid::from_raw(-(pid as libc::pid_t)), Signal::SIGKILL) { + if error != Errno::ESRCH { warn!(%pid, %name, %error, "failed to kill child process group"); } } @@ -350,9 +345,7 @@ mod tests { } fn process_is_gone(pid: libc::pid_t) -> bool { - // SAFETY: signal 0 only probes whether the PID still exists. - let missing = unsafe { libc::kill(pid, 0) != 0 }; - missing && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) + matches!(kill(Pid::from_raw(pid), None), Err(Errno::ESRCH)) } async fn create_fake_socket(path: PathBuf) { From c943fa6d3a56ae7d675234ebbd6803b6c83d29ca Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 17 Aug 2026 01:40:38 -0700 Subject: [PATCH 08/11] refactor(vmm): name bridge preparation explicitly --- dstack/vmm/src/netd.rs | 17 ++++++++++++++++ dstack/vmm/src/vm_launcher.rs | 38 ++--------------------------------- 2 files changed, 19 insertions(+), 36 deletions(-) diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index cbd8e489d..0ea6646cc 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -736,6 +736,23 @@ mod tests { assert!(value.get("identity").is_none()); } + #[test] + fn bridge_prepare_has_a_dedicated_operation() { + let request = Request::PrepareBridge(PrepareBridgeRequest { + identity: identity("instance", "vm", 0), + bridge: "br0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + filter: "clean-traffic".into(), + parameters: BTreeMap::new(), + }); + let value = serde_json::to_value(request).unwrap(); + assert_eq!(value["operation"], "prepare_bridge"); + assert_eq!(value["instance_id"], "instance"); + assert_eq!(value["bridge"], "br0"); + assert!(value.get("identity").is_none()); + } + #[tokio::test] async fn disconnected_client_is_confined_to_one_connection() { let (mut server, client) = UnixStream::pair().unwrap(); diff --git a/dstack/vmm/src/vm_launcher.rs b/dstack/vmm/src/vm_launcher.rs index 4ce1bebba..eaff26440 100644 --- a/dstack/vmm/src/vm_launcher.rs +++ b/dstack/vmm/src/vm_launcher.rs @@ -71,12 +71,6 @@ impl Drop for SocketCleanup { } } -impl LaunchSpec { - fn is_single_process(&self) -> bool { - self.swtpm.is_none() - } -} - struct PreparedOpenFiles { // Retain the original device handles through the fork or exec handoff; // they close automatically when this prepared set leaves scope. @@ -120,7 +114,7 @@ fn prepare_open_files(open_files: &[OpenFile]) -> Result { }) } -fn prepare_exec(expected_parent: libc::pid_t, mappings: &[(i32, i32)]) -> std::io::Result<()> { +fn prepare_child(expected_parent: libc::pid_t, mappings: &[(i32, i32)]) -> std::io::Result<()> { setpgid(Pid::from_raw(0), Pid::from_raw(0))?; prctl::set_pdeathsig(Signal::SIGKILL)?; if getppid().as_raw() != expected_parent { @@ -153,23 +147,13 @@ fn spawn_child(spec: &ChildCommand, open_files: &[OpenFile]) -> Result { unsafe { command .as_std_mut() - .pre_exec(move || prepare_exec(parent, &mappings)); + .pre_exec(move || prepare_child(parent, &mappings)); } command .spawn() .with_context(|| format!("failed to start {}", spec.command)) } -fn exec_in_place(spec: &ChildCommand, open_files: &[OpenFile]) -> Result<()> { - let prepared = prepare_open_files(open_files)?; - let parent = getppid().as_raw(); - prepare_exec(parent, &prepared.mappings).context("failed to prepare in-place QEMU exec")?; - let mut command = std::process::Command::new(&spec.command); - command.args(&spec.args); - let error = command.exec(); - Err(error).with_context(|| format!("failed to exec {}", spec.command)) -} - async fn stop_child(child: &mut Child, name: &str, grace: Duration) { let Some(pid) = child.id() else { return; @@ -217,9 +201,6 @@ pub async fn run(spec_path: &Path) -> Result<()> { let raw = fs_err::read(spec_path) .with_context(|| format!("failed to read launch spec {}", spec_path.display()))?; let spec: LaunchSpec = serde_json::from_slice(&raw).context("failed to parse launch spec")?; - if spec.is_single_process() { - return exec_in_place(&spec.qemu, &spec.open_files); - } let _socket_cleanup = spec.swtpm_socket.clone().map(SocketCleanup); if let Some(socket) = &spec.swtpm_socket { if socket.exists() { @@ -369,21 +350,6 @@ mod tests { Ok(()) } - #[test] - fn single_process_requires_no_swtpm() { - let mut spec = LaunchSpec { - qemu: shell("exit 0".into()), - swtpm: None, - swtpm_socket: None, - open_files: vec![], - startup_timeout_ms: 1_000, - shutdown_timeout_ms: 1_000, - }; - assert!(spec.is_single_process()); - spec.swtpm = Some(shell("exit 0".into())); - assert!(!spec.is_single_process()); - } - #[tokio::test] async fn qemu_failure_stops_and_reaps_swtpm() { let dir = tempfile::tempdir().unwrap(); From 15c9556f7e881d58896318188370c51aabf944be Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 17 Aug 2026 06:16:16 -0700 Subject: [PATCH 09/11] fix(vmm): launch with prepared macvtap devices --- dstack/vmm/src/app.rs | 8 +++++++- dstack/vmm/src/app/qemu.rs | 8 +++++--- dstack/vmm/src/netd.rs | 6 ++++++ dstack/vmm/src/one_shot.rs | 3 ++- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 1954f5d63..eb6d53d2f 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -466,7 +466,12 @@ impl App { let _ = work_dir.clear_runtime_networks(); return Err(error); } - let processes = match vm_config.config_qemu(&work_dir, &self.config.cvm, &devices) { + let processes = match vm_config.config_qemu( + &work_dir, + &self.config.cvm, + &devices, + &runtime_networks, + ) { Ok(processes) => processes, Err(error) => { let _ = self @@ -576,6 +581,7 @@ impl App { identity: identity.clone(), parent: network.parent.clone(), mac, + qemu_uid, mode: network.macvtap_mode.clone(), }), NetworkingMode::User | NetworkingMode::Custom => continue, diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 6de1455a3..8a83c02b7 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -9,7 +9,7 @@ use super::{ hugepage_numa_nodes, image::Image, mr_config::{snp_host_data, tdx_mr_config_id}, - network::{mac_address_for_vm_index, resolved_networks, validate_resolved_networks}, + network::{mac_address_for_vm_index, validate_resolved_networks}, pci_numa_node, round_up, GpuConfig, VmWorkDir, }; use crate::{ @@ -199,13 +199,14 @@ impl PreparedQemuLaunch { workdir: impl AsRef, cfg: &CvmConfig, gpus: &GpuConfig, + networks: &[Networking], ) -> Result { let workdir = VmWorkDir::new(workdir); prepare_data_disk(vm, &workdir)?; prepare_shared_dir(&workdir)?; let app_compose = workdir.app_compose().context("failed to get app compose")?; let platform = cfg.resolved_platform(); - let networks = resolved_networks(&vm.manifest, cfg); + let networks = networks.to_vec(); validate_resolved_networks(&networks)?; let volumes = vm .manifest @@ -337,8 +338,9 @@ impl VmConfig { workdir: impl AsRef, cfg: &CvmConfig, gpus: &GpuConfig, + networks: &[Networking], ) -> Result> { - let prepared = PreparedQemuLaunch::prepare(self, workdir, cfg, gpus)?; + let prepared = PreparedQemuLaunch::prepare(self, workdir, cfg, gpus, networks)?; let process = QemuCommandBuilder { vm: self, cfg, diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 0ea6646cc..9fabf98a3 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -66,6 +66,7 @@ pub struct PrepareMacvtapRequest { pub identity: InterfaceIdentity, pub parent: String, pub mac: String, + pub qemu_uid: u32, #[serde(default)] pub mode: String, } @@ -256,6 +257,7 @@ fn handle_request(libvirt_uri: &str, request: Request) -> Result<(String, Option &request.identity, &request.parent, &request.mac, + request.qemu_uid, &request.mode, ) .map(|(tap, device)| (tap, Some(device))), @@ -284,6 +286,7 @@ fn prepare_macvtap( identity: &InterfaceIdentity, parent: &str, mac: &str, + qemu_uid: u32, mode: &str, ) -> Result<(String, String)> { validate_identity(identity)?; @@ -319,6 +322,8 @@ fn prepare_macvtap( if !metadata.file_type().is_char_device() { bail!("macvtap device {device} is not a character device"); } + std::os::unix::fs::chown(&device, Some(qemu_uid), None) + .with_context(|| format!("failed to set owner of macvtap device {device}"))?; ip(&["link", "set", "dev", &tap, "up"])?; Ok(device) })(); @@ -727,6 +732,7 @@ mod tests { identity: identity("instance", "vm", 1), parent: "eth0".into(), mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, mode: "private".into(), }); let value = serde_json::to_value(request).unwrap(); diff --git a/dstack/vmm/src/one_shot.rs b/dstack/vmm/src/one_shot.rs index 6ee26a470..dd7a23d79 100644 --- a/dstack/vmm/src/one_shot.rs +++ b/dstack/vmm/src/one_shot.rs @@ -290,8 +290,9 @@ Compose file content (first 200 chars): ); } + let runtime_networks = resolved_networks(&manifest, &config.cvm); let process_configs = vm_builder_config - .config_qemu(&workdir_path, &config.cvm, &gpus) + .config_qemu(&workdir_path, &config.cvm, &gpus, &runtime_networks) .context("Failed to build QEMU configuration")?; // Get the main QEMU process config (first in the list) From ded3e54ed41777a32d1d9ef46c475c6d70d4ff5b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 17 Aug 2026 06:16:16 -0700 Subject: [PATCH 10/11] fix(vmm): preserve single-process exec launch --- dstack/vmm/src/vm_launcher.rs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/dstack/vmm/src/vm_launcher.rs b/dstack/vmm/src/vm_launcher.rs index eaff26440..001d02018 100644 --- a/dstack/vmm/src/vm_launcher.rs +++ b/dstack/vmm/src/vm_launcher.rs @@ -71,6 +71,12 @@ impl Drop for SocketCleanup { } } +impl LaunchSpec { + fn is_single_process(&self) -> bool { + self.swtpm.is_none() + } +} + struct PreparedOpenFiles { // Retain the original device handles through the fork or exec handoff; // they close automatically when this prepared set leaves scope. @@ -154,6 +160,16 @@ fn spawn_child(spec: &ChildCommand, open_files: &[OpenFile]) -> Result { .with_context(|| format!("failed to start {}", spec.command)) } +fn exec_in_place(spec: &ChildCommand, open_files: &[OpenFile]) -> Result<()> { + let prepared = prepare_open_files(open_files)?; + let parent = getppid().as_raw(); + prepare_child(parent, &prepared.mappings).context("failed to prepare in-place QEMU exec")?; + let mut command = std::process::Command::new(&spec.command); + command.args(&spec.args); + let error = command.exec(); + Err(error).with_context(|| format!("failed to exec {}", spec.command)) +} + async fn stop_child(child: &mut Child, name: &str, grace: Duration) { let Some(pid) = child.id() else { return; @@ -201,6 +217,9 @@ pub async fn run(spec_path: &Path) -> Result<()> { let raw = fs_err::read(spec_path) .with_context(|| format!("failed to read launch spec {}", spec_path.display()))?; let spec: LaunchSpec = serde_json::from_slice(&raw).context("failed to parse launch spec")?; + if spec.is_single_process() { + return exec_in_place(&spec.qemu, &spec.open_files); + } let _socket_cleanup = spec.swtpm_socket.clone().map(SocketCleanup); if let Some(socket) = &spec.swtpm_socket { if socket.exists() { @@ -350,6 +369,21 @@ mod tests { Ok(()) } + #[test] + fn single_process_launch_omits_swtpm() { + let mut spec = LaunchSpec { + qemu: shell("exit 0".into()), + swtpm: None, + swtpm_socket: None, + open_files: vec![], + startup_timeout_ms: 1_000, + shutdown_timeout_ms: 1_000, + }; + assert!(spec.is_single_process()); + spec.swtpm = Some(shell("exit 0".into())); + assert!(!spec.is_single_process()); + } + #[tokio::test] async fn qemu_failure_stops_and_reaps_swtpm() { let dir = tempfile::tempdir().unwrap(); From cebc153cbb46d394635e0f8b46fed98d3b7383bb Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 17 Aug 2026 18:39:53 -0700 Subject: [PATCH 11/11] fix(vmm): avoid persisting macvtap device paths --- dstack/vmm/src/app.rs | 50 +++++++++++++++++++---------------- dstack/vmm/src/app/workdir.rs | 36 ++++++++++++++++++++++++- dstack/vmm/src/netd.rs | 2 +- 3 files changed, 63 insertions(+), 25 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index eb6d53d2f..ed628db71 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -586,33 +586,37 @@ impl App { }), NetworkingMode::User | NetworkingMode::Custom => continue, }; - let response = netd::request(&self.config.netd.socket, &request).await; - if let Err(error) = response { - // The client may have timed out while netd was still finishing - // this Prepare. Remove the in-flight identity first; netd's - // serialized accept loop processes it after Prepare completes. - if let Err(cleanup_error) = netd::request( - &self.config.netd.socket, - &NetdRequest::Remove { - identity: identity.clone(), - }, - ) - .await - { - warn!(%cleanup_error, "failed to roll back in-flight filtered network"); - } - for identity in prepared.into_iter().rev() { - if let Err(cleanup_error) = - netd::request(&self.config.netd.socket, &NetdRequest::Remove { identity }) - .await + let response = match netd::request(&self.config.netd.socket, &request).await { + Ok(response) => response, + Err(error) => { + // The client may have timed out while netd was still finishing + // this Prepare. Remove the in-flight identity first; netd's + // serialized accept loop processes it after Prepare completes. + if let Err(cleanup_error) = netd::request( + &self.config.netd.socket, + &NetdRequest::Remove { + identity: identity.clone(), + }, + ) + .await { - warn!(%cleanup_error, "failed to roll back prepared filtered network"); + warn!(%cleanup_error, "failed to roll back in-flight filtered network"); } + for identity in prepared.into_iter().rev() { + if let Err(cleanup_error) = netd::request( + &self.config.netd.socket, + &NetdRequest::Remove { identity }, + ) + .await + { + warn!(%cleanup_error, "failed to roll back prepared filtered network"); + } + } + return Err(error).context("failed to prepare libvirt-filtered networking"); } - return Err(error).context("failed to prepare libvirt-filtered networking"); - } + }; if network.mode == NetworkingMode::Macvtap { - network.device = response? + network.device = response .device .context("netd response omitted macvtap device")?; } diff --git a/dstack/vmm/src/app/workdir.rs b/dstack/vmm/src/app/workdir.rs index 198dc7c94..0b358229f 100644 --- a/dstack/vmm/src/app/workdir.rs +++ b/dstack/vmm/src/app/workdir.rs @@ -151,7 +151,14 @@ impl VmWorkDir { } pub fn set_runtime_networks(&self, networks: &[Networking]) -> Result<()> { - let serialized = serde_json::to_vec(networks)?; + // A macvtap device path is valid only while its host interface exists. + // Keep it in memory for launch preparation, but never persist it across + // VMM restarts where the same /dev/tapN may identify another device. + let mut persistent_networks = networks.to_vec(); + for network in &mut persistent_networks { + network.device.clear(); + } + let serialized = serde_json::to_vec(&persistent_networks)?; safe_write::safe_write(self.runtime_networks_path(), serialized) .context("failed to write runtime networks") } @@ -254,6 +261,7 @@ mod tests { use fs_err as fs; use super::VmWorkDir; + use crate::config::Networking; #[test] fn runtime_networks_snapshot_replaces_target_instead_of_following_it() -> Result<()> { @@ -275,4 +283,30 @@ mod tests { fs::remove_dir_all(temp)?; Ok(()) } + + #[test] + fn runtime_networks_snapshot_omits_ephemeral_device_paths() -> Result<()> { + let temp = std::env::temp_dir().join(format!( + "dstack-vmm-runtime-networks-device-test-{}", + SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos() + )); + let workdir = VmWorkDir::new(temp.join("vm")); + fs::create_dir_all(workdir.path())?; + let network: Networking = serde_json::from_value(serde_json::json!({ + "mode": "macvtap", + "parent": "br0", + "macvtap_mode": "private", + "device": "/dev/tap42" + }))?; + + workdir.set_runtime_networks(&[network])?; + + let persisted = workdir.runtime_networks(); + assert_eq!(persisted.len(), 1); + assert_eq!(persisted[0].parent, "br0"); + assert!(persisted[0].device.is_empty()); + assert!(!fs::read_to_string(workdir.runtime_networks_path())?.contains("/dev/tap42")); + fs::remove_dir_all(temp)?; + Ok(()) + } } diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index 9fabf98a3..44a8aff4b 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -405,7 +405,7 @@ fn remove_interface(libvirt_uri: &str, tap: &str) -> Result<()> { } if Path::new("/sys/class/net").join(tap).exists() { ip(&["link", "delete", "dev", tap])?; - info!(%tap, "removed filtered TAP"); + info!(%tap, "removed managed network interface"); } Ok(()) }