diff --git a/docs/macvtap-networking.md b/docs/macvtap-networking.md new file mode 100644 index 000000000..af6a2d0e6 --- /dev/null +++ b/docs/macvtap-networking.md @@ -0,0 +1,55 @@ +# Macvtap networking + +Macvtap mode gives each CVM a layer-2 identity on an existing host network +without adding the parent interface to a Linux bridge. The VMM delegates the +privileged interface lifecycle to `dstack-vmm netd`; manifests never contain +the unstable `/dev/tapN` device path. + +## Configuration + +Configure a NIC through the manifest or VMM RPC: + +```json +{ + "mode": "macvtap", + "parent": "eth0", + "macvtap_mode": "private" +} +``` + +`parent` must name an existing host interface. `macvtap_mode` may be +`private`, `bridge`, `vepa`, or `passthru`; an empty value selects `private`. +The configured netd socket and caller allowlist apply in the same way as for +libvirt-filtered bridge networking. + +## Lifecycle + +For every macvtap NIC, the VMM sends netd the VM identity, NIC index, parent, +and the same deterministic MAC address passed to QEMU. Netd then: + +1. derives the stable `dt` interface name; +2. replaces any stale interface with that name; +3. creates and activates the macvtap interface; +4. reads its kernel-assigned ifindex and waits for `/dev/tap`; and +5. returns that runtime device path to the VMM. + +The per-VM launcher opens the character device, places it at the fd referenced +by QEMU's `-netdev tap,fd=...` argument, and then execs QEMU. This keeps device +paths out of persistent VM +configuration, works with both Supervisor and systemd process managers, and +does not pass network fds through `sudo`. + +VM shutdown removes the interface by its deterministic identity. The device +node disappears with the interface; its numeric path is never reused as an +identity or cleanup key. + +## Limitations + +- The host and a macvtap guest do not communicate directly through the parent + interface by default. Add a host macvlan/macvtap endpoint if that path is + required. +- Libvirt nwfilter bindings apply only to bridge mode. Macvtap deployments + must enforce network policy in the physical network or with another host + mechanism. +- Real-host testing requires `CAP_NET_ADMIN`, a working udev setup for + `/dev/tapN`, and an upstream network that accepts multiple MAC addresses. diff --git a/docs/tutorials/vmm-configuration.md b/docs/tutorials/vmm-configuration.md index 332217c0c..53068e6a2 100644 --- a/docs/tutorials/vmm-configuration.md +++ b/docs/tutorials/vmm-configuration.md @@ -113,7 +113,6 @@ cid_pool_size = 1000 max_allocable_vcpu = 124 # Adjust: total cores - 4 max_allocable_memory_in_mb = 990616 # Adjust: total MB - 16384 qmp_socket = false -user = "" use_mrconfigid = true qemu_pci_hole64_size = 0 qemu_hotplug_off = false diff --git a/dstack/vmm/Cargo.toml b/dstack/vmm/Cargo.toml index 941802b05..0dfbf2fa4 100644 --- a/dstack/vmm/Cargo.toml +++ b/dstack/vmm/Cargo.toml @@ -25,7 +25,7 @@ sha2.workspace = true hex.workspace = true fs-err.workspace = true getrandom = { workspace = true, features = ["std"] } -nix = { workspace = true, features = ["user"] } +nix = { workspace = true, features = ["fs", "process", "signal", "user"] } dirs.workspace = true which.workspace = true clap = { workspace = true, features = ["derive", "string"] } diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 734086e2e..ef4786af9 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -126,10 +126,14 @@ message VmConfiguration { // Per-VM networking configuration. message NetworkingConfig { - // Networking mode: "bridge", "user" + // Networking mode: "bridge", "user", "macvtap" string mode = 1; // Per-VM bridge interface name. Empty = node default bridge. string bridge_name = 2; + // Parent host interface for macvtap mode. + string parent = 3; + // macvtap forwarding mode. Empty selects "private". + string macvtap_mode = 4; } // Requested GPU layout for a CVM. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index ec31b3e5b..ed628db71 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -5,7 +5,10 @@ use crate::{ config::{Config, NetworkFilterMode, Networking, NetworkingMode, ProcessAnnotation, Protocol}, logrotate, - netd::{self, InterfaceIdentity, PrepareBridgeRequest, Request as NetdRequest}, + netd::{ + self, InterfaceIdentity, PrepareBridgeRequest, PrepareMacvtapRequest, + Request as NetdRequest, + }, }; use anyhow::{bail, Context, Result}; @@ -21,7 +24,7 @@ use dstack_vmm_rpc::{ use fs_err as fs; use guest_api::client::DefaultClient as GuestClient; use id_pool::IdPool; -use nix::unistd::{Uid, User}; +use nix::unistd::Uid; use or_panic::ResultOrPanic; use ra_rpc::client::RaClient; use serde::{Deserialize, Serialize}; @@ -447,7 +450,7 @@ impl App { append_boot_separator(&path); } - let runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm); + let mut runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm); let devices = self.try_allocate_gpus(&vm_config.manifest)?; let gpu_host_config = self.config.cvm.gpu.clone(); let devices_to_sanitize = devices.clone(); @@ -456,15 +459,33 @@ impl App { }) .await .context("GPU sanitization task failed")??; - let processes = vm_config.config_qemu(&work_dir, &self.config.cvm, &devices)?; - work_dir.set_runtime_networks(&runtime_networks)?; if let Err(error) = self - .prepare_filtered_networks(&vm_config, &runtime_networks) + .prepare_filtered_networks(&vm_config, &mut runtime_networks) .await { let _ = work_dir.clear_runtime_networks(); return Err(error); } + let processes = match vm_config.config_qemu( + &work_dir, + &self.config.cvm, + &devices, + &runtime_networks, + ) { + Ok(processes) => processes, + Err(error) => { + let _ = self + .remove_filtered_networks(&vm_config.manifest.id, &runtime_networks) + .await; + return Err(error); + } + }; + if let Err(error) = work_dir.set_runtime_networks(&runtime_networks) { + let _ = self + .remove_filtered_networks(&vm_config.manifest.id, &runtime_networks) + .await; + return Err(error); + } { let mut state = self.lock(); let vm_state = state.get_mut(id).context("VM not found")?; @@ -520,23 +541,21 @@ impl App { async fn prepare_filtered_networks( &self, vm: &VmConfig, - networks: &[Networking], + networks: &mut [Networking], ) -> Result<()> { - if self.config.cvm.network_filter.mode == NetworkFilterMode::None { + if self.config.cvm.network_filter.mode == NetworkFilterMode::None + && !networks + .iter() + .any(|network| network.mode == NetworkingMode::Macvtap) + { return Ok(()); } - let qemu_uid = if self.config.cvm.user.is_empty() { - Uid::effective().as_raw() - } else { - User::from_name(&self.config.cvm.user) - .context("failed to resolve QEMU user")? - .with_context(|| format!("QEMU user {} does not exist", self.config.cvm.user))? - .uid - .as_raw() - }; + let qemu_uid = Uid::effective().as_raw(); let mut prepared = Vec::new(); - for (nic_index, network) in networks.iter().enumerate() { - if network.mode != NetworkingMode::Bridge { + for (nic_index, network) in networks.iter_mut().enumerate() { + if network.mode == NetworkingMode::Bridge + && self.config.cvm.network_filter.mode == NetworkFilterMode::None + { continue; } let identity = InterfaceIdentity { @@ -544,46 +563,62 @@ impl App { vm_id: vm.manifest.id.clone(), nic_index, }; - let request = PrepareBridgeRequest { - identity: identity.clone(), - bridge: network.bridge.clone(), - mac: network::mac_address_for_vm_index( - &vm.manifest.id, - &network.mac_prefix_bytes(), - nic_index, - ), - qemu_uid, - filter: self.config.cvm.network_filter.filter.clone(), - parameters: self.config.cvm.network_filter.parameters.clone(), + let mac = network::mac_address_for_vm_index( + &vm.manifest.id, + &network.mac_prefix_bytes(), + nic_index, + ); + let request = match network.mode { + NetworkingMode::Bridge => NetdRequest::PrepareBridge(PrepareBridgeRequest { + identity: identity.clone(), + bridge: network.bridge.clone(), + mac, + qemu_uid, + filter: self.config.cvm.network_filter.filter.clone(), + parameters: self.config.cvm.network_filter.parameters.clone(), + }), + NetworkingMode::Macvtap => NetdRequest::PrepareMacvtap(PrepareMacvtapRequest { + identity: identity.clone(), + parent: network.parent.clone(), + mac, + qemu_uid, + mode: network.macvtap_mode.clone(), + }), + NetworkingMode::User | NetworkingMode::Custom => continue, }; - if let Err(error) = netd::request( - &self.config.netd.socket, - &NetdRequest::PrepareBridge(request), - ) - .await - { - // The client may have timed out while netd was still finishing - // this Prepare. Remove the in-flight identity first; netd's - // serialized accept loop processes it after Prepare completes. - if let Err(cleanup_error) = netd::request( - &self.config.netd.socket, - &NetdRequest::Remove { - identity: identity.clone(), - }, - ) - .await - { - warn!(%cleanup_error, "failed to roll back in-flight filtered network"); - } - for identity in prepared.into_iter().rev() { - if let Err(cleanup_error) = - netd::request(&self.config.netd.socket, &NetdRequest::Remove { identity }) - .await + let response = match netd::request(&self.config.netd.socket, &request).await { + Ok(response) => response, + Err(error) => { + // The client may have timed out while netd was still finishing + // this Prepare. Remove the in-flight identity first; netd's + // serialized accept loop processes it after Prepare completes. + if let Err(cleanup_error) = netd::request( + &self.config.netd.socket, + &NetdRequest::Remove { + identity: identity.clone(), + }, + ) + .await { - warn!(%cleanup_error, "failed to roll back prepared filtered network"); + warn!(%cleanup_error, "failed to roll back in-flight filtered network"); } + for identity in prepared.into_iter().rev() { + if let Err(cleanup_error) = netd::request( + &self.config.netd.socket, + &NetdRequest::Remove { identity }, + ) + .await + { + warn!(%cleanup_error, "failed to roll back prepared filtered network"); + } + } + return Err(error).context("failed to prepare libvirt-filtered networking"); } - return Err(error).context("failed to prepare libvirt-filtered networking"); + }; + if network.mode == NetworkingMode::Macvtap { + network.device = response + .device + .context("netd response omitted macvtap device")?; } prepared.push(identity); } @@ -595,12 +630,24 @@ impl App { vm_id: &str, networks: &[Networking], ) -> Result<()> { - if self.config.cvm.network_filter.mode == NetworkFilterMode::None { + if self.config.cvm.network_filter.mode == NetworkFilterMode::None + && !networks + .iter() + .any(|network| network.mode == NetworkingMode::Macvtap) + { return Ok(()); } let mut first_error = None; for (nic_index, network) in networks.iter().enumerate().rev() { - if network.mode != NetworkingMode::Bridge { + if network.mode == NetworkingMode::Bridge + && self.config.cvm.network_filter.mode == NetworkFilterMode::None + { + continue; + } + if !matches!( + network.mode, + NetworkingMode::Bridge | NetworkingMode::Macvtap + ) { continue; } let identity = InterfaceIdentity { @@ -2166,6 +2213,9 @@ mod tests { manifest.networks = vec![Networking { mode: NetworkingMode::Bridge, bridge: "dstack-br0".to_string(), + parent: String::new(), + macvtap_mode: String::new(), + device: String::new(), mac_prefix: String::new(), net: String::new(), dhcp_start: String::new(), @@ -2428,6 +2478,9 @@ mod tests { bridge_manifest.networks = vec![Networking { mode: NetworkingMode::Bridge, bridge: "dstack-br0".to_string(), + parent: String::new(), + macvtap_mode: String::new(), + device: String::new(), mac_prefix: "02:aa:bb".to_string(), net: String::new(), dhcp_start: String::new(), diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 67115c0fe..8a83c02b7 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -9,7 +9,7 @@ use super::{ hugepage_numa_nodes, image::Image, mr_config::{snp_host_data, tdx_mr_config_id}, - network::{mac_address_for_vm_index, resolved_networks, validate_resolved_networks}, + network::{mac_address_for_vm_index, validate_resolved_networks}, pci_numa_node, round_up, GpuConfig, VmWorkDir, }; use crate::{ @@ -18,18 +18,16 @@ use crate::{ CvmConfig, CvmPlatform, NetworkFilterMode, Networking, NetworkingMode, ProcessAnnotation, }, netd::{tap_name, InterfaceIdentity}, - vm_launcher::{ChildCommand, LaunchSpec}, + vm_launcher::{ChildCommand, LaunchSpec, OpenFile}, }; use anyhow::{bail, Context, Result}; use bon::Builder; use dstack_types::shared_filenames::HOST_SHARED_DISK_LABEL; use fs_err as fs; -use nix::unistd::User; +use nix::unistd::{Gid, Uid}; use serde::Serialize; use std::collections::HashMap; -use std::os::unix::fs::PermissionsExt; use std::{ - fs::Permissions, io::Write, path::{Path, PathBuf}, process::{Command, Stdio}, @@ -201,13 +199,14 @@ impl PreparedQemuLaunch { workdir: impl AsRef, cfg: &CvmConfig, gpus: &GpuConfig, + networks: &[Networking], ) -> Result { let workdir = VmWorkDir::new(workdir); - prepare_data_disk(vm, &workdir, cfg)?; + prepare_data_disk(vm, &workdir)?; prepare_shared_dir(&workdir)?; let app_compose = workdir.app_compose().context("failed to get app compose")?; let platform = cfg.resolved_platform(); - let networks = resolved_networks(&vm.manifest, cfg); + let networks = networks.to_vec(); validate_resolved_networks(&networks)?; let volumes = vm .manifest @@ -293,7 +292,7 @@ impl PreparedQemuLaunch { } } -fn prepare_data_disk(vm: &VmConfig, workdir: &VmWorkDir, cfg: &CvmConfig) -> Result<()> { +fn prepare_data_disk(vm: &VmConfig, workdir: &VmWorkDir) -> Result<()> { let hda_path = workdir.hda_path(); if !hda_path.exists() { create_hd( @@ -302,9 +301,6 @@ fn prepare_data_disk(vm: &VmConfig, workdir: &VmWorkDir, cfg: &CvmConfig) -> Res &format!("{}G", vm.manifest.disk_size), )?; } - if !cfg.user.is_empty() { - fs::set_permissions(&hda_path, Permissions::from_mode(0o660))?; - } Ok(()) } @@ -342,8 +338,9 @@ impl VmConfig { workdir: impl AsRef, cfg: &CvmConfig, gpus: &GpuConfig, + networks: &[Networking], ) -> Result> { - let prepared = PreparedQemuLaunch::prepare(self, workdir, cfg, gpus)?; + let prepared = PreparedQemuLaunch::prepare(self, workdir, cfg, gpus, networks)?; let process = QemuCommandBuilder { vm: self, cfg, @@ -351,21 +348,21 @@ impl VmConfig { prepared: &prepared, } .build()?; + let has_macvtap = prepared + .networks + .iter() + .any(|network| network.mode == NetworkingMode::Macvtap); let Some(socket) = prepared.swtpm_socket.as_deref() else { + if has_macvtap { + return self.wrap_launcher(&prepared, process, None, None); + } return Ok(vec![process]); }; let swtpm_path = prepared .swtpm_path .as_ref() .context("missing swtpm executable for configured socket")?; - let (socket_uid, socket_gid) = if cfg.user.is_empty() { - (unsafe { libc::geteuid() }, unsafe { libc::getegid() }) - } else { - let user = User::from_name(&cfg.user) - .context("failed to resolve QEMU user")? - .with_context(|| format!("QEMU user {} does not exist", cfg.user))?; - (user.uid.as_raw(), user.gid.as_raw()) - }; + let (socket_uid, socket_gid) = (Uid::effective().as_raw(), Gid::effective().as_raw()); let swtpm_args = vec![ "socket".into(), @@ -380,16 +377,42 @@ impl VmConfig { "--flags".into(), "not-need-init,startup-clear".into(), ]; + self.wrap_launcher( + &prepared, + process, + Some(ChildCommand { + command: swtpm_path.to_string_lossy().into_owned(), + args: swtpm_args, + }), + Some(socket.to_path_buf()), + ) + } + + fn wrap_launcher( + &self, + prepared: &PreparedQemuLaunch, + process: ProcessConfig, + swtpm: Option, + swtpm_socket: Option, + ) -> Result> { + let open_files = prepared + .networks + .iter() + .enumerate() + .filter(|(_, network)| network.mode == NetworkingMode::Macvtap) + .map(|(index, network)| OpenFile { + fd: (3 + index) as i32, + path: network.device.clone().into(), + }) + .collect(); let spec = LaunchSpec { qemu: ChildCommand { command: process.command, args: process.args, }, - swtpm: ChildCommand { - command: swtpm_path.to_string_lossy().into_owned(), - args: swtpm_args, - }, - swtpm_socket: socket.to_path_buf(), + swtpm, + swtpm_socket, + open_files, startup_timeout_ms: 5_000, shutdown_timeout_ms: 10_000, }; @@ -635,6 +658,12 @@ impl QemuCommandBuilder<'_> { } networking.netdev.clone() } + NetworkingMode::Macvtap => { + if networking.device.is_empty() { + bail!("macvtap interface {index} has not been prepared by netd"); + } + format!("tap,id={net_id},fd={},vhost=off", 3 + index) + } }; command.arg("-netdev").arg(netdev); command.arg("-device").arg(net_device); @@ -790,12 +819,6 @@ impl QemuCommandBuilder<'_> { if let Some(cpus) = &self.prepared.numa_cpus { arguments.splice(0..0, ["taskset", "-c", cpus].into_iter().map(String::from)); } - if !self.cfg.user.is_empty() { - arguments.splice( - 0..0, - ["sudo", "-u", &self.cfg.user].into_iter().map(String::from), - ); - } let command = arguments.remove(0); let note = serde_json::to_string(&ProcessAnnotation { diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index 79d1a78b9..eab200244 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -38,6 +38,7 @@ fn networking_mode_name(mode: NetworkingMode) -> &'static str { NetworkingMode::Bridge => "bridge", NetworkingMode::User => "user", NetworkingMode::Custom => "custom", + NetworkingMode::Macvtap => "macvtap", } } @@ -46,6 +47,7 @@ fn networking_backend_name(mode: NetworkingMode) -> &'static str { NetworkingMode::Bridge => "tap_bridge", NetworkingMode::User => "slirp", NetworkingMode::Custom => "custom", + NetworkingMode::Macvtap => "macvtap", } } @@ -57,6 +59,8 @@ fn networking_to_proto(networking: &Networking) -> pb::NetworkingConfig { } else { String::new() }, + parent: networking.parent.clone(), + macvtap_mode: networking.macvtap_mode.clone(), } } diff --git a/dstack/vmm/src/app/workdir.rs b/dstack/vmm/src/app/workdir.rs index 198dc7c94..0b358229f 100644 --- a/dstack/vmm/src/app/workdir.rs +++ b/dstack/vmm/src/app/workdir.rs @@ -151,7 +151,14 @@ impl VmWorkDir { } pub fn set_runtime_networks(&self, networks: &[Networking]) -> Result<()> { - let serialized = serde_json::to_vec(networks)?; + // A macvtap device path is valid only while its host interface exists. + // Keep it in memory for launch preparation, but never persist it across + // VMM restarts where the same /dev/tapN may identify another device. + let mut persistent_networks = networks.to_vec(); + for network in &mut persistent_networks { + network.device.clear(); + } + let serialized = serde_json::to_vec(&persistent_networks)?; safe_write::safe_write(self.runtime_networks_path(), serialized) .context("failed to write runtime networks") } @@ -254,6 +261,7 @@ mod tests { use fs_err as fs; use super::VmWorkDir; + use crate::config::Networking; #[test] fn runtime_networks_snapshot_replaces_target_instead_of_following_it() -> Result<()> { @@ -275,4 +283,30 @@ mod tests { fs::remove_dir_all(temp)?; Ok(()) } + + #[test] + fn runtime_networks_snapshot_omits_ephemeral_device_paths() -> Result<()> { + let temp = std::env::temp_dir().join(format!( + "dstack-vmm-runtime-networks-device-test-{}", + SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos() + )); + let workdir = VmWorkDir::new(temp.join("vm")); + fs::create_dir_all(workdir.path())?; + let network: Networking = serde_json::from_value(serde_json::json!({ + "mode": "macvtap", + "parent": "br0", + "macvtap_mode": "private", + "device": "/dev/tap42" + }))?; + + workdir.set_runtime_networks(&[network])?; + + let persisted = workdir.runtime_networks(); + assert_eq!(persisted.len(), 1); + assert_eq!(persisted[0].parent, "br0"); + assert!(persisted[0].device.is_empty()); + assert!(!fs::read_to_string(workdir.runtime_networks_path())?.contains("/dev/tap42")); + fs::remove_dir_all(temp)?; + Ok(()) + } } diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index e13464af8..47ffc3672 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -334,9 +334,6 @@ pub struct CvmConfig { pub qmp_socket: bool, /// GPU configuration pub gpu: GpuConfig, - /// Use sudo to run the VM - pub user: String, - /// Auto restart configuration pub auto_restart: AutoRestartConfig, @@ -786,6 +783,19 @@ fn validate_networking(networking: &Networking) -> Result<()> { !networking.netdev.trim().is_empty(), "cvm.networking.netdev must not be empty in custom mode" ), + NetworkingMode::Macvtap => { + anyhow::ensure!( + !networking.parent.trim().is_empty(), + "cvm.networking.parent must not be empty in macvtap mode" + ); + anyhow::ensure!( + matches!( + networking.macvtap_mode.as_str(), + "" | "private" | "bridge" | "vepa" | "passthru" + ), + "cvm.networking.macvtap_mode must be private, bridge, vepa, or passthru" + ); + } NetworkingMode::User => {} } Ok(()) @@ -797,6 +807,7 @@ pub enum NetworkingMode { User, Bridge, Custom, + Macvtap, } /// Flat networking configuration. The `mode` field selects which backend is @@ -811,6 +822,17 @@ pub struct Networking { #[serde(default)] pub bridge: String, + // ── Macvtap fields ──────────────────────────────────────────── + /// Parent host interface for macvtap (e.g., "eth0"). + #[serde(default)] + pub parent: String, + /// macvtap forwarding mode. Empty selects "private". + #[serde(default)] + pub macvtap_mode: String, + /// Runtime-only character device returned by netd. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub device: String, + // ── MAC prefix ───────────────────────────────────────────────── /// Fixed MAC address prefix (0-3 colon-separated hex bytes, e.g. "02:ab:cd"). /// Remaining bytes are derived from the VM ID hash. diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 839fb990a..84066bf1d 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -354,6 +354,7 @@ fn networking_from_proto(proto: &rpc::NetworkingConfig) -> Result NetworkingMode::Bridge, "user" => NetworkingMode::User, + "macvtap" => NetworkingMode::Macvtap, "" if bridge.is_empty() => return Ok(None), "" => bail!("networking mode is required when bridge is set"), "custom" => bail!("custom networking mode is manifest-only"), @@ -362,9 +363,17 @@ fn networking_from_proto(proto: &rpc::NetworkingConfig) -> Result "user".to_string(), NetworkingMode::Bridge => "bridge".to_string(), NetworkingMode::Custom => String::new(), + NetworkingMode::Macvtap => "macvtap".to_string(), }, default_bridge: default_networking.bridge.clone(), }), @@ -1221,6 +1232,7 @@ mod tests { request.networks = vec![rpc::NetworkingConfig { mode: "user".to_string(), bridge_name: String::new(), + ..Default::default() }]; let manifest = create_manifest_from_vm_config(request, &test_cvm_config()).unwrap(); @@ -1230,11 +1242,27 @@ mod tests { assert!(!manifest.networks[0].net.is_empty()); } + #[test] + fn macvtap_networking_preserves_parent_and_mode() { + let networks = networks_from_proto(&[rpc::NetworkingConfig { + mode: "macvtap".to_string(), + parent: "eth0".to_string(), + macvtap_mode: "private".to_string(), + ..Default::default() + }]) + .unwrap(); + + assert_eq!(networks[0].mode, NetworkingMode::Macvtap); + assert_eq!(networks[0].parent, "eth0"); + assert_eq!(networks[0].macvtap_mode, "private"); + } + #[test] fn bridge_name_is_rejected_for_user_mode() { let err = networks_from_proto(&[rpc::NetworkingConfig { mode: "user".to_string(), bridge_name: "dstack-br0".to_string(), + ..Default::default() }]) .unwrap_err(); @@ -1246,6 +1274,7 @@ mod tests { let err = networks_from_proto(&[rpc::NetworkingConfig { mode: String::new(), bridge_name: String::new(), + ..Default::default() }]) .unwrap_err(); @@ -1257,6 +1286,7 @@ mod tests { let err = networks_from_proto(&[rpc::NetworkingConfig { mode: "custom".to_string(), bridge_name: String::new(), + ..Default::default() }]) .unwrap_err(); diff --git a/dstack/vmm/src/netd.rs b/dstack/vmm/src/netd.rs index e8ac8c820..44a8aff4b 100644 --- a/dstack/vmm/src/netd.rs +++ b/dstack/vmm/src/netd.rs @@ -10,7 +10,10 @@ use std::{ io::Write as _, os::{ fd::AsRawFd, - unix::{fs::PermissionsExt, net::UnixStream as StdUnixStream}, + unix::{ + fs::{FileTypeExt, PermissionsExt}, + net::UnixStream as StdUnixStream, + }, }, path::Path, process::{Command, Stdio}, @@ -57,10 +60,22 @@ pub struct PrepareBridgeRequest { pub parameters: BTreeMap, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PrepareMacvtapRequest { + #[serde(flatten)] + pub identity: InterfaceIdentity, + pub parent: String, + pub mac: String, + pub qemu_uid: u32, + #[serde(default)] + pub mode: String, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "operation", rename_all = "snake_case")] pub enum Request { PrepareBridge(PrepareBridgeRequest), + PrepareMacvtap(PrepareMacvtapRequest), Remove { #[serde(flatten)] identity: InterfaceIdentity, @@ -79,6 +94,8 @@ struct Response { #[serde(default, skip_serializing_if = "Option::is_none")] tap: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + device: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] error: Option, } @@ -99,9 +116,14 @@ pub fn instance_id(configured: &str, run_path: &Path) -> String { format!("path-{}", hex::encode(&digest[..8])) } -pub async fn request(socket: &Path, request: &Request) -> Result { +pub struct PreparedInterface { + pub device: Option, +} + +pub async fn request(socket: &Path, request: &Request) -> Result { let operation = match request { Request::PrepareBridge(_) => "prepare_bridge", + Request::PrepareMacvtap(_) => "prepare_macvtap", Request::Remove { .. } => "remove", Request::Check { .. } => "check", }; @@ -131,7 +153,12 @@ pub async fn request(socket: &Path, request: &Request) -> Result { response.error.as_deref().unwrap_or("unknown error") ); } - response.tap.context("netd response omitted TAP name") + Ok(PreparedInterface { + device: { + response.tap.context("netd response omitted TAP name")?; + response.device + }, + }) }; timeout(Duration::from_secs(30), exchange) .await @@ -176,9 +203,10 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul .await .and_then(|request| handle_request(&config.libvirt_uri, request)) { - Ok(tap) => Response { + Ok((tap, device)) => Response { ok: true, tap: Some(tap), + device, error: None, }, Err(error) => { @@ -186,6 +214,7 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul Response { ok: false, tap: None, + device: None, error: Some(format!("{error:#}")), } } @@ -195,6 +224,7 @@ async fn serve_connection(config: &NetdConfig, stream: &mut UnixStream) -> Resul Response { ok: false, tap: None, + device: None, error: Some("caller UID is not authorized".into()), } }; @@ -216,15 +246,26 @@ async fn read_request(stream: &mut UnixStream) -> Result { serde_json::from_slice(&message).context("invalid netd request") } -fn handle_request(libvirt_uri: &str, request: Request) -> Result { +fn handle_request(libvirt_uri: &str, request: Request) -> Result<(String, Option)> { let _lock = OperationLock::acquire()?; match request { - Request::PrepareBridge(request) => prepare_bridge(libvirt_uri, &request), + Request::PrepareBridge(request) => { + prepare_bridge(libvirt_uri, &request).map(|tap| (tap, None)) + } + Request::PrepareMacvtap(request) => prepare_macvtap( + libvirt_uri, + &request.identity, + &request.parent, + &request.mac, + request.qemu_uid, + &request.mode, + ) + .map(|(tap, device)| (tap, Some(device))), Request::Remove { identity } => { validate_identity(&identity)?; let tap = tap_name(&identity); remove_interface(libvirt_uri, &tap)?; - Ok(tap) + Ok((tap, None)) } Request::Check { identity } => { validate_identity(&identity)?; @@ -232,8 +273,68 @@ fn handle_request(libvirt_uri: &str, request: Request) -> Result { if !Path::new("/sys/class/net").join(&tap).exists() { bail!("TAP {tap} does not exist"); } - virsh(libvirt_uri, &["nwfilter-binding-dumpxml", &tap], None)?; - Ok(tap) + if !is_macvtap(&tap) { + virsh(libvirt_uri, &["nwfilter-binding-dumpxml", &tap], None)?; + } + Ok((tap, None)) + } + } +} + +fn prepare_macvtap( + libvirt_uri: &str, + identity: &InterfaceIdentity, + parent: &str, + mac: &str, + qemu_uid: u32, + mode: &str, +) -> Result<(String, String)> { + validate_identity(identity)?; + validate_name("parent", parent, 15, "_.-")?; + if !Path::new("/sys/class/net").join(parent).exists() { + bail!("parent interface {parent} does not exist"); + } + validate_mac(mac)?; + let mode = if mode.is_empty() { "private" } else { mode }; + if !matches!(mode, "private" | "bridge" | "vepa" | "passthru") { + bail!("invalid macvtap mode"); + } + let tap = tap_name(identity); + remove_interface(libvirt_uri, &tap)?; + ip(&[ + "link", "add", "link", parent, "name", &tap, "address", mac, "type", "macvtap", "mode", + mode, + ])?; + let result = (|| { + let ifindex = + std::fs::read_to_string(Path::new("/sys/class/net").join(&tap).join("ifindex")) + .context("failed to read macvtap ifindex")?; + let device = format!("/dev/tap{}", ifindex.trim()); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !Path::new(&device).exists() { + if std::time::Instant::now() >= deadline { + bail!("timed out waiting for macvtap device {device}"); + } + std::thread::sleep(Duration::from_millis(25)); + } + let metadata = std::fs::symlink_metadata(&device) + .with_context(|| format!("failed to inspect macvtap device {device}"))?; + if !metadata.file_type().is_char_device() { + bail!("macvtap device {device} is not a character device"); + } + std::os::unix::fs::chown(&device, Some(qemu_uid), None) + .with_context(|| format!("failed to set owner of macvtap device {device}"))?; + ip(&["link", "set", "dev", &tap, "up"])?; + Ok(device) + })(); + match result { + Ok(device) => { + info!(%tap, %parent, %mode, %device, "prepared macvtap"); + Ok((tap, device)) + } + Err(error) => { + let _ = remove_interface(libvirt_uri, &tap); + Err(error) } } } @@ -295,17 +396,27 @@ fn prepare_bridge(libvirt_uri: &str, request: &PrepareBridgeRequest) -> Result Result<()> { + let macvtap = is_macvtap(tap); if Path::new("/sys/class/net").join(tap).exists() { let _ = ip(&["link", "set", "dev", tap, "down"]); } - delete_binding(libvirt_uri, tap)?; + if !macvtap { + delete_binding(libvirt_uri, tap)?; + } if Path::new("/sys/class/net").join(tap).exists() { ip(&["link", "delete", "dev", tap])?; - info!(%tap, "removed filtered TAP"); + info!(%tap, "removed managed network interface"); } Ok(()) } +fn is_macvtap(interface: &str) -> bool { + Path::new("/sys/class/net") + .join(interface) + .join("macvtap") + .exists() +} + fn delete_binding(uri: &str, tap: &str) -> Result<()> { let output = Command::new(VIRSH_PATH) .args(["--connect", uri, "nwfilter-binding-delete", tap]) @@ -615,6 +726,39 @@ mod tests { assert!(value.get("identity").is_none()); } + #[test] + fn macvtap_prepare_has_a_dedicated_operation() { + let request = Request::PrepareMacvtap(PrepareMacvtapRequest { + identity: identity("instance", "vm", 1), + parent: "eth0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + mode: "private".into(), + }); + let value = serde_json::to_value(request).unwrap(); + assert_eq!(value["operation"], "prepare_macvtap"); + assert_eq!(value["instance_id"], "instance"); + assert_eq!(value["parent"], "eth0"); + assert!(value.get("identity").is_none()); + } + + #[test] + fn bridge_prepare_has_a_dedicated_operation() { + let request = Request::PrepareBridge(PrepareBridgeRequest { + identity: identity("instance", "vm", 0), + bridge: "br0".into(), + mac: "02:00:00:00:00:01".into(), + qemu_uid: 1000, + filter: "clean-traffic".into(), + parameters: BTreeMap::new(), + }); + let value = serde_json::to_value(request).unwrap(); + assert_eq!(value["operation"], "prepare_bridge"); + assert_eq!(value["instance_id"], "instance"); + assert_eq!(value["bridge"], "br0"); + assert!(value.get("identity").is_none()); + } + #[tokio::test] async fn disconnected_client_is_confined_to_one_connection() { let (mut server, client) = UnixStream::pair().unwrap(); diff --git a/dstack/vmm/src/one_shot.rs b/dstack/vmm/src/one_shot.rs index 6ee26a470..dd7a23d79 100644 --- a/dstack/vmm/src/one_shot.rs +++ b/dstack/vmm/src/one_shot.rs @@ -290,8 +290,9 @@ Compose file content (first 200 chars): ); } + let runtime_networks = resolved_networks(&manifest, &config.cvm); let process_configs = vm_builder_config - .config_qemu(&workdir_path, &config.cvm, &gpus) + .config_qemu(&workdir_path, &config.cvm, &gpus, &runtime_networks) .context("Failed to build QEMU configuration")?; // Get the main QEMU process config (first in the list) diff --git a/dstack/vmm/src/vm_launcher.rs b/dstack/vmm/src/vm_launcher.rs index 7f9cff7b9..001d02018 100644 --- a/dstack/vmm/src/vm_launcher.rs +++ b/dstack/vmm/src/vm_launcher.rs @@ -2,11 +2,21 @@ // // SPDX-License-Identifier: Apache-2.0 +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; use std::os::unix::process::CommandExt; use std::path::{Path, PathBuf}; use std::time::Duration; use anyhow::{bail, Context, Result}; +use nix::{ + errno::Errno, + fcntl::{fcntl, FcntlArg, FdFlag}, + sys::{ + prctl, + signal::{kill, raise, Signal}, + }, + unistd::{dup2, getpid, getppid, setpgid, Pid}, +}; use serde::{Deserialize, Serialize}; use tokio::process::{Child, Command}; use tokio::signal::unix::{signal, SignalKind}; @@ -23,14 +33,24 @@ pub struct ChildCommand { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct LaunchSpec { pub qemu: ChildCommand, - pub swtpm: ChildCommand, - pub swtpm_socket: PathBuf, + #[serde(default)] + pub swtpm: Option, + #[serde(default)] + pub swtpm_socket: Option, + #[serde(default)] + pub open_files: Vec, #[serde(default = "default_startup_timeout_ms")] pub startup_timeout_ms: u64, #[serde(default = "default_shutdown_timeout_ms")] pub shutdown_timeout_ms: u64, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OpenFile { + pub fd: i32, + pub path: PathBuf, +} + fn default_startup_timeout_ms() -> u64 { 5_000 } @@ -51,39 +71,111 @@ impl Drop for SocketCleanup { } } -fn spawn_child(spec: &ChildCommand) -> Result { - let parent = unsafe { libc::getpid() }; +impl LaunchSpec { + fn is_single_process(&self) -> bool { + self.swtpm.is_none() + } +} + +struct PreparedOpenFiles { + // Retain the original device handles through the fork or exec handoff; + // they close automatically when this prepared set leaves scope. + _opened: Vec, + // Own the collision-free source fds used by dup2. Dropping these before + // fork or exec would close the source fds while `mappings` still refers to them. + _inherited: Vec, + mappings: Vec<(i32, i32)>, +} + +fn prepare_open_files(open_files: &[OpenFile]) -> Result { + let opened = open_files + .iter() + .map(|file| { + fs_err::OpenOptions::new() + .read(true) + .write(true) + .open(&file.path) + .with_context(|| format!("failed to open {}", file.path.display())) + }) + .collect::>>()?; + let max_target = open_files.iter().map(|file| file.fd).max().unwrap_or(2); + let inherited = opened + .iter() + .map(|file| { + let fd = fcntl(file.as_raw_fd(), FcntlArg::F_DUPFD_CLOEXEC(max_target + 1)) + .context("failed to reserve inherited file descriptor")?; + // SAFETY: F_DUPFD_CLOEXEC returned a new descriptor owned by this process. + Ok(unsafe { OwnedFd::from_raw_fd(fd) }) + }) + .collect::>>()?; + let mappings = open_files + .iter() + .zip(&inherited) + .map(|(file, opened)| (file.fd, opened.as_raw_fd())) + .collect::>(); + Ok(PreparedOpenFiles { + _opened: opened, + _inherited: inherited, + mappings, + }) +} + +fn prepare_child(expected_parent: libc::pid_t, mappings: &[(i32, i32)]) -> std::io::Result<()> { + setpgid(Pid::from_raw(0), Pid::from_raw(0))?; + prctl::set_pdeathsig(Signal::SIGKILL)?; + if getppid().as_raw() != expected_parent { + raise(Signal::SIGKILL)?; + } + for &(target, source) in mappings { + if target < 3 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "open file target fd must be at least 3", + )); + } + if source == target { + fcntl(target, FcntlArg::F_SETFD(FdFlag::empty()))?; + } else { + dup2(source, target)?; + } + } + Ok(()) +} + +fn spawn_child(spec: &ChildCommand, open_files: &[OpenFile]) -> Result { + let parent = getpid().as_raw(); let mut command = Command::new(&spec.command); command.args(&spec.args); + let prepared = prepare_open_files(open_files)?; + let mappings = prepared.mappings.clone(); // SAFETY: pre_exec only invokes async-signal-safe libc operations. Checking // the parent after PR_SET_PDEATHSIG closes the fork/parent-exit race. unsafe { - command.as_std_mut().pre_exec(move || { - if libc::setpgid(0, 0) != 0 { - return Err(std::io::Error::last_os_error()); - } - if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) != 0 { - return Err(std::io::Error::last_os_error()); - } - if libc::getppid() != parent { - libc::raise(libc::SIGKILL); - } - Ok(()) - }); + command + .as_std_mut() + .pre_exec(move || prepare_child(parent, &mappings)); } command .spawn() .with_context(|| format!("failed to start {}", spec.command)) } +fn exec_in_place(spec: &ChildCommand, open_files: &[OpenFile]) -> Result<()> { + let prepared = prepare_open_files(open_files)?; + let parent = getppid().as_raw(); + prepare_child(parent, &prepared.mappings).context("failed to prepare in-place QEMU exec")?; + let mut command = std::process::Command::new(&spec.command); + command.args(&spec.args); + let error = command.exec(); + Err(error).with_context(|| format!("failed to exec {}", spec.command)) +} + async fn stop_child(child: &mut Child, name: &str, grace: Duration) { let Some(pid) = child.id() else { return; }; - // SAFETY: pid comes from the live Child handle and SIGTERM has no pointer arguments. - if unsafe { libc::kill(-(pid as libc::pid_t), libc::SIGTERM) } != 0 { - let error = std::io::Error::last_os_error(); - if error.raw_os_error() != Some(libc::ESRCH) { + if let Err(error) = kill(Pid::from_raw(-(pid as libc::pid_t)), Signal::SIGTERM) { + if error != Errno::ESRCH { warn!(%pid, %name, %error, "failed to terminate child"); } } @@ -92,9 +184,8 @@ async fn stop_child(child: &mut Child, name: &str, grace: Duration) { Ok(Err(error)) => warn!(%pid, %name, %error, "failed to wait for child"), Err(_) => { warn!(%pid, %name, "child did not stop gracefully; killing"); - if unsafe { libc::kill(-(pid as libc::pid_t), libc::SIGKILL) } != 0 { - let error = std::io::Error::last_os_error(); - if error.raw_os_error() != Some(libc::ESRCH) { + if let Err(error) = kill(Pid::from_raw(-(pid as libc::pid_t)), Signal::SIGKILL) { + if error != Errno::ESRCH { warn!(%pid, %name, %error, "failed to kill child process group"); } } @@ -126,24 +217,33 @@ pub async fn run(spec_path: &Path) -> Result<()> { let raw = fs_err::read(spec_path) .with_context(|| format!("failed to read launch spec {}", spec_path.display()))?; let spec: LaunchSpec = serde_json::from_slice(&raw).context("failed to parse launch spec")?; - let _socket_cleanup = SocketCleanup(spec.swtpm_socket.clone()); - if spec.swtpm_socket.exists() { - fs_err::remove_file(&spec.swtpm_socket).context("failed to remove stale swtpm socket")?; + if spec.is_single_process() { + return exec_in_place(&spec.qemu, &spec.open_files); + } + let _socket_cleanup = spec.swtpm_socket.clone().map(SocketCleanup); + if let Some(socket) = &spec.swtpm_socket { + if socket.exists() { + fs_err::remove_file(socket).context("failed to remove stale swtpm socket")?; + } } let grace = Duration::from_millis(spec.shutdown_timeout_ms); let mut terminate = signal(SignalKind::terminate()).context("failed to watch SIGTERM")?; let mut interrupt = signal(SignalKind::interrupt()).context("failed to watch SIGINT")?; - let mut swtpm = spawn_child(&spec.swtpm)?; + let mut swtpm = if let Some(command) = &spec.swtpm { + Some(spawn_child(command, &[])?) + } else { + None + }; enum StartupExit { Ready, Error(anyhow::Error), Signal, } - let startup_exit = { + let startup_exit = if let (Some(swtpm), Some(socket)) = (&mut swtpm, &spec.swtpm_socket) { let startup = wait_for_swtpm( - &mut swtpm, - &spec.swtpm_socket, + swtpm, + socket, Instant::now() + Duration::from_millis(spec.startup_timeout_ms), ); tokio::pin!(startup); @@ -155,29 +255,37 @@ pub async fn run(spec_path: &Path) -> Result<()> { _ = terminate.recv() => StartupExit::Signal, _ = interrupt.recv() => StartupExit::Signal, } + } else { + StartupExit::Ready }; match startup_exit { StartupExit::Ready => {} StartupExit::Error(error) => { - stop_child(&mut swtpm, "swtpm", grace).await; + if let Some(child) = &mut swtpm { + stop_child(child, "swtpm", grace).await; + } return Err(error); } StartupExit::Signal => { - stop_child(&mut swtpm, "swtpm", grace).await; + if let Some(child) = &mut swtpm { + stop_child(child, "swtpm", grace).await; + } return Ok(()); } } - let mut qemu = match spawn_child(&spec.qemu) { + let mut qemu = match spawn_child(&spec.qemu, &spec.open_files) { Ok(child) => child, Err(error) => { - stop_child(&mut swtpm, "swtpm", grace).await; + if let Some(child) = &mut swtpm { + stop_child(child, "swtpm", grace).await; + } return Err(error); } }; info!( qemu_pid = qemu.id(), - swtpm_pid = swtpm.id(), + swtpm_pid = swtpm.as_ref().and_then(|child| child.id()), "VM processes started" ); @@ -190,19 +298,26 @@ pub async fn run(spec_path: &Path) -> Result<()> { _ = terminate.recv() => Exit::Signal, _ = interrupt.recv() => Exit::Signal, status = qemu.wait() => Exit::Qemu(status.context("failed to wait for QEMU")?), - status = swtpm.wait() => Exit::Swtpm(status.context("failed to wait for swtpm")?), + status = async { + match &mut swtpm { + Some(child) => child.wait().await, + None => std::future::pending().await, + } + } => Exit::Swtpm(status.context("failed to wait for swtpm")?), }; match exit { Exit::Signal => { - tokio::join!( - stop_child(&mut qemu, "qemu", grace), - stop_child(&mut swtpm, "swtpm", grace) - ); + stop_child(&mut qemu, "qemu", grace).await; + if let Some(child) = &mut swtpm { + stop_child(child, "swtpm", grace).await; + } Ok(()) } Exit::Qemu(status) => { - stop_child(&mut swtpm, "swtpm", grace).await; + if let Some(child) = &mut swtpm { + stop_child(child, "swtpm", grace).await; + } if status.success() { Ok(()) } else { @@ -230,9 +345,7 @@ mod tests { } fn process_is_gone(pid: libc::pid_t) -> bool { - // SAFETY: signal 0 only probes whether the PID still exists. - let missing = unsafe { libc::kill(pid, 0) != 0 }; - missing && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) + matches!(kill(Pid::from_raw(pid), None), Err(Errno::ESRCH)) } async fn create_fake_socket(path: PathBuf) { @@ -241,6 +354,36 @@ mod tests { sleep(Duration::from_secs(5)).await; } + #[tokio::test] + async fn passes_open_file_to_child() -> Result<()> { + let dir = tempfile::tempdir()?; + let input = dir.path().join("input"); + let output = dir.path().join("output"); + fs_err::write(&input, b"macvtap-fd")?; + let command = shell(format!("cat <&3 > {}", output.display())); + let open_files = vec![OpenFile { fd: 3, path: input }]; + let mut child = spawn_child(&command, &open_files)?; + assert!(child.wait().await?.success()); + + assert_eq!(fs_err::read(output)?, b"macvtap-fd"); + Ok(()) + } + + #[test] + fn single_process_launch_omits_swtpm() { + let mut spec = LaunchSpec { + qemu: shell("exit 0".into()), + swtpm: None, + swtpm_socket: None, + open_files: vec![], + startup_timeout_ms: 1_000, + shutdown_timeout_ms: 1_000, + }; + assert!(spec.is_single_process()); + spec.swtpm = Some(shell("exit 0".into())); + assert!(!spec.is_single_process()); + } + #[tokio::test] async fn qemu_failure_stops_and_reaps_swtpm() { let dir = tempfile::tempdir().unwrap(); @@ -248,8 +391,12 @@ mod tests { let swtpm_pid = dir.path().join("swtpm.pid"); let spec = LaunchSpec { qemu: shell("exit 7".into()), - swtpm: shell(format!("echo $$ > {}; sleep 30", swtpm_pid.display())), - swtpm_socket: socket.clone(), + swtpm: Some(shell(format!( + "echo $$ > {}; sleep 30", + swtpm_pid.display() + ))), + swtpm_socket: Some(socket.clone()), + open_files: vec![], startup_timeout_ms: 2_000, shutdown_timeout_ms: 500, }; @@ -273,8 +420,9 @@ mod tests { let qemu_pid = dir.path().join("qemu.pid"); let spec = LaunchSpec { qemu: shell(format!("echo $$ > {}; sleep 30", qemu_pid.display())), - swtpm: shell("sleep 0.2; exit 9".into()), - swtpm_socket: socket.clone(), + swtpm: Some(shell("sleep 0.2; exit 9".into())), + swtpm_socket: Some(socket.clone()), + open_files: vec![], startup_timeout_ms: 2_000, shutdown_timeout_ms: 500, }; @@ -298,8 +446,12 @@ mod tests { let swtpm_pid = dir.path().join("swtpm.pid"); let spec = LaunchSpec { qemu: shell("exit 0".into()), - swtpm: shell(format!("echo $$ > {}; sleep 30", swtpm_pid.display())), - swtpm_socket: socket.clone(), + swtpm: Some(shell(format!( + "echo $$ > {}; sleep 30", + swtpm_pid.display() + ))), + swtpm_socket: Some(socket.clone()), + open_files: vec![], startup_timeout_ms: 100, shutdown_timeout_ms: 500, }; @@ -323,8 +475,12 @@ mod tests { let swtpm_pid = dir.path().join("swtpm.pid"); let spec = LaunchSpec { qemu: shell("sleep 0.1; exit 0".into()), - swtpm: shell(format!("echo $$ > {}; sleep 30", swtpm_pid.display())), - swtpm_socket: socket.clone(), + swtpm: Some(shell(format!( + "echo $$ > {}; sleep 30", + swtpm_pid.display() + ))), + swtpm_socket: Some(socket.clone()), + open_files: vec![], startup_timeout_ms: 2_000, shutdown_timeout_ms: 500, }; @@ -350,11 +506,12 @@ mod tests { let qemu_marker = dir.path().join("qemu-started"); let spec = LaunchSpec { qemu: shell(format!("touch {}", qemu_marker.display())), - swtpm: ChildCommand { + swtpm: Some(ChildCommand { command: dir.path().join("missing-swtpm").display().to_string(), args: vec![], - }, - swtpm_socket: socket.clone(), + }), + swtpm_socket: Some(socket.clone()), + open_files: vec![], startup_timeout_ms: 100, shutdown_timeout_ms: 100, }; diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index 599c3e5c6..fd27137d6 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -43,7 +43,6 @@ max_allocable_memory_in_mb = 100_000 # MB # Enable QMP socket qmp_socket = false # The user to run the VM as. If empty, the VM will be run as the current user. -user = "" # Unique namespace when multiple dstack-vmm instances share one host. When # empty, a stable value is derived from run_path. instance_id = ""