Integrate vendor VFIO vGPUs into the instance lifecycle

cmd/api/api/instances.go

@@ -362,7 +362,16 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 
 	inst, err := s.InstanceManager.CreateInstance(ctx, domainReq)
 	if err != nil {
+		var vgpuPending *instances.VGPUCleanupPendingError
 		switch {
+		case errors.As(err, &vgpuPending):
+			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
+			message, inner := vgpuCleanupPendingDetail(vgpuPending, "create", "delete it to retry")
+			return oapi.CreateInstance500JSONResponse{
+				Code:       "vgpu_cleanup_pending",
+				Message:    message,
+				InnerError: inner,
+			}, nil
 		case errors.Is(err, instances.ErrImageNotReady):
 			return oapi.CreateInstance400JSONResponse{
 				Code:    "image_not_ready",
@@ -424,6 +433,19 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 	return oapi.CreateInstance201JSONResponse(instanceToOAPI(*inst)), nil
 }
 
+func vgpuCleanupPendingDetail(pending *instances.VGPUCleanupPendingError, action, retainedGuidance string) (string, *oapi.ErrorDetail) {
+	message := fmt.Sprintf("failed to %s instance: %v", action, pending)
+	innerCode := "vgpu_unretained_instance"
+	if pending.Retained {
+		message += "; " + retainedGuidance
+		innerCode = "vgpu_retained_instance"
+	}
+	return message, &oapi.ErrorDetail{
+		Code:    lo.ToPtr(innerCode),
+		Message: lo.ToPtr(pending.InstanceID),
+	}
+}
+
 // GetInstance gets instance details
 // The id parameter can be an instance ID, name, or ID prefix
 // Note: Resolution is handled by ResolveResource middleware
@@ -829,7 +851,16 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan
 
 	result, err := s.InstanceManager.StartInstance(ctx, inst.Id, startReq)
 	if err != nil {
+		var vgpuPending *instances.VGPUCleanupPendingError
 		switch {
+		case errors.As(err, &vgpuPending):
+			log.ErrorContext(ctx, "failed to start instance", "error", err)
+			message, inner := vgpuCleanupPendingDetail(vgpuPending, "start", "delete it or retry start to release it")
+			return oapi.StartInstance500JSONResponse{
+				Code:       "vgpu_cleanup_pending",
+				Message:    message,
+				InnerError: inner,
+			}, nil
 		case errors.Is(err, instances.ErrInvalidState):
 			return oapi.StartInstance409JSONResponse{
 				Code:    "invalid_state",
@@ -1264,6 +1295,9 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance {
 		if inst.GPUMdevUUID != "" {
 			gpu.MdevUuid = lo.ToPtr(inst.GPUMdevUUID)
 		}
+		if inst.GPUDevicePath != "" {
+			gpu.DevicePath = lo.ToPtr(inst.GPUDevicePath)
+		}
 		oapiInst.Gpu = gpu
 	}
 

cmd/api/api/instances_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/kernel/hypeman/lib/instances"
 	"github.com/kernel/hypeman/lib/instances/phasetracking"
 	mw "github.com/kernel/hypeman/lib/middleware"
+	"github.com/kernel/hypeman/lib/network"
 	"github.com/kernel/hypeman/lib/oapi"
 	"github.com/kernel/hypeman/lib/paths"
 	restartpolicy "github.com/kernel/hypeman/lib/restart-policy"
@@ -47,6 +48,39 @@ func TestGetInstance_NotFound(t *testing.T) {
 	require.Error(t, err)
 }
 
+func TestVGPUCleanupPendingDetail(t *testing.T) {
+	t.Parallel()
+	for _, tt := range []struct {
+		name     string
+		retained bool
+		code     string
+		guidance string
+		exclude  string
+	}{
+		{name: "retained", retained: true, code: "vgpu_retained_instance", guidance: "delete it to retry"},
+		{name: "unretained", code: "vgpu_unretained_instance", guidance: "periodic vGPU reconcile", exclude: "delete"},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			message, inner := vgpuCleanupPendingDetail(&instances.VGPUCleanupPendingError{
+				InstanceID: "inst-1",
+				Retained:   tt.retained,
+				Err:        network.ErrNameExists,
+			}, "create", "delete it to retry")
+			assert.Contains(t, message, "inst-1")
+			assert.Contains(t, message, network.ErrNameExists.Error())
+			assert.Contains(t, message, tt.guidance)
+			if tt.exclude != "" {
+				assert.NotContains(t, message, tt.exclude)
+			}
+			require.NotNil(t, inner.Code)
+			assert.Equal(t, tt.code, *inner.Code)
+			require.NotNil(t, inner.Message)
+			assert.Equal(t, "inst-1", *inner.Message)
+		})
+	}
+}
+
 func TestCreateInstance_AutoPullImage(t *testing.T) {
 	t.Parallel()
 	if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) {
@@ -842,6 +876,16 @@ func TestCreateInstance_ErrorStatusMapping(t *testing.T) {
 		wantCode    string
 		wantMessage string
 	}{
+		{
+			name: "vGPU cleanup pending beats wrapped name conflict -> 500",
+			err: &instances.VGPUCleanupPendingError{
+				InstanceID: "inst-1",
+				Retained:   true,
+				Err:        network.ErrNameExists,
+			},
+			wantType: oapi.CreateInstance500JSONResponse{},
+			wantCode: "vgpu_cleanup_pending",
+		},
 		{
 			name:     "platform not available -> 404",
 			err:      fmt.Errorf("resolve image: %w", images.ErrPlatformNotAvailable),
@@ -966,6 +1010,27 @@ func TestRestoreInstance_ErrorMapping(t *testing.T) {
 	}
 }
 
+func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) {
+	t.Parallel()
+	svc := newTestService(t)
+	resolved := &instances.Instance{
+		StoredMetadata: instances.StoredMetadata{Id: "inst-1", Name: "inst-1"},
+		State:          instances.StateStopped,
+	}
+	svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{
+		InstanceID: resolved.Id,
+		Retained:   true,
+		Err:        fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources),
+	}}
+
+	resp, err := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id})
+	require.NoError(t, err)
+	pending, ok := resp.(oapi.StartInstance500JSONResponse)
+	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+	assert.Contains(t, pending.Message, "delete it or retry start")
+}
+
 func TestInstanceActions_ImageNotFoundMapsTo404(t *testing.T) {
 	t.Parallel()
 

cmd/api/main.go

@@ -384,11 +384,9 @@ func run() error {
 		return fmt.Errorf("reconcile device state: %w", err)
 	}
 
-	// Reconcile mdev devices (clears orphaned vGPUs from previous runs)
-	logger.Info("Reconciling mdev devices...")
-	if err := devices.ReconcileMdevs(app.Ctx, nil); err != nil {
-		// Log but don't fail - mdev cleanup is best-effort
-		logger.Warn("failed to reconcile mdev devices", "error", err)
+	logger.Info("Reconciling vGPU devices...")
+	if r, ok := app.InstanceManager.(interface{ StartVGPUReconciler(context.Context) }); ok {
+		r.StartVGPUReconciler(ctx)
 	}
 
 	// Wire up resource validator for aggregate limit checking

integration/vgpu_test.go

@@ -324,11 +324,6 @@ func checkVGPUTestPrerequisites() (string, string) {
 	if framework == devices.VGPUFrameworkNone {
 		return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", ""
 	}
-	if framework == devices.VGPUFrameworkVendorVFIO {
-		// CreateVGPU rejects vendor VFIO until the instance lifecycle
-		// integration lands.
-		return "vGPU test requires the vendor VFIO instance lifecycle integration", ""
-	}
 
 	// Check for available profiles
 	profiles, err := devices.ListGPUProfiles()

lib/devices/GPU.md

@@ -97,7 +97,7 @@ Instance Create → Assign profile to VF → Attach VF to VM → Instance Runnin
 Instance Stop/Delete → Release profile → VF available again
 ```
 
-Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM.
+Hypeman reconciles orphaned assignments with a periodic fail-closed pass: once at startup and every minute afterward (skipped entirely on hosts without GPUs). Each pass releases assignments whose owning instance is no longer live and clears their metadata, then sweeps device-level leftovers with no live metadata claim. Devices held open by a running VMM and assignments younger than five minutes are preserved, so a release that fails during stop or delete (typically because a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait) is simply retried on later passes until the device is free.
 
 ### Hypervisor Support
 
@@ -288,10 +288,17 @@ every request, so one wedged VF presents as all vGPU instances failing while
 `/resources` reports full capacity.
 
 The wedge itself leaves no host-side log: no kernel error, no XID, no plugin
-crash. In the observed case it followed a period of heavy attach/teardown
-churn on the VF, including QEMU processes that exited within seconds of
-opening the VFIO device (failed start attempts that were then retried), so
-suspect any workload that repeatedly kills the VMM mid-device-init.
+crash. The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is
+still initializing the VF (roughly the first seconds after process start):
+a single hard kill in that window wedges the VF near-deterministically,
+while QEMU processes that exit voluntarily — error exits, QMP quit, SIGTERM —
+run their VFIO teardown and never wedge, and hard kills of fully-initialized
+vGPU VMs are also safe. Hypeman therefore SIGTERMs a vGPU QEMU first and only
+escalates to SIGKILL after a grace period, both in start-failure cleanup and
+when force-killing any vGPU instance (the instance reports Running seconds
+before driver init completes, so no state reliably marks the window); a hard
+kill after an ignored SIGTERM logs `VF may wedge` with the device path.
+External SIGKILLs (OOM killer, manual `kill -9`) can still trigger it.
 
 Confirm by assigning the same profile on a different VF: if that guest
 initializes, the VF is wedged, not the driver stack. Remediate by cycling

lib/devices/mdev_darwin.go

@@ -58,7 +58,7 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error {
 	return nil
 }
 
-func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error {
+func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepDevices bool) error {
 	return nil
 }
 

lib/devices/mdev_linux.go

@@ -23,10 +23,9 @@ import (
 )
 
 const (
-	mdevBusPath             = "/sys/class/mdev_bus"
-	mdevDevices             = "/sys/bus/mdev/devices"
-	orphanedMdevGracePeriod = 5 * time.Minute
-	procPath                = "/proc"
+	mdevBusPath = "/sys/class/mdev_bus"
+	mdevDevices = "/sys/bus/mdev/devices"
+	procPath    = "/proc"
 )
 
 // mdevMu protects mdev creation/destruction to prevent race conditions
@@ -708,17 +707,27 @@ func mdevPastGracePeriod(mdevUUID string, gracePeriod time.Duration) (bool, time
 	return age >= gracePeriod, age, nil
 }
 
+func protectedMdevUUIDs(instanceInfos []MdevReconcileInfo) map[string]struct{} {
+	protected := make(map[string]struct{}, len(instanceInfos))
+	for _, info := range instanceInfos {
+		if info.MdevUUID != "" && info.IsRunning {
+			protected[info.MdevUUID] = struct{}{}
+		}
+	}
+	return protected
+}
+
 // ReconcileMdevs destroys orphaned mdevs on managed VFs.
-// This is called on server startup to clean up stale mdevs from previous runs.
 //
 // Policy:
 //   - Consider only mdevs whose parent VF is currently managed by hypeman (discoverable via /sys/class/mdev_bus)
+//   - Keep mdevs claimed by live instance metadata
 //   - Keep mdevs whose VFIO group has an open file handle (/dev/vfio/<group>)
 //   - Keep mdevs younger than a short grace period to avoid racing very recent state transitions
 //   - Delete all remaining mdevs
 func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error {
 	log := logger.FromContext(ctx)
-	_ = instanceInfos
+	protectedMdevs := protectedMdevUUIDs(instanceInfos)
 
 	vfs, err := discoverMdevVFs()
 	if err != nil {
@@ -747,17 +756,22 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro
 	log.InfoContext(ctx, "reconciling mdev devices",
 		"total_mdevs", len(mdevs),
 		"managed_vfs", len(managedVFs),
-		"grace_period", orphanedMdevGracePeriod.String(),
+		"grace_period", VGPUAssignmentGracePeriod.String(),
 	)
 
 	groupInUseCache := make(map[int]bool)
-	var destroyed, failedDestroy, skippedUnmanagedVF, skippedInUse, skippedGrace, skippedProbeError int
+	var destroyed, failedDestroy, skippedUnmanagedVF, skippedClaimed, skippedInUse, skippedGrace, skippedProbeError int
 	for _, mdev := range mdevs {
 		if _, ok := managedVFs[mdev.VFAddress]; !ok {
 			log.DebugContext(ctx, "skipping mdev on unmanaged VF", "uuid", mdev.UUID, "vf", mdev.VFAddress)
 			skippedUnmanagedVF++
 			continue
 		}
+		if _, ok := protectedMdevs[mdev.UUID]; ok {
+			log.DebugContext(ctx, "skipping mdev claimed by live instance", "uuid", mdev.UUID)
+			skippedClaimed++
+			continue
+		}
 
 		group, err := mdevIOMMUGroup(mdev.UUID)
 		if err != nil {
@@ -782,7 +796,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro
 			continue
 		}
 
-		pastGracePeriod, age, err := mdevPastGracePeriod(mdev.UUID, orphanedMdevGracePeriod)
+		pastGracePeriod, age, err := mdevPastGracePeriod(mdev.UUID, VGPUAssignmentGracePeriod)
 		if err != nil {
 			log.WarnContext(ctx, "failed to determine mdev age, skipping cleanup", "uuid", mdev.UUID, "error", err)
 			skippedProbeError++
@@ -792,7 +806,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro
 			log.DebugContext(ctx, "skipping recently created mdev during grace period",
 				"uuid", mdev.UUID,
 				"age", age.String(),
-				"grace_period", orphanedMdevGracePeriod.String(),
+				"grace_period", VGPUAssignmentGracePeriod.String(),
 			)
 			skippedGrace++
 			continue
@@ -818,6 +832,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro
 		"destroyed", destroyed,
 		"failed_destroy", failedDestroy,
 		"skipped_unmanaged_vf", skippedUnmanagedVF,
+		"skipped_claimed", skippedClaimed,
 		"skipped_in_use", skippedInUse,
 		"skipped_grace", skippedGrace,
 		"skipped_probe_error", skippedProbeError,

lib/devices/types.go

@@ -60,6 +60,11 @@ func ValidateDeviceName(name string) bool {
 // GPUMode represents the host's GPU configuration mode
 type GPUMode string
 
+// VGPUAssignmentGracePeriod protects a fresh vGPU assignment from cleanup
+// until its VM has had time to boot and become identifiable — by a persisted
+// hypervisor PID, a control-socket owner, or an open VFIO handle.
+const VGPUAssignmentGracePeriod = 5 * time.Minute
+
 type VGPUFramework string
 
 const (