From 8de01a5275b5fd0cdad7d0d08e29a4660ac9dd34 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 19:05:18 +0000 Subject: [PATCH 01/76] Guard vGPU releases with live-instance claims A vGPU assignment goes stale when its release succeeds but the metadata save does not (or start fails between the release and its first save). The backend's owner map only covers assignments created since the last restart and the VFIO handle scan only covers VMs that have opened the device, so after a restart a stale release could still clear a VF during another live instance's pre-open boot window. Consult live instance metadata on every release: when another instance with a live hypervisor process claims the same device path, drop the stale metadata without touching the device. Tag assignments with the owning instance ID, persist the assignment before booting a started instance, and retain assignment metadata when rollback release fails in create and start so later release paths can still find the device. --- lib/instances/create.go | 41 ++++++++++++++++++++- lib/instances/delete.go | 2 +- lib/instances/lifecycle_noop_test.go | 55 ++++++++++++++++++++++++++++ lib/instances/start.go | 9 ++++- lib/instances/stop.go | 2 +- lib/instances/vgpu.go | 44 +++++++++++++++++----- lib/instances/vgpu_test.go | 50 ++++++++++++++++++++----- 7 files changed, 180 insertions(+), 23 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index b4b0952da..2608b2bd2 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -277,11 +277,13 @@ func (m *manager) createInstance( var gpuFramework devices.VGPUFramework var gpuDevicePath string var gpuMdevUUID string + var stored *StoredMetadata + var retainedVGPU *StoredMetadata // Setup cleanup stack early so device attachment errors trigger cleanup cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) - m.deleteInstanceData(id) + m.cleanupFailedCreate(ctx, id, retainedVGPU) }) defer cu.Clean() @@ -320,6 +322,25 @@ func (m *manager) createInstance( } if err := devices.DestroyVGPU(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) + retainedVGPU = stored + if retainedVGPU == nil { + retainedVGPU = &StoredMetadata{ + Id: id, + Name: req.Name, + Image: req.Image, + ResolvedImage: resolvedImageRef, + Platform: imageInfo.Platform, + CreatedAt: time.Now(), + HypervisorType: hvType, + HypervisorVersion: hvVersion, + SocketPath: m.paths.InstanceSocket(id, starter.SocketName()), + DataDir: m.paths.InstanceDir(id), + GPUProfile: gpuDevice.ProfileName, + GPUFramework: gpuDevice.Framework, + GPUDevicePath: gpuDevice.SysfsPath, + GPUMdevUUID: gpuDevice.MdevUUID, + } + } } }) } @@ -360,7 +381,7 @@ func (m *manager) createInstance( if err != nil { return nil, err } - stored := &StoredMetadata{ + stored = &StoredMetadata{ Id: id, Name: req.Name, Image: req.Image, @@ -610,6 +631,22 @@ func resolveCreateExpiration(req CreateInstanceRequest, now time.Time) (*time.Ti return &expiresAt, nil } +func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) { + if retainedVGPU == nil { + m.deleteInstanceData(id) + return + } + + log := logger.FromContext(ctx) + if err := m.ensureDirectories(id); err != nil { + log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) + return + } + if err := m.saveMetadata(&metadata{StoredMetadata: *retainedVGPU}); err != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) + } +} + // validateCreateRequest validates the create instance request. // The request is mutated in-place to persist normalized egress/credential policy fields. func validateCreateRequest(req *CreateInstanceRequest) error { diff --git a/lib/instances/delete.go b/lib/instances/delete.go index ad4a192d1..b5f157d33 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -152,7 +152,7 @@ func (m *manager) deleteInstanceWithOptions( if hadVGPUAssignment { log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { // Log error but continue with cleanup. log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) } else if hadVGPUAssignment { diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index a9b918dfb..89468d970 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -3,6 +3,7 @@ package instances import ( "context" "errors" + "net" "os" "path/filepath" "sync" @@ -195,6 +196,46 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { assert.Equal(t, restartpolicy.BlockedReasonManualStop, persisted.RestartStatus.BlockedReason) } +func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { + now := time.Now().UTC() + m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, now) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + require.NoError(t, m.saveMetadata(meta)) + + claimantID := "inst-live-claimant" + require.NoError(t, m.ensureDirectories(claimantID)) + pid := os.Getpid() + socketPath := m.paths.InstanceSocket(claimantID, "noop.sock") + listener, err := net.Listen("unix", socketPath) + require.NoError(t, err) + defer listener.Close() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: claimantID, + Name: claimantID, + Image: "test-image", + CreatedAt: now, + HypervisorType: lifecycleNoopHypervisorType, + HypervisorPID: &pid, + SocketPath: socketPath, + DataDir: m.paths.InstanceDir(claimantID), + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFramework("future-framework"), + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + require.NoError(t, m.DeleteInstance(context.Background(), id)) + + _, err = m.loadMetadata(id) + require.Error(t, err, "deleted instance metadata should be gone") + claimant, err := m.loadMetadata(claimantID) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", claimant.GPUDevicePath, "live claimant keeps its assignment") +} + func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) deviceManager := &recordingDeviceManager{} @@ -300,6 +341,20 @@ func (m *recordingDeviceManager) UnbindFromVFIO(ctx context.Context, id string) return nil } +func TestLifecycleNoopStandbyRejectsVendorVFIOVGPU(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateRunning, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkVendorVFIO + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + require.NoError(t, m.saveMetadata(meta)) + + _, err = m.StandbyInstance(context.Background(), id, StandbyInstanceRequest{}) + require.ErrorIs(t, err, ErrInvalidState) + assert.ErrorContains(t, err, "standby is not supported for instances with vGPU attached") +} + func newLifecycleNoopManagerWithInstance(t *testing.T, state State, now time.Time) (*manager, string) { t.Helper() diff --git a/lib/instances/start.go b/lib/instances/start.go index a6c832450..ff8e94ea3 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -54,7 +54,7 @@ func (m *manager) startInstance( // cannot leave on-disk metadata pointing at a device that is already // gone (matching releaseRetainedVGPULocked). if storedVGPUDevicePath(stored) != "" { - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.ErrorContext(ctx, "failed to release stale vGPU before start", "instance_id", id, "error", err) return nil, fmt.Errorf("release stale vGPU before start: %w", err) } @@ -181,8 +181,15 @@ func (m *manager) startInstance( } if err := devices.DestroyVGPU(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) + if saveErr := m.saveMetadata(meta); saveErr != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr) + } } }) + if err := m.saveMetadata(meta); err != nil { + log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) + return nil, fmt.Errorf("save metadata after vGPU creation: %w", err) + } } // 5. Regenerate config disk with new network configuration diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 7eddeb034..ebcf58708 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -232,7 +232,7 @@ func (m *manager) stopInstance( // 7. Release the vGPU assignment if present (frees the vGPU slot for other VMs). if path := storedVGPUDevicePath(stored); path != "" { log.InfoContext(ctx, "destroying vGPU on stop", "instance_id", id, "device_path", path) - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on stop; retaining assignment metadata", "instance_id", id, "device_path", path, "error", err) } } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index a8ca6aceb..b7d275428 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -20,23 +20,49 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUMdevUUID = "" } -func releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { +func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { - assignment := devices.VGPUAssignment{ - Framework: stored.GPUFramework, - DevicePath: path, - MdevUUID: stored.GPUMdevUUID, - InstanceID: stored.Id, - } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) + if err != nil { return err } + if claimed { + logger.FromContext(ctx).WarnContext(ctx, "dropping stale vGPU assignment claimed by another live instance", + "instance_id", stored.Id, "device_path", path) + } else { + assignment := devices.VGPUAssignment{ + Framework: stored.GPUFramework, + DevicePath: path, + MdevUUID: stored.GPUMdevUUID, + InstanceID: stored.Id, + } + if err := devices.DestroyVGPU(ctx, assignment); err != nil { + return err + } + } } clearStoredVGPUDevice(stored) return nil } +func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { + instances, err := m.listInstances(ctx) + if err != nil { + return false, fmt.Errorf("list instances for vGPU release check: %w", err) + } + for i := range instances { + inst := &instances[i] + if inst.Id == excludeID || inst.GPUDevicePath != devicePath || inst.HypervisorPID == nil { + continue + } + if HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { + return true, nil + } + } + return false, nil +} + // releaseRetainedVGPULocked releases a vGPU assignment retained on a stopped // instance after a failed release during the original stop. It is a no-op // when no assignment is retained, and a failed retry only logs so the @@ -52,7 +78,7 @@ func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) { if storedVGPUDevicePath(stored) == "" { return } - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.WarnContext(ctx, "failed to destroy retained vGPU; retaining assignment metadata", "instance_id", id, "error", err) return } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 6f2c46819..2b7d84a94 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -5,14 +5,47 @@ import ( "testing" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestValidateVGPUHypervisor(t *testing.T) { + t.Parallel() + + assert.NoError(t, validateVGPUHypervisor(hypervisor.TypeQEMU)) + assert.EqualError(t, validateVGPUHypervisor(hypervisor.TypeCloudHypervisor), "vGPU is only supported with qemu, got cloud-hypervisor") +} + +func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + stored := &StoredMetadata{ + Id: "failed-create", + Name: "failed-create", + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + HypervisorType: "qemu", + DataDir: m.paths.InstanceDir("failed-create"), + } + + m.cleanupFailedCreate(context.Background(), stored.Id, stored) + + retained, err := m.loadMetadata(stored.Id) + require.NoError(t, err) + assert.Equal(t, stored.GPUProfile, retained.GPUProfile) + assert.Equal(t, stored.GPUFramework, retained.GPUFramework) + assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() - assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", storedVGPUDevicePath(&StoredMetadata{ - GPUDevicePath: "/sys/bus/mdev/devices/new-uuid", + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", storedVGPUDevicePath(&StoredMetadata{ + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUMdevUUID: "legacy-uuid", })) assert.Equal(t, "/sys/bus/mdev/devices/legacy-uuid", storedVGPUDevicePath(&StoredMetadata{ @@ -24,11 +57,12 @@ func TestStoredVGPUDevicePath(t *testing.T) { func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) { t.Parallel() + m := &manager{paths: paths.New(t.TempDir())} stored := &StoredMetadata{ GPUFramework: devices.VGPUFramework("future-framework"), GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } - err := releaseStoredVGPU(context.Background(), stored) + err := m.releaseStoredVGPU(context.Background(), stored) assert.Error(t, err) assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) @@ -39,13 +73,11 @@ func TestSetAndClearStoredVGPUDevice(t *testing.T) { stored := &StoredMetadata{} setStoredVGPUDevice(stored, &devices.VGPUDevice{ - Framework: devices.VGPUFrameworkMdev, - SysfsPath: "/sys/bus/mdev/devices/new-uuid", - MdevUUID: "new-uuid", + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", }) - assert.Equal(t, devices.VGPUFrameworkMdev, stored.GPUFramework) - assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", stored.GPUDevicePath) - assert.Equal(t, "new-uuid", stored.GPUMdevUUID) + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) clearStoredVGPUDevice(stored) assert.Empty(t, stored.GPUFramework) From 756c6bd70b5ed8de6e9821281ffaaed0b4d0f6c3 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 19:05:42 +0000 Subject: [PATCH 02/76] Reconcile vendor VFIO vGPUs against a fail-closed instance inventory Startup reconciliation protects the VFs of instances whose hypervisor survived the restart, verified by socket ownership so a reused PID cannot hold a VF. The inventory behind that protected set must not silently skip unreadable metadata: a skipped live claimant would leave its VF unprotected during the pre-VFIO-open boot window. Add ListInstancesForReconcile, which fails on any unreadable metadata, and skip vendor VFIO reconciliation when the inventory is unavailable while keeping mdev reconciliation running. --- cmd/api/main.go | 33 ++++++++++++++++++++++++++++----- lib/builds/manager_test.go | 4 ++++ lib/instances/manager.go | 6 ++++++ lib/instances/query.go | 13 +++++++++++-- lib/instances/query_test.go | 22 ++++++++++++++++++++++ lib/instances/storage.go | 8 +++++++- lib/instances/wait_test.go | 3 +++ 7 files changed, 81 insertions(+), 8 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index f1bbfcf3d..0479f1583 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -185,6 +185,24 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } +func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, error) { + allInstances, err := instanceManager.ListInstancesForReconcile(ctx) + if err != nil { + return nil, err + } + protected := make(map[string]struct{}) + for _, inst := range allInstances { + if inst.GPUDevicePath == "" || inst.HypervisorPID == nil { + continue + } + if !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { + continue + } + protected[inst.GPUDevicePath] = struct{}{} + } + return protected, nil +} + func run() error { startupStarted := time.Now() slog.Info("starting hypeman initialization") @@ -384,11 +402,16 @@ func run() error { return fmt.Errorf("reconcile device state: %w", err) } - // Reconcile mdev devices (clears orphaned vGPUs from previous runs) - logger.Info("Reconciling mdev devices...") - if err := devices.ReconcileMdevs(app.Ctx, nil); err != nil { - // Log but don't fail - mdev cleanup is best-effort - logger.Warn("failed to reconcile mdev devices", "error", err) + // Reconcile vGPU devices (clears orphaned vGPUs from previous runs) + logger.Info("Reconciling vGPU devices...") + protected, err := liveInstanceVGPUDevicePaths(app.Ctx, app.InstanceManager) + if err != nil { + logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err) + protected = nil + } + if err := devices.ReconcileVGPUs(app.Ctx, protected); err != nil { + // Log but don't fail - vGPU cleanup is best-effort + logger.Warn("failed to reconcile vGPU devices", "error", err) } // Wire up resource validator for aggregate limit checking diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index 7be58b0fc..b28383889 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -51,6 +51,10 @@ func (m *mockInstanceManager) ListInstances(ctx context.Context, filter *instanc return result, nil } +func (m *mockInstanceManager) ListInstancesForReconcile(ctx context.Context) ([]instances.Instance, error) { + return m.ListInstances(ctx, nil) +} + func (m *mockInstanceManager) ListSnapshots(ctx context.Context, filter *instances.ListSnapshotsFilter) ([]instances.Snapshot, error) { return nil, nil } diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 28ac78f61..51bedbf4d 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -27,6 +27,7 @@ import ( type Manager interface { ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) + ListInstancesForReconcile(ctx context.Context) ([]Instance, error) ListSnapshots(ctx context.Context, filter *ListSnapshotsFilter) ([]Snapshot, error) GetSnapshot(ctx context.Context, snapshotID string) (*Snapshot, error) CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) @@ -720,6 +721,11 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } +// ListInstancesForReconcile returns every instance or an invalid metadata error. +func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { + return m.loadInstances(ctx, false) +} + // ListInstances returns instances, optionally filtered by the given criteria. // Pass nil to return all instances. func (m *manager) ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) { diff --git a/lib/instances/query.go b/lib/instances/query.go index 8e3ed61f1..eea66ab13 100644 --- a/lib/instances/query.go +++ b/lib/instances/query.go @@ -784,14 +784,18 @@ func parseSentinelTimestamp(line, sentinelPrefix string) (time.Time, bool) { return time.Time{}, false } -// listInstances returns all instances +// listInstances returns all instances, skipping metadata files that cannot be loaded. func (m *manager) listInstances(ctx context.Context) ([]Instance, error) { + return m.loadInstances(ctx, true) +} + +func (m *manager) loadInstances(ctx context.Context, skipInvalid bool) ([]Instance, error) { ctx, span := m.tracerOrDefault().Start(ctx, "instances.list_metadata") defer span.End() log := logger.FromContext(ctx) log.DebugContext(ctx, "listing all instances") - files, err := m.listMetadataFiles() + files, err := m.listMetadataFilesWithStatErrors(!skipInvalid) if err != nil { log.ErrorContext(ctx, "failed to list metadata files", "error", err) return nil, err @@ -809,6 +813,11 @@ func (m *manager) listInstances(ctx context.Context) ([]Instance, error) { ) meta, err := m.loadMetadata(id) if err != nil { + if !skipInvalid { + hydrateSpan.RecordError(err) + hydrateSpan.End() + return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) + } // Skip instances with invalid metadata log.WarnContext(hydrateCtx, "skipping instance with invalid metadata", "instance_id", id, "error", err) hydrateSpan.End() diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 8bb0bb464..41aba54e8 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -14,6 +14,28 @@ import ( "github.com/stretchr/testify/require" ) +func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + + require.NoError(t, m.ensureDirectories("valid")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "valid", + Name: "valid", + CreatedAt: time.Now(), + DataDir: m.paths.InstanceDir("valid"), + }})) + require.NoError(t, m.ensureDirectories("invalid")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid"), []byte("{"), 0644)) + + listed, err := m.ListInstances(context.Background(), nil) + require.NoError(t, err) + require.Len(t, listed, 1) + + _, err = m.ListInstancesForReconcile(context.Background()) + require.Error(t, err) + assert.ErrorContains(t, err, "load metadata for instance invalid") +} + func TestParseExitSentinelLine(t *testing.T) { t.Parallel() tests := []struct { diff --git a/lib/instances/storage.go b/lib/instances/storage.go index a293fc6e1..dd932d41a 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -187,8 +187,12 @@ func removeAllWithRetry(path string, removeAll func(string) error, sleep func(ti } } -// listMetadataFiles returns paths to all instance metadata files +// listMetadataFiles returns paths to all instance metadata files. func (m *manager) listMetadataFiles() ([]string, error) { + return m.listMetadataFilesWithStatErrors(false) +} + +func (m *manager) listMetadataFilesWithStatErrors(failOnStatError bool) ([]string, error) { guestsDir := m.paths.GuestsDir() // Ensure guests directory exists @@ -210,6 +214,8 @@ func (m *manager) listMetadataFiles() ([]string, error) { metaPath := filepath.Join(guestsDir, entry.Name(), "metadata.json") if _, err := os.Stat(metaPath); err == nil { metaFiles = append(metaFiles, metaPath) + } else if failOnStatError && !os.IsNotExist(err) { + return nil, fmt.Errorf("stat metadata for instance %s: %w", entry.Name(), err) } } diff --git a/lib/instances/wait_test.go b/lib/instances/wait_test.go index bab6f06d7..a42464795 100644 --- a/lib/instances/wait_test.go +++ b/lib/instances/wait_test.go @@ -32,6 +32,9 @@ func (s *stubManager) GetInstance(ctx context.Context, id string) (*Instance, er func (s *stubManager) ListInstances(context.Context, *ListInstancesFilter) ([]Instance, error) { return nil, nil } +func (s *stubManager) ListInstancesForReconcile(context.Context) ([]Instance, error) { + return nil, nil +} func (s *stubManager) ListSnapshots(context.Context, *ListSnapshotsFilter) ([]Snapshot, error) { return nil, nil } From 47aa8bd92af42902ab46d53c004538afae5042fc Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:19:26 +0000 Subject: [PATCH 03/76] Fail closed on vGPU claim checks --- lib/instances/vgpu.go | 2 +- lib/instances/vgpu_test.go | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index b7d275428..29edfeab7 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -47,7 +47,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) } func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { - instances, err := m.listInstances(ctx) + instances, err := m.ListInstancesForReconcile(ctx) if err != nil { return false, fmt.Errorf("list instances for vGPU release check: %w", err) } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 2b7d84a94..29341d230 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -2,6 +2,7 @@ package instances import ( "context" + "os" "testing" "github.com/kernel/hypeman/lib/devices" @@ -41,6 +42,17 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) } +func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("invalid-instance")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) + + _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() From cd055b661c7c047694f0ef4a997bcf074b13668b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:19:50 +0000 Subject: [PATCH 04/76] Retain only vGPU assignment after failed create --- lib/instances/create.go | 8 +++++++- lib/instances/vgpu_test.go | 13 ++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index 2608b2bd2..d25bd91ed 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -642,7 +642,13 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) return } - if err := m.saveMetadata(&metadata{StoredMetadata: *retainedVGPU}); err != nil { + retained := StoredMetadata{ + Id: id, + GPUFramework: retainedVGPU.GPUFramework, + GPUDevicePath: retainedVGPU.GPUDevicePath, + GPUMdevUUID: retainedVGPU.GPUMdevUUID, + } + if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) } } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 29341d230..b408ccafe 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -29,6 +29,10 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { GPUProfile: "NVIDIA L40S-2Q", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUMdevUUID: "mdev-uuid", + NetworkEnabled: true, + IP: "192.0.2.1", + Volumes: []VolumeAttachment{{VolumeID: "volume"}}, HypervisorType: "qemu", DataDir: m.paths.InstanceDir("failed-create"), } @@ -37,9 +41,16 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { retained, err := m.loadMetadata(stored.Id) require.NoError(t, err) - assert.Equal(t, stored.GPUProfile, retained.GPUProfile) + assert.Equal(t, stored.Id, retained.Id) assert.Equal(t, stored.GPUFramework, retained.GPUFramework) assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) + assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) + assert.Empty(t, retained.Name) + assert.Empty(t, retained.GPUProfile) + assert.False(t, retained.NetworkEnabled) + assert.Empty(t, retained.IP) + assert.Empty(t, retained.Volumes) + assert.Empty(t, retained.DataDir) } func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { From 1ae268142fbc4a6a9dd341ccb4b9b6fb580f3810 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:20:05 +0000 Subject: [PATCH 05/76] Clear released vGPU assignment on start rollback --- lib/instances/start.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/instances/start.go b/lib/instances/start.go index ff8e94ea3..d9c8c9aaa 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -184,6 +184,11 @@ func (m *manager) startInstance( if saveErr := m.saveMetadata(meta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr) } + } else { + clearStoredVGPUDevice(stored) + if saveErr := m.saveMetadata(meta); saveErr != nil { + log.ErrorContext(ctx, "failed to save metadata after vGPU cleanup", "instance_id", id, "error", saveErr) + } } }) if err := m.saveMetadata(meta); err != nil { From 37802ce6ed166f2c82fa4f670e25530a05d84492 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:50:00 +0000 Subject: [PATCH 06/76] Test start rollback vGPU cleanup --- lib/instances/vgpu_test.go | 70 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index b408ccafe..d9aa02f02 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -3,7 +3,10 @@ package instances import ( "context" "os" + "path/filepath" + "sync" "testing" + _ "unsafe" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" @@ -53,6 +56,73 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Empty(t, retained.DataDir) } +//go:linkname hostVendorVFIO github.com/kernel/hypeman/lib/devices.hostVendorVFIO +var hostVendorVFIO vendorVFIOSysfs + +type vendorVFIOSysfs struct { + pciDevicesPath string + procPath string + vfioDevicesPath string + owners map[string]string +} + +func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { + root := t.TempDir() + pciDevicesPath := filepath.Join(root, "sys", "bus", "pci", "devices") + vfAddress := "0000:82:00.4" + nvidiaPath := filepath.Join(pciDevicesPath, vfAddress, "nvidia") + require.NoError(t, os.MkdirAll(nvidiaPath, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "current_vgpu_type"), []byte("0"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "creatable_vgpu_types"), []byte("ID : vGPU Name\n1148 : NVIDIA L40S-2Q\n"), 0o644)) + + originalVendorVFIO := hostVendorVFIO + hostVendorVFIO = vendorVFIOSysfs{ + pciDevicesPath: pciDevicesPath, + procPath: filepath.Join(root, "proc"), + vfioDevicesPath: filepath.Join(root, "dev", "vfio", "devices"), + owners: make(map[string]string), + } + t.Cleanup(func() { hostVendorVFIO = originalVendorVFIO }) + require.NoError(t, os.MkdirAll(hostVendorVFIO.procPath, 0o755)) + + m := &manager{ + paths: paths.New(t.TempDir()), + imageManager: readyFixtureImageManager{name: "test-image"}, + instanceLocks: sync.Map{}, + bootMarkerScans: sync.Map{}, + } + const id = "start-rollback" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + Image: "test-image", + GPUProfile: "NVIDIA L40S-2Q", + HypervisorType: lifecycleNoopHypervisorType, + SocketPath: m.paths.InstanceSocket(id, "noop.sock"), + DataDir: m.paths.InstanceDir(id), + }})) + + t.Setenv("TMPDIR", filepath.Join(root, "missing")) + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.Error(t, err) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile) + assert.Empty(t, stored.GPUFramework) + assert.Empty(t, stored.GPUDevicePath) + assert.Empty(t, stored.GPUMdevUUID) + assertFileContents(t, filepath.Join(nvidiaPath, "current_vgpu_type"), "0") +} + +func assertFileContents(t *testing.T, path, want string) { + t.Helper() + got, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, want, string(got)) +} + func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { t.Parallel() From 2161077bfe87f759151a4051bfd90c191a2e445c Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 7 Aug 2026 15:02:16 +0000 Subject: [PATCH 07/76] Normalize legacy mdev paths in live-claim check The claim guard compared raw GPUDevicePath, which is empty on records persisted before the framework migration; a live claimant with only a legacy GPUMdevUUID was invisible to the check. Normalize the inventory side with storedVGPUDevicePath, matching the release subject. --- lib/instances/vgpu.go | 2 +- lib/instances/vgpu_test.go | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 29edfeab7..23ea2d825 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -53,7 +53,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu } for i := range instances { inst := &instances[i] - if inst.Id == excludeID || inst.GPUDevicePath != devicePath || inst.HypervisorPID == nil { + if inst.Id == excludeID || storedVGPUDevicePath(&inst.StoredMetadata) != devicePath || inst.HypervisorPID == nil { continue } if HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d9aa02f02..79e6fe0c0 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -134,6 +134,24 @@ func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) require.Error(t, err) } +func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("legacy-claimant")) + pid := os.Getpid() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "legacy-claimant", + Name: "legacy-claimant", + GPUMdevUUID: "legacy-uuid", + HypervisorPID: &pid, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/mdev/devices/legacy-uuid") + require.NoError(t, err) + assert.True(t, claimed) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() From 0a799887822622e559549774ac16b08e43317418 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:45:49 +0000 Subject: [PATCH 08/76] Bind the live-claimant test socket under /tmp for macOS --- lib/instances/lifecycle_noop_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 89468d970..f3baa9dae 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -209,7 +209,14 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { claimantID := "inst-live-claimant" require.NoError(t, m.ensureDirectories(claimantID)) pid := os.Getpid() - socketPath := m.paths.InstanceSocket(claimantID, "noop.sock") + // Bind under /tmp: a t.TempDir()-derived path exceeds the macOS AF_UNIX + // path limit. + socketDir, err := os.MkdirTemp("/tmp", "hypeman-claimant-socket-") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(socketDir) + }) + socketPath := filepath.Join(socketDir, "noop.sock") listener, err := net.Listen("unix", socketPath) require.NoError(t, err) defer listener.Close() From efb0896090c7dee5cf432381478381bda41dfa7c Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:45:49 +0000 Subject: [PATCH 09/76] Surface retained vGPU cleanup through a typed create error and manager seam Replace the go:linkname shadow of devices.hostVendorVFIO with createVGPU/destroyVGPU manager fields, and wrap failed creates whose rollback release also failed in VGPUCleanupPendingError so the API can point callers at the retained instance record. --- cmd/api/api/instances.go | 7 +++ lib/instances/create.go | 28 +++++++++--- lib/instances/manager.go | 4 ++ lib/instances/start.go | 8 ++-- lib/instances/vgpu.go | 40 ++++++++++++++++- lib/instances/vgpu_test.go | 92 ++++++++++++++++++++++++-------------- 6 files changed, 134 insertions(+), 45 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 945ad8314..16a37a540 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -362,6 +362,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst inst, err := s.InstanceManager.CreateInstance(ctx, domainReq) if err != nil { + var vgpuPending *instances.VGPUCleanupPendingError switch { case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ @@ -413,6 +414,12 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Code: "not_found", Message: err.Error(), }, nil + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) + return oapi.CreateInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID), + }, nil default: log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) return oapi.CreateInstance500JSONResponse{ diff --git a/lib/instances/create.go b/lib/instances/create.go index d25bd91ed..dc70639fd 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -280,10 +280,20 @@ func (m *manager) createInstance( var stored *StoredMetadata var retainedVGPU *StoredMetadata - // Setup cleanup stack early so device attachment errors trigger cleanup + // Setup cleanup stack early so device attachment errors trigger cleanup. + // When rollback retains a vGPU assignment, surface the retained instance + // ID to the caller so the record is discoverable and can be deleted to + // retry the release. The wrapping defer is registered first so it runs + // after cu.Clean has decided whether metadata was retained. + vgpuRetained := false + defer func() { + if retErr != nil && vgpuRetained { + retErr = &VGPUCleanupPendingError{InstanceID: id, Err: retErr} + } + }() cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) - m.cleanupFailedCreate(ctx, id, retainedVGPU) + vgpuRetained = m.cleanupFailedCreate(ctx, id, retainedVGPU) }) defer cu.Clean() @@ -300,7 +310,7 @@ func (m *manager) createInstance( // Handle vGPU profile request if req.GPU != nil && req.GPU.Profile != "" { log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) - gpuDevice, err = devices.CreateVGPU(ctx, req.GPU.Profile, id) + gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) if err != nil { log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) @@ -320,7 +330,7 @@ func (m *manager) createInstance( MdevUUID: gpuDevice.MdevUUID, InstanceID: id, } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) retainedVGPU = stored if retainedVGPU == nil { @@ -631,16 +641,18 @@ func resolveCreateExpiration(req CreateInstanceRequest, now time.Time) (*time.Ti return &expiresAt, nil } -func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) { +// cleanupFailedCreate reports whether it retained instance metadata for a +// vGPU assignment whose release failed during rollback. +func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool { if retainedVGPU == nil { m.deleteInstanceData(id) - return + return false } log := logger.FromContext(ctx) if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return + return false } retained := StoredMetadata{ Id: id, @@ -650,7 +662,9 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) + return false } + return true } // validateCreateRequest validates the create instance request. diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 51bedbf4d..0027acff3 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -181,6 +181,8 @@ type manager struct { now func() time.Time writeFile func(string, []byte, os.FileMode) error deleteInstanceFn func(context.Context, string) error + createVGPU func(context.Context, string, string) (*devices.VGPUDevice, error) + destroyVGPU func(context.Context, devices.VGPUAssignment) error deleteSnapshotFn func(context.Context, string) error ttlReaperDeleteTimeout time.Duration egressProxy *egressproxy.Service @@ -281,6 +283,8 @@ func NewManagerWithConfigE(p *paths.Paths, imageManager images.Manager, systemMa defaultHypervisor: defaultHypervisor, now: time.Now, writeFile: os.WriteFile, + createVGPU: devices.CreateVGPU, + destroyVGPU: devices.DestroyVGPU, meter: meter, tracer: tracer, guestMemoryPolicy: policy, diff --git a/lib/instances/start.go b/lib/instances/start.go index d9c8c9aaa..b830bafb2 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -162,11 +162,11 @@ func (m *manager) startInstance( // 4b. Recreate the vGPU if this instance had a GPU profile // Note: GPU availability was already validated in step 2b if stored.GPUProfile != "" { - log.InfoContext(ctx, "creating vGPU mdev for start", "instance_id", id, "profile", stored.GPUProfile) - device, err := devices.CreateVGPU(ctx, stored.GPUProfile, id) + log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) + device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id) if err != nil { log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) - return nil, fmt.Errorf("create vGPU mdev for profile %s: %w", stored.GPUProfile, err) + return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } setStoredVGPUDevice(stored, device) log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) @@ -179,7 +179,7 @@ func (m *manager) startInstance( MdevUUID: device.MdevUUID, InstanceID: id, } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) if saveErr := m.saveMetadata(meta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 23ea2d825..ae100a39e 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -5,9 +5,47 @@ import ( "path/filepath" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) +func validateVGPUHypervisor(hvType hypervisor.Type) error { + if hvType != hypervisor.TypeQEMU { + return fmt.Errorf("vGPU is only supported with qemu, got %s", hvType) + } + return nil +} + +// VGPUCleanupPendingError reports a failed create whose vGPU release also +// failed during rollback. The instance record identified by InstanceID is +// retained so the release can be retried; deleting the instance retries it. +type VGPUCleanupPendingError struct { + InstanceID string + Err error +} + +func (e *VGPUCleanupPendingError) Error() string { + return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) +} + +func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } + +func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID string) (*devices.VGPUDevice, error) { + create := m.createVGPU + if create == nil { + create = devices.CreateVGPU + } + return create(ctx, profileName, instanceID) +} + +func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { + destroy := m.destroyVGPU + if destroy == nil { + destroy = devices.DestroyVGPU + } + return destroy(ctx, assignment) +} + func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice) { stored.GPUFramework = device.Framework stored.GPUDevicePath = device.SysfsPath @@ -37,7 +75,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) MdevUUID: stored.GPUMdevUUID, InstanceID: stored.Id, } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { return err } } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 79e6fe0c0..da8426efe 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -2,11 +2,11 @@ package instances import ( "context" + "errors" "os" "path/filepath" "sync" "testing" - _ "unsafe" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" @@ -40,7 +40,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { DataDir: m.paths.InstanceDir("failed-create"), } - m.cleanupFailedCreate(context.Background(), stored.Id, stored) + assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) retained, err := m.loadMetadata(stored.Id) require.NoError(t, err) @@ -56,40 +56,43 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Empty(t, retained.DataDir) } -//go:linkname hostVendorVFIO github.com/kernel/hypeman/lib/devices.hostVendorVFIO -var hostVendorVFIO vendorVFIOSysfs +func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("failed-create")) -type vendorVFIOSysfs struct { - pciDevicesPath string - procPath string - vfioDevicesPath string - owners map[string]string + assert.False(t, m.cleanupFailedCreate(context.Background(), "failed-create", nil)) + _, err := m.loadMetadata("failed-create") + require.Error(t, err) } -func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { - root := t.TempDir() - pciDevicesPath := filepath.Join(root, "sys", "bus", "pci", "devices") - vfAddress := "0000:82:00.4" - nvidiaPath := filepath.Join(pciDevicesPath, vfAddress, "nvidia") - require.NoError(t, os.MkdirAll(nvidiaPath, 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "current_vgpu_type"), []byte("0"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "creatable_vgpu_types"), []byte("ID : vGPU Name\n1148 : NVIDIA L40S-2Q\n"), 0o644)) - - originalVendorVFIO := hostVendorVFIO - hostVendorVFIO = vendorVFIOSysfs{ - pciDevicesPath: pciDevicesPath, - procPath: filepath.Join(root, "proc"), - vfioDevicesPath: filepath.Join(root, "dev", "vfio", "devices"), - owners: make(map[string]string), - } - t.Cleanup(func() { hostVendorVFIO = originalVendorVFIO }) - require.NoError(t, os.MkdirAll(hostVendorVFIO.procPath, 0o755)) +func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { + t.Parallel() + + cause := errors.New("boot failed") + err := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} + assert.ErrorIs(t, err, cause) + assert.Contains(t, err.Error(), "inst-1") +} +func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) { + t.Helper() m := &manager{ paths: paths.New(t.TempDir()), imageManager: readyFixtureImageManager{name: "test-image"}, instanceLocks: sync.Map{}, bootMarkerScans: sync.Map{}, + createVGPU: func(_ context.Context, profileName, _ string) (*devices.VGPUDevice, error) { + return &devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: profileName, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + }, nil + }, + destroyVGPU: destroy, } const id = "start-rollback" require.NoError(t, m.ensureDirectories(id)) @@ -102,25 +105,48 @@ func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { SocketPath: m.paths.InstanceSocket(id, "noop.sock"), DataDir: m.paths.InstanceDir(id), }})) + return m, id +} + +func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { + var destroyed []devices.VGPUAssignment + m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }) - t.Setenv("TMPDIR", filepath.Join(root, "missing")) + t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) require.Error(t, err) + require.Len(t, destroyed, 1) + assert.Equal(t, devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: "/sys/bus/pci/devices/0000:82:00.4", + InstanceID: id, + }, destroyed[0]) + stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile) assert.Empty(t, stored.GPUFramework) assert.Empty(t, stored.GPUDevicePath) assert.Empty(t, stored.GPUMdevUUID) - assertFileContents(t, filepath.Join(nvidiaPath, "current_vgpu_type"), "0") } -func assertFileContents(t *testing.T, path, want string) { - t.Helper() - got, err := os.ReadFile(path) +func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + + t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.Error(t, err) + + stored, err := m.loadMetadata(id) require.NoError(t, err) - assert.Equal(t, want, string(got)) + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) } func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { From df9a94e817bb755c9b0004e60371bd95bff3932b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:45:49 +0000 Subject: [PATCH 10/76] Generalize the create vGPU error text --- lib/instances/create.go | 2 +- lib/instances/create_mdev_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index dc70639fd..2cf34b5d9 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -56,7 +56,7 @@ func wrapCreateVGPUErr(profile string, err error) error { if errors.Is(err, devices.ErrVGPUNotSupportedOnMacOS) { return fmt.Errorf("%w: %w", ErrInvalidRequest, err) } - return fmt.Errorf("create vGPU mdev for profile %s: %w", profile, err) + return fmt.Errorf("create vGPU for profile %s: %w", profile, err) } // generateVsockCID converts first 8 chars of instance ID to a unique CID diff --git a/lib/instances/create_mdev_test.go b/lib/instances/create_mdev_test.go index e6e4f55c5..05b3e9d8c 100644 --- a/lib/instances/create_mdev_test.go +++ b/lib/instances/create_mdev_test.go @@ -63,7 +63,7 @@ func TestWrapCreateVGPUErr(t *testing.T) { { name: "other vGPU error", err: errors.New("boom"), - wantMessage: "create vGPU mdev for profile profile: boom", + wantMessage: "create vGPU for profile profile: boom", }, } { t.Run(tc.name, func(t *testing.T) { From cc3709357100d306bfd6d970ebe4c51ac9db983e Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:58:24 +0000 Subject: [PATCH 11/76] Scope vGPU claim scan to vendor VFIO and close reconcile gaps --- cmd/api/api/instances.go | 14 +++++++------ cmd/api/api/instances_test.go | 31 ++++++++++++++++++++++++++++ cmd/api/main.go | 8 +++++-- cmd/api/main_test.go | 30 +++++++++++++++++++++++++++ lib/instances/lifecycle_noop_test.go | 4 ++-- lib/instances/vgpu.go | 15 +++++++++++--- lib/instances/vgpu_test.go | 21 +++++++++++++++++++ 7 files changed, 110 insertions(+), 13 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 16a37a540..b7d85a6dd 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -364,6 +364,14 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst if err != nil { var vgpuPending *instances.VGPUCleanupPendingError switch { + // Checked first: it wraps the original create error, so a later + // errors.Is case would match the cause and hide the retained instance. + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) + return oapi.CreateInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID), + }, nil case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ Code: "image_not_ready", @@ -414,12 +422,6 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Code: "not_found", Message: err.Error(), }, nil - case errors.As(err, &vgpuPending): - log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) - return oapi.CreateInstance500JSONResponse{ - Code: "vgpu_cleanup_pending", - Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID), - }, nil default: log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) return oapi.CreateInstance500JSONResponse{ diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index fce9ee10a..d25b7615d 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -17,6 +17,7 @@ import ( "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/instances/phasetracking" mw "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/paths" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" @@ -47,6 +48,36 @@ func TestGetInstance_NotFound(t *testing.T) { require.Error(t, err) } +type createErrorInstanceManager struct { + instances.Manager + err error +} + +func (m createErrorInstanceManager) CreateInstance(context.Context, instances.CreateInstanceRequest) (*instances.Instance, error) { + return nil, m.err +} + +// A retained-assignment error must win over the mapping of the create error +// it wraps, or the response omits the instance the caller has to delete. +func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Err: network.ErrNameExists, + }} + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{Image: "test-image"}, + }) + require.NoError(t, err) + + pending, ok := resp.(oapi.CreateInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "inst-1") +} + func TestCreateInstance_AutoPullImage(t *testing.T) { t.Parallel() if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { diff --git a/cmd/api/main.go b/cmd/api/main.go index 0479f1583..3374d9682 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -192,10 +192,14 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. } protected := make(map[string]struct{}) for _, inst := range allInstances { - if inst.GPUDevicePath == "" || inst.HypervisorPID == nil { + if inst.GPUDevicePath == "" { continue } - if !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { + // A nil PID does not mean the assignment is orphaned: the PID is + // persisted only after the hypervisor starts, so a crash during boot + // leaves the device path without one. Only skip protection when the + // recorded hypervisor is known to be gone. + if inst.HypervisorPID != nil && !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { continue } protected[inst.GPUDevicePath] = struct{}{} diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 34dbba428..573a404c1 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -2,15 +2,18 @@ package main import ( "bytes" + "context" "net/http" "net/http/httptest" "net/url" + "os/exec" "testing" "time" "github.com/getkin/kin-openapi/openapi3filter" "github.com/go-chi/chi/v5" "github.com/golang-jwt/jwt/v5" + "github.com/kernel/hypeman/lib/instances" mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/oapi" nethttpmiddleware "github.com/oapi-codegen/nethttp-middleware" @@ -338,3 +341,30 @@ func TestImageNameWithSlashes_URLEncoding(t *testing.T) { }) } } + +type vgpuReconcileManagerStub struct { + instances.Manager + list []instances.Instance +} + +func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([]instances.Instance, error) { + return s.list, nil +} + +// The hypervisor PID is persisted only after boot, so an assignment without +// one may belong to a VM that is still starting and must stay protected. +func TestLiveInstanceVGPUDevicePathsProtectsAssignmentsWithoutPID(t *testing.T) { + dead := exec.Command("true") + require.NoError(t, dead.Run()) + deadPID := dead.Process.Pid + + manager := vgpuReconcileManagerStub{list: []instances.Instance{ + {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4"}}, + {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", HypervisorPID: &deadPID}}, + }} + + protected, err := liveInstanceVGPUDevicePaths(context.Background(), manager) + require.NoError(t, err) + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") +} diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index f3baa9dae..15f632b3a 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -202,7 +202,7 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUFramework = devices.VGPUFrameworkVendorVFIO meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" require.NoError(t, m.saveMetadata(meta)) @@ -230,7 +230,7 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { SocketPath: socketPath, DataDir: m.paths.InstanceDir(claimantID), GPUProfile: "NVIDIA L40S-2Q", - GPUFramework: devices.VGPUFramework("future-framework"), + GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", }})) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index ae100a39e..9b97fddb7 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -61,9 +61,18 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) - if err != nil { - return err + // Vendor VFIO VFs are reused across instances, so stale metadata can + // point at a path claimed by a live instance and the release must fail + // closed on an incomplete inventory. mdev UUIDs are unique and never + // reused, so skip the scan there — it would let one unreadable + // metadata file block every mdev release on the host. + claimed := false + if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { + var err error + claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) + if err != nil { + return err + } } if claimed { logger.FromContext(ctx).WarnContext(ctx, "dropping stale vGPU assignment claimed by another live instance", diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index da8426efe..6f9a20073 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -178,6 +178,27 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. assert.True(t, claimed) } +func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) { + t.Parallel() + + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + } + require.NoError(t, m.ensureDirectories("invalid-instance")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) + + stored := &StoredMetadata{ + Id: "mdev-instance", + GPUFramework: devices.VGPUFrameworkMdev, + GPUMdevUUID: "uuid-1", + GPUDevicePath: "/sys/bus/mdev/devices/uuid-1", + } + require.NoError(t, m.releaseStoredVGPU(context.Background(), stored), + "an unreadable metadata file must not block mdev releases") + assert.Empty(t, stored.GPUDevicePath) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() From 0aefc378db7bd9839dd89591a5151459f0cf4edd Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:25:10 +0000 Subject: [PATCH 12/76] Harden the vendor VFIO release path Enable vendor VFIO dispatch in CreateVGPU now that the lifecycle persists assignments durably and guards releases. Protect nil-PID claims in the release guard: the hypervisor PID is only persisted after the claimant boots, so a matching assignment without a PID must be treated as live, matching the startup reconcile protection. Scan raw metadata instead of hydrating instances for the claim check. Hydration derives state through hypervisor queries for every instance on the host, which every vendor VFIO release would pay; the guard only needs the stored assignment, PID, and socket. Unreadable metadata still fails the release closed. Report pending vGPU cleanup even when retaining the rollback record fails: the destroy already failed, so the caller must learn about the outstanding assignment either way. --- integration/vgpu_test.go | 5 ---- lib/devices/vgpu_linux.go | 5 +--- lib/instances/create.go | 11 +++++--- lib/instances/vgpu.go | 35 ++++++++++++++++++++++---- lib/instances/vgpu_test.go | 51 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 18 deletions(-) diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 7873aa35c..1f1a82776 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -324,11 +324,6 @@ func checkVGPUTestPrerequisites() (string, string) { if framework == devices.VGPUFrameworkNone { return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", "" } - if framework == devices.VGPUFrameworkVendorVFIO { - // CreateVGPU rejects vendor VFIO until the instance lifecycle - // integration lands. - return "vGPU test requires the vendor VFIO instance lifecycle integration", "" - } // Check for available profiles profiles, err := devices.ListGPUProfiles() diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index b3c497899..be92e8ee6 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -73,10 +73,7 @@ func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevic MdevUUID: mdev.UUID, }, nil case VGPUFrameworkVendorVFIO: - // The instance lifecycle does not yet persist vendor VFIO assignments - // durably or guard their release against live claims, so keep the - // backend out of the create path until that integration lands. - return nil, fmt.Errorf("vendor VFIO vGPU support is not yet integrated with the instance lifecycle") + return hostVendorVFIO.create(ctx, profileName, instanceID) default: return nil, fmt.Errorf("vGPU framework not available") } diff --git a/lib/instances/create.go b/lib/instances/create.go index 2cf34b5d9..f1edb8aaa 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -641,8 +641,11 @@ func resolveCreateExpiration(req CreateInstanceRequest, now time.Time) (*time.Ti return &expiresAt, nil } -// cleanupFailedCreate reports whether it retained instance metadata for a -// vGPU assignment whose release failed during rollback. +// cleanupFailedCreate reports whether a vGPU assignment is still outstanding +// after a failed create. The vGPU destroy already failed when retainedVGPU is +// set, so the pending cleanup is reported even when the retention record +// cannot be persisted — in that case the assignment is orphaned until the +// next startup reconcile, and the caller must still surface it. func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool { if retainedVGPU == nil { m.deleteInstanceData(id) @@ -652,7 +655,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG log := logger.FromContext(ctx) if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return false + return true } retained := StoredMetadata{ Id: id, @@ -662,7 +665,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - return false + return true } return true } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 9b97fddb7..d62fc0d52 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -93,19 +93,44 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) return nil } +// vgpuAssignmentClaimedByLiveInstance reports whether another instance's +// stored metadata claims devicePath. It reads raw metadata instead of +// hydrating full instances: the scan runs on every vendor VFIO release, and +// deriving state would query the hypervisor of every instance on the host. +// It fails closed: unreadable metadata is an error, and a matching claim +// without a persisted PID counts as live because the PID is only persisted +// after the claimant's hypervisor starts. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { - instances, err := m.ListInstancesForReconcile(ctx) + files, err := m.listMetadataFilesWithStatErrors(true) if err != nil { return false, fmt.Errorf("list instances for vGPU release check: %w", err) } - for i := range instances { - inst := &instances[i] - if inst.Id == excludeID || storedVGPUDevicePath(&inst.StoredMetadata) != devicePath || inst.HypervisorPID == nil { + for _, file := range files { + id := filepath.Base(filepath.Dir(file)) + if id == excludeID { continue } - if HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { + meta, err := m.loadMetadata(id) + if err != nil { + return false, fmt.Errorf("load metadata for vGPU release check: instance %s: %w", id, err) + } + stored := &meta.StoredMetadata + if storedVGPUDevicePath(stored) != devicePath { + continue + } + if stored.HypervisorPID == nil { return true, nil } + if HypervisorProcessExists(*stored.HypervisorPID, stored.SocketPath) { + return true, nil + } + // The stored PID can be stale after a hypeman restart; a live owner + // of the claimant's socket still marks the claim as live. + if stored.SocketPath != "" && !ProcessExists(*stored.HypervisorPID) { + if owner, _, err := hypervisor.ResolveProcessPID(stored.SocketPath); err == nil && ProcessExists(owner) { + return true, nil + } + } } return false, nil } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 6f9a20073..0253d83da 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -67,6 +67,23 @@ func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { require.Error(t, err) } +func TestCleanupFailedCreateReportsPendingWhenRetentionFails(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + // A file at the guests directory path makes ensureDirectories fail even + // when running as root. + require.NoError(t, os.WriteFile(m.paths.GuestsDir(), nil, 0o644)) + + stored := &StoredMetadata{ + Id: "failed-create", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + } + assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored), + "a failed retention must still report the outstanding vGPU assignment") +} + func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { t.Parallel() @@ -178,6 +195,40 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. assert.True(t, claimed) } +func TestVGPUAssignmentClaimedByLiveInstanceProtectsNilPIDClaim(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("booting-claimant")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "booting-claimant", + Name: "booting-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.True(t, claimed, "a matching claim without a persisted PID must be treated as live: the PID is only persisted after the claimant boots") +} + +func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("dead-claimant")) + deadPID := 1 << 30 + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "dead-claimant", + Name: "dead-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + HypervisorPID: &deadPID, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.False(t, claimed, "a claim whose hypervisor is gone must not block the release") +} + func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) { t.Parallel() From dcdd6d8b36aec90b39bf43dcce4dacb53a0d56a2 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 07:40:01 +0000 Subject: [PATCH 13/76] Fail closed on retained vGPU cleanup --- lib/instances/create.go | 26 +++++------- lib/instances/process_identity_linux_test.go | 43 ++++++++++++++++++++ lib/instances/vgpu.go | 21 ++++------ lib/instances/vgpu_test.go | 23 ++++++----- 4 files changed, 76 insertions(+), 37 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index f1edb8aaa..de00bdcf3 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -281,19 +281,18 @@ func (m *manager) createInstance( var retainedVGPU *StoredMetadata // Setup cleanup stack early so device attachment errors trigger cleanup. - // When rollback retains a vGPU assignment, surface the retained instance - // ID to the caller so the record is discoverable and can be deleted to - // retry the release. The wrapping defer is registered first so it runs - // after cu.Clean has decided whether metadata was retained. - vgpuRetained := false + // When rollback cannot release a vGPU assignment, report whether its + // retention record was persisted. The wrapping defer is registered first + // so it runs after cu.Clean has attempted to retain the metadata. + vgpuPersisted := false defer func() { - if retErr != nil && vgpuRetained { - retErr = &VGPUCleanupPendingError{InstanceID: id, Err: retErr} + if retErr != nil && retainedVGPU != nil { + retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuPersisted, Err: retErr} } }() cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) - vgpuRetained = m.cleanupFailedCreate(ctx, id, retainedVGPU) + vgpuPersisted = m.cleanupFailedCreate(ctx, id, retainedVGPU) }) defer cu.Clean() @@ -641,11 +640,8 @@ func resolveCreateExpiration(req CreateInstanceRequest, now time.Time) (*time.Ti return &expiresAt, nil } -// cleanupFailedCreate reports whether a vGPU assignment is still outstanding -// after a failed create. The vGPU destroy already failed when retainedVGPU is -// set, so the pending cleanup is reported even when the retention record -// cannot be persisted — in that case the assignment is orphaned until the -// next startup reconcile, and the caller must still surface it. +// cleanupFailedCreate reports whether the retention record for a vGPU +// assignment whose release failed during rollback was persisted. func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool { if retainedVGPU == nil { m.deleteInstanceData(id) @@ -655,7 +651,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG log := logger.FromContext(ctx) if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return true + return false } retained := StoredMetadata{ Id: id, @@ -665,7 +661,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - return true + return false } return true } diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 7450ed112..d5bd973d9 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -17,7 +17,9 @@ import ( "testing" "time" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -508,6 +510,47 @@ func TestRefreshHypervisorPIDResolvesSocketOwnerWhenStoredPIDIsDead(t *testing.T assert.Equal(t, hostBootID(), stored.HypervisorBootID) } +func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "test.sock") + owner := exec.Command(os.Args[0], "-test.run=^TestHypervisorProcessExistsWithReboundSocketPathHelper$") + owner.Env = append(os.Environ(), "HYPERVISOR_SOCKET_HELPER=1", "HYPERVISOR_SOCKET_PATH="+socketPath) + stdin, err := owner.StdinPipe() + require.NoError(t, err) + stdout, err := owner.StdoutPipe() + require.NoError(t, err) + require.NoError(t, owner.Start()) + t.Cleanup(func() { + _ = stdin.Close() + _ = owner.Process.Kill() + _ = owner.Wait() + }) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + + stale := exec.Command("sleep", "30") + require.NoError(t, stale.Start()) + t.Cleanup(func() { + _ = stale.Process.Kill() + _ = stale.Wait() + }) + + m := &manager{paths: paths.New(t.TempDir())} + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" + stalePID := stale.Process.Pid + require.NoError(t, m.ensureDirectories("live-claimant")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "live-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + HypervisorPID: &stalePID, + SocketPath: socketPath, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", devicePath) + require.NoError(t, err) + assert.True(t, claimed) +} + func TestKillHypervisorSurvivesConcurrentReaper(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "test.sock") process := exec.Command(os.Args[0], "-test.run=^TestSocketListenerHelper$") diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index d62fc0d52..e27cf7a54 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -5,7 +5,6 @@ import ( "path/filepath" "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -17,15 +16,19 @@ func validateVGPUHypervisor(hvType hypervisor.Type) error { } // VGPUCleanupPendingError reports a failed create whose vGPU release also -// failed during rollback. The instance record identified by InstanceID is -// retained so the release can be retried; deleting the instance retries it. +// failed during rollback. When Retained is true, deleting the retained instance +// retries the release; otherwise startup reconciliation recovers the assignment. type VGPUCleanupPendingError struct { InstanceID string + Retained bool Err error } func (e *VGPUCleanupPendingError) Error() string { - return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) + if e.Retained { + return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) + } + return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", e.Err, e.InstanceID) } func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } @@ -121,16 +124,10 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if stored.HypervisorPID == nil { return true, nil } - if HypervisorProcessExists(*stored.HypervisorPID, stored.SocketPath) { + pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.SocketPath) + if err != nil || pid > 0 { return true, nil } - // The stored PID can be stale after a hypeman restart; a live owner - // of the claimant's socket still marks the claim as live. - if stored.SocketPath != "" && !ProcessExists(*stored.HypervisorPID) { - if owner, _, err := hypervisor.ResolveProcessPID(stored.SocketPath); err == nil && ProcessExists(owner) { - return true, nil - } - } } return false, nil } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 0253d83da..a0d3ae9da 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -67,30 +67,33 @@ func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { require.Error(t, err) } -func TestCleanupFailedCreateReportsPendingWhenRetentionFails(t *testing.T) { +func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { t.Parallel() m := &manager{paths: paths.New(t.TempDir())} - // A file at the guests directory path makes ensureDirectories fail even - // when running as root. - require.NoError(t, os.WriteFile(m.paths.GuestsDir(), nil, 0o644)) + const id = "failed-create" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, os.Mkdir(m.paths.InstanceMetadata(id), 0o755)) stored := &StoredMetadata{ - Id: "failed-create", + Id: id, GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } - assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored), - "a failed retention must still report the outstanding vGPU assignment") + assert.False(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) } func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { t.Parallel() cause := errors.New("boot failed") - err := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} - assert.ErrorIs(t, err, cause) - assert.Contains(t, err.Error(), "inst-1") + retained := &VGPUCleanupPendingError{InstanceID: "inst-1", Retained: true, Err: cause} + assert.ErrorIs(t, retained, cause) + assert.Equal(t, "boot failed; vGPU release failed during rollback, instance inst-1 retains the assignment", retained.Error()) + + unpersisted := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} + assert.ErrorIs(t, unpersisted, cause) + assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the assignment is recovered on the next startup reconcile", unpersisted.Error()) } func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) { From 1b8ecd60a38aea5cf5ce650502f6a35722951fdb Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:02:02 +0000 Subject: [PATCH 14/76] Report surviving vGPU retention metadata --- lib/instances/create.go | 14 ++++++++++++-- lib/instances/vgpu_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index de00bdcf3..8af298b66 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -649,9 +649,19 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } log := logger.FromContext(ctx) + retentionFailed := func() bool { + meta, err := m.loadMetadata(id) + if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { + return true + } + if err := m.deleteInstanceData(id); err != nil { + log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) + } + return false + } if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return false + return retentionFailed() } retained := StoredMetadata{ Id: id, @@ -661,7 +671,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - return false + return retentionFailed() } return true } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index a0d3ae9da..b6831109c 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -81,6 +81,34 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } assert.False(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) + _, err := m.loadMetadata(id) + require.Error(t, err) +} + +func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m := &manager{paths: paths.New(t.TempDir())} + const id = "failed-create" + require.NoError(t, m.ensureDirectories(id)) + stored := &StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + } + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: *stored})) + + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + assert.True(t, m.cleanupFailedCreate(context.Background(), id, stored)) + retained, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, stored.GPUFramework, retained.GPUFramework) + assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) } func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { From 26b3e7a9206f78b56f261e4545d312d03dc8abc6 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:02:38 +0000 Subject: [PATCH 15/76] Return accurate vGPU cleanup guidance --- cmd/api/api/instances.go | 8 ++++++-- cmd/api/api/instances_test.go | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index b7d85a6dd..cd7f1c707 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -365,12 +365,16 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst var vgpuPending *instances.VGPUCleanupPendingError switch { // Checked first: it wraps the original create error, so a later - // errors.Is case would match the cause and hide the retained instance. + // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) + message := fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID) + if !vgpuPending.Retained { + message = fmt.Sprintf("failed to create instance; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.InstanceID) + } return oapi.CreateInstance500JSONResponse{ Code: "vgpu_cleanup_pending", - Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID), + Message: message, }, nil case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index d25b7615d..45d9275a9 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -64,6 +64,7 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) svc := newTestService(t) svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ InstanceID: "inst-1", + Retained: true, Err: network.ErrNameExists, }} @@ -76,6 +77,28 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) assert.Contains(t, pending.Message, "inst-1") + assert.Contains(t, pending.Message, "delete it to retry") +} + +func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Err: network.ErrNameExists, + }} + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{Image: "test-image"}, + }) + require.NoError(t, err) + + pending, ok := resp.(oapi.CreateInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") + assert.Contains(t, pending.Message, "startup reconcile") + assert.NotContains(t, pending.Message, "delete") } func TestCreateInstance_AutoPullImage(t *testing.T) { From 846640de1f6efe85c0db80c881fb7a574d45354e Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:26:00 +0000 Subject: [PATCH 16/76] Clarify vGPU retention fallback --- lib/instances/create.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index 8af298b66..42fdcac39 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -649,7 +649,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } log := logger.FromContext(ctx) - retentionFailed := func() bool { + retentionSurvives := func() bool { meta, err := m.loadMetadata(id) if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { return true @@ -661,7 +661,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return retentionFailed() + return retentionSurvives() } retained := StoredMetadata{ Id: id, @@ -671,7 +671,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - return retentionFailed() + return retentionSurvives() } return true } From 278724aa27bdb5a35983f5b66ab856b58d9fc9f8 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:43:17 +0000 Subject: [PATCH 17/76] Pass hypervisor identity token to vGPU claim check --- lib/instances/vgpu.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index e27cf7a54..acc5eb749 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -124,7 +124,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if stored.HypervisorPID == nil { return true, nil } - pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.SocketPath) + pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.SocketPath) if err != nil || pid > 0 { return true, nil } From 2a30a0bb9ff952689447e02844c55214eb30515f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:12:07 +0000 Subject: [PATCH 18/76] Fail safely on ambiguous vGPU claims --- lib/instances/vgpu.go | 15 +++++++++------ lib/instances/vgpu_test.go | 38 ++++++++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index acc5eb749..9bca44dbc 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -96,13 +96,13 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) return nil } -// vgpuAssignmentClaimedByLiveInstance reports whether another instance's +// vgpuAssignmentClaimedByLiveInstance reports whether another live instance's // stored metadata claims devicePath. It reads raw metadata instead of // hydrating full instances: the scan runs on every vendor VFIO release, and // deriving state would query the hypervisor of every instance on the host. -// It fails closed: unreadable metadata is an error, and a matching claim -// without a persisted PID counts as live because the PID is only persisted -// after the claimant's hypervisor starts. +// A confirmed live claimant returns true. Unreadable metadata, a missing PID, +// or unverifiable process ownership returns an error so the requester retains +// its assignment for a later retry. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { files, err := m.listMetadataFilesWithStatErrors(true) if err != nil { @@ -122,10 +122,13 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu continue } if stored.HypervisorPID == nil { - return true, nil + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) } pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.SocketPath) - if err != nil || pid > 0 { + if err != nil { + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) + } + if pid > 0 { return true, nil } } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index b6831109c..d4e0c6489 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -226,7 +226,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. assert.True(t, claimed) } -func TestVGPUAssignmentClaimedByLiveInstanceProtectsNilPIDClaim(t *testing.T) { +func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnNilPIDClaim(t *testing.T) { t.Parallel() m := &manager{paths: paths.New(t.TempDir())} @@ -237,9 +237,9 @@ func TestVGPUAssignmentClaimedByLiveInstanceProtectsNilPIDClaim(t *testing.T) { GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", }})) - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") - require.NoError(t, err) - assert.True(t, claimed, "a matching claim without a persisted PID must be treated as live: the PID is only persisted after the claimant boots") + _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) + assert.Contains(t, err.Error(), "booting-claimant") } func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { @@ -281,6 +281,36 @@ func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) { assert.Empty(t, stored.GPUDevicePath) } +func TestReleaseStoredVGPURetainsRequesterOnAmbiguousClaim(t *testing.T) { + t.Parallel() + + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + t.Fatal("destroyVGPU must not be called for an ambiguous claim") + return nil + }, + } + require.NoError(t, m.ensureDirectories("ambiguous-claimant")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "ambiguous-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + }})) + + stored := &StoredMetadata{ + Id: "requester", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + } + err := m.releaseStoredVGPU(context.Background(), stored) + require.Error(t, err) + assert.Contains(t, err.Error(), "ambiguous-claimant") + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, devicePath, stored.GPUDevicePath) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() From a0509d8afbd23eb05c375229501febb7f2eb72a5 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:14:21 +0000 Subject: [PATCH 19/76] Expose retained vGPU instance IDs --- cmd/api/api/instances.go | 6 ++++++ cmd/api/api/instances_test.go | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index cd7f1c707..333c1f24b 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -369,12 +369,18 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) message := fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID) + innerCode := "vgpu_retained_instance" if !vgpuPending.Retained { message = fmt.Sprintf("failed to create instance; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.InstanceID) + innerCode = "vgpu_unretained_instance" } return oapi.CreateInstance500JSONResponse{ Code: "vgpu_cleanup_pending", Message: message, + InnerError: &oapi.ErrorDetail{ + Code: lo.ToPtr(innerCode), + Message: lo.ToPtr(vgpuPending.InstanceID), + }, }, nil case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 45d9275a9..a222c488c 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -78,6 +78,11 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) assert.Contains(t, pending.Message, "inst-1") assert.Contains(t, pending.Message, "delete it to retry") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) } func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) { @@ -99,6 +104,11 @@ func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance( assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") assert.Contains(t, pending.Message, "startup reconcile") assert.NotContains(t, pending.Message, "delete") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) } func TestCreateInstance_AutoPullImage(t *testing.T) { From 32b92849a84f589969b4c9e9791fb95b227e684a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:31:22 +0000 Subject: [PATCH 20/76] Harden vGPU startup rollback recovery --- cmd/api/main.go | 62 +++++++++++++++++++++++++++---------- cmd/api/main_test.go | 18 +++++++---- lib/instances/create.go | 4 +++ lib/instances/start.go | 25 +++------------ lib/instances/types.go | 3 +- lib/instances/vgpu.go | 27 +++++++++++++++- lib/instances/vgpu_test.go | 63 ++++++++++++++++++++++++++++++++++++-- 7 files changed, 155 insertions(+), 47 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 3374d9682..46308e201 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -185,26 +185,62 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } -func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, error) { +const vgpuAssignmentStartupGracePeriod = 5 * time.Minute + +func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) { allInstances, err := instanceManager.ListInstancesForReconcile(ctx) if err != nil { - return nil, err + return nil, 0, err } protected := make(map[string]struct{}) + var retryAfter time.Duration for _, inst := range allInstances { if inst.GPUDevicePath == "" { continue } - // A nil PID does not mean the assignment is orphaned: the PID is - // persisted only after the hypervisor starts, so a crash during boot - // leaves the device path without one. Only skip protection when the - // recorded hypervisor is known to be gone. - if inst.HypervisorPID != nil && !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) { + if inst.HypervisorPID != nil { + if !instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.SocketPath) { + continue + } + protected[inst.GPUDevicePath] = struct{}{} + continue + } + if inst.GPUAssignedAt == nil { + continue + } + remaining := vgpuAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt) + if remaining <= 0 { continue } protected[inst.GPUDevicePath] = struct{}{} + if retryAfter == 0 || remaining < retryAfter { + retryAfter = remaining + } } - return protected, nil + return protected, retryAfter, nil +} + +func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) { + protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager) + if err != nil { + logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err) + return + } + if err := devices.ReconcileVGPUs(ctx, protected); err != nil { + logger.Warn("failed to reconcile vGPU devices", "error", err) + } + if retryAfter <= 0 { + return + } + go func() { + timer := time.NewTimer(retryAfter) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + reconcileVGPUs(ctx, instanceManager, logger) + } + }() } func run() error { @@ -408,15 +444,7 @@ func run() error { // Reconcile vGPU devices (clears orphaned vGPUs from previous runs) logger.Info("Reconciling vGPU devices...") - protected, err := liveInstanceVGPUDevicePaths(app.Ctx, app.InstanceManager) - if err != nil { - logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err) - protected = nil - } - if err := devices.ReconcileVGPUs(app.Ctx, protected); err != nil { - // Log but don't fail - vGPU cleanup is best-effort - logger.Warn("failed to reconcile vGPU devices", "error", err) - } + reconcileVGPUs(ctx, app.InstanceManager, logger) // Wire up resource validator for aggregate limit checking // This enables the instance manager to validate CPU, memory, network, and GPU diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 573a404c1..02909e6ad 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -351,20 +351,26 @@ func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([] return s.list, nil } -// The hypervisor PID is persisted only after boot, so an assignment without -// one may belong to a VM that is still starting and must stay protected. -func TestLiveInstanceVGPUDevicePathsProtectsAssignmentsWithoutPID(t *testing.T) { +func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { dead := exec.Command("true") require.NoError(t, dead.Run()) deadPID := dead.Process.Pid + recent := time.Now().Add(-time.Minute) + stale := time.Now().Add(-vgpuAssignmentStartupGracePeriod - time.Minute) manager := vgpuReconcileManagerStub{list: []instances.Instance{ - {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4"}}, - {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", HypervisorPID: &deadPID}}, + {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}}, + {StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}}, + {StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}}, + {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorPID: &deadPID}}, }} - protected, err := liveInstanceVGPUDevicePaths(context.Background(), manager) + protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) require.NoError(t, err) + require.Positive(t, retryAfter) + require.LessOrEqual(t, retryAfter, vgpuAssignmentStartupGracePeriod) assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") } diff --git a/lib/instances/create.go b/lib/instances/create.go index 42fdcac39..98e1a6f29 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -277,6 +277,7 @@ func (m *manager) createInstance( var gpuFramework devices.VGPUFramework var gpuDevicePath string var gpuMdevUUID string + var gpuAssignedAt *time.Time var stored *StoredMetadata var retainedVGPU *StoredMetadata @@ -318,6 +319,8 @@ func (m *manager) createInstance( gpuFramework = gpuDevice.Framework gpuDevicePath = gpuDevice.SysfsPath gpuMdevUUID = gpuDevice.MdevUUID + assignedAt := m.nowUTC() + gpuAssignedAt = &assignedAt log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", gpuProfile, "uuid", gpuMdevUUID) // Add vGPU cleanup to stack @@ -426,6 +429,7 @@ func (m *manager) createInstance( GPUFramework: gpuFramework, GPUDevicePath: gpuDevicePath, GPUMdevUUID: gpuMdevUUID, + GPUAssignedAt: gpuAssignedAt, Entrypoint: req.Entrypoint, Cmd: req.Cmd, SkipKernelHeaders: req.SkipKernelHeaders, diff --git a/lib/instances/start.go b/lib/instances/start.go index b830bafb2..775038559 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/egressproxy" "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/logger" @@ -64,6 +63,8 @@ func (m *manager) startInstance( } } + rollbackMeta := *meta + // 2a. Clear stale exit info from previous run and apply command overrides stored.ExitCode = nil stored.ExitMessage = "" @@ -168,28 +169,12 @@ func (m *manager) startInstance( log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } - setStoredVGPUDevice(stored, device) + assignedAt := m.nowUTC() + setStoredVGPUDevice(stored, device, assignedAt) log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) // Add vGPU cleanup to stack cu.Add(func() { - log.DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID) - assignment := devices.VGPUAssignment{ - Framework: device.Framework, - DevicePath: device.SysfsPath, - MdevUUID: device.MdevUUID, - InstanceID: id, - } - if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { - log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) - if saveErr := m.saveMetadata(meta); saveErr != nil { - log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr) - } - } else { - clearStoredVGPUDevice(stored) - if saveErr := m.saveMetadata(meta); saveErr != nil { - log.ErrorContext(ctx, "failed to save metadata after vGPU cleanup", "instance_id", id, "error", saveErr) - } - } + m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) }) if err := m.saveMetadata(meta); err != nil { log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) diff --git a/lib/instances/types.go b/lib/instances/types.go index 6aac15985..a264498fd 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -154,7 +154,8 @@ type StoredMetadata struct { GPUProfile string // vGPU profile name (e.g., "L40S-1Q") GPUFramework devices.VGPUFramework GPUDevicePath string - GPUMdevUUID string // populated for mdev-backed vGPUs + GPUMdevUUID string // populated for mdev-backed vGPUs + GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 9bca44dbc..3e7dc350c 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -3,6 +3,7 @@ package instances import ( "context" "path/filepath" + "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/logger" @@ -49,16 +50,40 @@ func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices. return destroy(ctx, assignment) } -func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice) { +func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) { stored.GPUFramework = device.Framework stored.GPUDevicePath = device.SysfsPath stored.GPUMdevUUID = device.MdevUUID + stored.GPUAssignedAt = &assignedAt } func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUFramework = devices.VGPUFrameworkNone stored.GPUDevicePath = "" stored.GPUMdevUUID = "" + stored.GPUAssignedAt = nil +} + +func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) { + assignment := devices.VGPUAssignment{ + Framework: device.Framework, + DevicePath: device.SysfsPath, + MdevUUID: device.MdevUUID, + InstanceID: instanceID, + } + cleanupMeta := rollbackMeta + releaseErr := m.destroyVGPUAssignment(ctx, assignment) + if releaseErr != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "error", releaseErr) + setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) + } + if err := m.saveMetadata(&cleanupMeta); err != nil { + message := "failed to save metadata after vGPU cleanup" + if releaseErr != nil { + message = "failed to retain vGPU assignment metadata after cleanup failure" + } + logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err) + } } func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d4e0c6489..366e343f2 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "sync" "testing" + "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" @@ -188,13 +189,68 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { }) t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) - _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}}) require.Error(t, err) stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.NotNil(t, stored.GPUAssignedAt) + assert.Empty(t, stored.Entrypoint) +} + +func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + return nil + }, + } + const id = "failed-start" + require.NoError(t, m.ensureDirectories(id)) + + previousStart := time.Now().Add(-time.Hour).UTC() + previousProgramStart := previousStart.Add(time.Second) + exitCode := 1 + rollbackMeta := metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUProfile: "NVIDIA L40S-2Q", + Entrypoint: []string{"old-entrypoint"}, + Cmd: []string{"old-command"}, + StartedAt: &previousStart, + ProgramStartedAt: &previousProgramStart, + ExitCode: &exitCode, + ExitMessage: "previous exit", + }} + + partial := rollbackMeta + partial.Entrypoint = []string{"new-entrypoint"} + partial.Cmd = []string{"new-command"} + partial.StartedAt = ptr(time.Now().UTC()) + partial.ProgramStartedAt = nil + partial.ExitCode = nil + partial.ExitMessage = "" + assignedAt := time.Now().UTC() + device := &devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + setStoredVGPUDevice(&partial.StoredMetadata, device, assignedAt) + require.NoError(t, m.saveMetadata(&partial)) + + m.cleanupStartVGPU(context.Background(), id, device, assignedAt, rollbackMeta) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, rollbackMeta.Entrypoint, stored.Entrypoint) + assert.Equal(t, rollbackMeta.Cmd, stored.Cmd) + assert.Equal(t, rollbackMeta.StartedAt, stored.StartedAt) + assert.Equal(t, rollbackMeta.ProgramStartedAt, stored.ProgramStartedAt) + assert.Equal(t, rollbackMeta.ExitCode, stored.ExitCode) + assert.Equal(t, rollbackMeta.ExitMessage, stored.ExitMessage) + assert.Empty(t, stored.GPUDevicePath) + assert.Nil(t, stored.GPUAssignedAt) } func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { @@ -341,16 +397,19 @@ func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) { func TestSetAndClearStoredVGPUDevice(t *testing.T) { t.Parallel() + assignedAt := time.Now().UTC() stored := &StoredMetadata{} setStoredVGPUDevice(stored, &devices.VGPUDevice{ Framework: devices.VGPUFrameworkVendorVFIO, SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - }) + }, assignedAt) assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.Equal(t, assignedAt, *stored.GPUAssignedAt) clearStoredVGPUDevice(stored) assert.Empty(t, stored.GPUFramework) assert.Empty(t, stored.GPUDevicePath) assert.Empty(t, stored.GPUMdevUUID) + assert.Nil(t, stored.GPUAssignedAt) } From 5a3e1a3835dcfaa395f534e8f358f9d4f7a337d9 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:12:47 +0000 Subject: [PATCH 21/76] Preserve the create failure cause in vGPU cleanup errors The vgpu_cleanup_pending response replaced the original create error with cleanup guidance, leaving the cause only in server logs. Prefix the message with the wrapped error so callers see why creation failed as well as how to recover. --- cmd/api/api/instances.go | 4 ++-- cmd/api/api/instances_test.go | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 333c1f24b..278b7e634 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -368,10 +368,10 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) - message := fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID) + message := fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.Err, vgpuPending.InstanceID) innerCode := "vgpu_retained_instance" if !vgpuPending.Retained { - message = fmt.Sprintf("failed to create instance; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.InstanceID) + message = fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID) innerCode = "vgpu_unretained_instance" } return oapi.CreateInstance500JSONResponse{ diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index a222c488c..bd957b5af 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -77,6 +77,8 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) assert.Contains(t, pending.Message, "inst-1") + assert.Contains(t, pending.Message, network.ErrNameExists.Error(), + "the underlying create failure must survive the cleanup guidance") assert.Contains(t, pending.Message, "delete it to retry") require.NotNil(t, pending.InnerError) require.NotNil(t, pending.InnerError.Code) @@ -102,6 +104,8 @@ func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance( require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") + assert.Contains(t, pending.Message, network.ErrNameExists.Error(), + "the underlying create failure must survive the cleanup guidance") assert.Contains(t, pending.Message, "startup reconcile") assert.NotContains(t, pending.Message, "delete") require.NotNil(t, pending.InnerError) From e5618885e3e6b68411596b55419c4ec2ac06f859 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:33:44 +0000 Subject: [PATCH 22/76] Fix vGPU reconciliation edge cases --- cmd/api/main.go | 9 ++++----- cmd/api/main_test.go | 4 ++-- lib/instances/create.go | 2 ++ lib/instances/vgpu.go | 16 ++++++++++++---- lib/instances/vgpu_test.go | 27 ++++++++++++++++++++++++++- 5 files changed, 46 insertions(+), 12 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 46308e201..eeb6e9fae 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -185,8 +185,6 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } -const vgpuAssignmentStartupGracePeriod = 5 * time.Minute - func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) { allInstances, err := instanceManager.ListInstancesForReconcile(ctx) if err != nil { @@ -208,7 +206,7 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. if inst.GPUAssignedAt == nil { continue } - remaining := vgpuAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt) + remaining := instances.VGPUAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt) if remaining <= 0 { continue } @@ -223,8 +221,9 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) { protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager) if err != nil { - logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err) - return + logger.Warn("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err) + protected = nil + retryAfter = 0 } if err := devices.ReconcileVGPUs(ctx, protected); err != nil { logger.Warn("failed to reconcile vGPU devices", "error", err) diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 02909e6ad..09a4a2419 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -356,7 +356,7 @@ func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { require.NoError(t, dead.Run()) deadPID := dead.Process.Pid recent := time.Now().Add(-time.Minute) - stale := time.Now().Add(-vgpuAssignmentStartupGracePeriod - time.Minute) + stale := time.Now().Add(-instances.VGPUAssignmentStartupGracePeriod - time.Minute) manager := vgpuReconcileManagerStub{list: []instances.Instance{ {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}}, @@ -368,7 +368,7 @@ func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) require.NoError(t, err) require.Positive(t, retryAfter) - require.LessOrEqual(t, retryAfter, vgpuAssignmentStartupGracePeriod) + require.LessOrEqual(t, retryAfter, instances.VGPUAssignmentStartupGracePeriod) assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") diff --git a/lib/instances/create.go b/lib/instances/create.go index 98e1a6f29..6ea250e55 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -351,6 +351,7 @@ func (m *manager) createInstance( GPUFramework: gpuDevice.Framework, GPUDevicePath: gpuDevice.SysfsPath, GPUMdevUUID: gpuDevice.MdevUUID, + GPUAssignedAt: gpuAssignedAt, } } } @@ -672,6 +673,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG GPUFramework: retainedVGPU.GPUFramework, GPUDevicePath: retainedVGPU.GPUDevicePath, GPUMdevUUID: retainedVGPU.GPUMdevUUID, + GPUAssignedAt: retainedVGPU.GPUAssignedAt, } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 3e7dc350c..0d12e5323 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -6,6 +6,7 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -16,6 +17,10 @@ func validateVGPUHypervisor(hvType hypervisor.Type) error { return nil } +// VGPUAssignmentStartupGracePeriod bounds how long an assignment without a +// persisted hypervisor PID is treated as potentially live. +const VGPUAssignmentStartupGracePeriod = 5 * time.Minute + // VGPUCleanupPendingError reports a failed create whose vGPU release also // failed during rollback. When Retained is true, deleting the retained instance // retries the release; otherwise startup reconciliation recovers the assignment. @@ -125,9 +130,9 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) // stored metadata claims devicePath. It reads raw metadata instead of // hydrating full instances: the scan runs on every vendor VFIO release, and // deriving state would query the hypervisor of every instance on the host. -// A confirmed live claimant returns true. Unreadable metadata, a missing PID, -// or unverifiable process ownership returns an error so the requester retains -// its assignment for a later retry. +// A confirmed live claimant returns true. Unreadable metadata, a recent +// assignment without a PID, or unverifiable process ownership returns an error +// so the requester retains its assignment for a later retry. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { files, err := m.listMetadataFilesWithStatErrors(true) if err != nil { @@ -147,7 +152,10 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu continue } if stored.HypervisorPID == nil { - return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) + if stored.GPUAssignedAt == nil || time.Since(*stored.GPUAssignedAt) >= VGPUAssignmentStartupGracePeriod { + continue + } + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) } pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.SocketPath) if err != nil { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 366e343f2..7eed17444 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -27,6 +27,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { t.Parallel() m := &manager{paths: paths.New(t.TempDir())} + assignedAt := time.Now().UTC() stored := &StoredMetadata{ Id: "failed-create", Name: "failed-create", @@ -34,6 +35,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUMdevUUID: "mdev-uuid", + GPUAssignedAt: &assignedAt, NetworkEnabled: true, IP: "192.0.2.1", Volumes: []VolumeAttachment{{VolumeID: "volume"}}, @@ -49,6 +51,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Equal(t, stored.GPUFramework, retained.GPUFramework) assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) + assert.Equal(t, stored.GPUAssignedAt, retained.GPUAssignedAt) assert.Empty(t, retained.Name) assert.Empty(t, retained.GPUProfile) assert.False(t, retained.NetworkEnabled) @@ -282,15 +285,17 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. assert.True(t, claimed) } -func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnNilPIDClaim(t *testing.T) { +func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnRecentNilPIDClaim(t *testing.T) { t.Parallel() m := &manager{paths: paths.New(t.TempDir())} require.NoError(t, m.ensureDirectories("booting-claimant")) + assignedAt := time.Now().UTC() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ Id: "booting-claimant", Name: "booting-claimant", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, }})) _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") @@ -298,6 +303,24 @@ func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnNilPIDClaim(t *testing.T) { assert.Contains(t, err.Error(), "booting-claimant") } +func TestVGPUAssignmentClaimedByLiveInstanceIgnoresStaleNilPIDClaim(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("stale-claimant")) + assignedAt := time.Now().Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "stale-claimant", + Name: "stale-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.False(t, claimed) +} + func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { t.Parallel() @@ -349,10 +372,12 @@ func TestReleaseStoredVGPURetainsRequesterOnAmbiguousClaim(t *testing.T) { }, } require.NoError(t, m.ensureDirectories("ambiguous-claimant")) + assignedAt := time.Now().UTC() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ Id: "ambiguous-claimant", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: devicePath, + GPUAssignedAt: &assignedAt, }})) stored := &StoredMetadata{ From aa950486754e18838c85ca5b21df9d49e73a43d2 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:26:01 +0000 Subject: [PATCH 23/76] Use boot-scoped hypervisor identities for vGPUs --- cmd/api/main.go | 2 +- lib/instances/vgpu.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index eeb6e9fae..026267a05 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -197,7 +197,7 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. continue } if inst.HypervisorPID != nil { - if !instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.SocketPath) { + if !instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.HypervisorBootID, inst.SocketPath) { continue } protected[inst.GPUDevicePath] = struct{}{} diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 0d12e5323..772d93e17 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -157,7 +157,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu } return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) } - pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.SocketPath) + pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.HypervisorBootID, stored.SocketPath) if err != nil { return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } From 9a23f3eb956de2b5d85bdf436a69efee7ac8f466 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:34:19 +0000 Subject: [PATCH 24/76] Run vGPU rollback tests with QEMU --- lib/instances/vgpu_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 7eed17444..56c6f8574 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -153,7 +153,7 @@ func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, dev Name: id, Image: "test-image", GPUProfile: "NVIDIA L40S-2Q", - HypervisorType: lifecycleNoopHypervisorType, + HypervisorType: hypervisor.TypeQEMU, SocketPath: m.paths.InstanceSocket(id, "noop.sock"), DataDir: m.paths.InstanceDir(id), }})) From 2cad3cf9dd6a10c81e08a8f3afcd303779e43663 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:07:05 +0000 Subject: [PATCH 25/76] Protect new vGPU assignments from stale PIDs --- cmd/api/main.go | 5 +---- cmd/api/main_test.go | 4 +++- lib/instances/create.go | 6 ++++++ lib/instances/start.go | 9 +++++++++ lib/instances/vgpu.go | 1 + lib/instances/vgpu_test.go | 12 +++++++++++- 6 files changed, 31 insertions(+), 6 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 026267a05..982e15c3f 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -196,10 +196,7 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. if inst.GPUDevicePath == "" { continue } - if inst.HypervisorPID != nil { - if !instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.HypervisorBootID, inst.SocketPath) { - continue - } + if inst.HypervisorPID != nil && instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.HypervisorBootID, inst.SocketPath) { protected[inst.GPUDevicePath] = struct{}{} continue } diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 09a4a2419..217e9db8d 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -351,7 +351,7 @@ func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([] return s.list, nil } -func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { +func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) { dead := exec.Command("true") require.NoError(t, dead.Run()) deadPID := dead.Process.Pid @@ -363,6 +363,7 @@ func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { {StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}}, {StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}}, {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorPID: &deadPID}}, + {StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorPID: &deadPID, GPUAssignedAt: &recent}}, }} protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) @@ -373,4 +374,5 @@ func TestLiveInstanceVGPUDevicePathsBoundsProtectionWithoutPID(t *testing.T) { assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") } diff --git a/lib/instances/create.go b/lib/instances/create.go index 6ea250e55..0e8ef4ce7 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -106,6 +106,11 @@ func (m *manager) createInstance( if hvType == "" { hvType = m.defaultHypervisor } + if req.GPU != nil && req.GPU.Profile != "" { + if err := validateVGPUHypervisor(hvType); err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) + } + } starter, starterErr := m.getVMStarter(hvType) if starterErr == nil { if err := m.validateCreateVMConfig(starter, req, hvType); err != nil { @@ -113,6 +118,7 @@ func (m *manager) createInstance( } } + // 2. Validate image exists and is ready; auto-pull if not found log.DebugContext(ctx, "validating image", "image", req.Image) imageCtx, imageSpanEnd := m.startLifecycleStep(ctx, "resolve_image", diff --git a/lib/instances/start.go b/lib/instances/start.go index 775038559..ecb33f11f 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -48,6 +48,11 @@ func (m *manager) startInstance( return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } + if stored.GPUProfile != "" { + if err := validateVGPUHypervisor(stored.HypervisorType); err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidState, err) + } + } // Release any assignment retained by an earlier failed release and // persist the cleared fields immediately, so a failure later in start // cannot leave on-disk metadata pointing at a device that is already @@ -63,6 +68,10 @@ func (m *manager) startInstance( } } + // Do not persist the previous VMM's identity with a new vGPU assignment. + stored.HypervisorPID = nil + stored.HypervisorStartTime = 0 + stored.HypervisorBootID = "" rollbackMeta := *meta // 2a. Clear stale exit info from previous run and apply command overrides diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 772d93e17..6fd02352a 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -2,6 +2,7 @@ package instances import ( "context" + "fmt" "path/filepath" "time" diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 56c6f8574..4491bdd17 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -190,9 +190,16 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { return errors.New("destroy failed") }) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + stalePID := os.Getpid() + meta.HypervisorPID = &stalePID + meta.HypervisorStartTime = 1 + meta.HypervisorBootID = "previous-boot" + require.NoError(t, m.saveMetadata(meta)) t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) - _, err := m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}}) + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}}) require.Error(t, err) stored, err := m.loadMetadata(id) @@ -200,6 +207,9 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) assert.NotNil(t, stored.GPUAssignedAt) + assert.Nil(t, stored.HypervisorPID) + assert.Zero(t, stored.HypervisorStartTime) + assert.Empty(t, stored.HypervisorBootID) assert.Empty(t, stored.Entrypoint) } From a9f092934bbb07c7497296339b010468db3f13a3 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:30:28 +0000 Subject: [PATCH 26/76] Persist vGPU assignments after create rollback failure --- lib/instances/create.go | 1 + lib/instances/start.go | 8 ++++++ lib/instances/vgpu.go | 23 ++++++++++++++++ lib/instances/vgpu_test.go | 56 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+) diff --git a/lib/instances/create.go b/lib/instances/create.go index 0e8ef4ce7..c221f698a 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -318,6 +318,7 @@ func (m *manager) createInstance( log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) if err != nil { + retainedVGPU = retainedVGPUFromCreateError(id, m.nowUTC(), err) log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } diff --git a/lib/instances/start.go b/lib/instances/start.go index ecb33f11f..39418ad7d 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -175,6 +175,14 @@ func (m *manager) startInstance( log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id) if err != nil { + if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok { + assignedAt := m.nowUTC() + setStoredVGPUDevice(stored, pendingDevice, assignedAt) + if saveErr := m.saveMetadata(meta); saveErr != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) + return nil, fmt.Errorf("create vGPU for profile %s: %w; retain assignment: %v", stored.GPUProfile, err, saveErr) + } + } log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 6fd02352a..a52cfa5d1 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -2,6 +2,7 @@ package instances import ( "context" + "errors" "fmt" "path/filepath" "time" @@ -48,6 +49,28 @@ func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID return create(ctx, profileName, instanceID) } +func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { + var pending *devices.VGPUCreateCleanupPendingError + if !errors.As(err, &pending) { + return nil, false + } + return &pending.Device, true +} + +func retainedVGPUFromCreateError(instanceID string, assignedAt time.Time, err error) *StoredMetadata { + device, ok := vgpuDevicePendingCleanup(err) + if !ok { + return nil + } + return &StoredMetadata{ + Id: instanceID, + GPUFramework: device.Framework, + GPUDevicePath: device.SysfsPath, + GPUMdevUUID: device.MdevUUID, + GPUAssignedAt: &assignedAt, + } +} + func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { destroy := m.destroyVGPU if destroy == nil { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 4491bdd17..3cfd04430 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -3,6 +3,7 @@ package instances import ( "context" "errors" + "fmt" "os" "path/filepath" "sync" @@ -128,6 +129,35 @@ func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the assignment is recovered on the next startup reconcile", unpersisted.Error()) } +func TestVGPUDevicePendingCleanup(t *testing.T) { + t.Parallel() + + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + cause := errors.New("rollback failed") + pending := &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + + wrapped := fmt.Errorf("create failed: %w", pending) + actual, ok := vgpuDevicePendingCleanup(wrapped) + require.True(t, ok) + assert.Equal(t, device, *actual) + + assignedAt := time.Now().UTC() + retained := retainedVGPUFromCreateError("inst-1", assignedAt, wrapped) + require.NotNil(t, retained) + assert.Equal(t, "inst-1", retained.Id) + assert.Equal(t, device.Framework, retained.GPUFramework) + assert.Equal(t, device.SysfsPath, retained.GPUDevicePath) + assert.Equal(t, assignedAt, *retained.GPUAssignedAt) + + actual, ok = vgpuDevicePendingCleanup(cause) + assert.False(t, ok) + assert.Nil(t, actual) + assert.Nil(t, retainedVGPUFromCreateError("inst-1", assignedAt, cause)) +} + func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) { t.Helper() m := &manager{ @@ -160,6 +190,32 @@ func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, dev return m, id } +func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: "NVIDIA L40S-2Q", + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + cause := errors.New("create verification and rollback failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + } + + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, cause) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, device.Framework, stored.GPUFramework) + assert.Equal(t, device.SysfsPath, stored.GPUDevicePath) + assert.NotNil(t, stored.GPUAssignedAt) +} + func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { var destroyed []devices.VGPUAssignment m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { From 937ff9eb8fdfdfb5bb9f4671009efa699d0ea00d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:52:26 +0000 Subject: [PATCH 27/76] Preserve vGPU lifecycle compatibility --- lib/instances/create.go | 6 --- lib/instances/start.go | 10 ++--- lib/instances/vgpu.go | 8 ---- lib/instances/vgpu_test.go | 76 ++++++++++++++++++++++++++++++++++---- 4 files changed, 71 insertions(+), 29 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index c221f698a..3ffa322e6 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -106,11 +106,6 @@ func (m *manager) createInstance( if hvType == "" { hvType = m.defaultHypervisor } - if req.GPU != nil && req.GPU.Profile != "" { - if err := validateVGPUHypervisor(hvType); err != nil { - return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) - } - } starter, starterErr := m.getVMStarter(hvType) if starterErr == nil { if err := m.validateCreateVMConfig(starter, req, hvType); err != nil { @@ -118,7 +113,6 @@ func (m *manager) createInstance( } } - // 2. Validate image exists and is ready; auto-pull if not found log.DebugContext(ctx, "validating image", "image", req.Image) imageCtx, imageSpanEnd := m.startLifecycleStep(ctx, "resolve_image", diff --git a/lib/instances/start.go b/lib/instances/start.go index 39418ad7d..f95967699 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -48,11 +48,6 @@ func (m *manager) startInstance( return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } - if stored.GPUProfile != "" { - if err := validateVGPUHypervisor(stored.HypervisorType); err != nil { - return nil, fmt.Errorf("%w: %w", ErrInvalidState, err) - } - } // Release any assignment retained by an earlier failed release and // persist the cleared fields immediately, so a failure later in start // cannot leave on-disk metadata pointing at a device that is already @@ -177,8 +172,9 @@ func (m *manager) startInstance( if err != nil { if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok { assignedAt := m.nowUTC() - setStoredVGPUDevice(stored, pendingDevice, assignedAt) - if saveErr := m.saveMetadata(meta); saveErr != nil { + retentionMeta := rollbackMeta + setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, assignedAt) + if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) return nil, fmt.Errorf("create vGPU for profile %s: %w; retain assignment: %v", stored.GPUProfile, err, saveErr) } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index a52cfa5d1..290c474bb 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -8,17 +8,9 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) -func validateVGPUHypervisor(hvType hypervisor.Type) error { - if hvType != hypervisor.TypeQEMU { - return fmt.Errorf("vGPU is only supported with qemu, got %s", hvType) - } - return nil -} - // VGPUAssignmentStartupGracePeriod bounds how long an assignment without a // persisted hypervisor PID is treated as potentially live. const VGPUAssignmentStartupGracePeriod = 5 * time.Minute diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 3cfd04430..1ff759363 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -12,18 +12,12 @@ import ( "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestValidateVGPUHypervisor(t *testing.T) { - t.Parallel() - - assert.NoError(t, validateVGPUHypervisor(hypervisor.TypeQEMU)) - assert.EqualError(t, validateVGPUHypervisor(hypervisor.TypeCloudHypervisor), "vGPU is only supported with qemu, got cloud-hypervisor") -} - func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { t.Parallel() @@ -158,6 +152,22 @@ func TestVGPUDevicePendingCleanup(t *testing.T) { assert.Nil(t, retainedVGPUFromCreateError("inst-1", assignedAt, cause)) } +type startRetentionNetworkManager struct { + network.Manager + config network.NetworkConfig + releaseCalls int +} + +func (m *startRetentionNetworkManager) CreateAllocation(context.Context, network.AllocateRequest) (*network.NetworkConfig, error) { + config := m.config + return &config, nil +} + +func (m *startRetentionNetworkManager) ReleaseAllocation(context.Context, *network.Allocation) error { + m.releaseCalls++ + return nil +} + func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) { t.Helper() m := &manager{ @@ -194,6 +204,27 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { return nil }) + networkManager := &startRetentionNetworkManager{config: network.NetworkConfig{ + IP: "192.0.2.20", + MAC: "02:00:00:00:00:20", + TAPDevice: "tap-new", + }} + m.networkManager = networkManager + + previousProgramStart := time.Now().Add(-time.Hour).UTC() + previousExitCode := 23 + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.NetworkEnabled = true + meta.IP = "192.0.2.10" + meta.MAC = "02:00:00:00:00:10" + meta.Entrypoint = []string{"old-entrypoint"} + meta.Cmd = []string{"old-command"} + meta.ProgramStartedAt = &previousProgramStart + meta.ExitCode = &previousExitCode + meta.ExitMessage = "previous exit" + require.NoError(t, m.saveMetadata(meta)) + device := devices.VGPUDevice{ Framework: devices.VGPUFrameworkVendorVFIO, VFAddress: "0000:82:00.4", @@ -206,7 +237,10 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} } - _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{ + Entrypoint: []string{"new-entrypoint"}, + Cmd: []string{"new-command"}, + }) require.ErrorIs(t, err, cause) stored, err := m.loadMetadata(id) @@ -214,6 +248,32 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { assert.Equal(t, device.Framework, stored.GPUFramework) assert.Equal(t, device.SysfsPath, stored.GPUDevicePath) assert.NotNil(t, stored.GPUAssignedAt) + assert.Equal(t, []string{"old-entrypoint"}, stored.Entrypoint) + assert.Equal(t, []string{"old-command"}, stored.Cmd) + assert.Equal(t, previousProgramStart, *stored.ProgramStartedAt) + assert.Equal(t, previousExitCode, *stored.ExitCode) + assert.Equal(t, "previous exit", stored.ExitMessage) + assert.Equal(t, "192.0.2.10", stored.IP) + assert.Equal(t, "02:00:00:00:00:10", stored.MAC) + assert.Equal(t, 1, networkManager.releaseCalls) +} + +func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.HypervisorType = hypervisor.TypeCloudHypervisor + require.NoError(t, m.saveMetadata(meta)) + + cause := errors.New("create failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + return nil, cause + } + + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) + assert.ErrorIs(t, err, cause) } func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { From 76950f7838dc998c1b55af53b13467eceefaf6a8 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 21:11:08 +0000 Subject: [PATCH 28/76] Reconcile vGPU protection from raw metadata and restore GPUAssignedAt ListInstancesForReconcile hydrated every instance (socket stat, UFFD health, /vm.info per instance) before the API served and again on each grace retry, while the protected-set scan only reads stored metadata fields. List raw metadata fail-closed instead, matching the release claim scan, and drop the now-unused loadInstances parameterization. Snapshot restore preserved the source's vGPU assignment path fields but not GPUAssignedAt, so a retained assignment lost its crash-recovery grace timestamp across a restore. Carry the timestamp with the rest of the assignment. --- lib/instances/manager.go | 21 +++++++++++++++++++-- lib/instances/query.go | 11 +---------- lib/instances/query_test.go | 6 ++++++ lib/instances/snapshot.go | 1 + lib/instances/snapshot_test.go | 4 ++++ 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 0027acff3..a0622da19 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path/filepath" "strings" "sync" "time" @@ -725,9 +726,25 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } -// ListInstancesForReconcile returns every instance or an invalid metadata error. +// ListInstancesForReconcile returns every instance's stored metadata or an +// invalid metadata error. It does not derive state: reconcile protection only +// needs raw metadata fields, and hydration would query the hypervisor of +// every instance on the host before the API serves. func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { - return m.loadInstances(ctx, false) + files, err := m.listMetadataFilesWithStatErrors(true) + if err != nil { + return nil, err + } + result := make([]Instance, 0, len(files)) + for _, file := range files { + id := filepath.Base(filepath.Dir(file)) + meta, err := m.loadMetadata(id) + if err != nil { + return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) + } + result = append(result, Instance{StoredMetadata: meta.StoredMetadata}) + } + return result, nil } // ListInstances returns instances, optionally filtered by the given criteria. diff --git a/lib/instances/query.go b/lib/instances/query.go index eea66ab13..0621460da 100644 --- a/lib/instances/query.go +++ b/lib/instances/query.go @@ -786,16 +786,12 @@ func parseSentinelTimestamp(line, sentinelPrefix string) (time.Time, bool) { // listInstances returns all instances, skipping metadata files that cannot be loaded. func (m *manager) listInstances(ctx context.Context) ([]Instance, error) { - return m.loadInstances(ctx, true) -} - -func (m *manager) loadInstances(ctx context.Context, skipInvalid bool) ([]Instance, error) { ctx, span := m.tracerOrDefault().Start(ctx, "instances.list_metadata") defer span.End() log := logger.FromContext(ctx) log.DebugContext(ctx, "listing all instances") - files, err := m.listMetadataFilesWithStatErrors(!skipInvalid) + files, err := m.listMetadataFiles() if err != nil { log.ErrorContext(ctx, "failed to list metadata files", "error", err) return nil, err @@ -813,11 +809,6 @@ func (m *manager) loadInstances(ctx context.Context, skipInvalid bool) ([]Instan ) meta, err := m.loadMetadata(id) if err != nil { - if !skipInvalid { - hydrateSpan.RecordError(err) - hydrateSpan.End() - return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) - } // Skip instances with invalid metadata log.WarnContext(hydrateCtx, "skipping instance with invalid metadata", "instance_id", id, "error", err) hydrateSpan.End() diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 41aba54e8..ab3db29c3 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -34,6 +34,12 @@ func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { _, err = m.ListInstancesForReconcile(context.Background()) require.Error(t, err) assert.ErrorContains(t, err, "load metadata for instance invalid") + + require.NoError(t, os.Remove(m.paths.InstanceMetadata("invalid"))) + listed, err = m.ListInstancesForReconcile(context.Background()) + require.NoError(t, err) + require.Len(t, listed, 1) + assert.Equal(t, "valid", listed[0].Id) } func TestParseExitSentinelLine(t *testing.T) { diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 2d2676c72..48c51328a 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -312,6 +312,7 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str restored.GPUFramework = sourceMeta.GPUFramework restored.GPUDevicePath = sourceMeta.GPUDevicePath restored.GPUMdevUUID = sourceMeta.GPUMdevUUID + restored.GPUAssignedAt = sourceMeta.GPUAssignedAt restored.HypervisorType = targetHypervisor restored.HypervisorVersion = targetHypervisorVersion restored.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index b23e364ee..a92763b28 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -113,6 +113,8 @@ func TestRestoreSnapshotKeepsCurrentVGPUAssignment(t *testing.T) { meta.GPUFramework = devices.VGPUFramework("future-framework") meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" meta.GPUMdevUUID = "retained-uuid" + assignedAt := time.Now().UTC().Truncate(time.Second) + meta.GPUAssignedAt = &assignedAt require.NoError(t, mgr.saveMetadata(meta)) _, err = mgr.RestoreSnapshot(ctx, sourceID, snapshot.Id, RestoreSnapshotRequest{ @@ -126,6 +128,8 @@ func TestRestoreSnapshotKeepsCurrentVGPUAssignment(t *testing.T) { assert.Equal(t, devices.VGPUFramework("future-framework"), restored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", restored.GPUDevicePath) assert.Equal(t, "retained-uuid", restored.GPUMdevUUID) + require.NotNil(t, restored.GPUAssignedAt) + assert.True(t, assignedAt.Equal(*restored.GPUAssignedAt)) } func TestStoppedSnapshotLifecycleAndForkAfterSourceDeletion(t *testing.T) { From 7b20b431d7ed7805cfac84e1fee493ae62261751 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:46:50 +0000 Subject: [PATCH 29/76] Surface pending vGPU cleanup from start as a typed error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When start's vGPU create fails with a pending device-layer cleanup, the error was returned untyped, so the API mapped it to a generic internal_error. Create already wraps the same condition in VGPUCleanupPendingError and surfaces vgpu_cleanup_pending with retained/unretained guidance. Wrap start's pending-cleanup error the same way — Retained reflects whether the retention record was persisted — and map it in the StartInstance handler ahead of the errors.Is cases so the wrapped cause cannot hide the pending cleanup. --- cmd/api/api/instances.go | 19 +++++++++++ cmd/api/api/instances_test.go | 62 +++++++++++++++++++++++++++++++++++ lib/instances/start.go | 6 ++-- lib/instances/vgpu_test.go | 39 ++++++++++++++++++++++ 4 files changed, 124 insertions(+), 2 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 278b7e634..d3e7467be 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -848,7 +848,26 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan result, err := s.InstanceManager.StartInstance(ctx, inst.Id, startReq) if err != nil { + var vgpuPending *instances.VGPUCleanupPendingError switch { + // Checked first: it wraps the original start error, so a later + // errors.Is case would match the cause and hide the pending vGPU cleanup. + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to start instance", "error", err) + message := fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it or retry start to release it", vgpuPending.Err, vgpuPending.InstanceID) + innerCode := "vgpu_retained_instance" + if !vgpuPending.Retained { + message = fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID) + innerCode = "vgpu_unretained_instance" + } + return oapi.StartInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: &oapi.ErrorDetail{ + Code: lo.ToPtr(innerCode), + Message: lo.ToPtr(vgpuPending.InstanceID), + }, + }, nil case errors.Is(err, instances.ErrInvalidState): return oapi.StartInstance409JSONResponse{ Code: "invalid_state", diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index bd957b5af..8a6e52518 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -1034,6 +1034,68 @@ func TestRestoreInstance_ErrorMapping(t *testing.T) { } } +// A retained-assignment error must win over the mapping of the start error +// it wraps, or the response omits the pending vGPU cleanup the caller has to +// resolve. +func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { + t.Parallel() + + resolved := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{Id: "inst-1", Name: "inst-1"}, + State: instances.StateStopped, + } + + t.Run("retained", func(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: true, + Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), + }} + + resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) + require.NoError(t, rerr) + + pending, ok := resp.(oapi.StartInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "inst-1") + assert.Contains(t, pending.Message, instances.ErrInsufficientResources.Error(), + "the underlying start failure must survive the cleanup guidance") + assert.Contains(t, pending.Message, "delete it or retry start") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) + }) + + t.Run("unretained", func(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), + }} + + resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) + require.NoError(t, rerr) + + pending, ok := resp.(oapi.StartInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") + assert.Contains(t, pending.Message, "startup reconcile") + assert.NotContains(t, pending.Message, "delete") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) + }) +} + func TestInstanceActions_ImageNotFoundMapsTo404(t *testing.T) { t.Parallel() diff --git a/lib/instances/start.go b/lib/instances/start.go index f95967699..2b7712af7 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -170,16 +170,18 @@ func (m *manager) startInstance( log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id) if err != nil { + log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok { assignedAt := m.nowUTC() retentionMeta := rollbackMeta setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, assignedAt) + wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) - return nil, fmt.Errorf("create vGPU for profile %s: %w; retain assignment: %v", stored.GPUProfile, err, saveErr) + return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} } + return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} } - log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } assignedAt := m.nowUTC() diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 1ff759363..d9e599d9b 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -242,6 +242,10 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { Cmd: []string{"new-command"}, }) require.ErrorIs(t, err, cause) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, id, pending.InstanceID) + assert.True(t, pending.Retained) stored, err := m.loadMetadata(id) require.NoError(t, err) @@ -258,6 +262,41 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { assert.Equal(t, 1, networkManager.releaseCalls) } +func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: "NVIDIA L40S-2Q", + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + cause := errors.New("create verification and rollback failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + } + + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, cause) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, id, pending.InstanceID) + assert.False(t, pending.Retained) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded") +} + func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { return nil From b6cfb61e8185bf00533946862ed56970063319df Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:41:14 +0000 Subject: [PATCH 30/76] Cover retained-stub delete recovery and flag reconcile inventory failures The vgpu_cleanup_pending guidance tells callers to delete the retained instance to retry a failed vGPU release, but no test exercised delete against the minimal GPU-fields-only stub cleanupFailedCreate writes. Add one. Losing the reconcile inventory disables vendor VFIO reconciliation host-wide while releases fail closed on the same inventory, so log it at error level instead of warn. Also document the wholesale-restore assumption in cleanupStartVGPU. --- cmd/api/main.go | 5 +++- lib/instances/lifecycle_noop_test.go | 37 ++++++++++++++++++++++++++++ lib/instances/vgpu.go | 4 +++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 982e15c3f..f38a6a7c3 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -218,7 +218,10 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) { protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager) if err != nil { - logger.Warn("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err) + // Operator-actionable: vendor VFIO reconciliation stays disabled + // host-wide (and releases fail closed on the same inventory) until + // the unreadable instance metadata is repaired. + logger.Error("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err) protected = nil retryAfter = 0 } diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 15f632b3a..c74d352fd 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -196,6 +196,43 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { assert.Equal(t, restartpolicy.BlockedReasonManualStop, persisted.RestartStatus.BlockedReason) } +// A failed create whose vGPU release also failed retains a minimal +// GPU-fields-only stub, and the API tells the caller to delete it to retry +// the release. Exercise that recovery path against the exact stub shape +// cleanupFailedCreate writes. +func TestDeleteReleasesRetainedCreateStub(t *testing.T) { + p := paths.New(t.TempDir()) + var destroyed []devices.VGPUAssignment + m := &manager{ + paths: p, + instanceLocks: sync.Map{}, + bootMarkerScans: sync.Map{}, + now: time.Now, + lifecycleEvents: newLifecycleSubscribers(), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + } + const id = "retained-stub" + require.NoError(t, m.ensureDirectories(id)) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + require.NoError(t, m.DeleteInstance(context.Background(), id)) + + require.Len(t, destroyed, 1) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", destroyed[0].DevicePath) + assert.Equal(t, id, destroyed[0].InstanceID) + _, err := m.loadMetadata(id) + require.Error(t, err, "retained stub must be fully deleted") +} + func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { now := time.Now().UTC() m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, now) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 290c474bb..f25c7c63e 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -85,6 +85,10 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUAssignedAt = nil } +// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. That +// is safe while the instance lock serializes start and no cleanup registered +// after the vGPU one persists metadata; a future cleanup that writes metadata +// must switch this to targeted field restores. func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) { assignment := devices.VGPUAssignment{ Framework: device.Framework, From 97a0d1507ee08cf3e9466705e31f9bb133718968 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:16:52 +0000 Subject: [PATCH 31/76] Reject vendor VFIO vGPUs on Cloud Hypervisor and improve wedge forensics Vendor VFIO vGPUs boot but are non-functional on Cloud Hypervisor (upstream cloud-hypervisor#7572), and the wedged VM then blocks the VF release until startup reconcile. Reject the combination at create and start after the rollback handler is registered, so the rejected device is released through the normal cleanup path. Hypervisor selection otherwise stays caller policy and mdev on Cloud Hypervisor keeps working. Retain identity fields (name, image, hypervisor, data dir) on the failed-create retention record so it lists as a recognizable, deletable instance instead of a nameless phantom; resource claims released by rollback stay dropped. Expose the assigned vGPU device_path in the instance API - on vendor VFIO hosts mdev_uuid is empty and the sysfs path is the identity an operator needs when a release wedges. --- cmd/api/api/instances.go | 3 + lib/instances/create.go | 29 ++- lib/instances/start.go | 6 + lib/instances/vgpu.go | 24 ++- lib/instances/vgpu_test.go | 39 +++- lib/oapi/oapi.go | 402 +++++++++++++++++++------------------ openapi.yaml | 6 +- 7 files changed, 297 insertions(+), 212 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index d3e7467be..888bc13fb 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -1302,6 +1302,9 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { if inst.GPUMdevUUID != "" { gpu.MdevUuid = lo.ToPtr(inst.GPUMdevUUID) } + if inst.GPUDevicePath != "" { + gpu.DevicePath = lo.ToPtr(inst.GPUDevicePath) + } oapiInst.Gpu = gpu } diff --git a/lib/instances/create.go b/lib/instances/create.go index 3ffa322e6..23fd534b1 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -357,6 +357,12 @@ func (m *manager) createInstance( } } }) + // Checked after the cleanup handler is registered so rejection + // releases the device through the normal rollback. + if err := validateVGPUHypervisorCompat(gpuDevice.Framework, hvType); err != nil { + log.ErrorContext(ctx, "unsupported vGPU hypervisor combination", "instance_id", id, "framework", gpuDevice.Framework, "hypervisor", hvType) + return nil, err + } } if len(req.Devices) > 0 && m.deviceManager != nil { @@ -669,12 +675,25 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) return retentionSurvives() } + // Retain identity fields so the instance lists as a recognizable, + // deletable record rather than a nameless phantom, but drop resource + // claims (network, volumes, devices) that rollback already released. retained := StoredMetadata{ - Id: id, - GPUFramework: retainedVGPU.GPUFramework, - GPUDevicePath: retainedVGPU.GPUDevicePath, - GPUMdevUUID: retainedVGPU.GPUMdevUUID, - GPUAssignedAt: retainedVGPU.GPUAssignedAt, + Id: id, + Name: retainedVGPU.Name, + Image: retainedVGPU.Image, + ResolvedImage: retainedVGPU.ResolvedImage, + Platform: retainedVGPU.Platform, + CreatedAt: retainedVGPU.CreatedAt, + HypervisorType: retainedVGPU.HypervisorType, + HypervisorVersion: retainedVGPU.HypervisorVersion, + SocketPath: retainedVGPU.SocketPath, + DataDir: retainedVGPU.DataDir, + GPUProfile: retainedVGPU.GPUProfile, + GPUFramework: retainedVGPU.GPUFramework, + GPUDevicePath: retainedVGPU.GPUDevicePath, + GPUMdevUUID: retainedVGPU.GPUMdevUUID, + GPUAssignedAt: retainedVGPU.GPUAssignedAt, } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) diff --git a/lib/instances/start.go b/lib/instances/start.go index 2b7712af7..b44a68d50 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -191,6 +191,12 @@ func (m *manager) startInstance( cu.Add(func() { m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) }) + // Checked after the cleanup handler is registered so rejection + // releases the device through the normal rollback. + if err := validateVGPUHypervisorCompat(device.Framework, stored.HypervisorType); err != nil { + log.ErrorContext(ctx, "unsupported vGPU hypervisor combination", "instance_id", id, "framework", device.Framework, "hypervisor", stored.HypervisorType) + return nil, err + } if err := m.saveMetadata(meta); err != nil { log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) return nil, fmt.Errorf("save metadata after vGPU creation: %w", err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index f25c7c63e..8c6351742 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -8,6 +8,7 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -63,6 +64,18 @@ func retainedVGPUFromCreateError(instanceID string, assignedAt time.Time, err er } } +// validateVGPUHypervisorCompat rejects the one proven-broken combination: +// vendor VFIO vGPUs boot but are non-functional on Cloud Hypervisor (upstream +// cloud-hypervisor#7572), and the wedged VM then blocks the VF release until +// startup reconcile. Hypervisor selection otherwise remains caller policy; +// mdev on Cloud Hypervisor keeps working. See lib/devices/GPU.md. +func validateVGPUHypervisorCompat(framework devices.VGPUFramework, hvType hypervisor.Type) error { + if framework == devices.VGPUFrameworkVendorVFIO && hvType == hypervisor.TypeCloudHypervisor { + return fmt.Errorf("%w: vendor VFIO vGPUs are not functional on cloud-hypervisor, use qemu", ErrInvalidRequest) + } + return nil +} + func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { destroy := m.destroyVGPU if destroy == nil { @@ -85,10 +98,13 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUAssignedAt = nil } -// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. That -// is safe while the instance lock serializes start and no cleanup registered -// after the vGPU one persists metadata; a future cleanup that writes metadata -// must switch this to targeted field restores. +// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. The +// cleanup stack is LIFO, so cleanups registered after this one run before it +// and this restore would clobber anything they persisted; it is safe only +// while no such cleanup writes metadata and the instance lock serializes +// start. The snapshot is also a shallow copy (Phases shares its map), so it +// must be persisted before any Phases.Record on the live struct. Violating +// either invariant requires switching to targeted field restores. func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) { assignment := devices.VGPUAssignment{ Framework: device.Framework, diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d9e599d9b..d466f1883 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -47,12 +47,16 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) assert.Equal(t, stored.GPUAssignedAt, retained.GPUAssignedAt) - assert.Empty(t, retained.Name) - assert.Empty(t, retained.GPUProfile) + // Identity fields survive so the retained record lists as a + // recognizable, deletable instance instead of a nameless phantom. + assert.Equal(t, stored.Name, retained.Name) + assert.Equal(t, stored.GPUProfile, retained.GPUProfile) + assert.Equal(t, stored.HypervisorType, retained.HypervisorType) + assert.Equal(t, stored.DataDir, retained.DataDir) + // Resource claims released by rollback stay dropped. assert.False(t, retained.NetworkEnabled) assert.Empty(t, retained.IP) assert.Empty(t, retained.Volumes) - assert.Empty(t, retained.DataDir) } func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { @@ -315,6 +319,35 @@ func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { assert.ErrorIs(t, err, cause) } +func TestValidateVGPUHypervisorCompat(t *testing.T) { + t.Parallel() + + err := validateVGPUHypervisorCompat(devices.VGPUFrameworkVendorVFIO, hypervisor.TypeCloudHypervisor) + require.ErrorIs(t, err, ErrInvalidRequest) + assert.NoError(t, validateVGPUHypervisorCompat(devices.VGPUFrameworkVendorVFIO, hypervisor.TypeQEMU)) + assert.NoError(t, validateVGPUHypervisorCompat(devices.VGPUFrameworkMdev, hypervisor.TypeCloudHypervisor)) +} + +func TestStartRejectsVendorVFIOOnCloudHypervisor(t *testing.T) { + var destroyed []devices.VGPUAssignment + m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.HypervisorType = hypervisor.TypeCloudHypervisor + require.NoError(t, m.saveMetadata(meta)) + + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, ErrInvalidRequest) + + require.Len(t, destroyed, 1, "the rejected vGPU must be released by rollback") + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "no assignment may be persisted for a rejected combination") +} + func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { var destroyed []devices.VGPUAssignment m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 76e7e0a64..ba5df2db0 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1354,7 +1354,10 @@ type InstanceHypervisor string // InstanceGPU GPU information attached to the instance type InstanceGPU struct { - // MdevUuid mdev device UUID + // DevicePath sysfs path of the assigned vGPU device + DevicePath *string `json:"device_path,omitempty"` + + // MdevUuid mdev device UUID (mdev hosts only) MdevUuid *string `json:"mdev_uuid,omitempty"` // Profile vGPU profile name @@ -19032,204 +19035,205 @@ var swaggerSpec = []string{ "1VhohilpTMSQ6dcCWKt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtlrDJ+QW0P2ZRhQFGUOQM5Swmoqj/", "TQw51EPkbug4/UwQYfd5O/HKfl7KVeGDuQ5zuB2Y8OciuMJQG/RziA69hXK+f3sRvVXOkZNfbbaR/Wp0", "kwgGgiKeJ7HW+Mb6tjMGORJbM6QkynBn8y6V6J2pvVmdug09Vhz9nhOxQO9PTythD4JMNA9oN3HgEg37", - "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJlgz", - "5cdkPsrzkFaiHzkclHfvTo4rlILxk+1ng2fPe8/G2096e/Fgu4e3d5/0dvbxYLIbPd3d3tldkXHSIm3t", - "9ploQdU0EDBchIePXJh6KHq4KUmgJgTYwOcrymJ+VblngpGofu82ynVd98sx7K2HEMx8SbBUxk7QwDJO", - "4TYlkW7bRH7b1MiiplHYovjk7WD7c80sMLgGZvxW5Mz4L00yf2GrT70B+5tVHefteCsMyGWYrFstv/P2", - "izY42H9+sP+5i+ayJNaNsU5O97i5TaFXDuy4lobhUgE9g42zBHas8GHM5zZro9PtFIkl8DfcurWg5eJx", - "q2yppgPbDbORVfy7IUv5pKIYQEiIAbuLD7RI4AR/KJ1Q5MJrWeMo4XmMPKOXwf4Ch9eJpyToZsD/ZG1h", - "BsvTZD1oZQLAo6FEA2WaEYOjTzdiU5oP0Et4Fx7h1OhPdhCmUIjv48LxwgSm6PPlujbazOohn1tFBr7R", - "Wg3S/4Jp62WwttHVTRgx6AD9yuGbQq1ivG5kNa+DPrP8et0gu2FxsR1EBXRmZboD9FMhxxWSoJX8NiSx", - "f44swyqRYTYr+fl2xzuaWsqd83LNux2zop1uxy0U5KQvZ6e/K6l+6fz5pBiK2CI4gbNcJuPmiiYWDxtm", - "QqWikbRZGnpzm+QLW8OIxCOjpTQFf5oMT6vJFB858eX9KdoAyMO/IGtB1v/aLAJFK3fdzvO950+e7jx/", - "0grYqBzgehn0CPKPlwe3ViCNsnxkjRBNUz86e2eMDJFR34sgk/enPo5EJrhmPXrmrkG/8+f95z6eU8zz", - "ceJ58Cz4m4GPhQ0LQpYVvKgh4PB3mszpZMJ+/xhd7vxd0HT7+oncGW834NSajsL2rRPfi79kDCbjnqlH", - "FIbcAYISshGV6g2RMAN0ThQC+ukhHIEeUaQNW5Jz2FV2xYOEtbe7u/vs6f5OK7qyo/MOzgisXYFL2Y7A", - "O2LwJtp4c36OtjyCM2068AaAEmdWxwyfM2SLCQ+qAml/e7AbopKGi7ukGtv2PG1c8vdWT7OTsosO2c+F", - "Drd0yoOrvbs7eLq3/2y/3TG2dtiRuF7NYVxukFkei3jv7/wGSJNvD88QZN5OcFQ1orhQrBuNSt1oVFCt", - "waCs32Bgz54+2d/b3dluB68Wiu6wwIGVA1vlXYFDFyCKwG4ElmKZ9XabbouQOGUI7A2JEkzTw8jlMtRu", - "H4OmPhLmtXIT2lwM1vS/dHG1+LaVFamwDZlMGCMacIFyVtTw6K/3fX4RF2Yz1zbXw3quHsp/YXr1LA6Q", - "qVV2i6XMBJlTnssv0BBXJjl1knAubvRtk8Lyhsg8UcbPSCV6f/od8BRNa0gqklV1KEuNK9CSbjm5G53n", - "ComEibxpsVrtRputXzXhbsOp7a5Crqhwg0aMslhzrpytj7I8wkmUQ9UaXOynnhWAbUHufZYlCxNEnySc", - "MxTNMANvhPCghdCMJ3E/GHKqn4wmwfAFfoUSbtCVLwnJbEEXMwj9mRZh6JygDb+UmSGlWoHR/dQwGVuy", - "o0qN+2m4UiKWoaywIudcrydW3AP+NZ9UTI4Jn0pQChWkB/TrePMZFibqHzNToGieGl0yENkcGGKNmYdu", - "VHOT8olVcK3IARndZiVxJLiUiCR0CsVw3p/WEoVXJJcV6cLrIyerg21BusZzGLjKDOxU6zpmofsxkDjz", - "OTck0DAk562ISXTGyRSzHEq8eIRsLd791nGHMy7VqACAuuFgpRpB3YZckBKWrkhvL+xB7p3gvehY222W", - "ywb43urrJaoKN9U0wGaeGlzR8Gp1CxoMkfEyBNZK1K0SxquO2XQTVLgS6J9KaJV6+GBoA5JLPLbkYb1t", - "tolGCausup8lbdWW4Xy1Nzhvi5+2Gi7tDKvZCZvwAMjGDVyUzhJtw0IzIlIKlUtQTBglsdMlC1+lNXVB", - "ZnYiCYpzYlfOyKcC2wXH5ngDUAZzNjLKpjVeX++wjXnYjGF1WQfo177YJq5IhjNX34oc1soEBkqEyxzW", - "VtGWVI7C7qzlhgWZ5gkWyCIfthmyXKQJZZdtWpeLdMwTGiH9Qd0BPeFJwq9G+pH8Aeay2Wp2+oNRUw2g", - "czM4m4BnNqTWbzmFH/QsN2vpv2CJ2TLfbwE0S5swrWBI9k80IRZG7x2j1x6hV3HP93YGTWnpDY1WEtKX", - "IRhvyrktyQZPfC4DSXwrpRxXvYjEFozeiD1ZLk0hlRa3kkM7dS7A23l0qhkan4cBcmT4dQ0BBI0JJNi4", - "qS1zjRZssc1UgjUccjlDf+fjqkG0bXxtoDLYBiuxKASZBAPpYUdXGqTNG0tr4u3uTcAegK3qicJHN8RQ", - "WFdDrQxkauInb5bKic2IXTLq5mhKi7UoleECLQqcANtre8CAeuG3QGAwwMFItYASqlC3ZuFVM5RozIUA", - "qGct4XDmZgP4Jlrm0WvtAKbQ2xlZIEFSTNmQUVYYSQG1jCBG5kR46ahcaCVrSuI++pun4gE4dpqphUVd", - "B+P5dxLxK1aMccj8QerGc6nbOWTGsihyqMpfvqSbBa1PEwqkB4MTTAkoVEjVDE0EkTN/7qHilFrGu+Ii", - "bqz6s0DuFSgmAz5WpPglYT4rK5oJqoamoZH5ajlczlSWhadW/0SVYq+oXsx1dX+5JCIsJBZTKl5pFbri", - "HRVPOTFoKwA9AkX87F+GxRdwIy3ARcrm/+qaLH86Kxqv/lZ7zQMQcXi+h8ZsGzTBRiZfphbsU/WkrQ1V", - "gXyzVbAxy74EtOFilV0llKok4FUkaXVPtkt5q0flu9FsSRJVe997tv/0ScuSMJ/lrDMwWV/aNTdPV7jk", - "GnbqtI3f59n+s+fPd/f2n+/cyMPiEjga9qcpicPfH7RBrpU+rMm//vHP96c1r88+BDsPbjQok8IRHlJD", - "Gkd1QO9P//WPf7pR3XpAIUazDMXd4LdvjNJJ/J10gQJVF147J9kK/f6wYiTABZtBG2QyIWAGHZl165WD", - "qeFttJOCcYYjqhYBRo6vTFh58UoNUrqNO6g62JDIa9q28KOac8l8XGZ3brjO0X8a33CNFp61riwl83GT", - "H/p1vVfjhS69Fn6MQ4sQA1kUNV82cBfzucKyEjmt/44gwcGlci2ntZg3VsPb1nMOIIrFFlDzQgFDsOg1", - "edJ+5G9/bTs9v2XFrFNf8Q8rzmHzEbyR1TdwIweMvtH6HNYaf7AX4O2+Go39mm8ri+pVCsSVt+7N+22R", - "prtckKC4wW7en5eZeJMP6+C7QI92DHbJy7a7FZJooCYv6SRgQOMJ6RWBejYjBcnceAT1mbd47oFUyeiS", - "TyZVUNn9ZhBywNeBrCrXC1ZKayZdRK6dzaKOYG3AdIadfTnsaBVg2NlOh52a2yqYp5ji65HtoAqiMliF", - "Cl7mmdcGKd0MxgmPLk05M6iS3UcDlBLMJMoZHP6aV217sNo71O1k3t4UGNzEhDgtsS0Y05jM8JxC6Qfr", - "U5lWAjHJNVUSAkahnQMUcwOrVKnlameoXzNZhAflpOHSwWxhG9YN6vc4cxGt5btg4JtABVn2kQjetagA", - "mmO/fn3aNQEMEHpoBlaJb3QTNSPQDLLoolbHoPw9HD88TsgIxl3HxU+X19FP/gbPqiCSKGmBsktyqBEB", - "injOVB0wP22nyFXzt5avpJxBsJ8N/wAANNu7IRAUkwhOpFw+i1VCvwVx1/IG7EqHEgd2QyQMhwJ8SWFf", - "8RvrEK4PwBgbvDLMph0/rtt4CUdScVu3qzjVI3IdERLXkTXDr7SNlbdfBmPlX2ELxlNUSLZvQ7zz8uz6", - "d5dJBWNtWm0/pp9x1gMYELelFrLDYPBZUJgqoVUwvj3siFEIxzT0QpvUZnK9eq1/JdcKgMjjPDHocmHS", - "tazKXkbrVvzWKYRNB5oLsrYO3h3UhzPx5reqEGdD1R+iSJx9604Kwy3tzjlR7t1zS0aNO1StqFJxabmA", - "f/dKNcbGkFIX2QsebaebNRLcm4WtIhb9tmUyJMMpGWWCTOj1CuIxLxjFuIofUh6kIoPBAHlupPga7T1F", - "0QwLWRs7o9OZShbVAJy9AGjRZ1VPFEQR5gyFbXa+3E334XK0m91Ov/WQcHzuYfAs1Q6xIuloFUD1Uelt", - "s9b5DC/AitPoJHy6uzcY7O4MboVQ7YZ1g+U6Kj+xtQar7TSl1HnfWUd/JUrVb6HIZl4uYHslKCRFF8sk", - "lSA4PYDEmwxHBCVkAmh0ReHw9Z7FeterB28FKov8UtC/2yi7b84HX61NU3Rlwb3dNDrOuVgF+/Gfr3GI", - "NrCZaAm7LpBzt9sbPHm7vXuw/+Rge/suUKWLRWrK9nj6cfvqabKDJ3vJs8XT37dnT6c76W5QD7ukpgRP", - "G1r9Rb/bGGVTXpJV0KAKS0Mbdg4ZEfXKxPWK3pIklJGeLDKk1qcpruAFxv++9vzfzM5vZrBSdjivTtIX", - "IbAqF6dCWQ8DdGUns9J3UZ/NyfHqWdwqA6k+kDC91YcC5NVuMFAKYrvzmRAIOWt5Db3zXmx9Ea3Milt3", - "FYU87HDSg7vcsOIh8q4hIHizXnWBL19yAdvplAuqZunq26J4rcDrhrjpj1LFVWClPjqZMihL7v9chMn5", - "SpT+uNPtJB/3qmfG/t4eYstC/RYEaLfalwpahJFB1fvVqwCvlIqHMJHsWlfXY/5hu7f9HOIQko97Pwx6", - "z6sRB12zWv7ybbu3K78O2qyhX2vP1Wjafn6jiGu3nqso6BcaqhRX3ssWBNjSeFkE2l0dLuG2ssHl46U9", - "rkHmNAqgnyvp2ctt5AtNMUnwIgQC7xlqZU179IkMjcmUMtnGbrs7KAy3++mw00eHFokbdNmy5H+leSj2", - "7tEJTVMSUy1jGtW/OYNhp6Utrq5L3KwIiPsqIK31w+La8/UQCesSrtZdk/3PyMf9LO23nca7Cr0D7GpO", - "RQWwLnixi+gEYVarBErZHCc0ton0kBgJ8WoHDhGtJFnLA2QpBzo7SRdNuUJlCn1Le1vOmu2CxfjJNdhb", - "V2BmGILY+SKAKAVSF13Fvk6OUSZ4nEdl/mgCgy4RP0Rew0JbIeSvD8m9S/sGJGZPuEDr7RtNBo129smm", - "/a7ZJjXBNm/19mD9Vt+JUaTbybN4PQ8zL7XjYDeCSF+Tghgw0VSXvSYJepP50IKjv/FXcFnnNbbkSItE", - "eeYcLJqmlikp4G4BF0MorveYJERfU8uNIJ7EZZYElSUXXc9St588mzW5OMEjtTyQXwjJtK4C+EfQX4rZ", - "IjgwV9+zuEs2Bg5WWxqHV8/UBbKrVR3c07WSWONW+SbcploFhsvXbN4GL+XSM38XYNq+aLaMgOIYfkVI", - "e9OMtW+/dGFvjfbjuzDLPaSQ9tq6Hmr4qA69t4Ahd/2XscBarKsS717IPR8ii7dWM27CfK1ngfpW58Pe", - "/xgrMxr1D7Z++Mv/3fvwn0Frc01vlkT0YjKBQKNLsuiZKj9aR+9XEU+hxIAWpqeWVAhOwYYEaOL2MPrj", - "3R8UTGPxK06XpgARWl6Jnu21E/rLfzTHN3nL+A745FqS/ewKHHdRqVRxdx1tpERMXSy5SyTb7A8Z5KZd", - "koVEXuEvK9I4Qv1OFp94EejowoiBfcLmF2hMoZKiHDKt1eIoIpnWJmwtGWrKgXPgPoLgxG/HFiBzid/W", - "IWniCQh6f7oEl/v63dsfX7/79Xj0+uzFr4cno19e/DeEeFz1TA9xT9Pe3v4TWwTcX8ntYCGKm9dT6KNT", - "G6ZvXf2THBRawOmSKM1VDkEh5DpKcknnzkGokttXTlhO1r19JYLPhNpVKglFJVhI6IROCPj14TqxQTVU", - "OmKkEqqnW+MGZWj5xjaEM+wAJ/WK34fqVuitCK92ubHVRX8ya8dCDQhp4LBDxiuUuQ9oL1QCXoWL/fBe", - "RhuQOeJKvLrE2c2bgaIeFg0GIw+/cCWfwfMvUW3z3crymnOe9LR601CSIGhNNmsRjJyHpkxGQqfJ6TAd", - "B2R4a9qd0ikO+BlC/oQvUhXTDWhtxtTS/jeWBwvnMRzX6zWYY2mWqlZfoGYkkKrXnOaQaql2VJb+rwbP", - "5MzmrlIvtq6aqJoytWWr14bwMmIOqOGrspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzN", - "iCDeRsAHJQ7+DZfM5uW0QGEx1f8yIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1XUOTvF10QO4UrBc8j/C", - "PMo6S9svfwRM+jeutiSduCZgGDXlLozAXqWiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLH", - "v2GqfuIC1MFmzJM7B3KHyz8mAjDg6jDtrTDOaUriEc/V6vNvS9fbK7+oP1rWr3WqLwYijirpvE28wKFy", - "lGNYXmm9HCTKBVWLc71eNpgb0iBd0VhYSOgIfi47hkKdnz6B0XgSSBh5SRgRNIIyqPo8ppiBxoTen3rV", - "8ExhxCW8VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT", - "3ILS9TZ/2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfA66/u1AP", - "zK3cNcve2mYKacaQ/UKy13ZzP4CgDGcHprkzGFhgc2WvX8jdMQkDW3+30aNlv62kOrtEAZj7JTXPyZbF", - "0n/qdvYG2zca06qhwNkNdfyOYZvES0A737/hQtyq0xNm0vJskrUNh/JPHBCSf9Z++6D3TOZpisXCLZi/", - "WhmXTYIxkQi7d40epySKNKuAYjx99JoR8xxhhbCJXBY5gxrG7kNNodVTYNp2m1yAFP3I48UXW8JKH85G", - "8anKzvRx+bREz1+OdgoyXt5I+8ghbBuqvQcC+hEXBbgf7KTsDZ7ffadHnE0SGinUKwjYxiNTCSE/CeCF", - "O+whLtDvOVcYFeH8j+hIW5l1XJBbt7yKtv6g8SdzvBMSMoOfEZFiZpIjzDtrDv3ScTYuifI4r7zVHOFD", - "aQ+4qRwIj7moQJCrHlH/2qoLg8vX0V4AgcH2aaYXPyDh793DCbeTLWqwPuSRg8qXKJfkMR0n62Ibl0JI", - "UJZ7SdTXQvOD+7yybBGBP+EpeiwE/JIUEl65W0uXwlYmcmYU4KAE+KZMWLTffVcV/t6WT7woGfBr6Kah", - "nIUyflUcL/rIralR+tUCIJYEgXnGy9fKmR7e13LCdu7jhMGMC0/Rt2vq2zW16pQbanFTgIPpnfIWNogb", - "WSD+fPaHG1sfvtke2tseWlkeGLmy1oW/83Ef2YjUiMcEyRnPkxiNCTJ4Ry72RGHRn35EWEQzOicAagdF", - "2vJE0QwLiCxJUYwVNj70RsPESrNE0dyWbq7n4hDLBa7jWEgyAhy+URP+ZBmBSBkjMdKfWOi+Ek5wqW63", - "OftBA3vRYHk1oqsZl6TA82PKu80hvVka7Ria7Q/ZWwv0qhcQgqkdr5EkAbjaFfYfzhAeMvvB946FuEAw", - "idOSc2EBmIHUIFOabVlObdMjHcmIh7B23hKGmerJjER0QiM7rUuysPGcwQZb1V3SA3bjfH9aJGygnc0w", - "XhvAM4bBeY+LZ8hSUtV/wyAIOkryuHRyOQghLMY4SYKFOaYJH+NkZNbnkgR8gi/hDbsofn1/501iPCam", - "Vnu2UDPOzN/5OGcqN3+PBb+SRAw7m/0hg0QMu9Yk7pYCIrqCQm5pxvU5Ezw1fW6ZIW79cUkWn/pDdhin", - "lDmKgE9wIjki1/Ad1LcCzAzDvRrowZymsB/8KJeKpz7yqaM7M0yeqyxXNqNEEtUNoX4OmeLoD4ft+Gnr", - "j7LHT+AsJjjWdOK9YqYEsnXTqOUI69mP4NWAu53AAgw7+iI1YR5TgZkysJ0FOCWa+lu6UVRH0Id0s77C", - "EWYo45mpLAFENcOa5CptAFYDThKk4Ci5b7XgDjvZMB8LvZeOG3H3DFBa7RhRhk5/9A7TYO9Z+DxJEgkS", - "iij5r/PXvyK4lfUemNfKcC2T0sG0wIDiHFynjqe9wNEMGUcVFBMcdmg87BTu3HgTxppLGy7T64FP8Qc9", - "tB9MN10a/9Dv66aMu/IA/faHaeVAn6UsNTigw86nLvIeTKma5ePi2YfwgjbBl51XGAHaMNfcJnASTAFp", - "xrvxzRWJWYy4vQWSBcKo5EB+4MqYMiwWqxIJA0tvV5BPTCSjtxh/DCFycdg5GLrYxWGnO+wQNoffbIDj", - "sPMpvALWa9lcuQ7us8K5WRDRk8Fgcz0Stl3fgM+yhWPgC+uAjVpRUXZT76CFYf1z+Qf+rfXPwvWDme68", - "hCYyir8zvj9CB4QnsfuaaMAFURO7MYtI4sTu9Yae+3ce6M2KSJLcN4E+FHkW7rECqf9RkSNsVnmMVprv", - "H5jiBvd1qVTM9g9Dv4/Ofh6wnlvbOZm7UOdwnRLAoLGqNDIvIyzROYypd66V7xfwa9/+1+l+gKl4kfDp", - "xYFR3VHCpyihzOYDeIHKWjywawkfGRia4juLSuOKxG0YSeJf//gnDIqy6b/+8U+L7f6vf/wTjvuWgVeD", - "GtMXM4KFGhOsLg7QL4RkPZzQOXGTgSqwZE7EAu0OrM0fHiGv1L2V0uSQDdkbonLBvLwJU69N2gatq0DP", - "h7KcSAvjo1+kE1tMxsQ2Buw27iybpbzXE90NwCHCDLwJ6FvR0QBgyVFTaNtqop2wydTMuWI0rYdpLgXr", - "recvilwrQ709M8AbMhhY4tC5gwd20mjj/PzFZh+BtmWoAgoGge5QNmPViP43nrSeJxmOUmUosMqGN0U4", - "w2OaUGdybKh2Yo5giqMZZaSMLy6wxl0TB26kmsccnp0gGwjZhVeH7PX5FphYFYlULkjXcgJhEUbLcmjc", - "5rlAD8C/qILosJ59d8gmBEOe0MmxYQIeCHeRD1g0zADIA2JcqapUXusOmUGStcjF+uClPCYJfAT9T7Ei", - "V3jRRUWtW1cdJcFKK8Syq18eMoP1ategB1AlyBtmH/iZGVLPRfLanC1BJolWjSEC35T9hr43JlwgG+Hs", - "Vfl33ZkkSzMsvWgpjl6f6/lNQRPkxh4ILb0+d7ux2UWSoyihQA0RZkM2hUAgB97LWWVXi4SyGRZxL+L6", - "EvDBnC4Zv0pIPG3isUc+kd2hJFPpJ3Ccfq6T62MTLmbLE9CH2ADUrfbcHdt32rnubIt/Jt+dLQR5A+ed", - "seASw2/M6n5z5LVw5IXXzTn1Qp61Y4fAeHcRv6aLBwr4dbS3vObmibdkD2HRQxsO2ga8Ilygs6MThONY", - "ECk3/73tfXqmhkpL+U/fj5oVP0ToiR0LFxb0z9pbqgTyWNjBGztqhN286vV1/fttq1J8p/GmK+rwlFfe", - "3d8etU5vco2UQm9Ja99ukrXBtlRGHMoMltTSA9EoIYX4UpxTn4rWWZVNGG9x5awUlyx7Pjl2B/L+7Mu2", - "65zV74Z7YIrHNYb4gIywmmrtV81+TNT8rthFhza9wvz8dZHm4P6koPs2RYfI/DGpi3Ft2TQXNEAnjRfo", - "S6IMvMld6um2h8DEz4lwp9oMdGFmXUzLfIoMTgtMCCwxq3XfE/NKO9XXtPdn0nxheW4isdgl/yaitFB2", - "y7VapeCe2BLQd6ffQg83Um+/XNiKJbDAIoMVdezcTmBZ3cBywaLNb5ErX5yiTVxjqcQKN28SF5Zsg6ZU", - "6Fn3JdcdMr/euJbprF5LGZokdDqzToCYTiBWT/n1u2GUO/cwyqJOtsCK2BDFx5j3e6YX2XqB50Qo9Pro", - "xKy/f6Vu/QFBq+tVJce8Vt6u79686hEW8bhwnjTLpPbJF1aYDP1Xcnnv/9Q9wnxW6sSDJoHxM/bfBJMj", - "E//ep/x/7fyU0LHAYvG/dn7CSUYZ+V+7hwlWRKrNOyOWwX3ddPetwDxi4tP6C60uGrAmNgXI2DUCf/FW", - "S5nfvf+nEvvNpG8k+Bfr+k32byP7+8u1Uvy3W3GnCoDp44E8XAWxhVYbHn2DtLkHo6mlSA/SpuJFKkFt", - "ZlwqePT48pttUDktKM6/Nlpa/8sDufL6cKR7ctyFhYSK0lDRwqYP3pMvwI3j3oVb2+/9OwIO0zGd5jyX", - "fmZiilU0I9Jm7SakyoAfm9hdXs+NgvdXTKWD+7w67l2u/kb3dyTx1zfUMG/j0Fsn87u32sr89n0t8xtE", - "U5vZbMtudF1Jps2GQGuHadqWjCvQr8sB4KFxhXQR9E4rKqW6gECDOBiy/631j98UwemHH1wKZT4Y7DyB", - "3wmbf/jBZVGyU0cqhClBbQW9w1+PwYs6hUBZKLJXJmzXx2FqdgPpubIC/3YKUulIbq8hOSr8piG10pC8", - "5VqtIdm9uFsVqVqa5N51JEdvoQW3mOJ/Ti3pT+4eqWhwMp9MaEQJgwIvkJgul+IBjSb3zTNyy4RkZv2R", - "XjBRRRJprUYWXGuNhF7WlL73QLKTsojWfWuPrnz140ys4pmtB2v1tVJaaFbYvjZ6GNzv7XX/itpjJjGj", - "ES0vXaaF7kBlIFMTKs1NFlrxZQkq1kdv375yGWda4heuzpTirriUq7s5ZH5xqT56UVbtMi+4FrRETmKb", - "oQp5eLZcU0xwnFBGIESXyFByWLUk3IMeiy8vVIbr3bUSKu/5WNoKpg8nVD4YK7gX8e2kUhialzq+Xwqv", - "OC1OhINT86j4lWVAAcYTEp+2cK54z+awbs24ATYLYzueJTgCaEf9mkEds7ABBmbQbwqwAARPEiIMmlyW", - "K1cDcciKwVHm1Xi3NW4udPOjnCmaXHRNhAxAgkiE2cJCKg1ZpTOsFEkzBam9kLYOIxQkMyOuFX/Ug6Y8", - "l/AWZOH6XSKcXOGFHDKbDGw+h0K5gkQGeDFJ+uhnDjgMCE8xZR7jNRUIv5NDdkHjhIwsjMIFohLJGReK", - "MBKjlM+JrPZLsEgoETCJI6xXTqIULwDPzEA7mvXhGTGYYRWwBq7/jVlMoZad7rmY8sGQYbQzGKCUYCZt", - "6rXEE7hwbBsIBlEZ0PcIo73Bc/tVbd8Ac9ct/4Y+TUKQOY/wOFkgoqkYwB/UJmxgamtLmhq9evsmVEiz", - "X4XJ0BYNq2wsla5UYtxFOSuTy8F8nrMiF1xvl8oFg3laxxqhorgGLZ7GmERYryfj1X4AyZBHUS5CF6Te", - "aq/I6b+j4OhN7xyWKpy6nYAWHpEY9pxxNYMzzeEobX7fQFUlUf05LprgIeECYeTRdWkkIFEOrHEDkP8u", - "yop9zFXgvdj83p0dfXwtI3DH32DvPZb7CYiITyaVA7j+ajIHeFXKxDIJ/1nP6ZEr1eqzuJjiKeNS0cgx", - "w3pl928KYWuFcPXKBql5wsWlL1tV6fcnLi7bamAWT5Q+LkXMn+FXaNvXwwPs5oc38YOB2SgrmmjuXUmr", - "01dxSkHookq62GGOEs6m+hSVhu57t8T7Wt2GwWHTl6kw/uMCNUcrISP7o6n2qidja2mC1T6yrT40L9K9", - "34N/51euEE2zhKQEqsH2DLHpzS4RlqByPpUeztDNeKU+VX46sNEFpXHpd504BHTlNmwDpPfl7Qoy1YRP", - "1+P4FZ070LoAkN+Qmcr+BF0Yb84FKniwFmgNaj66mtFoBqB+oLfq9g3mH86yiwLPePMAvYSD7MM6Q+cb", - "Bitf05rkCTFYffM0vThYrnf6/vQUPjJ4fqay6cUBcjVOi/tD6rd8kD49iwRLhX610IMbhTIOO3qhsNY3", - "i/ltWvi+Em96yEJQfoxc2QbpBF14qH4XDZBTjt++4tMHE8a6zVUCzFwUR1Z1BNokLO40xU3QJAzotz0Y", - "hMCrW4ILmmHcMbbg0mBe8WlRoaBCyjjL2pKvHSZQ8TxNV9Aw2vCQyqSKea7+IlVMhICPLXU3ETfawJGt", - "ToUvNaFaXDp3sDeB/ILRQQYyPLhUmql2uh3C8rRz8Jv91zxNO92OHY8HNX4D4X4NSGO9weUoFr0zHhLj", - "N7H8JhiLVWbvgSzWbg6rTjdL5G/MC396b6Gz2T0gGYJ8UDPifk0iqDfeqsGH8QIsEkb2/D5GBoiSKEq4", - "JBUHz+PBo7KGrprM2Gwocmvc08OLc1fAp01QyLn99Nx9+RXo3utiRdyYkZvuvQeNLI/gMefWyqXZTLio", - "gxitiyb56gnpy23J0lTbUMg32ry5lbEVYWo9YZlF2A9iU9AN54qnWNEIiglFM86lR/YF4rAp+2WNxwVl", - "gmnFaLk2KP9Ck+qFNUNfWDXiwJrMEPYf2T768LkN5Q9/4R6VX/zkWQUKjt91oj8A7kO1c0HJBGU4l0RL", - "dXlKULSINFc01aMIjmYowpnKBYHCeASllNE0T30oab1jcwywFxfb6UUXjXOFEiymoJWZhy7YJuJpSlhM", - "wD43ZDOC51SrlAIlWBEWLXqSQEHdOUFXXFwmHMdgYshiDJ4eKMgniKZAwOVOicIxVhgEnQt94kcmL+ii", - "qLFr1HpGrktqiIdM5Ox7UyRAN3vhBnqBCKBgUzkrajFGOCYsCqJDn3/dbOzL26LPiapP9IEig27FSx8y", - "VMi3ubrhfB1RRI8svJkLu41t2PwKoVc2q7DVhApHRv+eR9rM1c3xgRxMxRKvOsVfh2epILqvxrv08O4j", - "LlCcm+68Uwlk/mf1CRUMxQ+2gmRNs423dQwVReeKZb4Rz9v6w/15cgtb3lfCCbuNin1TeaNy0l8Dy7Wr", - "eiue+0BGTGtL8m1yD8eCXUTXg4lPXHhc7rEYWy3DNkez4Ns+d1ICg/bF2Te2XWfbNuDhtmzb2WaXXPoe", - "I6esBzGiYQ5uzbiNrNqaDv5Ns1Fqs/NY5oOzyNJzcW9s8aRghIY1ZniRcBz/GYKEV/iPIi6EQZQAjIrH", - "hGjqWQ399ACwzZV107ouW/P96elmE5cQaiWPEOoRcwgvJUd/lsbLBtzXcyIEjS3wJzo6PbbhulQikbM+", - "ep1ShRRHl4RkZUYLZBX29fwctsZyjfYKiEa3Q5gSi4xTptaOonz1bgbz6VaV3e+ZT1qI7G/u8NbucLDs", - "Pz52BlwGcjbMBFZrpgqrtaU7KZtwkRq5DI95rlvXPEgvk95PUwJzQhMiF1KR1EQlTvIEjhuUW7Alde13", - "Zpe7EJOrT45Jl8uISKmUlDM5ZDZXJCNC960/1+17AVZBh4DCBX89M0zy6wje04Mx8WpYNa0aoCBBqc7O", - "QWcLZ9lWjBVuCBCzw/uMIf0E0XhILtIxT2iEEsouJdpI6KVRT9BcokT/sbkynG8E333pgsG3P1l6pU/Y", - "hAfLsRmaLYj5T5XVZdmac0w+Orb2kviHxfEf2OgwW1tfklgQnPSgxK/DwkG5ogn9aFidboRKRSOTcoSL", - "tXt/WjDV/pCdEiX0OxhS25LEIBqAdrmVCR5tDfPBYDfKKACq7RIYHDC85scp9Hh09s6koZKUi0V3yPQ/", - "oOG3h2fGuzvB1prgDdTWIkYnW6/XBDifwzL9G0cImgmuRC8Ibvg3l+DNMUYaz5BsOKI8W6Uq8exPH8Jq", - "JbhvdoXHaVcAkKdiNhtTgSMQiuUsVzG/YmEbwpwnear/Yf44WQcVpnA0ew+vfjXSrhnO2m7cBB/FobRz", - "iokpF/kgTg+zYI81ZlUvnJsCCDGVaMDgLXCo/ozU/eXN9/46foXuTruirhTrV3O27vvms2NwCBv+ejyW", - "Y24ozc1E8dXWpytMm61PPyY8upQWisU3G2q9DSDL9Y8lxLR1EYKYAJmhyEIYGaAsIrtDVjNAGsQfiTBS", - "RKSU4WQL5mwaAbBsZ8XCc04hQTuCPJWepDFgJiWAiA3wd3o2YKhyDXgeXWmLVfnv+M5IxdGYRDwlDkB8", - "M6S6/Q1T9RMXVTTwr4UvvvXWHyABMQV7+xoA9OYePwsQ/RRfQ6h0nFuHshvRxkte/mhMQV0EezPs7A7k", - "sNNFw85OOuzoHTjCYELFCu2jlLJcEdlHx8a+BSm4TwZIkoizWDocc2fB2x3IpoRcQ5YN2Z1P4Lv7FHss", - "VcFSvrGdhNiDfg/p7yFpB234B86eybgLhy5GPFfG3G/PlX0rJgrMI5v37qv1zsg33b4NJ/+bPb4VHgW7", - "rNmlt/WGs2e5nJFmk9srUxsoV2PAx3b1OuUM/Z2PZRcxcmWs4UKq/hLf01+fmQ7uA7tfd3UT3H4792+g", - "/S1A+8u1CoM1mgBLfSU76jCIjeQ640IBiqPNtTc0BJoEIEfwCCfo9dHJkEWaFRloQUFSDtzJQoybW/jw", - "b+foxdGbLjqG2pHo53y82UevWbJwFbyNj2bIjCRmmFeEGRobqiVx6Ho2Ywfquctgcd3BAxVjNicj4Flx", - "e+WCxLudGcExSCR/dF5x01kAdfjNK32AAPjXfFlse2el8NF5Q5RY9A4niojlZk9tnhQrMDPsJe0g6Kzg", - "ZoAvdYfSIa+VfRrZwEBj7O50AkgZn77VUbj7mqP34yUzcSKmgt04B6RRBkkGOF48rlgmOUMFcwyxQP+6", - "LioRNGUJW162UsGALpsiv78ik/tK3lXBlv93PV0w00fraMoq+6SJuKhgstbT65KDZwYO2TqqIpzhiKpF", - "F+EksXeUvQmKiJReIf6OBcGXMb9i/SF7U9ROsQm96OjsXdc5alFM5aVpwfpi++j1nAiZj4vBIThoxmsM", - "a07iIVMcRTiJ8kSLG2QyIRHk4kJJFNngyy2G0rnDs1N2Eqzf4kW154+ubFyYJmD3SrKoU9yW2eotQaIE", - "07QZfNwKahBwCKEGY90oZ4iySWJDqiLBpUS2qR5J6JSOExsgJPvo7YwgiVMyZFmCGSMC5dJExeuh9zJB", - "pMxNgrduAEB6DUV1UQksmAmubGhCwrmQJppAU/j7UyQVyVaQ2RvT8inM+Y5kW9O47emBjNS1MTSbQuwr", - "SG+IoRSz4JqO8sQFMN5rKLoZ0ENLiY/l4L8VdDolQp8KbJisCcczx9otpzn0lYzlxhKS58Vb7UpIFq16", - "WYlext5KYLhRibUdd24W9Rfo/JI2YgfaRzfLIv5Ff9Sy72q2angQ9tFnzvLPUpn/3EsSbGvAKin8sZmT", - "vJFXjmol0XY9rFbrzNq7zHRtjZ/1YLBZjxktC1fSZ5sU3q+PEAb3i/Jw30XWHjdtVdCuKrppQ8r/ejT9", - "r4IC7wZG/4FRTm4Bo/9V5d0DzvnD4Z8ED+pD5dFXfM+ufu2fHgn/rtLnDRw+wLE1pc8brmeDV1cqSu/t", - "O+3UJNvin0mCt/GON5Df3bJ/0/pbqAzeYq1zQWuCJ2mmFi6gzfoqy6AzST+SfoMjuIhbvTtX8C1COr8c", - "eTg6bQzo/HOWm3+QmFFbOpBKdHIcqOP+yDAG/TNXuVi29K3TwyKa0TlpNrpXT7BdokyQXsYzcK7EZsHs", - "eri7TGHRn35EtnmLuWr/BbUnAaqfxCimgkQqWZg6oJojmD6+k0hwrQnAcy4WzVEi5oj8JHh6aGez5j60", - "Z8oaw8o4w3TRi7HCvbnjNitMaJ8R3eniKTXDQ5Shlz+iDXKthKlwgSZa80F0UiypKeAvgSY3/QFvDxos", - "m/QjGU3HbUa5olbJa1sLBkW5VDx1e39yjDag9tmUML0XWtSfgCSbCT6nMYkrY+zMeWJWdbthQW9qd9VC", - "RVG4zikXZnAPIsO0uZCmH2lWZQtFSMyYMgyDW1sVpHqmTBK/7g9T5gJw7B65UXy7wqzmt+GUHU2JUIfT", - "LqLi3EA8b3675h7zNecnQ7k7rXLbufCc1cbrdvlRLdOW7qLwQ5E7d79m6/dfT0oPlY8ym8eazueFQtpk", - "Nv+6SHBwf/fDfZvL3z/iFNCXxCnfnqkcGtAthgjmFcR0x2ROEp6lUA8d3u10O7lIOgedmVLZwdYWxH7P", - "uFQHe8+f7nY+ffj0/wcAAP//uZ+MGF3vAQA=", + "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJhhK", + "kRmVgZF+q3IhJxbf0YrS2MlDBm+6gH8onfpyIbfGudzKIrpl82+2AJvjGWBz7AWTp2MyH+V5SDXSjxwY", + "y7t3J8doA34BbFlIoawSMMZPtp8Nnj3vPRtvP+ntxYPtHt7efdLb2ceDyW70dHd7Z3dFIkyLbLrbJ8gF", + "NeZAHHMRtT5y0fOhoOam3IWabGLjsa8oi/lV5foLBsj6vdvg23XdL4fWtx5CMCEnwVIZ80UDJzuFS55E", + "um0TkG4zNotSS2FD55O3g+3Ptf7A4BruiLciZ8atajAGChdC6g3Y36zqOG/H8mFALvFl3Wr5nbdftMHB", + "/vOD/c9dNJe8sW6MdXK6x81tighzGMy17BCXoejZkZyBsmNlImPVt8kknW6nyHeBv0EYqMVSF49bJXE1", + "HdhumI2sulYakqdPKvoKRKoYDL74QEsqTh+Big5Fir4WgY4SnsfIs8UZSDLww514uotuBtxi1kRnIEZN", + "MobWcQDTGipHUKYZMfgfdSM20/oAvYR34RFOjVpnB2Hql/iuNxwvTLyMPl+ua6NkrR7yudWv4ButbCH9", + "L5i2XgZrsl3dhJHODtCvHL4ptD3G67Zf8zqoWcuv1+3EGxau2yFnQGdW1DxAPxXiZSGgWoF0QxL758gy", + "rBKwZrMCG2B3vKOppdw5LwW+2zEr2ul23EJBqvxy0vy7kuqXzp9PiqFAMoITOMtljnCuaGJhumEmVCoa", + "SZs8oje3SeyxpZVIPDLKU1NMqkk8tQpW8ZGTqt6fog1AYvwLsoZt/a/NIn61ctftPN97/uTpzvMnrfCW", + "ygGuF42PIC16eXBr5eQoy0fWNtI09aOzd8b2ERmrQhH78v7Uh7fIBNesR8/cNeh3/rz/3IeZink+TjzH", + "osWkM6i2sGFBJLWCFzXEQf5OkzmdTNjvH6PLnb8Lmm5fP5E74+0G+FzTUdjsduIHFyzZqMm4Z8okhZGA", + "gKCEbATLekMkzACdE4WAfnoIR6DeFNnMluQcpJZd8SBh7e3u7j57ur/Tiq7s6LyDMwIjXOBStiPwjhi8", + "iTbenJ+jLY/gTJsOUwIQzplVfcPnDNkax4OqQNrfHuyGqKTh4i6pxrY9TxuX/L1VH+2k7KJDUnahWi6d", + "8uBq7+4Onu7tP9tvd4yteXgkrldzGJeyZJbHAvH7O78B0uTbwzMECcETHFVtOy5C7EajUjcaFRSRMODv", + "NxjYs6dP9vd2d7bbob6Fgk4snmHlwFZ5V+DQBYgisBuBpVhmvd2m2yIkThkCe0OiBNP0MHIpFrXbx4C8", + "j4R5rdyENheD1cCXLq4W37YybhUmK5OgY0QDLlDOitIi/fUu2S/iWW3m2uZ6WM/VQ2k5TK+ehScyJdRu", + "sZSZIHPKc/kFGuLK5MxOEs7Fjb5tUljeEJknythsqETvT78DnqJpDUlFsqoOZalxBYjTLSd3o/NcIZEw", + "kTctVqvdaLP1qybcbTi13VWAGhVu0AidFmvOlbP1wZ9HOIlyKKaDi/3UswIMMIAEyLJkYWL7k4RzhqIZ", + "ZuAkER7iEZrxJO4HI2H1k9EkGFXBr1DCDejzJSGZrTNjBqE/0yIMnRO04VdYM6RUq3u6nxomYyuJVKlx", + "Pw0XcMQylKxWpMLr9cSKe3jE5pOKJTThUwlKoYKshX4dBj/DwiQjYGbqJs1To0sGAq4DQ6wx89CNam5S", + "PrEKrhU5INHcrCSOBJcSkYROoUbP+9Na/vKKnLcii3l9QGd1sC1I1zg0A1eZQcNqXV4tdD8G8nk+54YE", + "GoacwRWhks44mWKWQ+UZj5CtIb7fOhxyxqUaFbhUNxysVCMoJ5ELUqLlFVn3hT3IvRO8Fx1ru81y2bjj", + "W329RFXhppoG2MxTgysaXq1uQYMhMl5G5loJBlaii9WhpG4CVlfWH6ASWqUebBnagJwXjy15EHSbbYJk", + "wiqr7mdJW7XVQV/tDc7bwrqtRnE7w2p2wiY8gP1xA8+ps0TbaNWMiJRCQRUUE0ZJ7HTJwoVqTV2QMJ5I", + "guKc2JUz8qnAdsGxOd7gs2LORkbZtMbr6x22MQ+bMayuNgH92hfbhDvJcELtW5HDWpl4RYlwmVrbKgiU", + "ylHYnbXcsCDTPMECWUDGNkOWizSh7LJN63KRjnlCI6Q/qPvFJzxJ+NVIP5I/wFw2W81OfzBqKk10bgZn", + "8wLNhtT6Lafwg57lZi0rGSwxW+b7LXCMtokeC0aK/0QTYtH93jF67RF6FY59b2fQlC3f0GglT34ZGfKm", + "nNuSbPDE5zKQW7hSynFFlUhsMfKN2JPl0tR3aXErORBW5wK8nUenmjjyedAkR4Zf14BJ0JhA3o+b2jLX", + "aMEW20wlWFoilzP0dz6uGkTbhv0GCpZtsBIiQ5BJML4fdnSlQdq8sbQm3u7eBIMC2KqeKHx0Q2iHdaXd", + "yviqJn7yZqnK2YzYJaNujqbiWYsKHi7+o4AvsL22xzGo16MLxCsDSo1UC6jsCuV0Fl6RRYnGXAhAoNYS", + "DmduNgC7omUevdYO9wq9nZEFEiTFlA0ZZYWRFMDUCGJkToSXJcuFVrKmJO6jv3kqHmB2p5laWDB4MJ5/", + "JxG/YsUYh8wfpG48l7qdQ2YsiyLPVKVcpG4WtD5NKJC1DE4wJaB+IlUzNBFEzvy5h2pmahnviou4sRjR", + "ArlXoMYN+FiR4peE+aysaCaoGpqGRuar5Sg+U/AWnlr9E1Vq0KJ6jdnV/eWSiLCQWEypeKVV6Ip3VDzl", + "xIDAACIK1Ba0fxkWX6CgtMA8KZv/q2uy/OmsaLz6W+01D9fEwQwfGrNt0AQbmTSeWrBP1ZO2NlQF0uBW", + "odks+xLQhguhdgVaqpKAVyil1T3ZLhOvnizgRrMlSVTtfe/Z/tMnLSvVfJazzqB3fWnX3Dxd4ZJr2KnT", + "Nn6fZ/vPnj/f3dt/vnMjD4vLK2nYn6bcEn9/0Aa5VvqwJv/6xz/fn9a8PvsQgz240aBMZkl4SA3ZJdUB", + "vT/91z/+6UZ16wGFGM0yQniD374xSifxd9IFClRdeO2cZCv0+8OKkQAXbAZtkMmEgBl0ZNatVw6mBgPS", + "TgrGGY6oWgQYOb4y0e7FKzWk6zbuoOpgQyKvaduiomrOJfNxmXS64TpH/2l8wzVaeNa64JXMx01+6Nf1", + "Xo0XuvRa+DEOLUIMZFFrfdnAXcznCstKQLf+O4K8C5dhtpxtY95YjbpbT4WAKBZb180LBQyhtdfkSfuR", + "v/217fT8lhWzTn3FP6w4h81H8EZW38CNHDD6RutTa2v8wV6At/tqNPZL0a2s9VepW1feujfvt0X28HKd", + "hOIGu3l/XsLkTT6sYwIDPdox2CUv2+5WSKKBmrxcmIABjSekVwTq2UQZJHPjEdRn3sLMBzI4o0s+mVSx", + "bvebsdEB9geSvVwvWCmtmXQRuXY2izqwtsH4GXb25bCjVYBhZzsddmpuq2D6ZIqvR7aDKrbLYBVYeZn+", + "XhukdDMYJzy6NFXWoHh3Hw1QSjCTKGdw+Gtete3Bau9Qt5N5e1NAgxMT4rTEtmBMYzLDcwoVKaxPZVoJ", + "xCTXVEkIGIV2DlDMDdpTpcSsnaF+zSQ3HpSThksHs4VtWDeo3+PMRbSW74KBbwKFbdlHInjXghVojv36", + "9WnXBDBA6KEZWCW+0U3UjEAzyKKLWnmF8vdw/PA4ISMYdx2uP11eRz8nHTyrgkiipMXvLsmhRgQo4jlT", + "dRz/tJ0iV00rW76ScgbBfjb8A3DZbO+GQFBMIjiRcvksVgn9FsRdyxuwKx1KHNgNkTAcCvAlhX3Fb6xD", + "uD4AY2zwqkObdvy4buMlHEnFbTmx4lSPyHVESFwH/Ay/0jZW3n4ZjJV/hS1GUFG42b4N8c7Ls+vfXYIX", + "jLVptf2YfsZZD9BJ3JZaJBEDDWixaqqEVoEe9yAtRiF41dALbTKuyfXqtf6VXCvAR4/zxIDehUnXsip7", + "Ga1b8VtnNjYdaC7I2vJ8d1C2zsSb36pwnQ1Vf4jadfatO6lXt7Q750S5d88tGTXuULXQS8Wl5QL+3SvV", + "GBtDSl1kL3i0nW7WSHBvFraKWFDeljmaDKdklAkyodcriMe8YBTjKqxJeZCKDAaDL7qR4mu09xRFMyxk", + "beyMTmcqWVQDcPYCWEqfVdRREEWYMxS22flyN92Hy9Fudjv91kPC8bkHDbRU0sSKpKNVuNlHpbfNWucz", + "vAArTqOT8Onu3mCwuzO4FXC2G9YNluuo/MSWQKy205RS531nHf2VKFW/hSLJermu7pWgkKtdLJNUguD0", + "ABJvMhwRlJAJgOQVCa3rPYv1rlcP3gpUNou2oH+3UXbfnA++WjKn6MpijrtpdJxzsYpB5D9f4xBtYDPR", + "EqReIOdutzd48nZ792D/ycH29l2AXReL1JTt8fTj9tXTZAdP9pJni6e/b8+eTnfS3aAedklNZaA2tPqL", + "frcxyqa8JKtYRhWWhjbsHDIi6gWT64XGJUkoIz1ZZEitT1NcwQuM/33t+b+Znd/MYKXscF6dpC9CYFUu", + "ToWyHgZ/y05mpe+iPpuT49WzuFUGUn0gYXqrDwXIq91goELFduczkRly1vIaeue92PoiWpkVt+4qCnnY", + "4aQHd7lhxUPkXQNm8Ga96gJfvuQCttMpF1TN0tW3RfFaASMOcdMfpYqreE99dDJlUC3d/7kIk/OVKP1x", + "p9tJPu5Vz4z9vT3yl0UgLgjQbrUvFbQII4Ni/KtXAV4pFQ9hItm1rq7H/MN2b/s5xCEkH/d+GPSeVyMO", + "uma1/OXbdm9Xfh20WUO/BKArHbX9/EYR1249V1HQLzRUwK68ly02saXxsja1uzpcwm1lg8vHS3tcQ/Jp", + "FEA/V9Kzl9vIF5pikuBFCJveM9TKmvboExkakyllso3ddndQGG7302Gnjw4tQDjosooX/fjNQw16j05o", + "mpKYahnTqP7NGQw7LW1xdV3iZrVJ3FcBaa0fFteer4dIWJdwte6a7H9GPu5nab/tNN5V6B1gV3MqKmCI", + "wYtdRCcIs1qBUsrmOKGxTaSHxEiIVztwQG0lyVoeIEs50NlJumjKFSpT6Fva23LWbBcsxk+uwd66AjPD", + "EMTOFwFEKQDE6Cr2dXKMMsHjPCrzRxMYdIn4IfIaRNsKIX99SO5d2jcgMXvCBVpv32gyaLSzTzbtd802", + "qQm2eau3B+u3+k6MIt1OnsXreZh5qR0HuxFy+5oUxICJprrsNUnQm8yHFhz9jb+CyzqvsSVHWiTKM+dg", + "0TS1TEkBdwu4GEJxvcckIfqaWm4E8SQusySoLLnoepa6/eTZrMnFCR6p5YH8QkimdRXAP4L+UswWwYG5", + "sqPFXbIxcGjf0ji8eqZckV2t6uCerpXEGrfKN+E2lVAwXL5m8zZ4KZee+bvA+PZFs2UEFMfwK0Lam+YS", + "APZLF/bWaD++C7PcQwppr63roQbb6kCFC3R0138ZC6zFuirx7oXc8yGyeGs14yYo2noWqG91Puz9j7Ey", + "o1H/YOuHv/zfvQ//GbQ21/RmSUQvJhMINLoki54pPqR19H4ViBUqH2hhempJheAUbEgAcm4Poz/e/UHB", + "NBa/4nRpChCh5VUO2l47ob/8R3N8k7eM74BPriXZzy4MchcFVBV319FGSsTUxZK7RLLN/pBBbtolWUjk", + "1SOzIo0j1O9k8YkXgY4ujBjYJ2x+gcYUCjzKIdNaLY4ikmltwpa4oaZKOQfuIwhO/HZsXTSX+G0dkiae", + "gKD3p0sovq/fvf3x9btfj0evz178engy+uXFf0OIx1XP9BD3NO3t7T+xtcn9ldwO1se4eZmHPjq1YfrW", + "1T/JQaEFnC6J0lzlEBRCrqMkl3TuHIQquX1Bh+Vk3dsXSPhMBGClklBUgkWqTuiEgF8frhMbVEOlI0Yq", + "oai7NW5QhpZvbEM4ww5wUq8mf6icht6K8GqXG1td9CezdizUYKMGDjtkvEL1/YD2QiXgVbjYD+9ltAGZ", + "I67yrEuc3bwZVuth0WAw8vALFxgaPP8SRUDfraz6OedJT6s3DZUSgtZksxbByHloymQkdJqcDtNxQIa3", + "pt0pneKAnyHkT/gixTrdgNZmTC3tf2PVsnAew3G9jIQ5lmapamUPakYCqXrNaQ6plmobgHcBWdjkrlIv", + "tq6aqJoytWWL6obwMmIOYOarspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzNiCDeRsAH", + "JTz/DZfM5uW0QGExRQkzIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1eUXTvF10QO4UrBc8j/CPMryT9sv", + "fwSo/Deu5CWduCZgGDXlLgwMX6WiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLHv2GqfuIC", + "1MFmzJM7x5eHyz8mAjDg6ujxraDXaUriEc/V6vNvK+rbK78oi1qW1XWqLwYijirpvE28wKFylGNYXmm9", + "HCTKBVWLc71eNpgb0iBdLVtYSOgIfi47hvqhnz6B0XgSSBh5SRgRNILqrPo8ppiBxoTen3pF+ky9xiW8", + "VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT3IKK+jZ/", + "2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfQ7kBd6EemFu5a5a9", + "tc0U0owh+4Vkr+3mfgBBGc4OTHNnMLDA5spev5C7YxIGtv5uo0fLfltJdXaJAuj7S2qeky2Lpf/U7ewN", + "tm80plVDgbMb6vgdwzaJl4B2vn/DhbhVpyfMpOXZJGsbDuWfOCAk/6z99kHvmczTFIuFWzB/tTIumwRj", + "IhF27xo9TkkUaVYBNYL66DUj5jnCCmETuSxyBqWV3YeaQqunwLTtNrkAKfqRx4svtoSVPpyN4lOVnenj", + "8mmJnr8c7RRkvLyR9pFD2DZUew8E9CMu6oI/2EnZGzy/+06POJskNFKoVxCwjUemEkJ+EsALd9hDXKDf", + "c64wKsL5H9GRtjLruCC3bnkVbf1B40/meCckZAY/IyLFzCRHmHfWHPql42xcEuVxXnmrOcI/Oe7Ym8qB", + "8JiLCgS56hH1r626MLh8He0FEBhsn2Z68QMS/t49nHA72aI07EMeOSjIiXJJHtNxsi62cSmEBGW5l0R9", + "LTQ/uM8ryxYR+BOeosdCwC9JIeGVu7V0KWxlImdGAQ5KgG/KhEX73XdV4e9t+cSLkgG/hm4aylko41fF", + "8aKP3JoapV8tAGJJEJhnvHytnOnhfS0nbOc+ThjMuPAUfbumvl1Tq065oRY3BTiY3ilvYYO4kQXiz2d/", + "uLH14Zvtob3toZXlgZEra134Ox/3kY1IjXhMkJzxPInRmCCDd+RiTxQW/elHhEU0o3MCoHZQpC1PFM2w", + "gMiSFMVYYeNDbzRMrDRLFM1t6eZ6Lg6xXOA6joUkI8DhGzXhT5YRiJQxEiP9iYXuK+EEl8qJm7MfNLAX", + "DZZXI7qacUkKPD+mvNsc0pul0Y6h2f6QvbVAr3oBIZja8RpJEoCrXWH/4QzhIbMffO9YiAsEkzgtORcW", + "gBlIDTKl2Zbl1DY90pGMeAhr5y1hmKmezEhEJzSy07okCxvPGWywVd0lPWA3zvenRcIG2tkM47UBPGMY", + "nPe4eIYsJVX9NwyCoKMkj0snl4MQwmKMkyRYmGOa8DFORmZ9LknAJ/gS3rCLUjpcSm8S4zExJeSzhZpx", + "Zv7OxzlTufl7LPiVJGLY2ewPGSRi2LUmcbcUENEVFHJLM67PmeCp6XPLDHHrj0uy+NQfssM4pcxRBHyC", + "E8kRuYbvoL4VYGYY7tVAD+Y0hf3gR7lUPPWRTx3dmWHyXGW5shklkqhuCPVzyBRHfzhsx09bf5Q9fgJn", + "McGxphPvFTMlkK2bRi1HWM9+BK8G3O0EFmDY0RepCfOYCsyUge0swCnR1N/SjaI6AlRMra9whBnKeGYq", + "SwBRzbAmuUobgNWAkwQpOEruWy24w042zMdC76XjRtw9A5RWO0aUodMfvcM02HsWPk+SRIKEIkr+6/z1", + "rwhuZb0H5rUyXMukdDAtMKA4B9ep42kvcDRDxlEFxQSHHRoPO4U7N96EsebShsv0euBT/EEP7QfTTZfG", + "P/T7uinjrjxAv/1hWjnQZylLDQ7osPOpi7wHU6pm+bh49iG8oE3wZecVRoA2zDW3CZwEU0Ca8W58c0Vi", + "FiNub4FkgTAqOZAfuDKmDIvFqkTCwNLbFeQTE8noLcYfQ4hcHHYOhi52cdjpDjuEzeE3G+A47HwKr4D1", + "WjZXroP7rHBuFkT0ZDDYXI+Ebdc34LNs4Rj4wjpgo1ZUlN3UO2hhWP9c/oF/a/2zcP1gpjsvoYmM4u+M", + "74/QAeFJ7L4mGnBB1MRuzCKSOLF7vaHn/p0HerMikiT3TaAPRZ6Fe6xA6n9U5AibVR6jleb7B6a4wX1d", + "KhWz/cPQ76Oznwes59Z2TuYu1DlcpwQwaKwqjczLCEt0DmPqnWvl+wX82rf/dbofYCpeJHx6cWBUd5Tw", + "KUoos/kAXqCyFg/sWsJHBoam+M6i0rgicRtGkvjXP/4Jg6Js+q9//NNiu//rH/+E475l4NWgxvTFjGCh", + "xgSriwP0CyFZDyd0TtxkoAosmROxQLsDa/OHR8grdW+lNDlkQ/aGqFwwL2/C1GuTtkHrKtDzoSwn0sL4", + "6BfpxBaTMbGNAbuNO8tmKe/1RHcDcIgwA28C+lZ0NABYctQU2raaaCdsMjVzrhhN62GaS8F66/mLItfK", + "UG/PDPCGDAaWOHTu4IGdNNo4P3+x2UegbRmqgIJBoDuUzVg1ov+NJ63nSYajVBkKrLLhTRHO8Jgm1Jkc", + "G6qdmCOY4mhGGSnjiwuscdfEgRup5jGHZyfIBkJ24dUhe32+BSZWRSKVC9K1nEBYhNGyHBq3eS7QA/Av", + "qiA6rGffHbIJwZAndHJsmIAHwl3kAxYNMwDygBhXqiqV17pDZpBkLXKxPngpj0kCH0H/U6zIFV50UVHr", + "1lVHSbDSCrHs6peHzGC92jXoAVQJ8obZB35mhtRzkbw2Z0uQSaJVY4jAN2W/oe+NCRfIRjh7Vf5ddybJ", + "0gxLL1qKo9fnen5T0AS5sQdCS6/P3W5sdpHkKEooUEOE2ZBNIRDIgfdyVtnVIqFshkXci7i+BHwwp0vG", + "rxIST5t47JFPZHcoyVT6CRynn+vk+tiEi9nyBPQhNgB1qz13x/addq472+KfyXdnC0HewHlnLLjE8Buz", + "ut8ceS0ceeF1c069kGft2CEw3l3Er+nigQJ+He0tr7l54i3ZQ1j00IaDtgGvCBfo7OgE4TgWRMrNf297", + "n56podJS/tP3o2bFDxF6YsfChQX9s/aWKoE8Fnbwxo4aYTeven1d/37bqhTfabzpijo85ZV397dHrdOb", + "XCOl0FvS2rebZG2wLZURhzKDJbX0QDRKSCG+FOfUp6J1VmUTxltcOSvFJcueT47dgbw/+7LtOmf1u+Ee", + "mOJxjSE+ICOsplr7VbMfEzW/K3bRoU2vMD9/XaQ5uD8p6L5N0SEyf0zqYlxbNs0FDdBJ4wX6kigDb3KX", + "errtITDxcyLcqTYDXZhZF9MynyKD0wITAkvMat33xLzSTvU17f2ZNF9YnptILHbJv4koLZTdcq1WKbgn", + "tgT03em30MON1NsvF7ZiCSywyGBFHTu3E1hWN7BcsGjzW+TKF6doE9dYKrHCzZvEhSXboCkVetZ9yXWH", + "zK83rmU6q9dShiYJnc6sEyCmE4jVU379bhjlzj2MsqiTLbAiNkTxMeb9nulFtl7gOREKvT46MevvX6lb", + "f0DQ6npVyTGvlbfruzeveoRFPC6cJ80yqX3yhRUmQ/+VXN77P3WPMJ+VOvGgSWD8jP03weTIxL/3Kf9f", + "Oz8ldCywWPyvnZ9wklFG/tfuYYIVkWrzzohlcF833X0rMI+Y+LT+QquLBqyJTQEydo3AX7zVUuZ37/+p", + "xH4z6RsJ/sW6fpP928j+/nKtFP/tVtypAmD6eCAPV0FsodWGR98gbe7BaGop0oO0qXiRSlCbGZcKHj2+", + "/GYbVE4LivOvjZbW//JArrw+HOmeHHdhIaGiNFS0sOmD9+QLcOO4d+HW9nv/joDDdEynOc+ln5mYYhXN", + "iLRZuwmpMuDHJnaX13Oj4P0VU+ngPq+Oe5erv9H9HUn89Q01zNs49NbJ/O6ttjK/fV/L/AbR1GY227Ib", + "XVeSabMh0NphmrYl4wr063IAeGhcIV0EvdOKSqkuINAgDobsf2v94zdFcPrhB5dCmQ8GO0/gd8LmH35w", + "WZTs1JEKYUpQW0Hv8Ndj8KJOIVAWiuyVCdv1cZia3UB6rqzAv52CVDqS22tIjgq/aUitNCRvuVZrSHYv", + "7lZFqpYmuXcdydFbaMEtpvifU0v6k7tHKhqczCcTGlHCoMALJKbLpXhAo8l984zcMiGZWX+kF0xUkURa", + "q5EF11ojoZc1pe89kOykLKJ139qjK1/9OBOreGbrwVp9rZQWmhW2r40eBvd7e92/ovaYScxoRMtLl2mh", + "O1AZyNSESnOThVZ8WYKK9dHbt69cxpmW+IWrM6W4Ky7l6m4OmV9cqo9elFW7zAuuBS2Rk9hmqEIeni3X", + "FBMcJ5QRCNElMpQcVi0J96DH4ssLleF6d62Eyns+lraC6cMJlQ/GCu5FfDupFIbmpY7vl8IrTosT4eDU", + "PCp+ZRlQgPGExKctnCveszmsWzNugM3C2I5nCY4A2lG/ZlDHLGyAgRn0mwIsAMGThAiDJpflytVAHLJi", + "cJR5Nd5tjZsL3fwoZ4omF10TIQOQIBJhtrCQSkNW6QwrRdJMQWovpK3DCAXJzIhrxR/1oCnPJbwFWbh+", + "lwgnV3ghh8wmA5vPoVCuIJEBXkySPvqZAw4DwlNMmcd4TQXC7+SQXdA4ISMLo3CBqERyxoUijMQo5XMi", + "q/0SLBJKBEziCOuVkyjFC8AzM9COZn14RgxmWAWsget/YxZTqGWney6mfDBkGO0MBiglmEmbei3xBC4c", + "2waCQVQG9D3CaG/w3H5V2zfA3HXLv6FPkxBkziM8ThaIaCoG8Ae1CRuY2tqSpkav3r4JFdLsV2EytEXD", + "KhtLpSuVGHdRzsrkcjCf56zIBdfbpXLBYJ7WsUaoKK5Bi6cxJhHW68l4tR9AMuRRlIvQBam32ity+u8o", + "OHrTO4elCqduJ6CFRySGPWdczeBMczhKm983UFVJVH+OiyZ4SLhAGHl0XRoJSJQDa9wA5L+LsmIfcxV4", + "Lza/d2dHH1/LCNzxN9h7j+V+AiLik0nlAK6/mswBXpUysUzCf9ZzeuRKtfosLqZ4yrhUNHLMsF7Z/ZtC", + "2FohXL2yQWqecHHpy1ZV+v2Ji8u2GpjFE6WPSxHzZ/gV2vb18AC7+eFN/GBgNsqKJpp7V9Lq9FWcUhC6", + "qJIudpijhLOpPkWlofveLfG+VrdhcNj0ZSqM/7hAzdFKyMj+aKq96snYWppgtY9sqw/Ni3Tv9+Df+ZUr", + "RNMsISmBarA9Q2x6s0uEJaicT6WHM3QzXqlPlZ8ObHRBaVz6XScOAV25DdsA6X15u4JMNeHT9Th+RecO", + "tC4A5DdkprI/QRfGm3OBCh6sBVqDmo+uZjSaAagf6K26fYP5h7PsosAz3jxAL+Eg+7DO0PmGwcrXtCZ5", + "QgxW3zxNLw6W652+Pz2Fjwyen6lsenGAXI3T4v6Q+i0fpE/PIsFSoV8t9OBGoYzDjl4orPXNYn6bFr6v", + "xJseshCUHyNXtkE6QRceqt9FA+SU47ev+PTBhLFuc5UAMxfFkVUdgTYJiztNcRM0CQP6bQ8GIfDqluCC", + "Zhh3jC24NJhXfFpUKKiQMs6ytuRrhwlUPE/TFTSMNjykMqlinqu/SBUTIeBjS91NxI02cGSrU+FLTagW", + "l84d7E0gv2B0kIEMDy6VZqqdboewPO0c/Gb/NU/TTrdjx+NBjd9AuF8D0lhvcDmKRe+Mh8T4TSy/CcZi", + "ldl7IIu1m8Oq080S+Rvzwp/eW+hsdg9IhiAf1Iy4X5MI6o23avBhvACLhJE9v4+RAaIkihIuScXB83jw", + "qKyhqyYzNhuK3Br39PDi3BXwaRMUcm4/PXdffgW697pYETdm5KZ770EjyyN4zLm1cmk2Ey7qIEbrokm+", + "ekL6cluyNNU2FPKNNm9uZWxFmFpPWGYR9oPYFHTDueIpVjSCYkLRjHPpkX2BOGzKflnjcUGZYFoxWq4N", + "yr/QpHphzdAXVo04sCYzhP1Hto8+fG5D+cNfuEflFz95VoGC43ed6A+A+1DtXFAyQRnOJdFSXZ4SFC0i", + "zRVN9SiCoxmKcKZyQaAwHkEpZTTNUx9KWu/YHAPsxcV2etFF41yhBIspaGXmoQu2iXiaEhYTsM8N2Yzg", + "OdUqpUAJVoRFi54kUFB3TtAVF5cJxzGYGLIYg6cHCvIJoikQcLlTonCMFQZB50Kf+JHJC7ooauwatZ6R", + "65Ia4iETOfveFAnQzV64gV4gAijYVM6KWowRjgmLgujQ5183G/vytuhzouoTfaDIoFvx0ocMFfJtrm44", + "X0cU0SMLb+bCbmMbNr9C6JXNKmw1ocKR0b/nkTZzdXN8IAdTscSrTvHX4VkqiO6r8S49vPuICxTnpjvv", + "VAKZ/1l9QgVD8YOtIFnTbONtHUNF0blimW/E87b+cH+e3MKW95Vwwm6jYt9U3qic9NfAcu2q3ornPpAR", + "09qSfJvcw7FgF9H1YOITFx6XeyzGVsuwzdEs+LbPnZTAoH1x9o1t19m2DXi4Ldt2ttkll77HyCnrQYxo", + "mINbM24jq7amg3/TbJTa7DyW+eAssvRc3BtbPCkYoWGNGV4kHMd/hiDhFf6jiAthECUAo+IxIZp6VkM/", + "PQBsc2XdtK7L1nx/errZxCWEWskjhHrEHMJLydGfpfGyAff1nAhBYwv8iY5Oj224LpVI5KyPXqdUIcXR", + "JSFZmdECWYV9PT+HrbFco70CotHtEKbEIuOUqbWjKF+9m8F8ulVl93vmkxYi+5s7vLU7HCz7j4+dAZeB", + "nA0zgdWaqcJqbelOyiZcpEYuw2Oe69Y1D9LLpPfTlMCc0ITIhVQkNVGJkzyB4wblFmxJXfud2eUuxOTq", + "k2PS5TIiUiol5UwOmc0VyYjQfevPdftegFXQIaBwwV/PDJP8OoL39GBMvBpWTasGKEhQqrNz0NnCWbYV", + "Y4UbAsTs8D5jSD9BNB6Si3TMExqhhLJLiTYSemnUEzSXKNF/bK4M5xvBd1+6YPDtT5Ze6RM24cFybIZm", + "C2L+U2V1WbbmHJOPjq29JP5hcfwHNjrM1taXJBYEJz0o8euwcFCuaEI/GlanG6FS0cikHOFi7d6fFky1", + "P2SnRAn9DobUtiQxiAagXW5lgkdbw3ww2I0yCoBquwQGBwyv+XEKPR6dvTNpqCTlYtEdMv0PaPjt4Znx", + "7k6wtSZ4A7W1iNHJ1us1Ac7nsEz/xhGCZoIr0QuCG/7NJXhzjJHGMyQbjijPVqlKPPvTh7BaCe6bXeFx", + "2hUA5KmYzcZU4AiEYjnLVcyvWNiGMOdJnup/mD9O1kGFKRzN3sOrX420a4azths3wUdxKO2cYmLKRT6I", + "08Ms2GONWdUL56YAQkwlGjB4CxyqPyN1f3nzvb+OX6G7066oK8X61Zyt+7757Bgcwoa/Ho/lmBtKczNR", + "fLX16QrTZuvTjwmPLqWFYvHNhlpvA8hy/WMJMW1dhCAmQGYoshBGBiiLyO6Q1QyQBvFHIowUESllONmC", + "OZtGACzbWbHwnFNI0I4gT6UnaQyYSQkgYgP8nZ4NGKpcA55HV9piVf47vjNScTQmEU+JAxDfDKluf8NU", + "/cRFFQ38a+GLb731B0hATMHevgYAvbnHzwJEP8XXECod59ah7Ea08ZKXPxpTUBfB3gw7uwM57HTRsLOT", + "Djt6B44wmFCxQvsopSxXRPbRsbFvQQrukwGSJOIslg7H3FnwdgeyKSHXkGVDducT+O4+xR5LVbCUb2wn", + "Ifag30P6e0jaQRv+gbNnMu7CoYsRz5Ux99tzZd+KiQLzyOa9+2q9M/JNt2/Dyf9mj2+FR8Eua3bpbb3h", + "7FkuZ6TZ5PbK1AbK1RjwsV29TjlDf+dj2UWMXBlruJCqv8T39NdnpoP7wO7XXd0Et9/O/RtofwvQ/nKt", + "wmCNJsBSX8mOOgxiI7nOuFCA4mhz7Q0NgSYByBE8wgl6fXQyZJFmRQZaUJCUA3eyEOPmFj782zl6cfSm", + "i46hdiT6OR9v9tFrlixcBW/joxkyI4kZ5hVhhsaGakkcup7N2IF67jJYXHfwQMWYzckIeFbcXrkg8W5n", + "RnAMEskfnVfcdBZAHX7zSh8gAP41Xxbb3lkpfHTeECUWvcOJImK52VObJ8UKzAx7STsIOiu4GeBL3aF0", + "yGtln0Y2MNAYuzudAFLGp291FO6+5uj9eMlMnIipYDfOAWmUQZIBjhePK5ZJzlDBHEMs0L+ui0oETVnC", + "lpetVDCgy6bI76/I5L6Sd1Ww5f9dTxfM9NE6mrLKPmkiLiqYrPX0uuTgmYFDto6qCGc4omrRRThJ7B1l", + "b4IiIqVXiL9jQfBlzK9Yf8jeFLVTbEIvOjp713WOWhRTeWlasL7YPno9J0Lm42JwCA6a8RrDmpN4yBRH", + "EU6iPNHiBplMSAS5uFASRTb4couhdO7w7JSdBOu3eFHt+aMrGxemCdi9kizqFLdltnpLkCjBNG0GH7eC", + "GgQcQqjBWDfKGaJsktiQqkhwKZFtqkcSOqXjxAYIyT56OyNI4pQMWZZgxohAuTRR8XrovUwQKXOT4K0b", + "AJBeQ1FdVAILZoIrG5qQcC6kiSbQFP7+FElFshVk9sa0fApzviPZ1jRue3ogI3VtDM2mEPsK0htiKMUs", + "uKajPHEBjPcaim4G9NBS4mM5+G8FnU6J0KcCGyZrwvHMsXbLaQ59JWO5sYTkefFWuxKSRateVqKXsbcS", + "GG5UYm3HnZtF/QU6v6SN2IH20c2yiH/RH7Xsu5qtGh6EffSZs/yzVOY/95IE2xqwSgp/bOYkb+SVo1pJ", + "tF0Pq9U6s/YuM11b42c9GGzWY0bLwpX02SaF9+sjhMH9ojzcd5G1x01bFbSrim7akPK/Hk3/q6DAu4HR", + "f2CUk1vA6H9VefeAc/5w+CfBg/pQefQV37OrX/unR8K/q/R5A4cPcGxN6fOG69ng1ZWK0nv7Tjs1ybb4", + "Z5LgbbzjDeR3t+zftP4WKoO3WOtc0JrgSZqphQtos77KMuhM0o+k3+AILuJW784VfIuQzi9HHo5OGwM6", + "/5zl5h8kZtSWDqQSnRwH6rg/MoxB/8xVLpYtfev0sIhmdE6aje7VE2yXKBOkl/EMnCuxWTC7Hu4uU1j0", + "px+Rbd5irtp/Qe1JgOonMYqpIJFKFqYOqOYIpo/vJBJcawLwnItFc5SIOSI/CZ4e2tmsuQ/tmbLGsDLO", + "MF30Yqxwb+64zQoT2mdEd7p4Ss3wEGXo5Y9og1wrYSpcoInWfBCdFEtqCvhLoMlNf8DbgwbLJv1IRtNx", + "m1GuqFXy2taCQVEuFU/d3p8cow2ofTYlTO+FFvUnIMlmgs9pTOLKGDtznphV3W5Y0JvaXbVQURSuc8qF", + "GdyDyDBtLqTpR5pV2UIREjOmDMPg1lYFqZ4pk8Sv+8OUuQAcu0duFN+uMKv5bThlR1Mi1OG0i6g4NxDP", + "m9+uucd8zfnJUO5Oq9x2LjxntfG6XX5Uy7Sluyj8UOTO3a/Z+v3Xk9JD5aPM5rGm83mhkDaZzb8uEhzc", + "3/1w3+by9484BfQlccq3ZyqHBnSLIYJ5BTHdMZmThGcp1EOHdzvdTi6SzkFnplR2sLUFsd8zLtXB3vOn", + "u51PHz79/wEAAP//zu89IfTvAQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index 48174e105..002c9fcce 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1757,8 +1757,12 @@ components: example: "L40S-1Q" mdev_uuid: type: string - description: mdev device UUID + description: mdev device UUID (mdev hosts only) example: "aa618089-8b16-4d01-a136-25a0f3c73123" + device_path: + type: string + description: sysfs path of the assigned vGPU device + example: "/sys/bus/pci/devices/0000:82:00.4" GPUProfile: type: object From c226b26c9dcb498d348eadb687448745f0c833f8 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:53:21 +0000 Subject: [PATCH 32/76] Surface retained rollback assignments from start as vgpu_cleanup_pending When a later start step failed and rollback could not destroy the freshly created vGPU, cleanupStartVGPU retained the assignment on disk but startInstance returned the original failure untyped, so the API reported a generic error instead of vgpu_cleanup_pending with the retained-assignment guidance. Mirror create's named-return wrap: cleanupStartVGPU reports retention state and start wraps the returned error in VGPUCleanupPendingError. --- lib/instances/start.go | 12 +++++++++++- lib/instances/vgpu.go | 10 +++++++++- lib/instances/vgpu_test.go | 4 ++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/lib/instances/start.go b/lib/instances/start.go index b44a68d50..1045829a1 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -116,6 +116,16 @@ func (m *manager) startInstance( } // Setup cleanup stack for automatic rollback on errors + // Registered before cu.Clean so it runs after cleanup and can report a + // vGPU assignment that rollback failed to destroy, matching create's + // vgpu_cleanup_pending contract. + vgpuRetained := false + vgpuRetentionPersisted := false + defer func() { + if retErr != nil && vgpuRetained { + retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuRetentionPersisted, Err: retErr} + } + }() cu := cleanup.Make(func() {}) defer cu.Clean() @@ -189,7 +199,7 @@ func (m *manager) startInstance( log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) // Add vGPU cleanup to stack cu.Add(func() { - m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) + vgpuRetained, vgpuRetentionPersisted = m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) }) // Checked after the cleanup handler is registered so rejection // releases the device through the normal rollback. diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 8c6351742..1e3dcc356 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -105,7 +105,12 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { // start. The snapshot is also a shallow copy (Phases shares its map), so it // must be persisted before any Phases.Record on the live struct. Violating // either invariant requires switching to targeted field restores. -func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) { +// +// It reports whether the assignment was retained after a failed destroy and +// whether that retention record was persisted, so start can surface the +// pending cleanup as a typed error like create does. +func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) { + logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID) assignment := devices.VGPUAssignment{ Framework: device.Framework, DevicePath: device.SysfsPath, @@ -117,6 +122,7 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if releaseErr != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "error", releaseErr) setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) + retained = true } if err := m.saveMetadata(&cleanupMeta); err != nil { message := "failed to save metadata after vGPU cleanup" @@ -124,7 +130,9 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic message = "failed to retain vGPU assignment metadata after cleanup failure" } logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err) + return retained, false } + return retained, retained } func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d466f1883..7668ff90c 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -389,6 +389,10 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) _, err = m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}}) require.Error(t, err) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending, "a retained rollback assignment must surface as vgpu_cleanup_pending") + assert.Equal(t, id, pending.InstanceID) + assert.True(t, pending.Retained) stored, err := m.loadMetadata(id) require.NoError(t, err) From e8730af1e4d875a56670d83958c8f001f86b9fd1 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 18:07:37 +0000 Subject: [PATCH 33/76] Report retention as persisted when the mid-start save survives When start rollback fails to destroy a vGPU and the cleanup metadata save also fails, the assignment may still be on disk from the mid-start save. Reporting Retained: false then misdirects callers to wait for startup reconcile when delete or a retried start can already release it. Check whether the surviving record still points at the device, matching create's retention-survives check. --- lib/instances/vgpu.go | 12 +++++++++++- lib/instances/vgpu_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 1e3dcc356..330e2d820 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -130,7 +130,17 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic message = "failed to retain vGPU assignment metadata after cleanup failure" } logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err) - return retained, false + if !retained { + return false, false + } + // The mid-start save may already have persisted this assignment, in + // which case the on-disk record still points at the device and + // delete or a retried start can release it (matching create's + // retention-survives check). + if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { + return true, true + } + return true, false } return retained, retained } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 7668ff90c..b34fe242a 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -405,6 +405,38 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { assert.Empty(t, stored.Entrypoint) } +func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + assignedAt := time.Now().UTC() + + // The mid-start save already persisted the assignment. + meta, err := m.loadMetadata(id) + require.NoError(t, err) + rollbackMeta := *meta + setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt) + require.NoError(t, m.saveMetadata(meta)) + + // The cleanup save fails, but the surviving on-disk record still points + // at the device, so retention must be reported as persisted. + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta) + assert.True(t, retained) + assert.True(t, persisted, "a surviving mid-start save keeps the assignment recoverable via delete") +} + func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { m := &manager{ paths: paths.New(t.TempDir()), From 81716523cc3628df411b15c1ef3b1f770ad1d06b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:07:40 +0000 Subject: [PATCH 34/76] Leave vGPU hypervisor selection to callers Drop the vendor-VFIO-on-Cloud-Hypervisor rejection from create and start, restoring the phase-0 decision that hypervisor selection is caller policy: production callers pin vGPU instances to QEMU, and the Cloud Hypervisor limitation stays documented in lib/devices/GPU.md. --- lib/instances/create.go | 6 ------ lib/instances/start.go | 6 ------ lib/instances/vgpu.go | 13 ------------- lib/instances/vgpu_test.go | 29 ----------------------------- 4 files changed, 54 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index 23fd534b1..a6cce101a 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -357,12 +357,6 @@ func (m *manager) createInstance( } } }) - // Checked after the cleanup handler is registered so rejection - // releases the device through the normal rollback. - if err := validateVGPUHypervisorCompat(gpuDevice.Framework, hvType); err != nil { - log.ErrorContext(ctx, "unsupported vGPU hypervisor combination", "instance_id", id, "framework", gpuDevice.Framework, "hypervisor", hvType) - return nil, err - } } if len(req.Devices) > 0 && m.deviceManager != nil { diff --git a/lib/instances/start.go b/lib/instances/start.go index 1045829a1..3e164b398 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -201,12 +201,6 @@ func (m *manager) startInstance( cu.Add(func() { vgpuRetained, vgpuRetentionPersisted = m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) }) - // Checked after the cleanup handler is registered so rejection - // releases the device through the normal rollback. - if err := validateVGPUHypervisorCompat(device.Framework, stored.HypervisorType); err != nil { - log.ErrorContext(ctx, "unsupported vGPU hypervisor combination", "instance_id", id, "framework", device.Framework, "hypervisor", stored.HypervisorType) - return nil, err - } if err := m.saveMetadata(meta); err != nil { log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) return nil, fmt.Errorf("save metadata after vGPU creation: %w", err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 330e2d820..88ae217da 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -8,7 +8,6 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -64,18 +63,6 @@ func retainedVGPUFromCreateError(instanceID string, assignedAt time.Time, err er } } -// validateVGPUHypervisorCompat rejects the one proven-broken combination: -// vendor VFIO vGPUs boot but are non-functional on Cloud Hypervisor (upstream -// cloud-hypervisor#7572), and the wedged VM then blocks the VF release until -// startup reconcile. Hypervisor selection otherwise remains caller policy; -// mdev on Cloud Hypervisor keeps working. See lib/devices/GPU.md. -func validateVGPUHypervisorCompat(framework devices.VGPUFramework, hvType hypervisor.Type) error { - if framework == devices.VGPUFrameworkVendorVFIO && hvType == hypervisor.TypeCloudHypervisor { - return fmt.Errorf("%w: vendor VFIO vGPUs are not functional on cloud-hypervisor, use qemu", ErrInvalidRequest) - } - return nil -} - func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { destroy := m.destroyVGPU if destroy == nil { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index b34fe242a..1bdb0fb97 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -319,35 +319,6 @@ func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { assert.ErrorIs(t, err, cause) } -func TestValidateVGPUHypervisorCompat(t *testing.T) { - t.Parallel() - - err := validateVGPUHypervisorCompat(devices.VGPUFrameworkVendorVFIO, hypervisor.TypeCloudHypervisor) - require.ErrorIs(t, err, ErrInvalidRequest) - assert.NoError(t, validateVGPUHypervisorCompat(devices.VGPUFrameworkVendorVFIO, hypervisor.TypeQEMU)) - assert.NoError(t, validateVGPUHypervisorCompat(devices.VGPUFrameworkMdev, hypervisor.TypeCloudHypervisor)) -} - -func TestStartRejectsVendorVFIOOnCloudHypervisor(t *testing.T) { - var destroyed []devices.VGPUAssignment - m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { - destroyed = append(destroyed, assignment) - return nil - }) - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.HypervisorType = hypervisor.TypeCloudHypervisor - require.NoError(t, m.saveMetadata(meta)) - - _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) - require.ErrorIs(t, err, ErrInvalidRequest) - - require.Len(t, destroyed, 1, "the rejected vGPU must be released by rollback") - stored, err := m.loadMetadata(id) - require.NoError(t, err) - assert.Empty(t, stored.GPUDevicePath, "no assignment may be persisted for a rejected combination") -} - func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { var destroyed []devices.VGPUAssignment m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { From 74470eb8d44b8f56cc57b3716a1b240e72ebda2d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:21:39 +0000 Subject: [PATCH 35/76] Carry identity fields into the create-pending retention stub retainedVGPUFromCreateError built a GPU-fields-only stub, so the retained record from a failed device-layer cleanup listed nameless and, with GPUProfile empty, the API hid its gpu block including device_path. The caller now supplies the identity fields and the stub picks up the pending device's profile. --- lib/instances/create.go | 16 +++++++++++++++- lib/instances/vgpu.go | 19 +++++++++++-------- lib/instances/vgpu_test.go | 6 ++++-- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index a6cce101a..07abfd4b6 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -312,7 +312,21 @@ func (m *manager) createInstance( log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) if err != nil { - retainedVGPU = retainedVGPUFromCreateError(id, m.nowUTC(), err) + stub := StoredMetadata{ + Id: id, + Name: req.Name, + Image: req.Image, + ResolvedImage: resolvedImageRef, + Platform: imageInfo.Platform, + CreatedAt: time.Now(), + HypervisorType: hvType, + HypervisorVersion: hvVersion, + DataDir: m.paths.InstanceDir(id), + } + if starterErr == nil { + stub.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) + } + retainedVGPU = retainedVGPUFromCreateError(stub, m.nowUTC(), err) log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 88ae217da..4caa365bb 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -49,18 +49,21 @@ func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { return &pending.Device, true } -func retainedVGPUFromCreateError(instanceID string, assignedAt time.Time, err error) *StoredMetadata { +// retainedVGPUFromCreateError fills stub with the pending device's assignment +// fields when err carries a failed device-layer cleanup. The caller provides +// identity fields on stub so the retained record lists as a recognizable, +// deletable instance. +func retainedVGPUFromCreateError(stub StoredMetadata, assignedAt time.Time, err error) *StoredMetadata { device, ok := vgpuDevicePendingCleanup(err) if !ok { return nil } - return &StoredMetadata{ - Id: instanceID, - GPUFramework: device.Framework, - GPUDevicePath: device.SysfsPath, - GPUMdevUUID: device.MdevUUID, - GPUAssignedAt: &assignedAt, - } + stub.GPUProfile = device.ProfileName + stub.GPUFramework = device.Framework + stub.GPUDevicePath = device.SysfsPath + stub.GPUMdevUUID = device.MdevUUID + stub.GPUAssignedAt = &assignedAt + return &stub } func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 1bdb0fb97..e5694ef3f 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -143,9 +143,11 @@ func TestVGPUDevicePendingCleanup(t *testing.T) { assert.Equal(t, device, *actual) assignedAt := time.Now().UTC() - retained := retainedVGPUFromCreateError("inst-1", assignedAt, wrapped) + retained := retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1", Name: "named", Image: "img"}, assignedAt, wrapped) require.NotNil(t, retained) assert.Equal(t, "inst-1", retained.Id) + assert.Equal(t, "named", retained.Name, "identity fields must survive into the retention stub") + assert.Equal(t, "img", retained.Image) assert.Equal(t, device.Framework, retained.GPUFramework) assert.Equal(t, device.SysfsPath, retained.GPUDevicePath) assert.Equal(t, assignedAt, *retained.GPUAssignedAt) @@ -153,7 +155,7 @@ func TestVGPUDevicePendingCleanup(t *testing.T) { actual, ok = vgpuDevicePendingCleanup(cause) assert.False(t, ok) assert.Nil(t, actual) - assert.Nil(t, retainedVGPUFromCreateError("inst-1", assignedAt, cause)) + assert.Nil(t, retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1"}, assignedAt, cause)) } type startRetentionNetworkManager struct { From f4d2d9bffcbcca174a4c2a11008894b19ec4120d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:39:32 +0000 Subject: [PATCH 36/76] Grace recent dead-PID claims in the release scan like reconcile does Startup reconcile protects an assignment whose PID is absent or stale for a bounded grace window, but the release-side claim scan treated a dead PID as unclaimed immediately. Align the two guards: a recent assignment whose recorded hypervisor is not running fails the scan closed so the requester retains and retries, and past the grace window the dead claim no longer blocks the release. --- lib/instances/vgpu.go | 8 +++++++- lib/instances/vgpu_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 4caa365bb..2f9cfa7d1 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -110,7 +110,7 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic cleanupMeta := rollbackMeta releaseErr := m.destroyVGPUAssignment(ctx, assignment) if releaseErr != nil { - logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "error", releaseErr) + logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID, "error", releaseErr) setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) retained = true } @@ -208,6 +208,12 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if pid > 0 { return true, nil } + // A dead PID with a recent assignment gets the same bounded grace as + // startup reconcile protection, so the two guards agree in the + // fail-closed direction while a mid-boot claimant hydrates. + if stored.GPUAssignedAt != nil && time.Since(*stored.GPUAssignedAt) < VGPUAssignmentStartupGracePeriod { + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", id, devicePath) + } } return false, nil } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index e5694ef3f..d5e988a43 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -528,6 +528,38 @@ func TestVGPUAssignmentClaimedByLiveInstanceIgnoresStaleNilPIDClaim(t *testing.T assert.False(t, claimed) } +func TestVGPUAssignmentClaimedByLiveInstanceGracesRecentDeadPIDClaim(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + claimantID := "claimant-dead-pid" + require.NoError(t, m.ensureDirectories(claimantID)) + deadPID := 1<<22 - 1 + require.False(t, ProcessExists(deadPID)) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: claimantID, + HypervisorPID: &deadPID, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + // Same bounded grace as startup reconcile: a recent claim whose PID is + // dead fails closed instead of being treated as unclaimed. + _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) + + // Past the grace period the dead claim no longer blocks the release. + stale := assignedAt.Add(-2 * VGPUAssignmentStartupGracePeriod) + meta, err := m.loadMetadata(claimantID) + require.NoError(t, err) + meta.GPUAssignedAt = &stale + require.NoError(t, m.saveMetadata(meta)) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.False(t, claimed) +} + func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { t.Parallel() From 09730c2f06105eeb7298944b6ab7c18eb7ca287b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 13 Aug 2026 18:57:32 +0000 Subject: [PATCH 37/76] Reject start on vGPU retention records A failed create whose vGPU release also failed persists a delete-only retention stub with no boot configuration. The stub derives as Stopped, so start would release the retained VF and then try to boot the incomplete record. Mark the stub with GPURetainedForCleanup and reject start with invalid_state guidance pointing at delete, which retries the release. --- lib/instances/create.go | 31 ++++++++++++++-------------- lib/instances/lifecycle_noop_test.go | 20 ++++++++++++++++++ lib/instances/start.go | 7 +++++++ lib/instances/types.go | 4 ++++ lib/instances/vgpu_test.go | 2 ++ 5 files changed, 49 insertions(+), 15 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index 07abfd4b6..e7ab3b6ad 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -687,21 +687,22 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG // deletable record rather than a nameless phantom, but drop resource // claims (network, volumes, devices) that rollback already released. retained := StoredMetadata{ - Id: id, - Name: retainedVGPU.Name, - Image: retainedVGPU.Image, - ResolvedImage: retainedVGPU.ResolvedImage, - Platform: retainedVGPU.Platform, - CreatedAt: retainedVGPU.CreatedAt, - HypervisorType: retainedVGPU.HypervisorType, - HypervisorVersion: retainedVGPU.HypervisorVersion, - SocketPath: retainedVGPU.SocketPath, - DataDir: retainedVGPU.DataDir, - GPUProfile: retainedVGPU.GPUProfile, - GPUFramework: retainedVGPU.GPUFramework, - GPUDevicePath: retainedVGPU.GPUDevicePath, - GPUMdevUUID: retainedVGPU.GPUMdevUUID, - GPUAssignedAt: retainedVGPU.GPUAssignedAt, + Id: id, + Name: retainedVGPU.Name, + Image: retainedVGPU.Image, + ResolvedImage: retainedVGPU.ResolvedImage, + Platform: retainedVGPU.Platform, + CreatedAt: retainedVGPU.CreatedAt, + HypervisorType: retainedVGPU.HypervisorType, + HypervisorVersion: retainedVGPU.HypervisorVersion, + SocketPath: retainedVGPU.SocketPath, + DataDir: retainedVGPU.DataDir, + GPUProfile: retainedVGPU.GPUProfile, + GPUFramework: retainedVGPU.GPUFramework, + GPUDevicePath: retainedVGPU.GPUDevicePath, + GPUMdevUUID: retainedVGPU.GPUMdevUUID, + GPUAssignedAt: retainedVGPU.GPUAssignedAt, + GPURetainedForCleanup: true, } if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index c74d352fd..e440e7e4a 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -324,6 +324,26 @@ func TestStartPersistsStaleVGPUReleaseImmediately(t *testing.T) { assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") } +func TestStartRejectsVGPURetentionRecord(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkNone + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, m.saveMetadata(meta)) + + _, err = m.StartInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") + + // The retained assignment must survive the rejected start for delete. + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) +} + func TestStopStoppedInstanceReleasesRetainedVGPU(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) meta, err := m.loadMetadata(id) diff --git a/lib/instances/start.go b/lib/instances/start.go index 3e164b398..8914f2ae1 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -47,6 +47,13 @@ func (m *manager) startInstance( log.ErrorContext(ctx, "invalid state for start", "instance_id", id, "state", inst.State) return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } + if stored.GPURetainedForCleanup { + // A delete-only retention stub from a failed create: it carries no + // boot configuration, so starting it would release the retained VF + // and then boot an incomplete record. Delete retries the release. + log.ErrorContext(ctx, "refusing to start vGPU retention record", "instance_id", id) + return nil, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + } // Release any assignment retained by an earlier failed release and // persist the cleared fields immediately, so a failure later in start diff --git a/lib/instances/types.go b/lib/instances/types.go index a264498fd..ab41178cd 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -156,6 +156,10 @@ type StoredMetadata struct { GPUDevicePath string GPUMdevUUID string // populated for mdev-backed vGPUs GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection + // GPURetainedForCleanup marks a delete-only retention stub written when a + // failed create could not release its vGPU: the record has no boot + // configuration, so only delete (which retries the release) may act on it. + GPURetainedForCleanup bool // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d5e988a43..a6fe4e49a 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -57,6 +57,8 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.False(t, retained.NetworkEnabled) assert.Empty(t, retained.IP) assert.Empty(t, retained.Volumes) + // The stub has no boot configuration, so it is marked delete-only. + assert.True(t, retained.GPURetainedForCleanup) } func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { From ae2bd429238ea3abaf9cf2f48e3d1db5d770afd6 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 13 Aug 2026 19:05:38 +0000 Subject: [PATCH 38/76] Make vGPU retention records fully delete-only A fork or snapshot of a failed-create retention stub could never boot: the stub has no boot configuration, and clearing the delete-only marker on the child would only produce a startable-but-broken record that recreates a vGPU from GPUProfile with incomplete metadata. Reject fork and snapshot of retention stubs with the same invalid_state guidance as start, so delete (which retries the release) is the only action on them. --- lib/instances/fork.go | 6 ++++++ lib/instances/fork_test.go | 25 +++++++++++++++++++++++++ lib/instances/snapshot.go | 7 +++++++ lib/instances/snapshot_test.go | 25 +++++++++++++++++++++++++ 4 files changed, 63 insertions(+) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index e0c778860..266299e96 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -219,6 +219,12 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin default: return nil, false, fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby)", ErrInvalidState, source.State) } + if stored.GPURetainedForCleanup { + // A delete-only retention stub from a failed create has no boot + // configuration, so a fork of it could never boot. Delete the stub to + // release its retained vGPU assignment. + return nil, false, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + } if !supportValidated { if err := m.validateForkSupport(ctx, stored.HypervisorType); err != nil { diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 26bc6cfcb..ef802d1ad 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -63,6 +63,31 @@ func TestForkInstanceClearsVGPUAssignment(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", source.GPUDevicePath) } +func TestForkInstanceRejectsVGPURetentionRecord(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + hvType := hypervisor.Type("fork-vgpu-retention-test") + hypervisor.RegisterCapabilities(hvType, hypervisor.Capabilities{SupportsConcurrentForkPrepare: true}) + manager.vmStarters[hvType] = concurrentForkPrepareTestStarter{} + + sourceID := "fork-vgpu-retention-source" + createStoppedSnapshotSourceFixture(t, manager, sourceID, sourceID, hvType) + + meta, err := manager.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, manager.saveMetadata(meta)) + + // The delete-only retention stub has no boot configuration, so a fork of + // it could never boot; only delete may act on it. + _, err = manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fork-vgpu-retention-copy"}) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") +} + func TestForkInstance_VZStoppedSourceSupported(t *testing.T) { t.Parallel() manager, _ := setupTestManager(t) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 48c51328a..297c7ef03 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -66,6 +66,13 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata + if stored.GPURetainedForCleanup { + // A delete-only retention stub from a failed create has no boot + // configuration, so a snapshot of it could never be restored or + // forked into a bootable instance. Delete the stub to release its + // retained vGPU assignment. + return nil, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + } if err := validateForkVolumeSafety(stored.Volumes); err != nil { return nil, fmt.Errorf("%w: snapshot requires readonly volume attachments: %v", ErrNotSupported, err) } diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index a92763b28..f8c022fe9 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -52,6 +52,31 @@ func TestForkSnapshotClearsVGPUAssignment(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", source.GPUDevicePath) } +func TestCreateSnapshotRejectsVGPURetentionRecord(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "snapshot-vgpu-retention" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, mgr.defaultHypervisor) + + meta, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, mgr.saveMetadata(meta)) + + // The delete-only retention stub has no boot configuration, so a snapshot + // of it could never be restored or forked into a bootable instance. + _, err = mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "snapshot-vgpu-retention", + }) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") +} + func TestRestoreSnapshotDoesNotResurrectStaleVGPUAssignment(t *testing.T) { mgr, _ := setupTestManager(t) ctx := context.Background() From a27cbe44ed7d7b8007438aa2af090aa59b7982e7 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:56:18 +0000 Subject: [PATCH 39/76] Adapt vGPU liveness guards to the identity struct resolver The claim guard and startup reconcile protection predate the HypervisorProcessIdentity struct and the removal of the standalone identity-exists helpers. Route both through resolveLiveHypervisorPID: the claim guard keeps failing closed on unresolvable ownership, and reconcile protection gets a fail-open HypervisorMayBeAlive wrapper so unresolvable ownership still protects the device. --- cmd/api/main.go | 2 +- cmd/api/main_test.go | 4 +-- lib/instances/lifecycle_noop_test.go | 22 ++++++++--------- lib/instances/process_identity.go | 10 ++++++++ lib/instances/process_identity_linux_test.go | 12 ++++----- lib/instances/vgpu.go | 2 +- lib/instances/vgpu_test.go | 26 ++++++++++---------- 7 files changed, 44 insertions(+), 34 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index f38a6a7c3..71e147ecf 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -196,7 +196,7 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances. if inst.GPUDevicePath == "" { continue } - if inst.HypervisorPID != nil && instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.HypervisorBootID, inst.SocketPath) { + if inst.HypervisorPID != nil && instances.HypervisorMayBeAlive(inst.HypervisorProcessIdentity, inst.SocketPath) { protected[inst.GPUDevicePath] = struct{}{} continue } diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 217e9db8d..85c132814 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -362,8 +362,8 @@ func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) { {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}}, {StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}}, {StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}}, - {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorPID: &deadPID}}, - {StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorPID: &deadPID, GPUAssignedAt: &recent}}, + {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}}}, + {StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}}, }} protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index e440e7e4a..f5f495988 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -258,17 +258,17 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { require.NoError(t, err) defer listener.Close() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: claimantID, - Name: claimantID, - Image: "test-image", - CreatedAt: now, - HypervisorType: lifecycleNoopHypervisorType, - HypervisorPID: &pid, - SocketPath: socketPath, - DataDir: m.paths.InstanceDir(claimantID), - GPUProfile: "NVIDIA L40S-2Q", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + Id: claimantID, + Name: claimantID, + Image: "test-image", + CreatedAt: now, + HypervisorType: lifecycleNoopHypervisorType, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, + SocketPath: socketPath, + DataDir: m.paths.InstanceDir(claimantID), + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", }})) require.NoError(t, m.DeleteInstance(context.Background(), id)) diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 1d885d0b5..be645e111 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -180,6 +180,16 @@ func classifyResolvedHypervisorOwner(socketPath string, stored, resolved int, er return 0, fmt.Errorf("cannot confirm ownership of socket %s: %w", socketPath, err) } +// HypervisorMayBeAlive reports whether the recorded hypervisor process may +// still be running. It fails open: when ownership cannot be resolved it +// returns true, which is the safe direction for its callers (reconcile +// protection and claim checks, where true means "protect"). Do not use it to +// authorize teardown. +func HypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { + pid, err := resolveLiveHypervisorPID(id, socketPath) + return err != nil || pid > 0 +} + // ProcessExists reports whether pid belongs to a live, non-zombie process. func ProcessExists(pid int) bool { if pid <= 0 { diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index d5bd973d9..7ad259c41 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -512,7 +512,7 @@ func TestRefreshHypervisorPIDResolvesSocketOwnerWhenStoredPIDIsDead(t *testing.T func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "test.sock") - owner := exec.Command(os.Args[0], "-test.run=^TestHypervisorProcessExistsWithReboundSocketPathHelper$") + owner := exec.Command(os.Args[0], "-test.run=^TestSocketListenerHelper$") owner.Env = append(os.Environ(), "HYPERVISOR_SOCKET_HELPER=1", "HYPERVISOR_SOCKET_PATH="+socketPath) stdin, err := owner.StdinPipe() require.NoError(t, err) @@ -539,11 +539,11 @@ func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) stalePID := stale.Process.Pid require.NoError(t, m.ensureDirectories("live-claimant")) require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "live-claimant", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: devicePath, - HypervisorPID: &stalePID, - SocketPath: socketPath, + Id: "live-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &stalePID}, + SocketPath: socketPath, }})) claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", devicePath) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 2f9cfa7d1..54a0bf9c6 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -201,7 +201,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu } return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) } - pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.HypervisorBootID, stored.SocketPath) + pid, err := resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) if err != nil { return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index a6fe4e49a..1d0e4e3bf 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -483,10 +483,10 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. require.NoError(t, m.ensureDirectories("legacy-claimant")) pid := os.Getpid() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "legacy-claimant", - Name: "legacy-claimant", - GPUMdevUUID: "legacy-uuid", - HypervisorPID: &pid, + Id: "legacy-claimant", + Name: "legacy-claimant", + GPUMdevUUID: "legacy-uuid", + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, }})) claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/mdev/devices/legacy-uuid") @@ -538,11 +538,11 @@ func TestVGPUAssignmentClaimedByLiveInstanceGracesRecentDeadPIDClaim(t *testing. require.False(t, ProcessExists(deadPID)) assignedAt := time.Now().UTC() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: claimantID, - HypervisorPID: &deadPID, - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, + Id: claimantID, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, }})) // Same bounded grace as startup reconcile: a recent claim whose PID is @@ -569,10 +569,10 @@ func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { require.NoError(t, m.ensureDirectories("dead-claimant")) deadPID := 1 << 30 require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "dead-claimant", - Name: "dead-claimant", - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - HypervisorPID: &deadPID, + Id: "dead-claimant", + Name: "dead-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, }})) claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") From 798e4a15e6db91584a13fb914a98cf76f66984e1 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 17 Aug 2026 21:10:56 +0000 Subject: [PATCH 40/76] Deep-copy the phase tracker into the start rollback snapshot --- lib/instances/start.go | 1 + lib/instances/vgpu.go | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/instances/start.go b/lib/instances/start.go index 8914f2ae1..44dc50216 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -75,6 +75,7 @@ func (m *manager) startInstance( stored.HypervisorStartTime = 0 stored.HypervisorBootID = "" rollbackMeta := *meta + rollbackMeta.Phases = meta.Phases.Clone() // 2a. Clear stale exit info from previous run and apply command overrides stored.ExitCode = nil diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 54a0bf9c6..d0cd9b226 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -92,9 +92,7 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { // cleanup stack is LIFO, so cleanups registered after this one run before it // and this restore would clobber anything they persisted; it is safe only // while no such cleanup writes metadata and the instance lock serializes -// start. The snapshot is also a shallow copy (Phases shares its map), so it -// must be persisted before any Phases.Record on the live struct. Violating -// either invariant requires switching to targeted field restores. +// start. Violating that requires switching to targeted field restores. // // It reports whether the assignment was retained after a failed destroy and // whether that retention record was persisted, so start can surface the From a5d5d19dc738519f8f8c70d5e2f7b08b120f6093 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 18 Aug 2026 18:59:15 +0000 Subject: [PATCH 41/76] Retry orphaned vGPU releases in the background after delete MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A vGPU release during delete routinely fails when a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait. Delete's log-and-continue contract then deleted the metadata, stranding the VF until the next server restart and silently shrinking host GPU capacity. Hand the failed assignment to a bounded background retry (30s interval, 20 attempts) that re-runs the full release path — claim scan and destroy guards included — off the request path. The in-memory queue dedupes by device path; a restart abandons it and startup reconciliation sweeps the VF as before. --- lib/devices/GPU.md | 2 +- lib/instances/delete.go | 9 +- lib/instances/lifecycle_noop_test.go | 4 +- lib/instances/manager.go | 8 ++ lib/instances/vgpu_orphan.go | 78 ++++++++++++++ lib/instances/vgpu_orphan_test.go | 146 +++++++++++++++++++++++++++ 6 files changed, 241 insertions(+), 6 deletions(-) create mode 100644 lib/instances/vgpu_orphan.go create mode 100644 lib/instances/vgpu_orphan_test.go diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 3a1504428..ea5764ee0 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -97,7 +97,7 @@ Instance Create → Assign profile to VF → Attach VF to VM → Instance Runnin Instance Stop/Delete → Release profile → VF available again ``` -Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. +Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. A release that fails during delete (typically because a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait) is retried in the background for up to ten minutes, so a completed delete does not strand the VF until the next restart. ### Hypervisor Support diff --git a/lib/instances/delete.go b/lib/instances/delete.go index b5f157d33..baf672da8 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -146,15 +146,18 @@ func (m *manager) deleteInstanceWithOptions( // or volume teardown. Release failure is logged and the delete continues, // matching the pre-refactor contract: the VMM is already confirmed dead, // the guards inside the release never destroy a device they cannot prove - // is unowned, and a skipped release is recovered by startup - // reconciliation. + // is unowned, and a skipped release is recovered by the background retry + // below or, after a restart, by startup reconciliation. hadVGPUAssignment := storedVGPUDevicePath(stored) != "" if hadVGPUAssignment { log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } if err := m.releaseStoredVGPU(ctx, stored); err != nil { - // Log error but continue with cleanup. + // Log error but continue with cleanup. The metadata is about to be + // deleted, so hand the assignment to the background retry — otherwise + // the VF stays allocated until the next startup reconciliation. log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) + m.scheduleOrphanedVGPURelease(ctx, *stored) } else if hadVGPUAssignment { if err := m.saveMetadata(meta); err != nil { log.WarnContext(ctx, "failed to save metadata after vGPU release", "instance_id", id, "error", err) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index f5f495988..7868534a3 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -160,8 +160,8 @@ func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { require.NoError(t, m.saveMetadata(meta)) // A failed release is logged and the delete continues, matching the - // pre-refactor contract; the leaked assignment is recovered by startup - // reconciliation. + // pre-refactor contract; the leaked assignment is recovered by the + // background retry or startup reconciliation. require.NoError(t, m.DeleteInstance(context.Background(), id)) _, err = m.loadMetadata(id) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index a0622da19..600f90888 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -212,6 +212,14 @@ type manager struct { // Periodic TAP garbage collection reconciler. tapGCOnce sync.Once + // vGPU assignments that survived a completed delete, keyed by device + // path, each with a background release retry in flight. + // orphanedVGPURetryDelay overrides the retry delay in tests; zero means + // the default. + orphanedVGPUMu sync.Mutex + orphanedVGPUs map[string]struct{} + orphanedVGPURetryDelay time.Duration + // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter defaultHypervisor hypervisor.Type // Default hypervisor type when not specified in request diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go new file mode 100644 index 000000000..69a3545c5 --- /dev/null +++ b/lib/instances/vgpu_orphan.go @@ -0,0 +1,78 @@ +package instances + +import ( + "context" + "time" + + "github.com/kernel/hypeman/lib/logger" +) + +const ( + // orphanedVGPUReleaseMaxAttempts bounds the retry loop so a genuinely + // wedged VF degrades to one operator-actionable error instead of + // indefinite log churn. At the default interval this covers ten minutes, + // far beyond the seconds a dying VMM normally needs to finish kernel-side + // VFIO teardown. + orphanedVGPUReleaseMaxAttempts = 20 + defaultOrphanedVGPUReleaseRetryDelay = 30 * time.Second +) + +// scheduleOrphanedVGPURelease retries a vGPU release that failed during a +// completed delete, off the request path. Delete's log-and-continue contract +// is untouched — the caller already has its success — but without a retry the +// VF would stay allocated until the next startup reconciliation, silently +// shrinking host GPU capacity. A GPU-busy VMM routinely outlives delete's +// force-kill wait while the kernel finishes VFIO teardown, so this is the +// common case under load, not a tail case. +// +// Each attempt re-runs releaseStoredVGPU on a copy of the deleted instance's +// stored metadata, so the vendor VFIO claim scan and the destroy-side owner +// and open-handle guards apply on every retry exactly as they did on the +// original release. The queue is in-memory only: a restart abandons it and +// startup reconciliation sweeps the VF instead. +func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { + path := storedVGPUDevicePath(&stored) + if path == "" { + return + } + m.orphanedVGPUMu.Lock() + if m.orphanedVGPUs == nil { + m.orphanedVGPUs = make(map[string]struct{}) + } + if _, pending := m.orphanedVGPUs[path]; pending { + m.orphanedVGPUMu.Unlock() + return + } + m.orphanedVGPUs[path] = struct{}{} + m.orphanedVGPUMu.Unlock() + + delay := m.orphanedVGPURetryDelay + if delay <= 0 { + delay = defaultOrphanedVGPUReleaseRetryDelay + } + // The request context ends with the delete; keep its values for logging + // but detach from its cancellation. + go m.retryOrphanedVGPURelease(context.WithoutCancel(ctx), stored, path, delay) +} + +func (m *manager) retryOrphanedVGPURelease(ctx context.Context, stored StoredMetadata, path string, delay time.Duration) { + log := logger.FromContext(ctx) + defer func() { + m.orphanedVGPUMu.Lock() + delete(m.orphanedVGPUs, path) + m.orphanedVGPUMu.Unlock() + }() + for attempt := 1; attempt <= orphanedVGPUReleaseMaxAttempts; attempt++ { + time.Sleep(delay) + if err := m.releaseStoredVGPU(ctx, &stored); err != nil { + log.WarnContext(ctx, "orphaned vGPU release retry failed", + "instance_id", stored.Id, "device_path", path, "attempt", attempt, "error", err) + continue + } + log.InfoContext(ctx, "released orphaned vGPU after delete", + "instance_id", stored.Id, "device_path", path, "attempt", attempt) + return + } + log.ErrorContext(ctx, "giving up on orphaned vGPU release; VF stays allocated until startup reconciliation or manual remediation", + "instance_id", stored.Id, "device_path", path, "attempts", orphanedVGPUReleaseMaxAttempts) +} diff --git a/lib/instances/vgpu_orphan_test.go b/lib/instances/vgpu_orphan_test.go new file mode 100644 index 000000000..25a350064 --- /dev/null +++ b/lib/instances/vgpu_orphan_test.go @@ -0,0 +1,146 @@ +package instances + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func waitForOrphanQueueEmpty(t *testing.T, m *manager) { + t.Helper() + require.Eventually(t, func() bool { + m.orphanedVGPUMu.Lock() + defer m.orphanedVGPUMu.Unlock() + return len(m.orphanedVGPUs) == 0 + }, 5*time.Second, 5*time.Millisecond, "orphan retry should finish and clear its queue entry") +} + +func TestScheduleOrphanedVGPUReleaseRetriesUntilSuccess(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + orphanedVGPURetryDelay: time.Millisecond, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + if attempts.Add(1) < 3 { + return errors.New("operation not permitted") + } + return nil + }, + } + m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ + Id: "deleted-instance", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }) + + waitForOrphanQueueEmpty(t, m) + assert.Equal(t, int32(3), attempts.Load(), "release should succeed on the third attempt and stop retrying") +} + +func TestScheduleOrphanedVGPUReleaseGivesUpAfterMaxAttempts(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + orphanedVGPURetryDelay: time.Millisecond, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + attempts.Add(1) + return errors.New("vGPU destroy failed: 0xffffffff") + }, + } + m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ + Id: "deleted-instance", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }) + + waitForOrphanQueueEmpty(t, m) + assert.Equal(t, int32(orphanedVGPUReleaseMaxAttempts), attempts.Load(), + "a wedged VF should get exactly the bounded number of attempts") +} + +func TestScheduleOrphanedVGPUReleaseDeduplicatesByDevicePath(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + release := make(chan struct{}) + m := &manager{ + paths: paths.New(t.TempDir()), + orphanedVGPURetryDelay: 20 * time.Millisecond, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + attempts.Add(1) + <-release + return nil + }, + } + stored := StoredMetadata{ + Id: "deleted-instance", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + } + m.scheduleOrphanedVGPURelease(context.Background(), stored) + m.scheduleOrphanedVGPURelease(context.Background(), stored) + + require.Eventually(t, func() bool { return attempts.Load() == 1 }, 5*time.Second, 5*time.Millisecond) + close(release) + waitForOrphanQueueEmpty(t, m) + assert.Equal(t, int32(1), attempts.Load(), "the second schedule for the same path must be dropped") +} + +func TestScheduleOrphanedVGPUReleaseIgnoresEmptyAssignment(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{Id: "no-gpu"}) + + m.orphanedVGPUMu.Lock() + defer m.orphanedVGPUMu.Unlock() + assert.Empty(t, m.orphanedVGPUs) +} + +// TestOrphanedVGPUReleaseReappliesClaimScan pins that the background retry +// goes through releaseStoredVGPU, not a raw destroy: a live claimant found by +// the vendor VFIO claim scan must keep blocking the release on every retry. +func TestOrphanedVGPUReleaseReappliesClaimScan(t *testing.T) { + t.Parallel() + + var destroys atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + orphanedVGPURetryDelay: time.Millisecond, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + destroys.Add(1) + return nil + }, + } + // A claimant with a recent assignment and no persisted PID makes the scan + // fail closed, exactly like the synchronous release path. + require.NoError(t, m.ensureDirectories("mid-boot-claimant")) + assignedAt := time.Now() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "mid-boot-claimant", + Name: "mid-boot-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ + Id: "deleted-instance", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }) + + waitForOrphanQueueEmpty(t, m) + assert.Zero(t, destroys.Load(), "no destroy may fire while the claim scan cannot clear the path") +} From b8b25bdce6d484d8ebdef08e81bb6e7503368267 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:40:11 +0000 Subject: [PATCH 42/76] Bound orphan retries in delete tests and tighten comments The two delete-continues tests triggered the new orphan retry with the default 30s delay, leaving a goroutine running ~10 minutes past the test. Use a millisecond delay and drain the queue before returning. --- lib/instances/delete.go | 5 ++--- lib/instances/lifecycle_noop_test.go | 4 ++++ lib/instances/vgpu_orphan.go | 18 ++++++------------ 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/lib/instances/delete.go b/lib/instances/delete.go index baf672da8..fdb285b5c 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -153,9 +153,8 @@ func (m *manager) deleteInstanceWithOptions( log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } if err := m.releaseStoredVGPU(ctx, stored); err != nil { - // Log error but continue with cleanup. The metadata is about to be - // deleted, so hand the assignment to the background retry — otherwise - // the VF stays allocated until the next startup reconciliation. + // Log error but continue with cleanup; the background retry releases + // the VF once the metadata is gone. log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) m.scheduleOrphanedVGPURelease(ctx, *stored) } else if hadVGPUAssignment { diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 7868534a3..ebb8a8da1 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -152,6 +152,7 @@ func TestLifecycleNoopStandbyWithOptionsStillRejectsStandbyInstance(t *testing.T func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + m.orphanedVGPURetryDelay = time.Millisecond meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" @@ -163,6 +164,7 @@ func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { // pre-refactor contract; the leaked assignment is recovered by the // background retry or startup reconciliation. require.NoError(t, m.DeleteInstance(context.Background(), id)) + waitForOrphanQueueEmpty(t, m) _, err = m.loadMetadata(id) require.Error(t, err, "instance data must be deleted despite the failed release") @@ -282,6 +284,7 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + m.orphanedVGPURetryDelay = time.Millisecond deviceManager := &recordingDeviceManager{} m.deviceManager = deviceManager meta, err := m.loadMetadata(id) @@ -295,6 +298,7 @@ func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { // The failed release must not block the rest of the teardown: devices // are detached and the instance is fully deleted. require.NoError(t, m.DeleteInstance(context.Background(), id)) + waitForOrphanQueueEmpty(t, m) assert.Equal(t, []string{"dev-1"}, deviceManager.detached) _, err = m.loadMetadata(id) diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go index 69a3545c5..5f3fc2b38 100644 --- a/lib/instances/vgpu_orphan.go +++ b/lib/instances/vgpu_orphan.go @@ -18,18 +18,12 @@ const ( ) // scheduleOrphanedVGPURelease retries a vGPU release that failed during a -// completed delete, off the request path. Delete's log-and-continue contract -// is untouched — the caller already has its success — but without a retry the -// VF would stay allocated until the next startup reconciliation, silently -// shrinking host GPU capacity. A GPU-busy VMM routinely outlives delete's -// force-kill wait while the kernel finishes VFIO teardown, so this is the -// common case under load, not a tail case. -// -// Each attempt re-runs releaseStoredVGPU on a copy of the deleted instance's -// stored metadata, so the vendor VFIO claim scan and the destroy-side owner -// and open-handle guards apply on every retry exactly as they did on the -// original release. The queue is in-memory only: a restart abandons it and -// startup reconciliation sweeps the VF instead. +// completed delete, off the request path. A GPU-busy VMM routinely outlives +// delete's force-kill wait while the kernel finishes VFIO teardown, and once +// the metadata is deleted nothing else releases the VF until the next +// startup reconciliation. Each attempt re-runs releaseStoredVGPU, so the +// claim scan and destroy guards apply on every retry. The queue is in-memory +// only: a restart abandons it and startup reconciliation sweeps the VF. func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { path := storedVGPUDevicePath(&stored) if path == "" { From dd5332acbef4a443972140935c38e47987aa5a59 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 19 Aug 2026 03:54:42 +0000 Subject: [PATCH 43/76] SIGTERM vGPU QEMU before SIGKILL during driver init A SIGKILL delivered to QEMU while the NVIDIA vGPU plugin is still initializing the VF wedges it near-deterministically: the guest driver loops on RmInitAdapter timeouts with no host-side signal, and only an SR-IOV cycle of the parent GPU recovers it. Voluntary QEMU exits run their VFIO teardown and are safe, as are hard kills after init. Start-failure cleanup and the force-kill fallback for initializing vGPU instances now send SIGTERM and wait a bounded grace before SIGKILL, and a hard kill inside the init window logs the affected device path. Clean creates, graceful stops, and running-instance deletes are unchanged. --- lib/devices/GPU.md | 14 +++- lib/hypervisor/qemu/process.go | 55 ++++++++++++- lib/hypervisor/qemu/process_test.go | 44 ++++++++++ lib/instances/delete.go | 11 +++ lib/instances/manager.go | 4 + lib/instances/process_identity.go | 14 ++++ lib/instances/process_identity_linux_test.go | 85 ++++++++++++++++++++ 7 files changed, 219 insertions(+), 8 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index ea5764ee0..2841d51c6 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -288,10 +288,16 @@ every request, so one wedged VF presents as all vGPU instances failing while `/resources` reports full capacity. The wedge itself leaves no host-side log: no kernel error, no XID, no plugin -crash. In the observed case it followed a period of heavy attach/teardown -churn on the VF, including QEMU processes that exited within seconds of -opening the VFIO device (failed start attempts that were then retried), so -suspect any workload that repeatedly kills the VMM mid-device-init. +crash. The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is +still initializing the VF (roughly the first seconds after process start): +a single hard kill in that window wedges the VF near-deterministically, +while QEMU processes that exit voluntarily — error exits, QMP quit, SIGTERM — +run their VFIO teardown and never wedge, and hard kills of fully-initialized +vGPU VMs are also safe. Hypeman therefore SIGTERMs a vGPU QEMU first and only +escalates to SIGKILL after a grace period, both in start-failure cleanup and +when force-killing an instance that is still initializing; a hard kill in the +init window logs `VF may wedge` with the device path. External SIGKILLs +(OOM killer, manual `kill -9`) can still trigger it. Confirm by assigning the same profile on a different VF: if that guest initializes, the VF is wedged, not the driver stack. Remediate by cycling diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index c02c5beef..e7f8c84cc 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -40,6 +40,11 @@ const ( // socketDialTimeout is timeout for individual socket connection attempts socketDialTimeout = 100 * time.Millisecond + // vfioTermGrace is how long start-failure cleanup waits for a + // VFIO-attached QEMU to exit on SIGTERM before SIGKILL. Only failed + // starts pay it. + vfioTermGrace = 10 * time.Second + // clientCreateTimeout is how long to retry QMP client creation after the // socket appears. Under high parallel load the socket can accept connections // slightly later than file creation/availability. @@ -225,8 +230,14 @@ func buildQMPArgs(socketPath string) []string { } type startedProcess struct { - pid int - socketPath string + pid int + socketPath string + // termGrace, when non-zero, makes cleanup send SIGTERM and wait this long + // before SIGKILL. Set for VFIO-attached processes: hard-killing QEMU while + // the NVIDIA vGPU plugin is initializing can silently wedge the VF until + // its parent GPU's SR-IOV is cycled, while a terminating QEMU runs its + // device teardown and leaves the VF reusable. + termGrace time.Duration waitDone chan error waitConsumed bool waitErr error @@ -277,14 +288,47 @@ func (p *startedProcess) wait() error { return err } +// waitFor waits up to d for the process to exit, returning whether it did. +func (p *startedProcess) waitFor(d time.Duration) bool { + if _, exited := p.checkExited(); exited { + return true + } + select { + case err := <-p.waitDone: + p.waitConsumed = true + p.waitErr = err + return true + case <-time.After(d): + return false + } +} + func (p *startedProcess) cleanup() { if _, exited := p.checkExited(); !exited { - _ = syscall.Kill(p.pid, syscall.SIGKILL) - _ = p.wait() + terminated := false + if p.termGrace > 0 { + if syscall.Kill(p.pid, syscall.SIGTERM) == nil { + terminated = p.waitFor(p.termGrace) + } + } + if !terminated { + _ = syscall.Kill(p.pid, syscall.SIGKILL) + _ = p.wait() + } } _ = os.Remove(p.socketPath) } +// hasVFIODevice reports whether the QEMU command line attaches a VFIO device. +func hasVFIODevice(args []string) bool { + for _, arg := range args { + if strings.Contains(arg, "vfio-pci") { + return true + } + } + return false +} + // startQEMUProcess handles the common QEMU process startup logic. // Returns the PID, hypervisor client, and a cleanup function. // The cleanup function must be called on error; call cleanup.Release() on success. @@ -358,6 +402,9 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version } pid := proc.pid + if hasVFIODevice(args) { + proc.termGrace = vfioTermGrace + } log.DebugContext(processCtx, "QEMU process started", "pid", pid, "duration_ms", time.Since(processStartTime).Milliseconds()) // Setup cleanup to kill, reap, and remove the socket if subsequent steps fail. diff --git a/lib/hypervisor/qemu/process_test.go b/lib/hypervisor/qemu/process_test.go index e8be0dda2..b334887e3 100644 --- a/lib/hypervisor/qemu/process_test.go +++ b/lib/hypervisor/qemu/process_test.go @@ -1,6 +1,7 @@ package qemu import ( + "bufio" "context" "errors" "os" @@ -409,3 +410,46 @@ func TestWaitForSocketOrExitReturnsEarlyWhenProcessDies(t *testing.T) { require.NotNil(t, cmd.ProcessState) assert.True(t, cmd.ProcessState.Exited()) } + +func TestCleanupSIGTERMsProcessWithTermGrace(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "qemu.sock") + markerPath := filepath.Join(t.TempDir(), "terminated") + + cmd := exec.Command("sh", "-c", "trap 'touch "+markerPath+"; exit 0' TERM; echo ready; sleep 30 & wait") + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + proc, err := startManagedProcess(cmd, socketPath) + require.NoError(t, err) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + proc.termGrace = 5 * time.Second + + proc.cleanup() + + assert.FileExists(t, markerPath, "process must get SIGTERM, not SIGKILL, when termGrace is set") + require.NoFileExists(t, socketPath) + require.NotNil(t, cmd.ProcessState) +} + +func TestCleanupEscalatesToSIGKILLAfterTermGrace(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "qemu.sock") + + cmd := exec.Command("sh", "-c", "trap '' TERM; echo ready; sleep 30 & wait") + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + proc, err := startManagedProcess(cmd, socketPath) + require.NoError(t, err) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + proc.termGrace = 50 * time.Millisecond + + proc.cleanup() + + assert.ErrorIs(t, syscall.Kill(proc.pid, 0), syscall.ESRCH, "SIGTERM-ignoring process must still be hard-killed") + require.NoFileExists(t, socketPath) +} + +func TestHasVFIODevice(t *testing.T) { + assert.True(t, hasVFIODevice([]string{"-device", "vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:82:00.4"})) + assert.False(t, hasVFIODevice([]string{"-device", "virtio-balloon-pci,id=balloon0"})) +} diff --git a/lib/instances/delete.go b/lib/instances/delete.go index fdb285b5c..556db8433 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -243,6 +243,17 @@ func (m *manager) killHypervisor(ctx context.Context, inst *Instance) error { log.WarnContext(ctx, "stored hypervisor PID does not own the instance socket, killing the socket owner", "instance_id", inst.Id, "stored_pid", *inst.HypervisorPID, "owner_pid", pid) } + if inst.GPUProfile != "" && inst.State == StateInitializing { + // SIGKILL during vGPU driver init can silently wedge the VF until + // its parent GPU's SR-IOV is cycled, so ask the VMM to exit and + // run its VFIO teardown first. + if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { + os.Remove(inst.SocketPath) + return nil + } + log.WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM during driver init; hard-killing, VF may wedge", + "instance_id", inst.Id, "device_path", inst.GPUDevicePath) + } log.DebugContext(ctx, "killing hypervisor process", "instance_id", inst.Id, "pid", pid) if err := killProcessAndWait(pid); err != nil { return err diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 600f90888..cc21f3f54 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -220,6 +220,10 @@ type manager struct { orphanedVGPUs map[string]struct{} orphanedVGPURetryDelay time.Duration + // vgpuInitTermGrace overrides killHypervisor's SIGTERM wait for vGPU + // instances still initializing; zero means the default. + vgpuInitTermGrace time.Duration + // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter defaultHypervisor hypervisor.Type // Default hypervisor type when not specified in request diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index be645e111..cca38e916 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -25,6 +25,20 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second +// defaultVGPUInitTermGrace is how long killHypervisor waits for a vGPU +// hypervisor still in driver init to exit on SIGTERM before SIGKILL. Only +// the force-kill fallback pays it, and only for initializing vGPU instances. +const defaultVGPUInitTermGrace = 10 * time.Second + +// vgpuTermGrace returns the SIGTERM wait used before hard-killing an +// initializing vGPU hypervisor. +func (m *manager) vgpuTermGrace() time.Duration { + if m.vgpuInitTermGrace > 0 { + return m.vgpuInitTermGrace + } + return defaultVGPUInitTermGrace +} + // killProcessAndWait SIGKILLs pid and waits for it to exit. A process that // survives the first wait gets its process group killed too (the hypervisor // may have spawned children in its own group) and a short grace period. An diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 7ad259c41..2121d151b 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -691,3 +691,88 @@ func TestResolveRuntimeHypervisorPIDMintsIdentityOnlyWhenConfirmed(t *testing.T) assert.Empty(t, stored.HypervisorBootID, "a dead fallback must not mint the identity token") }) } + +// startTrapProcess starts a shell with the given TERM trap action (empty +// ignores the signal) and blocks until the trap is installed. It returns the +// PID and its boot-scoped identity. +func startTrapProcess(t *testing.T, trapAction string) (int, HypervisorProcessIdentity) { + t.Helper() + script := fmt.Sprintf("trap '%s' TERM; echo ready; sleep 30 & wait", trapAction) + process := exec.Command("sh", "-c", script) + stdout, err := process.StdoutPipe() + require.NoError(t, err) + require.NoError(t, process.Start()) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + waitDone := make(chan error, 1) + go func() { waitDone <- process.Wait() }() + t.Cleanup(func() { + _ = process.Process.Kill() + <-waitDone + }) + + pid := process.Process.Pid + startTime := processStartTime(pid) + require.NotZero(t, startTime) + return pid, HypervisorProcessIdentity{HypervisorPID: &pid, HypervisorStartTime: startTime, HypervisorBootID: hostBootID()} +} + +func TestKillHypervisorSIGTERMsInitializingVGPUHypervisor(t *testing.T) { + markerPath := filepath.Join(t.TempDir(), "terminated") + pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateInitializing, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + GPUProfile: "NVIDIA L40S-1Q", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + + require.Eventually(t, func() bool { + return syscall.Kill(pid, 0) == syscall.ESRCH + }, 5*time.Second, 10*time.Millisecond) + assert.FileExists(t, markerPath, "hypervisor must be given SIGTERM, not SIGKILL, during vGPU driver init") +} + +func TestKillHypervisorEscalatesToSIGKILLWhenSIGTERMIgnored(t *testing.T) { + pid, identity := startTrapProcess(t, "") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{vgpuInitTermGrace: 50 * time.Millisecond} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateInitializing, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + GPUProfile: "NVIDIA L40S-1Q", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + + assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH, "SIGTERM-ignoring hypervisor must still be hard-killed") +} + +func TestKillHypervisorHardKillsVGPUHypervisorPastInit(t *testing.T) { + markerPath := filepath.Join(t.TempDir(), "terminated") + pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateRunning, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + GPUProfile: "NVIDIA L40S-1Q", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + + assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH) + assert.NoFileExists(t, markerPath, "running vGPU hypervisors keep the direct SIGKILL path") +} From cda26955ced01f1c5616a60a7e9cad4d3ac34fef Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:07:44 +0000 Subject: [PATCH 44/76] Apply vGPU SIGTERM grace on stop's direct kill paths shutdownHypervisor (stop) could still SIGKILL an initializing vGPU QEMU directly: on QMP connect failure, on graceful-quit timeout, and when the hypervisor lacks graceful shutdown, bypassing the grace killHypervisor applies. Extract the SIGTERM-then-SIGKILL escalation into terminateThenKill and use it at all four force-kill sites. --- lib/hypervisor/qemu/process_test.go | 5 ----- lib/instances/delete.go | 13 +------------ lib/instances/manager.go | 2 +- lib/instances/process_identity.go | 21 +++++++++++++++++++-- lib/instances/standby.go | 6 +++--- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/lib/hypervisor/qemu/process_test.go b/lib/hypervisor/qemu/process_test.go index b334887e3..2bdf27d4f 100644 --- a/lib/hypervisor/qemu/process_test.go +++ b/lib/hypervisor/qemu/process_test.go @@ -448,8 +448,3 @@ func TestCleanupEscalatesToSIGKILLAfterTermGrace(t *testing.T) { assert.ErrorIs(t, syscall.Kill(proc.pid, 0), syscall.ESRCH, "SIGTERM-ignoring process must still be hard-killed") require.NoFileExists(t, socketPath) } - -func TestHasVFIODevice(t *testing.T) { - assert.True(t, hasVFIODevice([]string{"-device", "vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:82:00.4"})) - assert.False(t, hasVFIODevice([]string{"-device", "virtio-balloon-pci,id=balloon0"})) -} diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 556db8433..322e8f139 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -243,19 +243,8 @@ func (m *manager) killHypervisor(ctx context.Context, inst *Instance) error { log.WarnContext(ctx, "stored hypervisor PID does not own the instance socket, killing the socket owner", "instance_id", inst.Id, "stored_pid", *inst.HypervisorPID, "owner_pid", pid) } - if inst.GPUProfile != "" && inst.State == StateInitializing { - // SIGKILL during vGPU driver init can silently wedge the VF until - // its parent GPU's SR-IOV is cycled, so ask the VMM to exit and - // run its VFIO teardown first. - if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { - os.Remove(inst.SocketPath) - return nil - } - log.WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM during driver init; hard-killing, VF may wedge", - "instance_id", inst.Id, "device_path", inst.GPUDevicePath) - } log.DebugContext(ctx, "killing hypervisor process", "instance_id", inst.Id, "pid", pid) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } diff --git a/lib/instances/manager.go b/lib/instances/manager.go index cc21f3f54..2ecc59fcc 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -220,7 +220,7 @@ type manager struct { orphanedVGPUs map[string]struct{} orphanedVGPURetryDelay time.Duration - // vgpuInitTermGrace overrides killHypervisor's SIGTERM wait for vGPU + // vgpuInitTermGrace overrides terminateThenKill's SIGTERM wait for vGPU // instances still initializing; zero means the default. vgpuInitTermGrace time.Duration diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index cca38e916..9bfe2cdaa 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -1,6 +1,7 @@ package instances import ( + "context" "errors" "fmt" "os" @@ -13,6 +14,7 @@ import ( "time" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/logger" ) // linuxBootIDPath is the kernel-provided boot ID used to scope process @@ -25,9 +27,9 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second -// defaultVGPUInitTermGrace is how long killHypervisor waits for a vGPU +// defaultVGPUInitTermGrace is how long terminateThenKill waits for a vGPU // hypervisor still in driver init to exit on SIGTERM before SIGKILL. Only -// the force-kill fallback pays it, and only for initializing vGPU instances. +// force-kill paths pay it, and only for initializing vGPU instances. const defaultVGPUInitTermGrace = 10 * time.Second // vgpuTermGrace returns the SIGTERM wait used before hard-killing an @@ -39,6 +41,21 @@ func (m *manager) vgpuTermGrace() time.Duration { return defaultVGPUInitTermGrace } +// terminateThenKill hard-kills the hypervisor process, first giving a vGPU +// instance still in driver init a SIGTERM grace: SIGKILL in that window can +// silently wedge the VF until its parent GPU's SR-IOV is cycled (see +// lib/devices/GPU.md), while a terminating QEMU runs its VFIO teardown. +func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { + if inst.GPUProfile != "" && inst.State == StateInitializing { + if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { + return nil + } + logger.FromContext(ctx).WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM during driver init; hard-killing, VF may wedge", + "instance_id", inst.Id, "device_path", inst.GPUDevicePath) + } + return killProcessAndWait(pid) +} + // killProcessAndWait SIGKILLs pid and waits for it to exit. A process that // survives the first wait gets its process group killed too (the hypervisor // may have spawned children in its own group) and a short grace period. An diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 4c6fd7d33..5c6e0173e 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -376,7 +376,7 @@ func (m *manager) shutdownHypervisor(ctx context.Context, inst *Instance) error // alive; teardown is committed, so kill it rather than report a // completed shutdown for a VMM that is still running. log.WarnContext(ctx, "could not connect to hypervisor, force killing resolved owner", "instance_id", inst.Id, "pid", pid, "error", err) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } @@ -405,13 +405,13 @@ func (m *manager) shutdownHypervisor(ctx context.Context, inst *Instance) error log.DebugContext(ctx, "hypervisor shutdown gracefully", "instance_id", inst.Id, "pid", pid) } else { log.WarnContext(ctx, "hypervisor did not exit gracefully in time, force killing process", "instance_id", inst.Id, "pid", pid) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } } else { log.DebugContext(ctx, "skipping graceful exit wait; force killing hypervisor process", "instance_id", inst.Id, "pid", pid) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } From 2b584a40d853c2ae6f331524c69310a27dfe0561 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:38:06 +0000 Subject: [PATCH 45/76] Lower vGPU SIGTERM grace to 5s Observed mid-init VFIO teardown completes in 1-2s, so 5s keeps 2-3x margin while halving the worst-case delay for a SIGTERM-ignoring process. --- lib/hypervisor/qemu/process.go | 5 +++-- lib/instances/process_identity.go | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index e7f8c84cc..cddd51824 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -42,8 +42,9 @@ const ( // vfioTermGrace is how long start-failure cleanup waits for a // VFIO-attached QEMU to exit on SIGTERM before SIGKILL. Only failed - // starts pay it. - vfioTermGrace = 10 * time.Second + // starts pay it, and only when the process ignores SIGTERM; observed + // mid-init VFIO teardown takes 1-2s. + vfioTermGrace = 5 * time.Second // clientCreateTimeout is how long to retry QMP client creation after the // socket appears. Under high parallel load the socket can accept connections diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 9bfe2cdaa..2f68b6c92 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -29,8 +29,10 @@ const hypervisorSIGKILLWaitTimeout = 2 * time.Second // defaultVGPUInitTermGrace is how long terminateThenKill waits for a vGPU // hypervisor still in driver init to exit on SIGTERM before SIGKILL. Only -// force-kill paths pay it, and only for initializing vGPU instances. -const defaultVGPUInitTermGrace = 10 * time.Second +// force-kill paths pay it, only for initializing vGPU instances, and only +// when the process ignores SIGTERM; observed mid-init VFIO teardown takes +// 1-2s. +const defaultVGPUInitTermGrace = 5 * time.Second // vgpuTermGrace returns the SIGTERM wait used before hard-killing an // initializing vGPU hypervisor. From b92ba06f4a32e5883790096f06152583e5fd3fdc Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 20 Aug 2026 18:52:27 +0000 Subject: [PATCH 46/76] Apply vGPU SIGTERM grace in every instance state Running reports true ~4s before the guest driver finishes initializing and no host-side signal observes that boundary, so gating the SIGTERM grace on StateInitializing left a window where a failed stop or delete could SIGKILL QEMU mid-driver-init and wedge the VF. Post-init the SIGTERM is proven harmless and costs the grace only when the process ignores it. --- lib/devices/GPU.md | 7 ++--- lib/instances/process_identity.go | 27 +++++++++++--------- lib/instances/process_identity_linux_test.go | 25 ++++++++++++++++-- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 2841d51c6..ff11c5cde 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -295,9 +295,10 @@ while QEMU processes that exit voluntarily — error exits, QMP quit, SIGTERM run their VFIO teardown and never wedge, and hard kills of fully-initialized vGPU VMs are also safe. Hypeman therefore SIGTERMs a vGPU QEMU first and only escalates to SIGKILL after a grace period, both in start-failure cleanup and -when force-killing an instance that is still initializing; a hard kill in the -init window logs `VF may wedge` with the device path. External SIGKILLs -(OOM killer, manual `kill -9`) can still trigger it. +when force-killing any vGPU instance (the instance reports Running seconds +before driver init completes, so no state reliably marks the window); a hard +kill after an ignored SIGTERM logs `VF may wedge` with the device path. +External SIGKILLs (OOM killer, manual `kill -9`) can still trigger it. Confirm by assigning the same profile on a different VF: if that guest initializes, the VF is wedged, not the driver stack. Remediate by cycling diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 2f68b6c92..0801aca6c 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -28,14 +28,13 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" const hypervisorSIGKILLWaitTimeout = 2 * time.Second // defaultVGPUInitTermGrace is how long terminateThenKill waits for a vGPU -// hypervisor still in driver init to exit on SIGTERM before SIGKILL. Only -// force-kill paths pay it, only for initializing vGPU instances, and only -// when the process ignores SIGTERM; observed mid-init VFIO teardown takes -// 1-2s. +// hypervisor to exit on SIGTERM before SIGKILL. Only force-kill paths pay it, +// only for vGPU instances, and only when the process ignores SIGTERM; +// observed mid-init VFIO teardown takes 1-2s. const defaultVGPUInitTermGrace = 5 * time.Second -// vgpuTermGrace returns the SIGTERM wait used before hard-killing an -// initializing vGPU hypervisor. +// vgpuTermGrace returns the SIGTERM wait used before hard-killing a vGPU +// hypervisor. func (m *manager) vgpuTermGrace() time.Duration { if m.vgpuInitTermGrace > 0 { return m.vgpuInitTermGrace @@ -43,16 +42,20 @@ func (m *manager) vgpuTermGrace() time.Duration { return defaultVGPUInitTermGrace } -// terminateThenKill hard-kills the hypervisor process, first giving a vGPU -// instance still in driver init a SIGTERM grace: SIGKILL in that window can -// silently wedge the VF until its parent GPU's SR-IOV is cycled (see -// lib/devices/GPU.md), while a terminating QEMU runs its VFIO teardown. +// terminateThenKill hard-kills the hypervisor process, first giving any vGPU +// instance a SIGTERM grace: SIGKILL during guest driver init can silently +// wedge the VF until its parent GPU's SR-IOV is cycled (see +// lib/devices/GPU.md), while a terminating QEMU runs its VFIO teardown. The +// grace applies in every state, not just Initializing, because the instance +// reports Running seconds before the guest driver finishes initializing and +// nothing host-side observes that boundary; post-init the SIGTERM is proven +// harmless and costs the grace only when the process ignores it. func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { - if inst.GPUProfile != "" && inst.State == StateInitializing { + if inst.GPUProfile != "" { if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { return nil } - logger.FromContext(ctx).WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM during driver init; hard-killing, VF may wedge", + logger.FromContext(ctx).WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM; hard-killing, VF may wedge if the guest driver was initializing", "instance_id", inst.Id, "device_path", inst.GPUDevicePath) } return killProcessAndWait(pid) diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 2121d151b..e090b9a98 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -757,7 +757,7 @@ func TestKillHypervisorEscalatesToSIGKILLWhenSIGTERMIgnored(t *testing.T) { assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH, "SIGTERM-ignoring hypervisor must still be hard-killed") } -func TestKillHypervisorHardKillsVGPUHypervisorPastInit(t *testing.T) { +func TestKillHypervisorSIGTERMsRunningVGPUHypervisor(t *testing.T) { markerPath := filepath.Join(t.TempDir(), "terminated") pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") socketPath := filepath.Join(t.TempDir(), "missing.sock") @@ -773,6 +773,27 @@ func TestKillHypervisorHardKillsVGPUHypervisorPastInit(t *testing.T) { }, })) + require.Eventually(t, func() bool { + return syscall.Kill(pid, 0) == syscall.ESRCH + }, 5*time.Second, 10*time.Millisecond) + assert.FileExists(t, markerPath, "Running reports true before guest driver init completes, so vGPU hypervisors get SIGTERM in every state") +} + +func TestKillHypervisorHardKillsNonVGPUHypervisor(t *testing.T) { + markerPath := filepath.Join(t.TempDir(), "terminated") + pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateRunning, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH) - assert.NoFileExists(t, markerPath, "running vGPU hypervisors keep the direct SIGKILL path") + assert.NoFileExists(t, markerPath, "non-vGPU hypervisors keep the direct SIGKILL path") } From 54656832b5a65f610524fe12c3ee11eea585471e Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:12:33 +0000 Subject: [PATCH 47/76] Harden the vGPU claim scan and retention-stub stop path A concurrent instance deletion between the claim scan's metadata listing and load turned ErrNotFound into a host-wide fail-closed release error, even though a vanished record cannot be a live claimant; skip it. Stop on a delete-only retention stub released its VF while leaving GPURetainedForCleanup set, so the stub's start/fork/snapshot errors kept claiming an assignment that no longer existed. Retention stubs now release only through delete, as documented. --- lib/instances/lifecycle_noop_test.go | 22 ++++++++++++++++++++++ lib/instances/vgpu.go | 11 +++++++++++ 2 files changed, 33 insertions(+) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index ebb8a8da1..46f891379 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -367,6 +367,28 @@ func TestStopStoppedInstanceReleasesRetainedVGPU(t *testing.T) { assert.Empty(t, stored.GPUDevicePath) } +func TestStopStoppedInstanceLeavesRetentionStubForDelete(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkNone + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, m.saveMetadata(meta)) + + inst, err := m.StopInstance(context.Background(), id) + require.NoError(t, err) + require.NotNil(t, inst) + + // Retention stubs are delete-only: releasing on stop would leave a stub + // whose start/fork/snapshot errors still claim a retained assignment. + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.True(t, stored.GPURetainedForCleanup) +} + func TestStopStoppedInstanceVGPUReleaseFailureRemainsNoop(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) meta, err := m.loadMetadata(id) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index d0cd9b226..6b707e94c 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -187,6 +187,11 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu } meta, err := m.loadMetadata(id) if err != nil { + if errors.Is(err, ErrNotFound) { + // Deleted between listing and load; a vanished record cannot + // be a live claimant. + continue + } return false, fmt.Errorf("load metadata for vGPU release check: instance %s: %w", id, err) } stored := &meta.StoredMetadata @@ -228,6 +233,12 @@ func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) { return } stored := &meta.StoredMetadata + if stored.GPURetainedForCleanup { + // Delete-only retention stubs release through delete. Releasing here + // would leave a stub whose start/fork/snapshot errors still claim a + // retained assignment that no longer exists. + return + } if storedVGPUDevicePath(stored) == "" { return } From a9700a496b0a1821d6bd72ac88bc990e23a011e8 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:10:59 +0000 Subject: [PATCH 48/76] Share one vGPU retention stub in create The two rollback paths carried near-identical retention-stub literals that would drift as fields are added. Build both from one helper, use nowUTC like the rest of the file, and drop the starter guard that is dead since create fails on a nil starter long before the vGPU block. --- lib/instances/create.go | 45 +++++++++++++++++------------------------ 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index e7ab3b6ad..eefd0a7ce 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -309,24 +309,27 @@ func (m *manager) createInstance( // Handle vGPU profile request if req.GPU != nil && req.GPU.Profile != "" { - log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) - gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) - if err != nil { - stub := StoredMetadata{ + // Identity fields a retention record keeps when rollback cannot + // release the assignment, so it lists as a recognizable, deletable + // instance. Create has already failed on a nil starter by this point. + retentionStub := func() StoredMetadata { + return StoredMetadata{ Id: id, Name: req.Name, Image: req.Image, ResolvedImage: resolvedImageRef, Platform: imageInfo.Platform, - CreatedAt: time.Now(), + CreatedAt: m.nowUTC(), HypervisorType: hvType, HypervisorVersion: hvVersion, + SocketPath: m.paths.InstanceSocket(id, starter.SocketName()), DataDir: m.paths.InstanceDir(id), } - if starterErr == nil { - stub.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) - } - retainedVGPU = retainedVGPUFromCreateError(stub, m.nowUTC(), err) + } + log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) + gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) + if err != nil { + retainedVGPU = retainedVGPUFromCreateError(retentionStub(), m.nowUTC(), err) log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } @@ -351,23 +354,13 @@ func (m *manager) createInstance( log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) retainedVGPU = stored if retainedVGPU == nil { - retainedVGPU = &StoredMetadata{ - Id: id, - Name: req.Name, - Image: req.Image, - ResolvedImage: resolvedImageRef, - Platform: imageInfo.Platform, - CreatedAt: time.Now(), - HypervisorType: hvType, - HypervisorVersion: hvVersion, - SocketPath: m.paths.InstanceSocket(id, starter.SocketName()), - DataDir: m.paths.InstanceDir(id), - GPUProfile: gpuDevice.ProfileName, - GPUFramework: gpuDevice.Framework, - GPUDevicePath: gpuDevice.SysfsPath, - GPUMdevUUID: gpuDevice.MdevUUID, - GPUAssignedAt: gpuAssignedAt, - } + stub := retentionStub() + stub.GPUProfile = gpuDevice.ProfileName + stub.GPUFramework = gpuDevice.Framework + stub.GPUDevicePath = gpuDevice.SysfsPath + stub.GPUMdevUUID = gpuDevice.MdevUUID + stub.GPUAssignedAt = gpuAssignedAt + retainedVGPU = &stub } } }) From 623c121e44ff0fbb2ab05cfa805a8686d0f31987 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:31:49 +0000 Subject: [PATCH 49/76] Name the strict metadata listing variant Replace the listMetadataFilesWithStatErrors(bool) mode flag with listMetadataFiles / listMetadataFilesStrict so call sites say which failure semantics they rely on. --- lib/instances/storage.go | 15 ++++++++++++--- lib/instances/vgpu.go | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/instances/storage.go b/lib/instances/storage.go index dd932d41a..6a2354624 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -187,12 +187,21 @@ func removeAllWithRetry(path string, removeAll func(string) error, sleep func(ti } } -// listMetadataFiles returns paths to all instance metadata files. +// listMetadataFiles returns paths to all instance metadata files, skipping +// entries whose metadata cannot be statted. func (m *manager) listMetadataFiles() ([]string, error) { - return m.listMetadataFilesWithStatErrors(false) + return m.walkMetadataFiles(false) } -func (m *manager) listMetadataFilesWithStatErrors(failOnStatError bool) ([]string, error) { +// listMetadataFilesStrict returns paths to all instance metadata files, +// failing on any stat error other than absence. Fail-closed callers (the +// vGPU release claim scan and startup reconcile protection) use it so an +// unreadable instance is an error instead of silently missing. +func (m *manager) listMetadataFilesStrict() ([]string, error) { + return m.walkMetadataFiles(true) +} + +func (m *manager) walkMetadataFiles(failOnStatError bool) ([]string, error) { guestsDir := m.paths.GuestsDir() // Ensure guests directory exists diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 6b707e94c..b38fb5225 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -176,7 +176,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) // assignment without a PID, or unverifiable process ownership returns an error // so the requester retains its assignment for a later retry. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { - files, err := m.listMetadataFilesWithStatErrors(true) + files, err := m.listMetadataFilesStrict() if err != nil { return false, fmt.Errorf("list instances for vGPU release check: %w", err) } From 17cfdb499e8ce59dbfa7e71de194d7011dad1869 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:31:49 +0000 Subject: [PATCH 50/76] Skip instances deleted mid-listing in reconcile protection ListInstancesForReconcile failed hard when an instance was deleted between the metadata listing and its load. The startup call runs before the API serves, but the grace-period retry fires while deletes are in flight; one racing delete errored the whole list, which zeroed the retry and left vendor VFIO reconciliation disabled until the next restart. Skip ErrNotFound like the release claim scan does: a vanished record cannot claim a VF. --- lib/instances/manager.go | 10 ++++++++- lib/instances/query_test.go | 41 +++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 2ecc59fcc..c0d2f82c2 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -2,6 +2,7 @@ package instances import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -743,7 +744,7 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { // needs raw metadata fields, and hydration would query the hypervisor of // every instance on the host before the API serves. func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { - files, err := m.listMetadataFilesWithStatErrors(true) + files, err := m.listMetadataFilesStrict() if err != nil { return nil, err } @@ -752,6 +753,13 @@ func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, er id := filepath.Base(filepath.Dir(file)) meta, err := m.loadMetadata(id) if err != nil { + if errors.Is(err, ErrNotFound) { + // Deleted between listing and load; a vanished record cannot + // claim a VF. Failing here instead would zero the grace-period + // retry and disable the vendor VFIO sweep whenever it races a + // concurrent delete. + continue + } return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) } result = append(result, Instance{StoredMetadata: meta.StoredMetadata}) diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index ab3db29c3..b3dbfba41 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -42,6 +42,47 @@ func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { assert.Equal(t, "valid", listed[0].Id) } +// A concurrent delete can remove an instance between the reconcile listing +// and its metadata load. A vanished record cannot claim a VF, so it must be +// skipped like the release claim scan does — failing instead would zero the +// grace-period retry and silently disable the vendor VFIO sweep whenever it +// races a delete. +func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + + for _, id := range []string{"aaa-ghost", "zzz-live"} { + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + CreatedAt: time.Now(), + DataDir: m.paths.InstanceDir(id), + }})) + } + + // loadMetadata takes the snapshot-alias read lock, so holding the + // mutation lock parks the reconcile between listing and loading — the + // window a concurrent delete lands in. + unlock := hypervisor.LockSnapshotSourceAliasMutation() + type result struct { + listed []Instance + err error + } + done := make(chan result, 1) + go func() { + listed, err := m.ListInstancesForReconcile(context.Background()) + done <- result{listed, err} + }() + time.Sleep(100 * time.Millisecond) + require.NoError(t, os.Remove(m.paths.InstanceMetadata("aaa-ghost"))) + unlock() + + res := <-done + require.NoError(t, res.err) + require.Len(t, res.listed, 1) + assert.Equal(t, "zzz-live", res.listed[0].Id) +} + func TestParseExitSentinelLine(t *testing.T) { t.Parallel() tests := []struct { From 5c0292cfba0d429cee87204c96c51873e9c44e7f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:31:49 +0000 Subject: [PATCH 51/76] Count abandoned orphaned vGPU releases Giving up on an orphaned release leaves the VF allocated while /resources still advertises it, until startup reconciliation or manual remediation. That was visible only as a log line; count it so capacity leaks can alert. --- lib/instances/metrics.go | 21 +++++++++++++++++++++ lib/instances/vgpu_orphan.go | 1 + 2 files changed, 22 insertions(+) diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 1ada5ac1e..bdffbdbce 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -94,6 +94,7 @@ type Metrics struct { lifecycleEventsDroppedTotal metric.Int64Counter forkMemFileShareFallbacksTotal metric.Int64Counter ttlReaperDeletionsTotal metric.Int64Counter + vgpuOrphanReleasesAbandonedTotal metric.Int64Counter tracer trace.Tracer } @@ -270,6 +271,14 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + vgpuOrphanReleasesAbandonedTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_orphan_releases_abandoned_total", + metric.WithDescription("Total orphaned vGPU release retries that gave up, leaving the VF allocated until startup reconciliation or manual remediation"), + ) + if err != nil { + return nil, err + } + // Register observable gauge for instance counts by state instancesTotal, err := meter.Int64ObservableGauge( "hypeman_instances_total", @@ -464,6 +473,7 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M lifecycleEventsDroppedTotal: lifecycleEventsDroppedTotal, forkMemFileShareFallbacksTotal: forkMemFileShareFallbacksTotal, ttlReaperDeletionsTotal: ttlReaperDeletionsTotal, + vgpuOrphanReleasesAbandonedTotal: vgpuOrphanReleasesAbandonedTotal, tracer: tracer, }, nil } @@ -563,6 +573,17 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } +// recordVGPUOrphanReleaseAbandoned records an orphaned vGPU release retry +// loop giving up: the VF stays allocated (capacity silently reduced) until +// startup reconciliation or manual remediation, so it must be visible beyond +// a log line. +func (m *manager) recordVGPUOrphanReleaseAbandoned(ctx context.Context) { + if m.metrics == nil { + return + } + m.metrics.vgpuOrphanReleasesAbandonedTotal.Add(ctx, 1) +} + // recordStateTransition records a state transition with hypervisor label. func (m *manager) recordStateTransition(ctx context.Context, fromState, toState string, hvType hypervisor.Type) { if m.metrics == nil { diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go index 5f3fc2b38..b3ff2b33b 100644 --- a/lib/instances/vgpu_orphan.go +++ b/lib/instances/vgpu_orphan.go @@ -67,6 +67,7 @@ func (m *manager) retryOrphanedVGPURelease(ctx context.Context, stored StoredMet "instance_id", stored.Id, "device_path", path, "attempt", attempt) return } + m.recordVGPUOrphanReleaseAbandoned(ctx) log.ErrorContext(ctx, "giving up on orphaned vGPU release; VF stays allocated until startup reconciliation or manual remediation", "instance_id", stored.Id, "device_path", path, "attempts", orphanedVGPUReleaseMaxAttempts) } From 7b4621c888ad486e20d0eccbb74765bc69c522e7 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:52:25 +0000 Subject: [PATCH 52/76] Define the vGPU retention-stub rejection once Start, fork, and snapshot each carried a verbatim copy of the rejection error and its rationale; the stub fill in create's cleanup closure duplicated retainedVGPUFromCreateError field-for-field. One error value and one device-to-stub helper replace the copies. --- lib/instances/create.go | 8 +------- lib/instances/fork.go | 5 +---- lib/instances/snapshot.go | 6 +----- lib/instances/start.go | 5 +---- lib/instances/vgpu.go | 16 ++++++++++++---- 5 files changed, 16 insertions(+), 24 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index eefd0a7ce..66c988b4f 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -354,13 +354,7 @@ func (m *manager) createInstance( log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) retainedVGPU = stored if retainedVGPU == nil { - stub := retentionStub() - stub.GPUProfile = gpuDevice.ProfileName - stub.GPUFramework = gpuDevice.Framework - stub.GPUDevicePath = gpuDevice.SysfsPath - stub.GPUMdevUUID = gpuDevice.MdevUUID - stub.GPUAssignedAt = gpuAssignedAt - retainedVGPU = &stub + retainedVGPU = retainedVGPUFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) } } }) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 266299e96..08ce4014a 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -220,10 +220,7 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin return nil, false, fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby)", ErrInvalidState, source.State) } if stored.GPURetainedForCleanup { - // A delete-only retention stub from a failed create has no boot - // configuration, so a fork of it could never boot. Delete the stub to - // release its retained vGPU assignment. - return nil, false, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + return nil, false, errVGPURetentionStub } if !supportValidated { diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 297c7ef03..e30192ee9 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -67,11 +67,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps stored := &meta.StoredMetadata if stored.GPURetainedForCleanup { - // A delete-only retention stub from a failed create has no boot - // configuration, so a snapshot of it could never be restored or - // forked into a bootable instance. Delete the stub to release its - // retained vGPU assignment. - return nil, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + return nil, errVGPURetentionStub } if err := validateForkVolumeSafety(stored.Volumes); err != nil { return nil, fmt.Errorf("%w: snapshot requires readonly volume attachments: %v", ErrNotSupported, err) diff --git a/lib/instances/start.go b/lib/instances/start.go index 44dc50216..c7eb85493 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -48,11 +48,8 @@ func (m *manager) startInstance( return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } if stored.GPURetainedForCleanup { - // A delete-only retention stub from a failed create: it carries no - // boot configuration, so starting it would release the retained VF - // and then boot an incomplete record. Delete retries the release. log.ErrorContext(ctx, "refusing to start vGPU retention record", "instance_id", id) - return nil, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + return nil, errVGPURetentionStub } // Release any assignment retained by an earlier failed release and diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index b38fb5225..dcb6cff4d 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -33,6 +33,11 @@ func (e *VGPUCleanupPendingError) Error() string { func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } +// errVGPURetentionStub rejects every lifecycle verb except delete on a +// retention stub from a failed create: the record has no boot configuration, +// and only delete retries the release of its retained assignment. +var errVGPURetentionStub = fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID string) (*devices.VGPUDevice, error) { create := m.createVGPU if create == nil { @@ -58,11 +63,14 @@ func retainedVGPUFromCreateError(stub StoredMetadata, assignedAt time.Time, err if !ok { return nil } + return retainedVGPUFromDevice(stub, device, assignedAt) +} + +// retainedVGPUFromDevice fills stub with device's assignment fields so a +// failed rollback release retains a recognizable, deletable record. +func retainedVGPUFromDevice(stub StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) *StoredMetadata { stub.GPUProfile = device.ProfileName - stub.GPUFramework = device.Framework - stub.GPUDevicePath = device.SysfsPath - stub.GPUMdevUUID = device.MdevUUID - stub.GPUAssignedAt = &assignedAt + setStoredVGPUDevice(&stub, device, assignedAt) return &stub } From 456bb21864c91b25aa5415928c0825bfbbff5dff Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:52:25 +0000 Subject: [PATCH 53/76] Render pending vGPU cleanup responses through one helper The create and start handlers carried near-identical 20-line blocks deriving the vgpu_cleanup_pending message and inner error detail, differing only in the verb and release guidance. --- cmd/api/api/instances.go | 48 ++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 888bc13fb..8b9cef579 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -368,19 +368,11 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) - message := fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.Err, vgpuPending.InstanceID) - innerCode := "vgpu_retained_instance" - if !vgpuPending.Retained { - message = fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID) - innerCode = "vgpu_unretained_instance" - } + message, inner := vgpuCleanupPendingDetail(vgpuPending, "create", "delete it to retry") return oapi.CreateInstance500JSONResponse{ - Code: "vgpu_cleanup_pending", - Message: message, - InnerError: &oapi.ErrorDetail{ - Code: lo.ToPtr(innerCode), - Message: lo.ToPtr(vgpuPending.InstanceID), - }, + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: inner, }, nil case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ @@ -443,6 +435,22 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst return oapi.CreateInstance201JSONResponse(instanceToOAPI(*inst)), nil } +// vgpuCleanupPendingDetail renders a pending vGPU cleanup into the message +// and inner error detail shared by the create and start handlers. The +// retained guidance names the verb-specific way to release the assignment. +func vgpuCleanupPendingDetail(pending *instances.VGPUCleanupPendingError, action, retainedGuidance string) (string, *oapi.ErrorDetail) { + message := fmt.Sprintf("failed to %s instance: %v; vGPU release failed during rollback and instance %s retains the assignment, %s", action, pending.Err, pending.InstanceID, retainedGuidance) + innerCode := "vgpu_retained_instance" + if !pending.Retained { + message = fmt.Sprintf("failed to %s instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", action, pending.Err, pending.InstanceID) + innerCode = "vgpu_unretained_instance" + } + return message, &oapi.ErrorDetail{ + Code: lo.ToPtr(innerCode), + Message: lo.ToPtr(pending.InstanceID), + } +} + // GetInstance gets instance details // The id parameter can be an instance ID, name, or ID prefix // Note: Resolution is handled by ResolveResource middleware @@ -854,19 +862,11 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to start instance", "error", err) - message := fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it or retry start to release it", vgpuPending.Err, vgpuPending.InstanceID) - innerCode := "vgpu_retained_instance" - if !vgpuPending.Retained { - message = fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID) - innerCode = "vgpu_unretained_instance" - } + message, inner := vgpuCleanupPendingDetail(vgpuPending, "start", "delete it or retry start to release it") return oapi.StartInstance500JSONResponse{ - Code: "vgpu_cleanup_pending", - Message: message, - InnerError: &oapi.ErrorDetail{ - Code: lo.ToPtr(innerCode), - Message: lo.ToPtr(vgpuPending.InstanceID), - }, + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: inner, }, nil case errors.Is(err, instances.ErrInvalidState): return oapi.StartInstance409JSONResponse{ From 8403c1058b67ec8caacc3d01aa69c2e572931192 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 01:27:54 +0000 Subject: [PATCH 54/76] Narrow the vGPU reconcile interface --- cmd/api/main.go | 10 +++++++++- cmd/api/main_test.go | 5 +++++ lib/instances/manager.go | 1 - 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 71e147ecf..a4db0f3e9 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -185,8 +185,16 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } +type vgpuReconcileInstanceLister interface { + ListInstancesForReconcile(context.Context) ([]instances.Instance, error) +} + func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) { - allInstances, err := instanceManager.ListInstancesForReconcile(ctx) + lister, ok := instanceManager.(vgpuReconcileInstanceLister) + if !ok { + return nil, 0, errors.New("instance manager does not support vGPU reconcile inventory") + } + allInstances, err := lister.ListInstancesForReconcile(ctx) if err != nil { return nil, 0, err } diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 85c132814..0718b4cf3 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -351,6 +351,11 @@ func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([] return s.list, nil } +func TestLiveInstanceVGPUDevicePathsRequiresReconcileInventory(t *testing.T) { + _, _, err := liveInstanceVGPUDevicePaths(context.Background(), struct{ instances.Manager }{}) + require.ErrorContains(t, err, "does not support vGPU reconcile inventory") +} + func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) { dead := exec.Command("true") require.NoError(t, dead.Run()) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index c0d2f82c2..6605258bd 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -29,7 +29,6 @@ import ( type Manager interface { ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) - ListInstancesForReconcile(ctx context.Context) ([]Instance, error) ListSnapshots(ctx context.Context, filter *ListSnapshotsFilter) ([]Snapshot, error) GetSnapshot(ctx context.Context, snapshotID string) (*Snapshot, error) CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) From 48b7673c6e07e8a76478d306e82981f17f5ad9aa Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:54:32 +0000 Subject: [PATCH 55/76] Encapsulate vGPU rollback retention --- lib/instances/create.go | 73 ++----------------- lib/instances/lifecycle_noop_test.go | 2 +- lib/instances/start.go | 17 ++--- lib/instances/vgpu.go | 95 +++++++++++++----------- lib/instances/vgpu_retention.go | 104 +++++++++++++++++++++++++++ lib/instances/vgpu_test.go | 46 ++++++++++-- 6 files changed, 212 insertions(+), 125 deletions(-) create mode 100644 lib/instances/vgpu_retention.go diff --git a/lib/instances/create.go b/lib/instances/create.go index 66c988b4f..f4ece0100 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -279,21 +279,13 @@ func (m *manager) createInstance( var gpuMdevUUID string var gpuAssignedAt *time.Time var stored *StoredMetadata - var retainedVGPU *StoredMetadata + retention := vgpuRetention{instanceID: id} // Setup cleanup stack early so device attachment errors trigger cleanup. - // When rollback cannot release a vGPU assignment, report whether its - // retention record was persisted. The wrapping defer is registered first - // so it runs after cu.Clean has attempted to retain the metadata. - vgpuPersisted := false - defer func() { - if retErr != nil && retainedVGPU != nil { - retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuPersisted, Err: retErr} - } - }() + defer retention.deferWrapPending(&retErr) cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) - vgpuPersisted = m.cleanupFailedCreate(ctx, id, retainedVGPU) + m.persistVGPURetention(ctx, &retention) }) defer cu.Clean() @@ -329,7 +321,7 @@ func (m *manager) createInstance( log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) if err != nil { - retainedVGPU = retainedVGPUFromCreateError(retentionStub(), m.nowUTC(), err) + retention.retainFromCreateError(retentionStub(), m.nowUTC(), err) log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } @@ -352,9 +344,9 @@ func (m *manager) createInstance( } if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) - retainedVGPU = stored - if retainedVGPU == nil { - retainedVGPU = retainedVGPUFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) + retention.retain(stored) + if stored == nil { + retention.retainFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) } } }) @@ -647,57 +639,6 @@ func resolveCreateExpiration(req CreateInstanceRequest, now time.Time) (*time.Ti return &expiresAt, nil } -// cleanupFailedCreate reports whether the retention record for a vGPU -// assignment whose release failed during rollback was persisted. -func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool { - if retainedVGPU == nil { - m.deleteInstanceData(id) - return false - } - - log := logger.FromContext(ctx) - retentionSurvives := func() bool { - meta, err := m.loadMetadata(id) - if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { - return true - } - if err := m.deleteInstanceData(id); err != nil { - log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) - } - return false - } - if err := m.ensureDirectories(id); err != nil { - log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - return retentionSurvives() - } - // Retain identity fields so the instance lists as a recognizable, - // deletable record rather than a nameless phantom, but drop resource - // claims (network, volumes, devices) that rollback already released. - retained := StoredMetadata{ - Id: id, - Name: retainedVGPU.Name, - Image: retainedVGPU.Image, - ResolvedImage: retainedVGPU.ResolvedImage, - Platform: retainedVGPU.Platform, - CreatedAt: retainedVGPU.CreatedAt, - HypervisorType: retainedVGPU.HypervisorType, - HypervisorVersion: retainedVGPU.HypervisorVersion, - SocketPath: retainedVGPU.SocketPath, - DataDir: retainedVGPU.DataDir, - GPUProfile: retainedVGPU.GPUProfile, - GPUFramework: retainedVGPU.GPUFramework, - GPUDevicePath: retainedVGPU.GPUDevicePath, - GPUMdevUUID: retainedVGPU.GPUMdevUUID, - GPUAssignedAt: retainedVGPU.GPUAssignedAt, - GPURetainedForCleanup: true, - } - if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { - log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - return retentionSurvives() - } - return true -} - // validateCreateRequest validates the create instance request. // The request is mutated in-place to persist normalized egress/credential policy fields. func validateCreateRequest(req *CreateInstanceRequest) error { diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 46f891379..a43efabb0 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -201,7 +201,7 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { // A failed create whose vGPU release also failed retains a minimal // GPU-fields-only stub, and the API tells the caller to delete it to retry // the release. Exercise that recovery path against the exact stub shape -// cleanupFailedCreate writes. +// persistVGPURetention writes. func TestDeleteReleasesRetainedCreateStub(t *testing.T) { p := paths.New(t.TempDir()) var destroyed []devices.VGPUAssignment diff --git a/lib/instances/start.go b/lib/instances/start.go index c7eb85493..a79663953 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -121,16 +121,8 @@ func (m *manager) startInstance( } // Setup cleanup stack for automatic rollback on errors - // Registered before cu.Clean so it runs after cleanup and can report a - // vGPU assignment that rollback failed to destroy, matching create's - // vgpu_cleanup_pending contract. - vgpuRetained := false - vgpuRetentionPersisted := false - defer func() { - if retErr != nil && vgpuRetained { - retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuRetentionPersisted, Err: retErr} - } - }() + retention := vgpuRetention{instanceID: id} + defer retention.deferWrapPending(&retErr) cu := cleanup.Make(func() {}) defer cu.Clean() @@ -204,7 +196,10 @@ func (m *manager) startInstance( log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) // Add vGPU cleanup to stack cu.Add(func() { - vgpuRetained, vgpuRetentionPersisted = m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) + retained, persisted := m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) + if retained { + retention.markRetained(persisted) + } }) if err := m.saveMetadata(meta); err != nil { log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index dcb6cff4d..3561cbc5a 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -54,24 +54,18 @@ func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { return &pending.Device, true } -// retainedVGPUFromCreateError fills stub with the pending device's assignment -// fields when err carries a failed device-layer cleanup. The caller provides -// identity fields on stub so the retained record lists as a recognizable, -// deletable instance. -func retainedVGPUFromCreateError(stub StoredMetadata, assignedAt time.Time, err error) *StoredMetadata { - device, ok := vgpuDevicePendingCleanup(err) - if !ok { - return nil +func vgpuAssignmentLiveness(stored *StoredMetadata, now time.Time, livePID bool) (live bool, graceRemaining time.Duration) { + if stored.HypervisorPID != nil && livePID { + return true, 0 } - return retainedVGPUFromDevice(stub, device, assignedAt) -} - -// retainedVGPUFromDevice fills stub with device's assignment fields so a -// failed rollback release retains a recognizable, deletable record. -func retainedVGPUFromDevice(stub StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) *StoredMetadata { - stub.GPUProfile = device.ProfileName - setStoredVGPUDevice(&stub, device, assignedAt) - return &stub + if stored.GPUAssignedAt == nil { + return false, 0 + } + remaining := VGPUAssignmentStartupGracePeriod - now.Sub(*stored.GPUAssignedAt) + if remaining <= 0 { + return false, 0 + } + return true, remaining } func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { @@ -96,15 +90,9 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUAssignedAt = nil } -// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. The -// cleanup stack is LIFO, so cleanups registered after this one run before it -// and this restore would clobber anything they persisted; it is safe only -// while no such cleanup writes metadata and the instance lock serializes -// start. Violating that requires switching to targeted field restores. -// -// It reports whether the assignment was retained after a failed destroy and -// whether that retention record was persisted, so start can surface the -// pending cleanup as a typed error like create does. +// cleanupStartVGPU reports whether the assignment was retained after a failed +// destroy and whether that retention record was persisted, so start can surface +// the pending cleanup as a typed error like create does. func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) { logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID) assignment := devices.VGPUAssignment{ @@ -113,14 +101,20 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic MdevUUID: device.MdevUUID, InstanceID: instanceID, } - cleanupMeta := rollbackMeta + cleanupMeta, err := m.loadMetadata(instanceID) + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to load current metadata for vGPU cleanup; restoring rollback snapshot", "instance_id", instanceID, "error", err) + cleanupMeta = &rollbackMeta + } else { + restoreStartMutatedFields(&cleanupMeta.StoredMetadata, &rollbackMeta.StoredMetadata) + } releaseErr := m.destroyVGPUAssignment(ctx, assignment) if releaseErr != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID, "error", releaseErr) setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) retained = true } - if err := m.saveMetadata(&cleanupMeta); err != nil { + if err := m.saveMetadata(cleanupMeta); err != nil { message := "failed to save metadata after vGPU cleanup" if releaseErr != nil { message = "failed to retain vGPU assignment metadata after cleanup failure" @@ -141,6 +135,27 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic return retained, retained } +// restoreStartMutatedFields must cover every field start mutates before the +// vGPU cleanup runs. +func restoreStartMutatedFields(dst, src *StoredMetadata) { + dst.HypervisorPID = src.HypervisorPID + dst.HypervisorStartTime = src.HypervisorStartTime + dst.HypervisorBootID = src.HypervisorBootID + dst.ExitCode = src.ExitCode + dst.ExitMessage = src.ExitMessage + dst.ProgramStartedAt = src.ProgramStartedAt + dst.GuestAgentReadyAt = src.GuestAgentReadyAt + dst.Entrypoint = src.Entrypoint + dst.Cmd = src.Cmd + dst.IP = src.IP + dst.MAC = src.MAC + dst.GPUFramework = src.GPUFramework + dst.GPUDevicePath = src.GPUDevicePath + dst.GPUMdevUUID = src.GPUMdevUUID + dst.GPUAssignedAt = src.GPUAssignedAt + dst.StartedAt = src.StartedAt +} + func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { @@ -206,23 +221,21 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if storedVGPUDevicePath(stored) != devicePath { continue } - if stored.HypervisorPID == nil { - if stored.GPUAssignedAt == nil || time.Since(*stored.GPUAssignedAt) >= VGPUAssignmentStartupGracePeriod { - continue + pid := 0 + if stored.HypervisorPID != nil { + pid, err = resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) + if err != nil { + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } - return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) } - pid, err := resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) - if err != nil { - return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) - } - if pid > 0 { + live, remaining := vgpuAssignmentLiveness(stored, time.Now(), pid > 0) + if pid > 0 && live { return true, nil } - // A dead PID with a recent assignment gets the same bounded grace as - // startup reconcile protection, so the two guards agree in the - // fail-closed direction while a mid-boot claimant hydrates. - if stored.GPUAssignedAt != nil && time.Since(*stored.GPUAssignedAt) < VGPUAssignmentStartupGracePeriod { + if remaining > 0 { + if stored.HypervisorPID == nil { + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) + } return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", id, devicePath) } } diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go new file mode 100644 index 000000000..a1a1aae0d --- /dev/null +++ b/lib/instances/vgpu_retention.go @@ -0,0 +1,104 @@ +package instances + +import ( + "context" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" +) + +type vgpuRetention struct { + instanceID string + stub *StoredMetadata + retained bool + persisted bool +} + +func (r *vgpuRetention) retainFromCreateError(stub StoredMetadata, assignedAt time.Time, err error) { + device, ok := vgpuDevicePendingCleanup(err) + if !ok { + return + } + r.retainFromDevice(stub, device, assignedAt) +} + +func (r *vgpuRetention) retainFromDevice(stub StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) { + stub.GPUProfile = device.ProfileName + setStoredVGPUDevice(&stub, device, assignedAt) + r.stub = &stub + r.retained = true +} + +func (r *vgpuRetention) retain(stub *StoredMetadata) { + r.stub = stub + r.retained = stub != nil +} + +func (r *vgpuRetention) markRetained(persisted bool) { + r.retained = true + r.persisted = persisted +} + +func (r *vgpuRetention) wrapPending(err error) error { + if err == nil || !r.retained { + return err + } + return &VGPUCleanupPendingError{InstanceID: r.instanceID, Retained: r.persisted, Err: err} +} + +// deferWrapPending must be deferred before cleanup so it observes retention +// state recorded by rollback. +func (r *vgpuRetention) deferWrapPending(retErr *error) { + *retErr = r.wrapPending(*retErr) +} + +func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuRetention) { + if retention.stub == nil { + m.deleteInstanceData(retention.instanceID) + return + } + + id := retention.instanceID + retainedVGPU := retention.stub + log := logger.FromContext(ctx) + retentionSurvives := func() bool { + meta, err := m.loadMetadata(id) + if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { + return true + } + if err := m.deleteInstanceData(id); err != nil { + log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) + } + return false + } + if err := m.ensureDirectories(id); err != nil { + log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) + retention.persisted = retentionSurvives() + return + } + retained := StoredMetadata{ + Id: id, + Name: retainedVGPU.Name, + Image: retainedVGPU.Image, + ResolvedImage: retainedVGPU.ResolvedImage, + Platform: retainedVGPU.Platform, + CreatedAt: retainedVGPU.CreatedAt, + HypervisorType: retainedVGPU.HypervisorType, + HypervisorVersion: retainedVGPU.HypervisorVersion, + SocketPath: retainedVGPU.SocketPath, + DataDir: retainedVGPU.DataDir, + GPUProfile: retainedVGPU.GPUProfile, + GPUFramework: retainedVGPU.GPUFramework, + GPUDevicePath: retainedVGPU.GPUDevicePath, + GPUMdevUUID: retainedVGPU.GPUMdevUUID, + GPUAssignedAt: retainedVGPU.GPUAssignedAt, + GPURetainedForCleanup: true, + } + if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) + retention.persisted = retentionSurvives() + return + } + retention.persisted = true +} diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 1d0e4e3bf..c5ef4911b 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -18,6 +18,19 @@ import ( "github.com/stretchr/testify/require" ) +func persistTestVGPURetention(m *manager, ctx context.Context, id string, stub *StoredMetadata) bool { + retention := vgpuRetention{instanceID: id} + retention.retain(stub) + m.persistVGPURetention(ctx, &retention) + return retention.persisted +} + +func retainedVGPUFromCreateErrorForTest(stub StoredMetadata, assignedAt time.Time, err error) *StoredMetadata { + retention := vgpuRetention{} + retention.retainFromCreateError(stub, assignedAt, err) + return retention.stub +} + func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { t.Parallel() @@ -38,7 +51,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { DataDir: m.paths.InstanceDir("failed-create"), } - assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) + assert.True(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) retained, err := m.loadMetadata(stored.Id) require.NoError(t, err) @@ -67,7 +80,7 @@ func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { m := &manager{paths: paths.New(t.TempDir())} require.NoError(t, m.ensureDirectories("failed-create")) - assert.False(t, m.cleanupFailedCreate(context.Background(), "failed-create", nil)) + assert.False(t, persistTestVGPURetention(m, context.Background(), "failed-create", nil)) _, err := m.loadMetadata("failed-create") require.Error(t, err) } @@ -85,7 +98,7 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } - assert.False(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) + assert.False(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) _, err := m.loadMetadata(id) require.Error(t, err) } @@ -109,13 +122,31 @@ func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T require.NoError(t, os.Chmod(instanceDir, 0o555)) t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) - assert.True(t, m.cleanupFailedCreate(context.Background(), id, stored)) + assert.True(t, persistTestVGPURetention(m, context.Background(), id, stored)) retained, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, stored.GPUFramework, retained.GPUFramework) assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) } +func TestVGPURetentionWrapPending(t *testing.T) { + cause := errors.New("boot failed") + + retention := vgpuRetention{instanceID: "inst-1"} + assert.Same(t, cause, retention.wrapPending(cause)) + + retention.retained = true + pending := retention.wrapPending(cause) + var cleanupPending *VGPUCleanupPendingError + require.ErrorAs(t, pending, &cleanupPending) + assert.False(t, cleanupPending.Retained) + + retention.persisted = true + pending = retention.wrapPending(cause) + require.ErrorAs(t, pending, &cleanupPending) + assert.True(t, cleanupPending.Retained) +} + func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { t.Parallel() @@ -145,7 +176,7 @@ func TestVGPUDevicePendingCleanup(t *testing.T) { assert.Equal(t, device, *actual) assignedAt := time.Now().UTC() - retained := retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1", Name: "named", Image: "img"}, assignedAt, wrapped) + retained := retainedVGPUFromCreateErrorForTest(StoredMetadata{Id: "inst-1", Name: "named", Image: "img"}, assignedAt, wrapped) require.NotNil(t, retained) assert.Equal(t, "inst-1", retained.Id) assert.Equal(t, "named", retained.Name, "identity fields must survive into the retention stub") @@ -157,7 +188,7 @@ func TestVGPUDevicePendingCleanup(t *testing.T) { actual, ok = vgpuDevicePendingCleanup(cause) assert.False(t, ok) assert.Nil(t, actual) - assert.Nil(t, retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1"}, assignedAt, cause)) + assert.Nil(t, retainedVGPUFromCreateErrorForTest(StoredMetadata{Id: "inst-1"}, assignedAt, cause)) } type startRetentionNetworkManager struct { @@ -427,6 +458,7 @@ func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { exitCode := 1 rollbackMeta := metadata{StoredMetadata: StoredMetadata{ Id: id, + Name: "original name", GPUProfile: "NVIDIA L40S-2Q", Entrypoint: []string{"old-entrypoint"}, Cmd: []string{"old-command"}, @@ -437,6 +469,7 @@ func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { }} partial := rollbackMeta + partial.Name = "concurrent update" partial.Entrypoint = []string{"new-entrypoint"} partial.Cmd = []string{"new-command"} partial.StartedAt = ptr(time.Now().UTC()) @@ -455,6 +488,7 @@ func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { stored, err := m.loadMetadata(id) require.NoError(t, err) + assert.Equal(t, "concurrent update", stored.Name) assert.Equal(t, rollbackMeta.Entrypoint, stored.Entrypoint) assert.Equal(t, rollbackMeta.Cmd, stored.Cmd) assert.Equal(t, rollbackMeta.StartedAt, stored.StartedAt) From a44d855a56be202992f9281489e9b4ad3f3cc707 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:54:32 +0000 Subject: [PATCH 56/76] Move vGPU reconciliation into instance manager --- cmd/api/main.go | 67 +------------------------- cmd/api/main_test.go | 41 ---------------- lib/builds/manager_test.go | 4 +- lib/devices/mdev_darwin.go | 2 +- lib/devices/vgpu_linux.go | 26 ++++++++-- lib/devices/vgpu_linux_test.go | 16 +++++++ lib/instances/manager.go | 1 + lib/instances/vgpu_reconcile.go | 61 ++++++++++++++++++++++++ lib/instances/vgpu_reconcile_test.go | 71 ++++++++++++++++++++++++++++ lib/instances/wait_test.go | 4 +- 10 files changed, 175 insertions(+), 118 deletions(-) create mode 100644 lib/instances/vgpu_reconcile.go create mode 100644 lib/instances/vgpu_reconcile_test.go diff --git a/cmd/api/main.go b/cmd/api/main.go index a4db0f3e9..da2748103 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -185,71 +185,6 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } -type vgpuReconcileInstanceLister interface { - ListInstancesForReconcile(context.Context) ([]instances.Instance, error) -} - -func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) { - lister, ok := instanceManager.(vgpuReconcileInstanceLister) - if !ok { - return nil, 0, errors.New("instance manager does not support vGPU reconcile inventory") - } - allInstances, err := lister.ListInstancesForReconcile(ctx) - if err != nil { - return nil, 0, err - } - protected := make(map[string]struct{}) - var retryAfter time.Duration - for _, inst := range allInstances { - if inst.GPUDevicePath == "" { - continue - } - if inst.HypervisorPID != nil && instances.HypervisorMayBeAlive(inst.HypervisorProcessIdentity, inst.SocketPath) { - protected[inst.GPUDevicePath] = struct{}{} - continue - } - if inst.GPUAssignedAt == nil { - continue - } - remaining := instances.VGPUAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt) - if remaining <= 0 { - continue - } - protected[inst.GPUDevicePath] = struct{}{} - if retryAfter == 0 || remaining < retryAfter { - retryAfter = remaining - } - } - return protected, retryAfter, nil -} - -func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) { - protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager) - if err != nil { - // Operator-actionable: vendor VFIO reconciliation stays disabled - // host-wide (and releases fail closed on the same inventory) until - // the unreadable instance metadata is repaired. - logger.Error("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err) - protected = nil - retryAfter = 0 - } - if err := devices.ReconcileVGPUs(ctx, protected); err != nil { - logger.Warn("failed to reconcile vGPU devices", "error", err) - } - if retryAfter <= 0 { - return - } - go func() { - timer := time.NewTimer(retryAfter) - defer timer.Stop() - select { - case <-ctx.Done(): - case <-timer.C: - reconcileVGPUs(ctx, instanceManager, logger) - } - }() -} - func run() error { startupStarted := time.Now() slog.Info("starting hypeman initialization") @@ -451,7 +386,7 @@ func run() error { // Reconcile vGPU devices (clears orphaned vGPUs from previous runs) logger.Info("Reconciling vGPU devices...") - reconcileVGPUs(ctx, app.InstanceManager, logger) + app.InstanceManager.ReconcileVGPUs(ctx) // Wire up resource validator for aggregate limit checking // This enables the instance manager to validate CPU, memory, network, and GPU diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 0718b4cf3..b771e27fc 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -6,7 +6,6 @@ import ( "net/http" "net/http/httptest" "net/url" - "os/exec" "testing" "time" @@ -341,43 +340,3 @@ func TestImageNameWithSlashes_URLEncoding(t *testing.T) { }) } } - -type vgpuReconcileManagerStub struct { - instances.Manager - list []instances.Instance -} - -func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([]instances.Instance, error) { - return s.list, nil -} - -func TestLiveInstanceVGPUDevicePathsRequiresReconcileInventory(t *testing.T) { - _, _, err := liveInstanceVGPUDevicePaths(context.Background(), struct{ instances.Manager }{}) - require.ErrorContains(t, err, "does not support vGPU reconcile inventory") -} - -func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) { - dead := exec.Command("true") - require.NoError(t, dead.Run()) - deadPID := dead.Process.Pid - recent := time.Now().Add(-time.Minute) - stale := time.Now().Add(-instances.VGPUAssignmentStartupGracePeriod - time.Minute) - - manager := vgpuReconcileManagerStub{list: []instances.Instance{ - {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}}, - {StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}}, - {StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}}, - {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}}}, - {StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}}, - }} - - protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) - require.NoError(t, err) - require.Positive(t, retryAfter) - require.LessOrEqual(t, retryAfter, instances.VGPUAssignmentStartupGracePeriod) - assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") - assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") - assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") - assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") - assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") -} diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index b28383889..bbc426252 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -51,9 +51,7 @@ func (m *mockInstanceManager) ListInstances(ctx context.Context, filter *instanc return result, nil } -func (m *mockInstanceManager) ListInstancesForReconcile(ctx context.Context) ([]instances.Instance, error) { - return m.ListInstances(ctx, nil) -} +func (m *mockInstanceManager) ReconcileVGPUs(context.Context) {} func (m *mockInstanceManager) ListSnapshots(ctx context.Context, filter *instances.ListSnapshotsFilter) ([]instances.Snapshot, error) { return nil, nil diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 4274063ed..4b726bb08 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -58,7 +58,7 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { return nil } -func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error { +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepVendorVFIO bool) error { return nil } diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index be92e8ee6..72827f7b2 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -105,20 +105,38 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { } // ReconcileVGPUs releases orphaned vGPU assignments. -func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error { +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepVendorVFIO bool) error { framework, _, err := DiscoverVGPU() if err != nil { return err } + return reconcileDiscoveredVGPUs( + ctx, + framework, + protectedDevicePaths, + sweepVendorVFIO, + func(ctx context.Context) error { return ReconcileMdevs(ctx, nil) }, + hostVendorVFIO.reconcile, + ) +} + +func reconcileDiscoveredVGPUs( + ctx context.Context, + framework VGPUFramework, + protectedDevicePaths map[string]struct{}, + sweepVendorVFIO bool, + reconcileMdev func(context.Context) error, + reconcileVendorVFIO func(context.Context, map[string]struct{}) error, +) error { switch framework { case VGPUFrameworkMdev: - return ReconcileMdevs(ctx, nil) + return reconcileMdev(ctx) case VGPUFrameworkVendorVFIO: - if protectedDevicePaths == nil { + if !sweepVendorVFIO { return nil } - return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) + return reconcileVendorVFIO(ctx, protectedDevicePaths) default: return nil } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go index 7b03d9c26..32f9a30ea 100644 --- a/lib/devices/vgpu_linux_test.go +++ b/lib/devices/vgpu_linux_test.go @@ -3,6 +3,7 @@ package devices import ( + "context" "errors" "os" "path/filepath" @@ -12,6 +13,21 @@ import ( "github.com/stretchr/testify/require" ) +func TestReconcileDiscoveredVGPUsControlsVendorSweep(t *testing.T) { + protected := make(map[string]struct{}) + vendorCalls := 0 + reconcileVendor := func(context.Context, map[string]struct{}) error { + vendorCalls++ + return nil + } + + require.NoError(t, reconcileDiscoveredVGPUs(context.Background(), VGPUFrameworkVendorVFIO, protected, false, nil, reconcileVendor)) + assert.Zero(t, vendorCalls) + + require.NoError(t, reconcileDiscoveredVGPUs(context.Background(), VGPUFrameworkVendorVFIO, protected, true, nil, reconcileVendor)) + assert.Equal(t, 1, vendorCalls) +} + func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { t.Parallel() diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 6605258bd..cca794015 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -29,6 +29,7 @@ import ( type Manager interface { ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) + ReconcileVGPUs(ctx context.Context) ListSnapshots(ctx context.Context, filter *ListSnapshotsFilter) ([]Snapshot, error) GetSnapshot(ctx context.Context, snapshotID string) (*Snapshot, error) CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go new file mode 100644 index 000000000..85d3a6e0d --- /dev/null +++ b/lib/instances/vgpu_reconcile.go @@ -0,0 +1,61 @@ +package instances + +import ( + "context" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" +) + +func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]struct{}, time.Duration, error) { + allInstances, err := m.ListInstancesForReconcile(ctx) + if err != nil { + return nil, 0, err + } + protected := make(map[string]struct{}) + var retryAfter time.Duration + for i := range allInstances { + stored := &allInstances[i].StoredMetadata + if stored.GPUDevicePath == "" { + continue + } + livePID := stored.HypervisorPID != nil && HypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID) + if !live { + continue + } + protected[stored.GPUDevicePath] = struct{}{} + if remaining > 0 && (retryAfter == 0 || remaining < retryAfter) { + retryAfter = remaining + } + } + return protected, retryAfter, nil +} + +// ReconcileVGPUs releases orphaned vGPU assignments. +func (m *manager) ReconcileVGPUs(ctx context.Context) { + log := logger.FromContext(ctx) + protected, retryAfter, err := m.liveVGPUReconcileProtection(ctx) + sweepVendorVFIO := err == nil + if err != nil { + log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconcile, mdev reconcile still runs", "error", err) + protected = make(map[string]struct{}) + retryAfter = 0 + } + if err := devices.ReconcileVGPUs(ctx, protected, sweepVendorVFIO); err != nil { + log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) + } + if retryAfter <= 0 { + return + } + go func() { + timer := time.NewTimer(retryAfter) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + m.ReconcileVGPUs(ctx) + } + }() +} diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go new file mode 100644 index 000000000..1fbd90384 --- /dev/null +++ b/lib/instances/vgpu_reconcile_test.go @@ -0,0 +1,71 @@ +package instances + +import ( + "os/exec" + "testing" + "time" + + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLiveVGPUReconcileProtectionBoundsStartupProtection(t *testing.T) { + dead := exec.Command("true") + require.NoError(t, dead.Run()) + deadPID := dead.Process.Pid + now := time.Now().UTC() + recent := now.Add(-time.Minute) + stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + + m := &manager{paths: paths.New(t.TempDir()), now: func() time.Time { return now }} + instances := []StoredMetadata{ + {Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}, + {Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}, + {Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}, + {Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}}, + {Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}, + } + for i := range instances { + require.NoError(t, m.ensureDirectories(instances[i].Id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: instances[i]})) + } + + protected, retryAfter, err := m.liveVGPUReconcileProtection(t.Context()) + require.NoError(t, err) + assert.Equal(t, VGPUAssignmentStartupGracePeriod-time.Minute, retryAfter) + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") +} + +func TestVGPUAssignmentLiveness(t *testing.T) { + now := time.Now().UTC() + recent := now.Add(-time.Minute) + stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + pid := 123 + + tests := []struct { + name string + stored StoredMetadata + livePID bool + live bool + remaining time.Duration + }{ + {name: "live PID", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}}, livePID: true, live: true}, + {name: "dead PID recent assignment", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, GPUAssignedAt: &recent}, live: true, remaining: VGPUAssignmentStartupGracePeriod - time.Minute}, + {name: "dead PID stale assignment", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, GPUAssignedAt: &stale}}, + {name: "no PID recent assignment", stored: StoredMetadata{GPUAssignedAt: &recent}, live: true, remaining: VGPUAssignmentStartupGracePeriod - time.Minute}, + {name: "no PID stale assignment", stored: StoredMetadata{GPUAssignedAt: &stale}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + live, remaining := vgpuAssignmentLiveness(&tt.stored, now, tt.livePID) + assert.Equal(t, tt.live, live) + assert.Equal(t, tt.remaining, remaining) + }) + } +} diff --git a/lib/instances/wait_test.go b/lib/instances/wait_test.go index a42464795..a15c19c8c 100644 --- a/lib/instances/wait_test.go +++ b/lib/instances/wait_test.go @@ -32,9 +32,7 @@ func (s *stubManager) GetInstance(ctx context.Context, id string) (*Instance, er func (s *stubManager) ListInstances(context.Context, *ListInstancesFilter) ([]Instance, error) { return nil, nil } -func (s *stubManager) ListInstancesForReconcile(context.Context) ([]Instance, error) { - return nil, nil -} +func (s *stubManager) ReconcileVGPUs(context.Context) {} func (s *stubManager) ListSnapshots(context.Context, *ListSnapshotsFilter) ([]Snapshot, error) { return nil, nil } From 2bedc0592a3145d6a75137f3b09caa1351c8120e Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:54:32 +0000 Subject: [PATCH 57/76] Derive QEMU VFIO grace from VM config --- cmd/api/main_test.go | 2 -- lib/hypervisor/qemu/process.go | 28 +++++++++------------------- lib/hypervisor/qemu/process_test.go | 17 +++++++++++++++++ lib/hypervisor/vfio.go | 7 +++++++ lib/instances/process_identity.go | 8 +------- 5 files changed, 34 insertions(+), 28 deletions(-) create mode 100644 lib/hypervisor/vfio.go diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index b771e27fc..34dbba428 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "context" "net/http" "net/http/httptest" "net/url" @@ -12,7 +11,6 @@ import ( "github.com/getkin/kin-openapi/openapi3filter" "github.com/go-chi/chi/v5" "github.com/golang-jwt/jwt/v5" - "github.com/kernel/hypeman/lib/instances" mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/oapi" nethttpmiddleware "github.com/oapi-codegen/nethttp-middleware" diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index cddd51824..367bf14e3 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -40,12 +40,6 @@ const ( // socketDialTimeout is timeout for individual socket connection attempts socketDialTimeout = 100 * time.Millisecond - // vfioTermGrace is how long start-failure cleanup waits for a - // VFIO-attached QEMU to exit on SIGTERM before SIGKILL. Only failed - // starts pay it, and only when the process ignores SIGTERM; observed - // mid-init VFIO teardown takes 1-2s. - vfioTermGrace = 5 * time.Second - // clientCreateTimeout is how long to retry QMP client creation after the // socket appears. Under high parallel load the socket can accept connections // slightly later than file creation/availability. @@ -320,20 +314,17 @@ func (p *startedProcess) cleanup() { _ = os.Remove(p.socketPath) } -// hasVFIODevice reports whether the QEMU command line attaches a VFIO device. -func hasVFIODevice(args []string) bool { - for _, arg := range args { - if strings.Contains(arg, "vfio-pci") { - return true - } +func vfioTermGraceFor(cfg hypervisor.VMConfig) time.Duration { + if cfg.VGPUDevicePath != "" || len(cfg.PCIDevices) > 0 { + return hypervisor.VFIOTermGrace } - return false + return 0 } // startQEMUProcess handles the common QEMU process startup logic. // Returns the PID, hypervisor client, and a cleanup function. // The cleanup function must be called on error; call cleanup.Release() on success. -func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version string, socketPath string, args []string) (int, *QEMU, *cleanup.Cleanup, error) { +func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version string, socketPath string, args []string, termGrace time.Duration) (int, *QEMU, *cleanup.Cleanup, error) { log := logger.FromContext(ctx) processAttrs := hypervisor.TraceAttributesFromContext(ctx) processAttrs = append(processAttrs, @@ -403,9 +394,8 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version } pid := proc.pid - if hasVFIODevice(args) { - proc.termGrace = vfioTermGrace - } + // Only failed starts pay the VFIO termination grace. + proc.termGrace = termGrace log.DebugContext(processCtx, "QEMU process started", "pid", pid, "duration_ms", time.Since(processStartTime).Milliseconds()) // Setup cleanup to kill, reap, and remove the socket if subsequent steps fail. @@ -522,7 +512,7 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s // Build command arguments: QMP socket + VM configuration args := buildQMPArgs(socketPath) args = append(args, buildArgs(attempt, machineType)...) - pid, hv, cu, err = s.startQEMUProcess(ctx, p, version, socketPath, args) + pid, hv, cu, err = s.startQEMUProcess(ctx, p, version, socketPath, args, vfioTermGraceFor(attempt)) if err == nil { booted = attempt started = true @@ -657,7 +647,7 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, incomingURI := "exec:cat < " + memoryFile args = append(args, "-incoming", incomingURI) - pid, hv, cu, err := s.startQEMUProcess(ctx, p, version, socketPath, args) + pid, hv, cu, err := s.startQEMUProcess(ctx, p, version, socketPath, args, vfioTermGraceFor(config)) if err != nil { return 0, nil, err } diff --git a/lib/hypervisor/qemu/process_test.go b/lib/hypervisor/qemu/process_test.go index 2bdf27d4f..ad08aea24 100644 --- a/lib/hypervisor/qemu/process_test.go +++ b/lib/hypervisor/qemu/process_test.go @@ -411,6 +411,23 @@ func TestWaitForSocketOrExitReturnsEarlyWhenProcessDies(t *testing.T) { assert.True(t, cmd.ProcessState.Exited()) } +func TestVFIOTermGraceFor(t *testing.T) { + tests := []struct { + name string + cfg hypervisor.VMConfig + want time.Duration + }{ + {name: "vGPU", cfg: hypervisor.VMConfig{VGPUDevicePath: "/sys/bus/mdev/devices/test"}, want: hypervisor.VFIOTermGrace}, + {name: "PCI device", cfg: hypervisor.VMConfig{PCIDevices: []string{"0000:01:00.0"}}, want: hypervisor.VFIOTermGrace}, + {name: "no VFIO device", cfg: hypervisor.VMConfig{}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, vfioTermGraceFor(tt.cfg)) + }) + } +} + func TestCleanupSIGTERMsProcessWithTermGrace(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "qemu.sock") markerPath := filepath.Join(t.TempDir(), "terminated") diff --git a/lib/hypervisor/vfio.go b/lib/hypervisor/vfio.go new file mode 100644 index 000000000..1744cfc84 --- /dev/null +++ b/lib/hypervisor/vfio.go @@ -0,0 +1,7 @@ +package hypervisor + +import "time" + +// VFIOTermGrace allows VFIO teardown to finish after SIGTERM before SIGKILL; +// SIGKILL during initialization can wedge the VF, while teardown takes 1-2s. +const VFIOTermGrace = 5 * time.Second diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 0801aca6c..83aa682bc 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -27,19 +27,13 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second -// defaultVGPUInitTermGrace is how long terminateThenKill waits for a vGPU -// hypervisor to exit on SIGTERM before SIGKILL. Only force-kill paths pay it, -// only for vGPU instances, and only when the process ignores SIGTERM; -// observed mid-init VFIO teardown takes 1-2s. -const defaultVGPUInitTermGrace = 5 * time.Second - // vgpuTermGrace returns the SIGTERM wait used before hard-killing a vGPU // hypervisor. func (m *manager) vgpuTermGrace() time.Duration { if m.vgpuInitTermGrace > 0 { return m.vgpuInitTermGrace } - return defaultVGPUInitTermGrace + return hypervisor.VFIOTermGrace } // terminateThenKill hard-kills the hypervisor process, first giving any vGPU From 4652dcd6cbb1d0dde0ec13761265b9ca32272d95 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:25:09 +0000 Subject: [PATCH 58/76] Tighten vGPU lifecycle comments --- lib/hypervisor/qemu/process.go | 6 ++---- lib/instances/manager.go | 11 ++++------- lib/instances/metrics.go | 5 ++--- lib/instances/process_identity.go | 18 +++++++----------- lib/instances/storage.go | 5 ++--- lib/instances/vgpu.go | 24 +++++++++--------------- lib/instances/vgpu_orphan.go | 20 +++++++++----------- 7 files changed, 35 insertions(+), 54 deletions(-) diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index 367bf14e3..de560c10d 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -228,10 +228,8 @@ type startedProcess struct { pid int socketPath string // termGrace, when non-zero, makes cleanup send SIGTERM and wait this long - // before SIGKILL. Set for VFIO-attached processes: hard-killing QEMU while - // the NVIDIA vGPU plugin is initializing can silently wedge the VF until - // its parent GPU's SR-IOV is cycled, while a terminating QEMU runs its - // device teardown and leaves the VF reusable. + // before SIGKILL. Set for VFIO-attached processes: SIGKILL during vGPU + // plugin init can wedge the VF until its parent GPU is SR-IOV cycled. termGrace time.Duration waitDone chan error waitConsumed bool diff --git a/lib/instances/manager.go b/lib/instances/manager.go index cca794015..c499b1a82 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -740,9 +740,8 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { } // ListInstancesForReconcile returns every instance's stored metadata or an -// invalid metadata error. It does not derive state: reconcile protection only -// needs raw metadata fields, and hydration would query the hypervisor of -// every instance on the host before the API serves. +// invalid metadata error. It does not derive state: hydration would query +// every hypervisor on the host before the API serves. func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { files, err := m.listMetadataFilesStrict() if err != nil { @@ -754,10 +753,8 @@ func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, er meta, err := m.loadMetadata(id) if err != nil { if errors.Is(err, ErrNotFound) { - // Deleted between listing and load; a vanished record cannot - // claim a VF. Failing here instead would zero the grace-period - // retry and disable the vendor VFIO sweep whenever it races a - // concurrent delete. + // Deleted between listing and load; failing instead would + // disable the vendor VFIO sweep whenever it races a delete. continue } return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index bdffbdbce..085216fdf 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -574,9 +574,8 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat } // recordVGPUOrphanReleaseAbandoned records an orphaned vGPU release retry -// loop giving up: the VF stays allocated (capacity silently reduced) until -// startup reconciliation or manual remediation, so it must be visible beyond -// a log line. +// loop giving up: the VF stays allocated until startup reconciliation or +// manual remediation, so it must be visible beyond a log line. func (m *manager) recordVGPUOrphanReleaseAbandoned(ctx context.Context) { if m.metrics == nil { return diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 83aa682bc..dd546c420 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -37,13 +37,10 @@ func (m *manager) vgpuTermGrace() time.Duration { } // terminateThenKill hard-kills the hypervisor process, first giving any vGPU -// instance a SIGTERM grace: SIGKILL during guest driver init can silently -// wedge the VF until its parent GPU's SR-IOV is cycled (see -// lib/devices/GPU.md), while a terminating QEMU runs its VFIO teardown. The -// grace applies in every state, not just Initializing, because the instance -// reports Running seconds before the guest driver finishes initializing and -// nothing host-side observes that boundary; post-init the SIGTERM is proven -// harmless and costs the grace only when the process ignores it. +// instance a SIGTERM grace: SIGKILL during guest driver init can wedge the VF +// until its parent GPU is SR-IOV cycled (see lib/devices/GPU.md). The grace +// applies in every state because the instance reports Running seconds before +// driver init finishes and nothing host-side observes that boundary. func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { if inst.GPUProfile != "" { if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { @@ -211,10 +208,9 @@ func classifyResolvedHypervisorOwner(socketPath string, stored, resolved int, er } // HypervisorMayBeAlive reports whether the recorded hypervisor process may -// still be running. It fails open: when ownership cannot be resolved it -// returns true, which is the safe direction for its callers (reconcile -// protection and claim checks, where true means "protect"). Do not use it to -// authorize teardown. +// still be running. It fails open (unresolvable ownership returns true, the +// safe direction for reconcile protection and claim checks); do not use it +// to authorize teardown. func HypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { pid, err := resolveLiveHypervisorPID(id, socketPath) return err != nil || pid > 0 diff --git a/lib/instances/storage.go b/lib/instances/storage.go index 6a2354624..40bbba684 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -194,9 +194,8 @@ func (m *manager) listMetadataFiles() ([]string, error) { } // listMetadataFilesStrict returns paths to all instance metadata files, -// failing on any stat error other than absence. Fail-closed callers (the -// vGPU release claim scan and startup reconcile protection) use it so an -// unreadable instance is an error instead of silently missing. +// failing on any stat error other than absence, so fail-closed callers see +// an unreadable instance as an error instead of silently missing. func (m *manager) listMetadataFilesStrict() ([]string, error) { return m.walkMetadataFiles(true) } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 3561cbc5a..8ffe64dd8 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -123,10 +123,8 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if !retained { return false, false } - // The mid-start save may already have persisted this assignment, in - // which case the on-disk record still points at the device and - // delete or a retried start can release it (matching create's - // retention-survives check). + // The mid-start save may already have persisted this assignment, so + // delete or a retried start can still release it. if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { return true, true } @@ -159,11 +157,10 @@ func restoreStartMutatedFields(dst, src *StoredMetadata) { func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { - // Vendor VFIO VFs are reused across instances, so stale metadata can - // point at a path claimed by a live instance and the release must fail - // closed on an incomplete inventory. mdev UUIDs are unique and never - // reused, so skip the scan there — it would let one unreadable - // metadata file block every mdev release on the host. + // Vendor VFIO VFs are reused across instances, so the release must + // fail closed on an incomplete inventory. mdev UUIDs are never reused; + // scanning there would let one unreadable metadata file block every + // mdev release on the host. claimed := false if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { var err error @@ -192,12 +189,9 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) } // vgpuAssignmentClaimedByLiveInstance reports whether another live instance's -// stored metadata claims devicePath. It reads raw metadata instead of -// hydrating full instances: the scan runs on every vendor VFIO release, and -// deriving state would query the hypervisor of every instance on the host. -// A confirmed live claimant returns true. Unreadable metadata, a recent -// assignment without a PID, or unverifiable process ownership returns an error -// so the requester retains its assignment for a later retry. +// stored metadata claims devicePath. Unreadable metadata, a recent assignment +// without a PID, or unverifiable process ownership returns an error so the +// requester retains its assignment for a later retry. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { files, err := m.listMetadataFilesStrict() if err != nil { diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go index b3ff2b33b..d85a68bf3 100644 --- a/lib/instances/vgpu_orphan.go +++ b/lib/instances/vgpu_orphan.go @@ -8,22 +8,20 @@ import ( ) const ( - // orphanedVGPUReleaseMaxAttempts bounds the retry loop so a genuinely - // wedged VF degrades to one operator-actionable error instead of - // indefinite log churn. At the default interval this covers ten minutes, - // far beyond the seconds a dying VMM normally needs to finish kernel-side - // VFIO teardown. + // Bounds the retry loop (~10 minutes at the default interval, far beyond + // normal VFIO teardown) so a wedged VF degrades to one operator-actionable + // error instead of indefinite log churn. orphanedVGPUReleaseMaxAttempts = 20 defaultOrphanedVGPUReleaseRetryDelay = 30 * time.Second ) // scheduleOrphanedVGPURelease retries a vGPU release that failed during a -// completed delete, off the request path. A GPU-busy VMM routinely outlives -// delete's force-kill wait while the kernel finishes VFIO teardown, and once -// the metadata is deleted nothing else releases the VF until the next -// startup reconciliation. Each attempt re-runs releaseStoredVGPU, so the -// claim scan and destroy guards apply on every retry. The queue is in-memory -// only: a restart abandons it and startup reconciliation sweeps the VF. +// completed delete, off the request path: a GPU-busy VMM routinely outlives +// delete's force-kill wait, and once metadata is deleted nothing else +// releases the VF until startup reconciliation. Each attempt re-runs +// releaseStoredVGPU, so the claim scan and destroy guards apply on every +// retry. The queue is in-memory only; a restart abandons it and startup +// reconciliation sweeps the VF. func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { path := storedVGPUDevicePath(&stored) if path == "" { From a062d9c4f21d1c3db330ce7eb1d9ee747cb75dd8 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:25:12 +0000 Subject: [PATCH 59/76] Retry vGPU recovery paths that previously waited for restart - Retry the vendor VFIO reconcile sweep with a bounded delay when the startup instance listing fails, instead of disabling orphan recovery until the next process restart. One pending retry at a time. - Schedule the in-process orphaned-release retry when a rollback's retention record cannot be saved (create and start), instead of leaking the VF until restart. The retry scans claims without a self-exclusion because a restarted instance may hold the same VF. - Reject snapshot restore into a vGPU retention stub, matching start, fork, and snapshot. - Give passthrough PCI instances the same SIGTERM grace as vGPU instances on stop/delete, matching the QEMU-side vfioTermGraceFor. - Render the vgpu_cleanup_pending API detail from the error itself instead of duplicating its prose; use the manager clock in the claim scan; collapse the create-rollback retention branch. - Move ReconcileVGPUs off the Manager interface to a startup type assertion and unexport listInstancesForReconcile and hypervisorMayBeAlive. --- cmd/api/api/instances.go | 10 ++++---- cmd/api/main.go | 9 +++++-- lib/builds/manager_test.go | 2 -- lib/instances/create.go | 8 ++---- lib/instances/manager.go | 12 ++++++--- lib/instances/process_identity.go | 20 ++++++++------- lib/instances/query_test.go | 6 ++--- lib/instances/snapshot.go | 5 ++++ lib/instances/snapshot_test.go | 37 ++++++++++++++++++++++++++++ lib/instances/start.go | 4 +++ lib/instances/vgpu.go | 21 +++++++++++++--- lib/instances/vgpu_orphan.go | 17 +++++++------ lib/instances/vgpu_reconcile.go | 21 +++++++++++++--- lib/instances/vgpu_reconcile_test.go | 29 ++++++++++++++++++++++ lib/instances/vgpu_retention.go | 8 +++--- lib/instances/vgpu_test.go | 15 ++++++++--- lib/instances/wait_test.go | 1 - 17 files changed, 172 insertions(+), 53 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 8b9cef579..fb7aaab7d 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -439,11 +439,11 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst // and inner error detail shared by the create and start handlers. The // retained guidance names the verb-specific way to release the assignment. func vgpuCleanupPendingDetail(pending *instances.VGPUCleanupPendingError, action, retainedGuidance string) (string, *oapi.ErrorDetail) { - message := fmt.Sprintf("failed to %s instance: %v; vGPU release failed during rollback and instance %s retains the assignment, %s", action, pending.Err, pending.InstanceID, retainedGuidance) - innerCode := "vgpu_retained_instance" - if !pending.Retained { - message = fmt.Sprintf("failed to %s instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", action, pending.Err, pending.InstanceID) - innerCode = "vgpu_unretained_instance" + message := fmt.Sprintf("failed to %s instance: %v", action, pending) + innerCode := "vgpu_unretained_instance" + if pending.Retained { + message += "; " + retainedGuidance + innerCode = "vgpu_retained_instance" } return message, &oapi.ErrorDetail{ Code: lo.ToPtr(innerCode), diff --git a/cmd/api/main.go b/cmd/api/main.go index da2748103..20ac8dc60 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -384,9 +384,14 @@ func run() error { return fmt.Errorf("reconcile device state: %w", err) } - // Reconcile vGPU devices (clears orphaned vGPUs from previous runs) + // Reconcile vGPU devices (clears orphaned vGPUs from previous runs). + // Type-asserted rather than added to instances.Manager so alternate + // Manager implementations compiled against the public module keep + // building without this startup-only method. logger.Info("Reconciling vGPU devices...") - app.InstanceManager.ReconcileVGPUs(ctx) + if r, ok := app.InstanceManager.(interface{ ReconcileVGPUs(context.Context) }); ok { + r.ReconcileVGPUs(ctx) + } // Wire up resource validator for aggregate limit checking // This enables the instance manager to validate CPU, memory, network, and GPU diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index bbc426252..7be58b0fc 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -51,8 +51,6 @@ func (m *mockInstanceManager) ListInstances(ctx context.Context, filter *instanc return result, nil } -func (m *mockInstanceManager) ReconcileVGPUs(context.Context) {} - func (m *mockInstanceManager) ListSnapshots(ctx context.Context, filter *instances.ListSnapshotsFilter) ([]instances.Snapshot, error) { return nil, nil } diff --git a/lib/instances/create.go b/lib/instances/create.go index f4ece0100..de301a95d 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -278,7 +278,6 @@ func (m *manager) createInstance( var gpuDevicePath string var gpuMdevUUID string var gpuAssignedAt *time.Time - var stored *StoredMetadata retention := vgpuRetention{instanceID: id} // Setup cleanup stack early so device attachment errors trigger cleanup. @@ -344,10 +343,7 @@ func (m *manager) createInstance( } if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) - retention.retain(stored) - if stored == nil { - retention.retainFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) - } + retention.retainFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) } }) } @@ -388,7 +384,7 @@ func (m *manager) createInstance( if err != nil { return nil, err } - stored = &StoredMetadata{ + stored := &StoredMetadata{ Id: id, Name: req.Name, Image: req.Image, diff --git a/lib/instances/manager.go b/lib/instances/manager.go index c499b1a82..d7147173b 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "time" "github.com/kernel/hypeman/lib/devices" @@ -29,7 +30,6 @@ import ( type Manager interface { ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) - ReconcileVGPUs(ctx context.Context) ListSnapshots(ctx context.Context, filter *ListSnapshotsFilter) ([]Snapshot, error) GetSnapshot(ctx context.Context, snapshotID string) (*Snapshot, error) CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) @@ -221,6 +221,12 @@ type manager struct { orphanedVGPUs map[string]struct{} orphanedVGPURetryDelay time.Duration + // One pending vGPU reconcile retry at a time, for both the startup-grace + // and listing-failure paths. vgpuReconcileRetryDelay overrides the + // listing-failure delay in tests; zero means the default. + vgpuReconcileRetryPending atomic.Bool + vgpuReconcileRetryDelay time.Duration + // vgpuInitTermGrace overrides terminateThenKill's SIGTERM wait for vGPU // instances still initializing; zero means the default. vgpuInitTermGrace time.Duration @@ -739,10 +745,10 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } -// ListInstancesForReconcile returns every instance's stored metadata or an +// listInstancesForReconcile returns every instance's stored metadata or an // invalid metadata error. It does not derive state: hydration would query // every hypervisor on the host before the API serves. -func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { +func (m *manager) listInstancesForReconcile(ctx context.Context) ([]Instance, error) { files, err := m.listMetadataFilesStrict() if err != nil { return nil, err diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index dd546c420..cd8ff2fe1 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -36,17 +36,19 @@ func (m *manager) vgpuTermGrace() time.Duration { return hypervisor.VFIOTermGrace } -// terminateThenKill hard-kills the hypervisor process, first giving any vGPU -// instance a SIGTERM grace: SIGKILL during guest driver init can wedge the VF -// until its parent GPU is SR-IOV cycled (see lib/devices/GPU.md). The grace -// applies in every state because the instance reports Running seconds before -// driver init finishes and nothing host-side observes that boundary. +// terminateThenKill hard-kills the hypervisor process, first giving any +// instance with VFIO devices (a vGPU VF or passthrough PCI devices, matching +// the QEMU-side vfioTermGraceFor) a SIGTERM grace: SIGKILL during guest +// driver init can wedge the device until its parent GPU is SR-IOV cycled +// (see lib/devices/GPU.md). The grace applies in every state because the +// instance reports Running seconds before driver init finishes and nothing +// host-side observes that boundary. func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { - if inst.GPUProfile != "" { + if inst.GPUProfile != "" || len(inst.Devices) > 0 { if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { return nil } - logger.FromContext(ctx).WarnContext(ctx, "vGPU hypervisor did not exit on SIGTERM; hard-killing, VF may wedge if the guest driver was initializing", + logger.FromContext(ctx).WarnContext(ctx, "hypervisor with VFIO devices did not exit on SIGTERM; hard-killing, device may wedge if the guest driver was initializing", "instance_id", inst.Id, "device_path", inst.GPUDevicePath) } return killProcessAndWait(pid) @@ -207,11 +209,11 @@ func classifyResolvedHypervisorOwner(socketPath string, stored, resolved int, er return 0, fmt.Errorf("cannot confirm ownership of socket %s: %w", socketPath, err) } -// HypervisorMayBeAlive reports whether the recorded hypervisor process may +// hypervisorMayBeAlive reports whether the recorded hypervisor process may // still be running. It fails open (unresolvable ownership returns true, the // safe direction for reconcile protection and claim checks); do not use it // to authorize teardown. -func HypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { +func hypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { pid, err := resolveLiveHypervisorPID(id, socketPath) return err != nil || pid > 0 } diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index b3dbfba41..3d12c739d 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -31,12 +31,12 @@ func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { require.NoError(t, err) require.Len(t, listed, 1) - _, err = m.ListInstancesForReconcile(context.Background()) + _, err = m.listInstancesForReconcile(context.Background()) require.Error(t, err) assert.ErrorContains(t, err, "load metadata for instance invalid") require.NoError(t, os.Remove(m.paths.InstanceMetadata("invalid"))) - listed, err = m.ListInstancesForReconcile(context.Background()) + listed, err = m.listInstancesForReconcile(context.Background()) require.NoError(t, err) require.Len(t, listed, 1) assert.Equal(t, "valid", listed[0].Id) @@ -70,7 +70,7 @@ func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T } done := make(chan result, 1) go func() { - listed, err := m.ListInstancesForReconcile(context.Background()) + listed, err := m.listInstancesForReconcile(context.Background()) done <- result{listed, err} }() time.Sleep(100 * time.Millisecond) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index e30192ee9..1d8456ae5 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -266,6 +266,11 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str if sourceInst.State == StateRunning { return nil, fmt.Errorf("%w: cannot restore snapshot while source is %s", ErrInvalidState, sourceInst.State) } + if sourceMeta.GPURetainedForCleanup { + // Restoring would rebuild boot config from the snapshot record, whose + // retention flag is false, silently clearing the delete-only marker. + return nil, errVGPURetentionStub + } targetState, err := resolveSnapshotTargetState(rec.Snapshot.Kind, req.TargetState) if err != nil { diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index f8c022fe9..e4fcc048f 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -77,6 +77,43 @@ func TestCreateSnapshotRejectsVGPURetentionRecord(t *testing.T) { require.ErrorContains(t, err, "delete it to release the assignment") } +func TestRestoreSnapshotRejectsVGPURetentionRecord(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "snapshot-vgpu-restore-retention" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, mgr.defaultHypervisor) + + snapshot, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "snapshot-vgpu-restore-retention", + }) + require.NoError(t, err) + + meta, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, mgr.saveMetadata(meta)) + + // Restoring into the delete-only stub would rebuild boot config from the + // snapshot record, whose retention flag is false, clearing the marker. + _, err = mgr.RestoreSnapshot(ctx, sourceID, snapshot.Id, RestoreSnapshotRequest{ + TargetState: StateStopped, + TargetHypervisor: mgr.defaultHypervisor, + }) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") + + // The retained assignment must survive the rejected restore for delete. + stored, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + assert.True(t, stored.GPURetainedForCleanup) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) +} + func TestRestoreSnapshotDoesNotResurrectStaleVGPUAssignment(t *testing.T) { mgr, _ := setupTestManager(t) ctx := context.Background() diff --git a/lib/instances/start.go b/lib/instances/start.go index a79663953..1089a35a3 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -185,6 +185,10 @@ func (m *manager) startInstance( wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) + // No on-disk record points at the device; retry the release + // in the background instead of waiting for the next startup + // reconcile. + m.scheduleOrphanedVGPURelease(ctx, retentionMeta.StoredMetadata) return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} } return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 8ffe64dd8..b82a07d7d 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -17,7 +17,8 @@ const VGPUAssignmentStartupGracePeriod = 5 * time.Minute // VGPUCleanupPendingError reports a failed create whose vGPU release also // failed during rollback. When Retained is true, deleting the retained instance -// retries the release; otherwise startup reconciliation recovers the assignment. +// retries the release; otherwise a background retry and startup reconciliation +// recover the assignment. type VGPUCleanupPendingError struct { InstanceID string Retained bool @@ -28,7 +29,7 @@ func (e *VGPUCleanupPendingError) Error() string { if e.Retained { return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) } - return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", e.Err, e.InstanceID) + return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the release is retried in the background and by the next startup reconcile", e.Err, e.InstanceID) } func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } @@ -128,6 +129,9 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { return true, true } + // No on-disk record points at the device; retry the release in the + // background instead of waiting for the next startup reconcile. + m.scheduleOrphanedVGPURelease(ctx, cleanupMeta.StoredMetadata) return true, false } return retained, retained @@ -155,6 +159,15 @@ func restoreStartMutatedFields(dst, src *StoredMetadata) { } func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { + return m.releaseStoredVGPUExcluding(ctx, stored, stored.Id) +} + +// releaseStoredVGPUExcluding releases stored's assignment while treating +// excludeID's metadata as not a claimant. Callers releasing an instance's own +// persisted assignment exclude that instance; the orphan retry passes no +// exclusion because its instance may have been restarted onto the same VF, +// and that live claim must block the release. +func (m *manager) releaseStoredVGPUExcluding(ctx context.Context, stored *StoredMetadata, excludeID string) error { path := storedVGPUDevicePath(stored) if path != "" { // Vendor VFIO VFs are reused across instances, so the release must @@ -164,7 +177,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) claimed := false if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { var err error - claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) + claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, excludeID, path) if err != nil { return err } @@ -222,7 +235,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } } - live, remaining := vgpuAssignmentLiveness(stored, time.Now(), pid > 0) + live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), pid > 0) if pid > 0 && live { return true, nil } diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go index d85a68bf3..513cf9bc9 100644 --- a/lib/instances/vgpu_orphan.go +++ b/lib/instances/vgpu_orphan.go @@ -15,12 +15,13 @@ const ( defaultOrphanedVGPUReleaseRetryDelay = 30 * time.Second ) -// scheduleOrphanedVGPURelease retries a vGPU release that failed during a -// completed delete, off the request path: a GPU-busy VMM routinely outlives -// delete's force-kill wait, and once metadata is deleted nothing else -// releases the VF until startup reconciliation. Each attempt re-runs -// releaseStoredVGPU, so the claim scan and destroy guards apply on every -// retry. The queue is in-memory only; a restart abandons it and startup +// scheduleOrphanedVGPURelease retries a vGPU release for an assignment no +// on-disk metadata points at anymore: a release that failed during a +// completed delete (a GPU-busy VMM routinely outlives delete's force-kill +// wait), or a rollback whose retention record could not be saved. Without a +// record, nothing else releases the VF until startup reconciliation. Each +// attempt re-runs the release, so the claim scan and destroy guards apply on +// every retry. The queue is in-memory only; a restart abandons it and startup // reconciliation sweeps the VF. func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { path := storedVGPUDevicePath(&stored) @@ -56,7 +57,9 @@ func (m *manager) retryOrphanedVGPURelease(ctx context.Context, stored StoredMet }() for attempt := 1; attempt <= orphanedVGPUReleaseMaxAttempts; attempt++ { time.Sleep(delay) - if err := m.releaseStoredVGPU(ctx, &stored); err != nil { + // No claim-scan exclusion: unlike delete, a failed start keeps its + // instance record, and a restarted instance may hold this same VF. + if err := m.releaseStoredVGPUExcluding(ctx, &stored, ""); err != nil { log.WarnContext(ctx, "orphaned vGPU release retry failed", "instance_id", stored.Id, "device_path", path, "attempt", attempt, "error", err) continue diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 85d3a6e0d..4890b216a 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -8,8 +8,13 @@ import ( "github.com/kernel/hypeman/lib/logger" ) +// vgpuReconcileListRetryDelay spaces retries of the vendor VFIO sweep when +// the instance listing fails: without a retry, one transient stat error at +// startup would disable orphan recovery until the next restart. +const vgpuReconcileListRetryDelay = time.Minute + func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]struct{}, time.Duration, error) { - allInstances, err := m.ListInstancesForReconcile(ctx) + allInstances, err := m.listInstancesForReconcile(ctx) if err != nil { return nil, 0, err } @@ -20,7 +25,7 @@ func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]s if stored.GPUDevicePath == "" { continue } - livePID := stored.HypervisorPID != nil && HypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID) if !live { continue @@ -41,7 +46,10 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { if err != nil { log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconcile, mdev reconcile still runs", "error", err) protected = make(map[string]struct{}) - retryAfter = 0 + retryAfter = vgpuReconcileListRetryDelay + if m.vgpuReconcileRetryDelay > 0 { + retryAfter = m.vgpuReconcileRetryDelay + } } if err := devices.ReconcileVGPUs(ctx, protected, sweepVendorVFIO); err != nil { log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) @@ -49,12 +57,19 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { if retryAfter <= 0 { return } + // One pending retry at a time: overlapping calls would fork parallel + // retry chains. + if !m.vgpuReconcileRetryPending.CompareAndSwap(false, true) { + return + } go func() { timer := time.NewTimer(retryAfter) defer timer.Stop() select { case <-ctx.Done(): + m.vgpuReconcileRetryPending.Store(false) case <-timer.C: + m.vgpuReconcileRetryPending.Store(false) m.ReconcileVGPUs(ctx) } }() diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 1fbd90384..6fd299af7 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -1,7 +1,10 @@ package instances import ( + "context" + "os" "os/exec" + "path/filepath" "testing" "time" @@ -41,6 +44,32 @@ func TestLiveVGPUReconcileProtectionBoundsStartupProtection(t *testing.T) { assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") } +func TestReconcileVGPUsRetriesAfterListingFailure(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m := &manager{paths: paths.New(t.TempDir()), vgpuReconcileRetryDelay: 250 * time.Millisecond} + const id = "unreadable" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{Id: id}})) + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o000)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + m.ReconcileVGPUs(ctx) + require.True(t, m.vgpuReconcileRetryPending.Load(), + "a listing failure must schedule a retry instead of disabling the vendor sweep until restart") + + // Once the listing recovers, the retry runs the sweep and stops rearming. + require.NoError(t, os.Chmod(instanceDir, 0o755)) + require.Eventually(t, func() bool { + return !m.vgpuReconcileRetryPending.Load() + }, 5*time.Second, 10*time.Millisecond) +} + func TestVGPUAssignmentLiveness(t *testing.T) { now := time.Now().UTC() recent := now.Add(-time.Minute) diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index a1a1aae0d..f6cd4b02b 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -30,11 +30,6 @@ func (r *vgpuRetention) retainFromDevice(stub StoredMetadata, device *devices.VG r.retained = true } -func (r *vgpuRetention) retain(stub *StoredMetadata) { - r.stub = stub - r.retained = stub != nil -} - func (r *vgpuRetention) markRetained(persisted bool) { r.retained = true r.persisted = persisted @@ -70,6 +65,9 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) } + // No on-disk record points at the device; retry the release in the + // background instead of waiting for the next startup reconcile. + m.scheduleOrphanedVGPURelease(ctx, *retainedVGPU) return false } if err := m.ensureDirectories(id); err != nil { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index c5ef4911b..ea3b18429 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -19,8 +19,7 @@ import ( ) func persistTestVGPURetention(m *manager, ctx context.Context, id string, stub *StoredMetadata) bool { - retention := vgpuRetention{instanceID: id} - retention.retain(stub) + retention := vgpuRetention{instanceID: id, stub: stub, retained: stub != nil} m.persistVGPURetention(ctx, &retention) return retention.persisted } @@ -101,6 +100,11 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { assert.False(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) _, err := m.loadMetadata(id) require.Error(t, err) + + m.orphanedVGPUMu.Lock() + _, queued := m.orphanedVGPUs[stored.GPUDevicePath] + m.orphanedVGPUMu.Unlock() + assert.True(t, queued, "unpersisted retention must queue a background release") } func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T) { @@ -157,7 +161,7 @@ func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { unpersisted := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} assert.ErrorIs(t, unpersisted, cause) - assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the assignment is recovered on the next startup reconcile", unpersisted.Error()) + assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the release is retried in the background and by the next startup reconcile", unpersisted.Error()) } func TestVGPUDevicePendingCleanup(t *testing.T) { @@ -334,6 +338,11 @@ func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded") + + m.orphanedVGPUMu.Lock() + _, queued := m.orphanedVGPUs[device.SysfsPath] + m.orphanedVGPUMu.Unlock() + assert.True(t, queued, "unpersisted retention must queue a background release") } func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { diff --git a/lib/instances/wait_test.go b/lib/instances/wait_test.go index a15c19c8c..bab6f06d7 100644 --- a/lib/instances/wait_test.go +++ b/lib/instances/wait_test.go @@ -32,7 +32,6 @@ func (s *stubManager) GetInstance(ctx context.Context, id string) (*Instance, er func (s *stubManager) ListInstances(context.Context, *ListInstancesFilter) ([]Instance, error) { return nil, nil } -func (s *stubManager) ReconcileVGPUs(context.Context) {} func (s *stubManager) ListSnapshots(context.Context, *ListSnapshotsFilter) ([]Snapshot, error) { return nil, nil } From 4c149eaf6fe8d40f36ddd770a113ec3fcc0e933d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 20:08:05 +0000 Subject: [PATCH 60/76] Validate live vGPU claim test identity --- lib/instances/vgpu_test.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index ea3b18429..7ae6f3cdf 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -526,10 +526,14 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. require.NoError(t, m.ensureDirectories("legacy-claimant")) pid := os.Getpid() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "legacy-claimant", - Name: "legacy-claimant", - GPUMdevUUID: "legacy-uuid", - HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, + Id: "legacy-claimant", + Name: "legacy-claimant", + GPUMdevUUID: "legacy-uuid", + HypervisorProcessIdentity: HypervisorProcessIdentity{ + HypervisorPID: &pid, + HypervisorStartTime: processStartTime(pid), + HypervisorBootID: hostBootID(), + }, }})) claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/mdev/devices/legacy-uuid") From ee3c29674f777dd9f9da8a66fc0b96cbd0416881 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:00:23 +0000 Subject: [PATCH 61/76] Clean guest data before retaining vGPU assignment --- lib/instances/vgpu_retention.go | 5 +++++ lib/instances/vgpu_test.go | 17 ++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index f6cd4b02b..eaca96289 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -70,6 +70,11 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten m.scheduleOrphanedVGPURelease(ctx, *retainedVGPU) return false } + if err := m.deleteInstanceData(id); err != nil { + log.ErrorContext(ctx, "failed to clean instance data before retaining vGPU assignment", "instance_id", id, "error", err) + retention.persisted = retentionSurvives() + return + } if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) retention.persisted = retentionSurvives() diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 7ae6f3cdf..cb54c509b 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -49,8 +49,16 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { HypervisorType: "qemu", DataDir: m.paths.InstanceDir("failed-create"), } + require.NoError(t, m.ensureDirectories(stored.Id)) + require.NoError(t, os.WriteFile(m.paths.InstanceOverlay(stored.Id), []byte("overlay"), 0o644)) + require.NoError(t, os.WriteFile(m.paths.InstanceConfigDisk(stored.Id), []byte("config"), 0o644)) + require.NoError(t, os.MkdirAll(m.paths.InstanceVolumeOverlaysDir(stored.Id), 0o755)) + require.NoError(t, os.WriteFile(m.paths.InstanceVolumeOverlay(stored.Id, "volume"), []byte("volume overlay"), 0o644)) assert.True(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) + assert.NoFileExists(t, m.paths.InstanceOverlay(stored.Id)) + assert.NoFileExists(t, m.paths.InstanceConfigDisk(stored.Id)) + assert.NoDirExists(t, m.paths.InstanceVolumeOverlaysDir(stored.Id)) retained, err := m.loadMetadata(stored.Id) require.NoError(t, err) @@ -85,12 +93,15 @@ func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { } func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { - t.Parallel() + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } m := &manager{paths: paths.New(t.TempDir())} const id = "failed-create" - require.NoError(t, m.ensureDirectories(id)) - require.NoError(t, os.Mkdir(m.paths.InstanceMetadata(id), 0o755)) + require.NoError(t, os.MkdirAll(m.paths.GuestsDir(), 0o755)) + require.NoError(t, os.Chmod(m.paths.GuestsDir(), 0o555)) + t.Cleanup(func() { _ = os.Chmod(m.paths.GuestsDir(), 0o755) }) stored := &StoredMetadata{ Id: id, From c9f6e431acb7adb0954d7e710eaf60435caf2329 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:21:58 +0000 Subject: [PATCH 62/76] Retry failed vGPU reconciliation --- lib/instances/manager.go | 7 ++++--- lib/instances/vgpu_reconcile.go | 22 ++++++++++++++++------ lib/instances/vgpu_reconcile_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index d7147173b..187e94f37 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -185,6 +185,7 @@ type manager struct { deleteInstanceFn func(context.Context, string) error createVGPU func(context.Context, string, string) (*devices.VGPUDevice, error) destroyVGPU func(context.Context, devices.VGPUAssignment) error + reconcileVGPUDevices func(context.Context, map[string]struct{}, bool) error deleteSnapshotFn func(context.Context, string) error ttlReaperDeleteTimeout time.Duration egressProxy *egressproxy.Service @@ -221,9 +222,9 @@ type manager struct { orphanedVGPUs map[string]struct{} orphanedVGPURetryDelay time.Duration - // One pending vGPU reconcile retry at a time, for both the startup-grace - // and listing-failure paths. vgpuReconcileRetryDelay overrides the - // listing-failure delay in tests; zero means the default. + // One pending vGPU reconcile retry at a time, for startup grace and + // reconciliation failures. vgpuReconcileRetryDelay overrides the retry + // delay in tests; zero means the default. vgpuReconcileRetryPending atomic.Bool vgpuReconcileRetryDelay time.Duration diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 4890b216a..24be04fe3 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -8,10 +8,9 @@ import ( "github.com/kernel/hypeman/lib/logger" ) -// vgpuReconcileListRetryDelay spaces retries of the vendor VFIO sweep when -// the instance listing fails: without a retry, one transient stat error at -// startup would disable orphan recovery until the next restart. -const vgpuReconcileListRetryDelay = time.Minute +// vgpuReconcileFailureRetryDelay spaces retries after a transient metadata or +// device error would otherwise disable orphan recovery until the next restart. +const vgpuReconcileFailureRetryDelay = time.Minute func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]struct{}, time.Duration, error) { allInstances, err := m.listInstancesForReconcile(ctx) @@ -46,13 +45,24 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { if err != nil { log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconcile, mdev reconcile still runs", "error", err) protected = make(map[string]struct{}) - retryAfter = vgpuReconcileListRetryDelay + retryAfter = vgpuReconcileFailureRetryDelay if m.vgpuReconcileRetryDelay > 0 { retryAfter = m.vgpuReconcileRetryDelay } } - if err := devices.ReconcileVGPUs(ctx, protected, sweepVendorVFIO); err != nil { + reconcileDevices := m.reconcileVGPUDevices + if reconcileDevices == nil { + reconcileDevices = devices.ReconcileVGPUs + } + if err := reconcileDevices(ctx, protected, sweepVendorVFIO); err != nil { log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) + deviceRetryAfter := vgpuReconcileFailureRetryDelay + if m.vgpuReconcileRetryDelay > 0 { + deviceRetryAfter = m.vgpuReconcileRetryDelay + } + if retryAfter <= 0 || deviceRetryAfter < retryAfter { + retryAfter = deviceRetryAfter + } } if retryAfter <= 0 { return diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 6fd299af7..815415d09 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -2,9 +2,11 @@ package instances import ( "context" + "errors" "os" "os/exec" "path/filepath" + "sync/atomic" "testing" "time" @@ -70,6 +72,28 @@ func TestReconcileVGPUsRetriesAfterListingFailure(t *testing.T) { }, 5*time.Second, 10*time.Millisecond) } +func TestReconcileVGPUsRetriesAfterDeviceFailure(t *testing.T) { + var calls atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + vgpuReconcileRetryDelay: 10 * time.Millisecond, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { + if calls.Add(1) == 1 { + return errors.New("transient device error") + } + return nil + }, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + m.ReconcileVGPUs(ctx) + require.True(t, m.vgpuReconcileRetryPending.Load()) + require.Eventually(t, func() bool { + return calls.Load() >= 2 && !m.vgpuReconcileRetryPending.Load() + }, 5*time.Second, 10*time.Millisecond) +} + func TestVGPUAssignmentLiveness(t *testing.T) { now := time.Now().UTC() recent := now.Add(-time.Minute) From 9da500157f9cf071221847901ac068909b325233 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:19:57 +0000 Subject: [PATCH 63/76] Trim vGPU lifecycle comments and tests --- cmd/api/api/instances.go | 7 ---- cmd/api/api/instances_test.go | 5 --- cmd/api/main.go | 4 --- lib/hypervisor/qemu/process.go | 9 ++--- lib/instances/create.go | 4 --- lib/instances/delete.go | 9 +---- lib/instances/fork_test.go | 2 -- lib/instances/lifecycle_noop_test.go | 7 ---- lib/instances/manager.go | 14 -------- lib/instances/metrics.go | 3 -- lib/instances/process_identity.go | 15 ++------ lib/instances/process_identity_linux_test.go | 27 +-------------- lib/instances/query_test.go | 8 ----- lib/instances/snapshot.go | 2 -- lib/instances/snapshot_test.go | 5 --- lib/instances/start.go | 4 --- lib/instances/storage.go | 5 --- lib/instances/types.go | 15 ++++---- lib/instances/vgpu.go | 36 ++------------------ lib/instances/vgpu_orphan.go | 15 -------- lib/instances/vgpu_orphan_test.go | 16 --------- lib/instances/vgpu_reconcile.go | 4 --- lib/instances/vgpu_reconcile_test.go | 30 ---------------- lib/instances/vgpu_retention.go | 2 -- lib/instances/vgpu_test.go | 23 ------------- 25 files changed, 14 insertions(+), 257 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index fb7aaab7d..2d7f1bbb1 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -364,8 +364,6 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst if err != nil { var vgpuPending *instances.VGPUCleanupPendingError switch { - // Checked first: it wraps the original create error, so a later - // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) message, inner := vgpuCleanupPendingDetail(vgpuPending, "create", "delete it to retry") @@ -435,9 +433,6 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst return oapi.CreateInstance201JSONResponse(instanceToOAPI(*inst)), nil } -// vgpuCleanupPendingDetail renders a pending vGPU cleanup into the message -// and inner error detail shared by the create and start handlers. The -// retained guidance names the verb-specific way to release the assignment. func vgpuCleanupPendingDetail(pending *instances.VGPUCleanupPendingError, action, retainedGuidance string) (string, *oapi.ErrorDetail) { message := fmt.Sprintf("failed to %s instance: %v", action, pending) innerCode := "vgpu_unretained_instance" @@ -858,8 +853,6 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan if err != nil { var vgpuPending *instances.VGPUCleanupPendingError switch { - // Checked first: it wraps the original start error, so a later - // errors.Is case would match the cause and hide the pending vGPU cleanup. case errors.As(err, &vgpuPending): log.ErrorContext(ctx, "failed to start instance", "error", err) message, inner := vgpuCleanupPendingDetail(vgpuPending, "start", "delete it or retry start to release it") diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 8a6e52518..408cedc65 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -57,8 +57,6 @@ func (m createErrorInstanceManager) CreateInstance(context.Context, instances.Cr return nil, m.err } -// A retained-assignment error must win over the mapping of the create error -// it wraps, or the response omits the instance the caller has to delete. func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { t.Parallel() svc := newTestService(t) @@ -1034,9 +1032,6 @@ func TestRestoreInstance_ErrorMapping(t *testing.T) { } } -// A retained-assignment error must win over the mapping of the start error -// it wraps, or the response omits the pending vGPU cleanup the caller has to -// resolve. func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { t.Parallel() diff --git a/cmd/api/main.go b/cmd/api/main.go index 20ac8dc60..3bd0fff39 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -384,10 +384,6 @@ func run() error { return fmt.Errorf("reconcile device state: %w", err) } - // Reconcile vGPU devices (clears orphaned vGPUs from previous runs). - // Type-asserted rather than added to instances.Manager so alternate - // Manager implementations compiled against the public module keep - // building without this startup-only method. logger.Info("Reconciling vGPU devices...") if r, ok := app.InstanceManager.(interface{ ReconcileVGPUs(context.Context) }); ok { r.ReconcileVGPUs(ctx) diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index de560c10d..f16d606eb 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -225,11 +225,8 @@ func buildQMPArgs(socketPath string) []string { } type startedProcess struct { - pid int - socketPath string - // termGrace, when non-zero, makes cleanup send SIGTERM and wait this long - // before SIGKILL. Set for VFIO-attached processes: SIGKILL during vGPU - // plugin init can wedge the VF until its parent GPU is SR-IOV cycled. + pid int + socketPath string termGrace time.Duration waitDone chan error waitConsumed bool @@ -281,7 +278,6 @@ func (p *startedProcess) wait() error { return err } -// waitFor waits up to d for the process to exit, returning whether it did. func (p *startedProcess) waitFor(d time.Duration) bool { if _, exited := p.checkExited(); exited { return true @@ -392,7 +388,6 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version } pid := proc.pid - // Only failed starts pay the VFIO termination grace. proc.termGrace = termGrace log.DebugContext(processCtx, "QEMU process started", "pid", pid, "duration_ms", time.Since(processStartTime).Milliseconds()) diff --git a/lib/instances/create.go b/lib/instances/create.go index de301a95d..525f2c262 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -280,7 +280,6 @@ func (m *manager) createInstance( var gpuAssignedAt *time.Time retention := vgpuRetention{instanceID: id} - // Setup cleanup stack early so device attachment errors trigger cleanup. defer retention.deferWrapPending(&retErr) cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) @@ -300,9 +299,6 @@ func (m *manager) createInstance( // Handle vGPU profile request if req.GPU != nil && req.GPU.Profile != "" { - // Identity fields a retention record keeps when rollback cannot - // release the assignment, so it lists as a recognizable, deletable - // instance. Create has already failed on a nil starter by this point. retentionStub := func() StoredMetadata { return StoredMetadata{ Id: id, diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 322e8f139..9c5f6b055 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -142,19 +142,12 @@ func (m *manager) deleteInstanceWithOptions( } m.closeFirecrackerUFFDSession(ctx, stored) - // 5b. Release the vGPU assignment if present, before any network, device, - // or volume teardown. Release failure is logged and the delete continues, - // matching the pre-refactor contract: the VMM is already confirmed dead, - // the guards inside the release never destroy a device they cannot prove - // is unowned, and a skipped release is recovered by the background retry - // below or, after a restart, by startup reconciliation. + // Release before deleting metadata so a failed release can be retried safely. hadVGPUAssignment := storedVGPUDevicePath(stored) != "" if hadVGPUAssignment { log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } if err := m.releaseStoredVGPU(ctx, stored); err != nil { - // Log error but continue with cleanup; the background retry releases - // the VF once the metadata is gone. log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) m.scheduleOrphanedVGPURelease(ctx, *stored) } else if hadVGPUAssignment { diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index ef802d1ad..c0bed670e 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -81,8 +81,6 @@ func TestForkInstanceRejectsVGPURetentionRecord(t *testing.T) { meta.GPURetainedForCleanup = true require.NoError(t, manager.saveMetadata(meta)) - // The delete-only retention stub has no boot configuration, so a fork of - // it could never boot; only delete may act on it. _, err = manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fork-vgpu-retention-copy"}) require.ErrorIs(t, err, ErrInvalidState) require.ErrorContains(t, err, "delete it to release the assignment") diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index a43efabb0..90f2e7aa1 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -198,10 +198,6 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { assert.Equal(t, restartpolicy.BlockedReasonManualStop, persisted.RestartStatus.BlockedReason) } -// A failed create whose vGPU release also failed retains a minimal -// GPU-fields-only stub, and the API tells the caller to delete it to retry -// the release. Exercise that recovery path against the exact stub shape -// persistVGPURetention writes. func TestDeleteReleasesRetainedCreateStub(t *testing.T) { p := paths.New(t.TempDir()) var destroyed []devices.VGPUAssignment @@ -342,7 +338,6 @@ func TestStartRejectsVGPURetentionRecord(t *testing.T) { require.ErrorIs(t, err, ErrInvalidState) require.ErrorContains(t, err, "delete it to release the assignment") - // The retained assignment must survive the rejected start for delete. stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) @@ -381,8 +376,6 @@ func TestStopStoppedInstanceLeavesRetentionStubForDelete(t *testing.T) { require.NoError(t, err) require.NotNil(t, inst) - // Retention stubs are delete-only: releasing on stop would leave a stub - // whose start/fork/snapshot errors still claim a retained assignment. stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 187e94f37..a37049f05 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -214,22 +214,13 @@ type manager struct { // Periodic TAP garbage collection reconciler. tapGCOnce sync.Once - // vGPU assignments that survived a completed delete, keyed by device - // path, each with a background release retry in flight. - // orphanedVGPURetryDelay overrides the retry delay in tests; zero means - // the default. orphanedVGPUMu sync.Mutex orphanedVGPUs map[string]struct{} orphanedVGPURetryDelay time.Duration - // One pending vGPU reconcile retry at a time, for startup grace and - // reconciliation failures. vgpuReconcileRetryDelay overrides the retry - // delay in tests; zero means the default. vgpuReconcileRetryPending atomic.Bool vgpuReconcileRetryDelay time.Duration - // vgpuInitTermGrace overrides terminateThenKill's SIGTERM wait for vGPU - // instances still initializing; zero means the default. vgpuInitTermGrace time.Duration // Hypervisor support @@ -746,9 +737,6 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } -// listInstancesForReconcile returns every instance's stored metadata or an -// invalid metadata error. It does not derive state: hydration would query -// every hypervisor on the host before the API serves. func (m *manager) listInstancesForReconcile(ctx context.Context) ([]Instance, error) { files, err := m.listMetadataFilesStrict() if err != nil { @@ -760,8 +748,6 @@ func (m *manager) listInstancesForReconcile(ctx context.Context) ([]Instance, er meta, err := m.loadMetadata(id) if err != nil { if errors.Is(err, ErrNotFound) { - // Deleted between listing and load; failing instead would - // disable the vendor VFIO sweep whenever it races a delete. continue } return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 085216fdf..de71c2686 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -573,9 +573,6 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } -// recordVGPUOrphanReleaseAbandoned records an orphaned vGPU release retry -// loop giving up: the VF stays allocated until startup reconciliation or -// manual remediation, so it must be visible beyond a log line. func (m *manager) recordVGPUOrphanReleaseAbandoned(ctx context.Context) { if m.metrics == nil { return diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index cd8ff2fe1..02f43fcda 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -27,8 +27,6 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second -// vgpuTermGrace returns the SIGTERM wait used before hard-killing a vGPU -// hypervisor. func (m *manager) vgpuTermGrace() time.Duration { if m.vgpuInitTermGrace > 0 { return m.vgpuInitTermGrace @@ -36,13 +34,7 @@ func (m *manager) vgpuTermGrace() time.Duration { return hypervisor.VFIOTermGrace } -// terminateThenKill hard-kills the hypervisor process, first giving any -// instance with VFIO devices (a vGPU VF or passthrough PCI devices, matching -// the QEMU-side vfioTermGraceFor) a SIGTERM grace: SIGKILL during guest -// driver init can wedge the device until its parent GPU is SR-IOV cycled -// (see lib/devices/GPU.md). The grace applies in every state because the -// instance reports Running seconds before driver init finishes and nothing -// host-side observes that boundary. +// SIGKILL during guest driver init can wedge a VF until the parent GPU is reset. func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { if inst.GPUProfile != "" || len(inst.Devices) > 0 { if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { @@ -209,10 +201,7 @@ func classifyResolvedHypervisorOwner(socketPath string, stored, resolved int, er return 0, fmt.Errorf("cannot confirm ownership of socket %s: %w", socketPath, err) } -// hypervisorMayBeAlive reports whether the recorded hypervisor process may -// still be running. It fails open (unresolvable ownership returns true, the -// safe direction for reconcile protection and claim checks); do not use it -// to authorize teardown. +// Ambiguous ownership is treated as live; this must not authorize teardown. func hypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { pid, err := resolveLiveHypervisorPID(id, socketPath) return err != nil || pid > 0 diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index e090b9a98..b208a6ef3 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -692,9 +692,6 @@ func TestResolveRuntimeHypervisorPIDMintsIdentityOnlyWhenConfirmed(t *testing.T) }) } -// startTrapProcess starts a shell with the given TERM trap action (empty -// ignores the signal) and blocks until the trap is installed. It returns the -// PID and its boot-scoped identity. func startTrapProcess(t *testing.T, trapAction string) (int, HypervisorProcessIdentity) { t.Helper() script := fmt.Sprintf("trap '%s' TERM; echo ready; sleep 30 & wait", trapAction) @@ -717,7 +714,7 @@ func startTrapProcess(t *testing.T, trapAction string) (int, HypervisorProcessId return pid, HypervisorProcessIdentity{HypervisorPID: &pid, HypervisorStartTime: startTime, HypervisorBootID: hostBootID()} } -func TestKillHypervisorSIGTERMsInitializingVGPUHypervisor(t *testing.T) { +func TestKillHypervisorSIGTERMsVGPUHypervisor(t *testing.T) { markerPath := filepath.Join(t.TempDir(), "terminated") pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") socketPath := filepath.Join(t.TempDir(), "missing.sock") @@ -757,28 +754,6 @@ func TestKillHypervisorEscalatesToSIGKILLWhenSIGTERMIgnored(t *testing.T) { assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH, "SIGTERM-ignoring hypervisor must still be hard-killed") } -func TestKillHypervisorSIGTERMsRunningVGPUHypervisor(t *testing.T) { - markerPath := filepath.Join(t.TempDir(), "terminated") - pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") - socketPath := filepath.Join(t.TempDir(), "missing.sock") - - m := &manager{} - require.NoError(t, m.killHypervisor(context.Background(), &Instance{ - State: StateRunning, - StoredMetadata: StoredMetadata{ - Id: "kill-test", - GPUProfile: "NVIDIA L40S-1Q", - HypervisorProcessIdentity: identity, - SocketPath: socketPath, - }, - })) - - require.Eventually(t, func() bool { - return syscall.Kill(pid, 0) == syscall.ESRCH - }, 5*time.Second, 10*time.Millisecond) - assert.FileExists(t, markerPath, "Running reports true before guest driver init completes, so vGPU hypervisors get SIGTERM in every state") -} - func TestKillHypervisorHardKillsNonVGPUHypervisor(t *testing.T) { markerPath := filepath.Join(t.TempDir(), "terminated") pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 3d12c739d..795b00baf 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -42,11 +42,6 @@ func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { assert.Equal(t, "valid", listed[0].Id) } -// A concurrent delete can remove an instance between the reconcile listing -// and its metadata load. A vanished record cannot claim a VF, so it must be -// skipped like the release claim scan does — failing instead would zero the -// grace-period retry and silently disable the vendor VFIO sweep whenever it -// races a delete. func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T) { m := &manager{paths: paths.New(t.TempDir())} @@ -60,9 +55,6 @@ func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T }})) } - // loadMetadata takes the snapshot-alias read lock, so holding the - // mutation lock parks the reconcile between listing and loading — the - // window a concurrent delete lands in. unlock := hypervisor.LockSnapshotSourceAliasMutation() type result struct { listed []Instance diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 1d8456ae5..a87acb614 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -267,8 +267,6 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str return nil, fmt.Errorf("%w: cannot restore snapshot while source is %s", ErrInvalidState, sourceInst.State) } if sourceMeta.GPURetainedForCleanup { - // Restoring would rebuild boot config from the snapshot record, whose - // retention flag is false, silently clearing the delete-only marker. return nil, errVGPURetentionStub } diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index e4fcc048f..31f0a4343 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -67,8 +67,6 @@ func TestCreateSnapshotRejectsVGPURetentionRecord(t *testing.T) { meta.GPURetainedForCleanup = true require.NoError(t, mgr.saveMetadata(meta)) - // The delete-only retention stub has no boot configuration, so a snapshot - // of it could never be restored or forked into a bootable instance. _, err = mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ Kind: SnapshotKindStopped, Name: "snapshot-vgpu-retention", @@ -98,8 +96,6 @@ func TestRestoreSnapshotRejectsVGPURetentionRecord(t *testing.T) { meta.GPURetainedForCleanup = true require.NoError(t, mgr.saveMetadata(meta)) - // Restoring into the delete-only stub would rebuild boot config from the - // snapshot record, whose retention flag is false, clearing the marker. _, err = mgr.RestoreSnapshot(ctx, sourceID, snapshot.Id, RestoreSnapshotRequest{ TargetState: StateStopped, TargetHypervisor: mgr.defaultHypervisor, @@ -107,7 +103,6 @@ func TestRestoreSnapshotRejectsVGPURetentionRecord(t *testing.T) { require.ErrorIs(t, err, ErrInvalidState) require.ErrorContains(t, err, "delete it to release the assignment") - // The retained assignment must survive the rejected restore for delete. stored, err := mgr.loadMetadata(sourceID) require.NoError(t, err) assert.True(t, stored.GPURetainedForCleanup) diff --git a/lib/instances/start.go b/lib/instances/start.go index 1089a35a3..8986d7fb3 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -67,7 +67,6 @@ func (m *manager) startInstance( } } - // Do not persist the previous VMM's identity with a new vGPU assignment. stored.HypervisorPID = nil stored.HypervisorStartTime = 0 stored.HypervisorBootID = "" @@ -185,9 +184,6 @@ func (m *manager) startInstance( wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) - // No on-disk record points at the device; retry the release - // in the background instead of waiting for the next startup - // reconcile. m.scheduleOrphanedVGPURelease(ctx, retentionMeta.StoredMetadata) return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} } diff --git a/lib/instances/storage.go b/lib/instances/storage.go index 40bbba684..1a4d325b0 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -187,15 +187,10 @@ func removeAllWithRetry(path string, removeAll func(string) error, sleep func(ti } } -// listMetadataFiles returns paths to all instance metadata files, skipping -// entries whose metadata cannot be statted. func (m *manager) listMetadataFiles() ([]string, error) { return m.walkMetadataFiles(false) } -// listMetadataFilesStrict returns paths to all instance metadata files, -// failing on any stat error other than absence, so fail-closed callers see -// an unreadable instance as an error instead of silently missing. func (m *manager) listMetadataFilesStrict() ([]string, error) { return m.walkMetadataFiles(true) } diff --git a/lib/instances/types.go b/lib/instances/types.go index ab41178cd..ed32f5b5d 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -151,15 +151,12 @@ type StoredMetadata struct { Devices []string // Device IDs attached to this instance // GPU configuration (vGPU mode) - GPUProfile string // vGPU profile name (e.g., "L40S-1Q") - GPUFramework devices.VGPUFramework - GPUDevicePath string - GPUMdevUUID string // populated for mdev-backed vGPUs - GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection - // GPURetainedForCleanup marks a delete-only retention stub written when a - // failed create could not release its vGPU: the record has no boot - // configuration, so only delete (which retries the release) may act on it. - GPURetainedForCleanup bool + GPUProfile string // vGPU profile name (e.g., "L40S-1Q") + GPUFramework devices.VGPUFramework + GPUDevicePath string + GPUMdevUUID string // populated for mdev-backed vGPUs + GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection + GPURetainedForCleanup bool // delete-only stub holding a vGPU assignment // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index b82a07d7d..0e09f2f34 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -15,10 +15,7 @@ import ( // persisted hypervisor PID is treated as potentially live. const VGPUAssignmentStartupGracePeriod = 5 * time.Minute -// VGPUCleanupPendingError reports a failed create whose vGPU release also -// failed during rollback. When Retained is true, deleting the retained instance -// retries the release; otherwise a background retry and startup reconciliation -// recover the assignment. +// VGPUCleanupPendingError reports a failed rollback that left a vGPU assigned. type VGPUCleanupPendingError struct { InstanceID string Retained bool @@ -34,9 +31,6 @@ func (e *VGPUCleanupPendingError) Error() string { func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } -// errVGPURetentionStub rejects every lifecycle verb except delete on a -// retention stub from a failed create: the record has no boot configuration, -// and only delete retries the release of its retained assignment. var errVGPURetentionStub = fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID string) (*devices.VGPUDevice, error) { @@ -91,9 +85,6 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUAssignedAt = nil } -// cleanupStartVGPU reports whether the assignment was retained after a failed -// destroy and whether that retention record was persisted, so start can surface -// the pending cleanup as a typed error like create does. func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) { logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID) assignment := devices.VGPUAssignment{ @@ -124,21 +115,15 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if !retained { return false, false } - // The mid-start save may already have persisted this assignment, so - // delete or a retried start can still release it. if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { return true, true } - // No on-disk record points at the device; retry the release in the - // background instead of waiting for the next startup reconcile. m.scheduleOrphanedVGPURelease(ctx, cleanupMeta.StoredMetadata) return true, false } return retained, retained } -// restoreStartMutatedFields must cover every field start mutates before the -// vGPU cleanup runs. func restoreStartMutatedFields(dst, src *StoredMetadata) { dst.HypervisorPID = src.HypervisorPID dst.HypervisorStartTime = src.HypervisorStartTime @@ -162,18 +147,10 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) return m.releaseStoredVGPUExcluding(ctx, stored, stored.Id) } -// releaseStoredVGPUExcluding releases stored's assignment while treating -// excludeID's metadata as not a claimant. Callers releasing an instance's own -// persisted assignment exclude that instance; the orphan retry passes no -// exclusion because its instance may have been restarted onto the same VF, -// and that live claim must block the release. func (m *manager) releaseStoredVGPUExcluding(ctx context.Context, stored *StoredMetadata, excludeID string) error { path := storedVGPUDevicePath(stored) if path != "" { - // Vendor VFIO VFs are reused across instances, so the release must - // fail closed on an incomplete inventory. mdev UUIDs are never reused; - // scanning there would let one unreadable metadata file block every - // mdev release on the host. + // Vendor VFIO VFs are reusable, so release fails closed on an incomplete inventory. claimed := false if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { var err error @@ -201,10 +178,6 @@ func (m *manager) releaseStoredVGPUExcluding(ctx context.Context, stored *Stored return nil } -// vgpuAssignmentClaimedByLiveInstance reports whether another live instance's -// stored metadata claims devicePath. Unreadable metadata, a recent assignment -// without a PID, or unverifiable process ownership returns an error so the -// requester retains its assignment for a later retry. func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { files, err := m.listMetadataFilesStrict() if err != nil { @@ -218,8 +191,6 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu meta, err := m.loadMetadata(id) if err != nil { if errors.Is(err, ErrNotFound) { - // Deleted between listing and load; a vanished record cannot - // be a live claimant. continue } return false, fmt.Errorf("load metadata for vGPU release check: instance %s: %w", id, err) @@ -262,9 +233,6 @@ func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) { } stored := &meta.StoredMetadata if stored.GPURetainedForCleanup { - // Delete-only retention stubs release through delete. Releasing here - // would leave a stub whose start/fork/snapshot errors still claim a - // retained assignment that no longer exists. return } if storedVGPUDevicePath(stored) == "" { diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go index 513cf9bc9..041402642 100644 --- a/lib/instances/vgpu_orphan.go +++ b/lib/instances/vgpu_orphan.go @@ -8,21 +8,10 @@ import ( ) const ( - // Bounds the retry loop (~10 minutes at the default interval, far beyond - // normal VFIO teardown) so a wedged VF degrades to one operator-actionable - // error instead of indefinite log churn. orphanedVGPUReleaseMaxAttempts = 20 defaultOrphanedVGPUReleaseRetryDelay = 30 * time.Second ) -// scheduleOrphanedVGPURelease retries a vGPU release for an assignment no -// on-disk metadata points at anymore: a release that failed during a -// completed delete (a GPU-busy VMM routinely outlives delete's force-kill -// wait), or a rollback whose retention record could not be saved. Without a -// record, nothing else releases the VF until startup reconciliation. Each -// attempt re-runs the release, so the claim scan and destroy guards apply on -// every retry. The queue is in-memory only; a restart abandons it and startup -// reconciliation sweeps the VF. func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { path := storedVGPUDevicePath(&stored) if path == "" { @@ -43,8 +32,6 @@ func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored Stored if delay <= 0 { delay = defaultOrphanedVGPUReleaseRetryDelay } - // The request context ends with the delete; keep its values for logging - // but detach from its cancellation. go m.retryOrphanedVGPURelease(context.WithoutCancel(ctx), stored, path, delay) } @@ -57,8 +44,6 @@ func (m *manager) retryOrphanedVGPURelease(ctx context.Context, stored StoredMet }() for attempt := 1; attempt <= orphanedVGPUReleaseMaxAttempts; attempt++ { time.Sleep(delay) - // No claim-scan exclusion: unlike delete, a failed start keeps its - // instance record, and a restarted instance may hold this same VF. if err := m.releaseStoredVGPUExcluding(ctx, &stored, ""); err != nil { log.WarnContext(ctx, "orphaned vGPU release retry failed", "instance_id", stored.Id, "device_path", path, "attempt", attempt, "error", err) diff --git a/lib/instances/vgpu_orphan_test.go b/lib/instances/vgpu_orphan_test.go index 25a350064..ee8576a3b 100644 --- a/lib/instances/vgpu_orphan_test.go +++ b/lib/instances/vgpu_orphan_test.go @@ -97,20 +97,6 @@ func TestScheduleOrphanedVGPUReleaseDeduplicatesByDevicePath(t *testing.T) { assert.Equal(t, int32(1), attempts.Load(), "the second schedule for the same path must be dropped") } -func TestScheduleOrphanedVGPUReleaseIgnoresEmptyAssignment(t *testing.T) { - t.Parallel() - - m := &manager{paths: paths.New(t.TempDir())} - m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{Id: "no-gpu"}) - - m.orphanedVGPUMu.Lock() - defer m.orphanedVGPUMu.Unlock() - assert.Empty(t, m.orphanedVGPUs) -} - -// TestOrphanedVGPUReleaseReappliesClaimScan pins that the background retry -// goes through releaseStoredVGPU, not a raw destroy: a live claimant found by -// the vendor VFIO claim scan must keep blocking the release on every retry. func TestOrphanedVGPUReleaseReappliesClaimScan(t *testing.T) { t.Parallel() @@ -123,8 +109,6 @@ func TestOrphanedVGPUReleaseReappliesClaimScan(t *testing.T) { return nil }, } - // A claimant with a recent assignment and no persisted PID makes the scan - // fail closed, exactly like the synchronous release path. require.NoError(t, m.ensureDirectories("mid-boot-claimant")) assignedAt := time.Now() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 24be04fe3..aab2da2a1 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -8,8 +8,6 @@ import ( "github.com/kernel/hypeman/lib/logger" ) -// vgpuReconcileFailureRetryDelay spaces retries after a transient metadata or -// device error would otherwise disable orphan recovery until the next restart. const vgpuReconcileFailureRetryDelay = time.Minute func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]struct{}, time.Duration, error) { @@ -67,8 +65,6 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { if retryAfter <= 0 { return } - // One pending retry at a time: overlapping calls would fork parallel - // retry chains. if !m.vgpuReconcileRetryPending.CompareAndSwap(false, true) { return } diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 815415d09..03851f40d 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -65,7 +65,6 @@ func TestReconcileVGPUsRetriesAfterListingFailure(t *testing.T) { require.True(t, m.vgpuReconcileRetryPending.Load(), "a listing failure must schedule a retry instead of disabling the vendor sweep until restart") - // Once the listing recovers, the retry runs the sweep and stops rearming. require.NoError(t, os.Chmod(instanceDir, 0o755)) require.Eventually(t, func() bool { return !m.vgpuReconcileRetryPending.Load() @@ -93,32 +92,3 @@ func TestReconcileVGPUsRetriesAfterDeviceFailure(t *testing.T) { return calls.Load() >= 2 && !m.vgpuReconcileRetryPending.Load() }, 5*time.Second, 10*time.Millisecond) } - -func TestVGPUAssignmentLiveness(t *testing.T) { - now := time.Now().UTC() - recent := now.Add(-time.Minute) - stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) - pid := 123 - - tests := []struct { - name string - stored StoredMetadata - livePID bool - live bool - remaining time.Duration - }{ - {name: "live PID", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}}, livePID: true, live: true}, - {name: "dead PID recent assignment", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, GPUAssignedAt: &recent}, live: true, remaining: VGPUAssignmentStartupGracePeriod - time.Minute}, - {name: "dead PID stale assignment", stored: StoredMetadata{HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, GPUAssignedAt: &stale}}, - {name: "no PID recent assignment", stored: StoredMetadata{GPUAssignedAt: &recent}, live: true, remaining: VGPUAssignmentStartupGracePeriod - time.Minute}, - {name: "no PID stale assignment", stored: StoredMetadata{GPUAssignedAt: &stale}}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - live, remaining := vgpuAssignmentLiveness(&tt.stored, now, tt.livePID) - assert.Equal(t, tt.live, live) - assert.Equal(t, tt.remaining, remaining) - }) - } -} diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index eaca96289..feac72ed2 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -65,8 +65,6 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) } - // No on-disk record points at the device; retry the release in the - // background instead of waiting for the next startup reconcile. m.scheduleOrphanedVGPURelease(ctx, *retainedVGPU) return false } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index cb54c509b..4b31bf7c7 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -67,17 +67,13 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) assert.Equal(t, stored.GPUAssignedAt, retained.GPUAssignedAt) - // Identity fields survive so the retained record lists as a - // recognizable, deletable instance instead of a nameless phantom. assert.Equal(t, stored.Name, retained.Name) assert.Equal(t, stored.GPUProfile, retained.GPUProfile) assert.Equal(t, stored.HypervisorType, retained.HypervisorType) assert.Equal(t, stored.DataDir, retained.DataDir) - // Resource claims released by rollback stay dropped. assert.False(t, retained.NetworkEnabled) assert.Empty(t, retained.IP) assert.Empty(t, retained.Volumes) - // The stub has no boot configuration, so it is marked delete-only. assert.True(t, retained.GPURetainedForCleanup) } @@ -162,19 +158,6 @@ func TestVGPURetentionWrapPending(t *testing.T) { assert.True(t, cleanupPending.Retained) } -func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { - t.Parallel() - - cause := errors.New("boot failed") - retained := &VGPUCleanupPendingError{InstanceID: "inst-1", Retained: true, Err: cause} - assert.ErrorIs(t, retained, cause) - assert.Equal(t, "boot failed; vGPU release failed during rollback, instance inst-1 retains the assignment", retained.Error()) - - unpersisted := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} - assert.ErrorIs(t, unpersisted, cause) - assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the release is retried in the background and by the next startup reconcile", unpersisted.Error()) -} - func TestVGPUDevicePendingCleanup(t *testing.T) { t.Parallel() @@ -445,15 +428,12 @@ func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { } assignedAt := time.Now().UTC() - // The mid-start save already persisted the assignment. meta, err := m.loadMetadata(id) require.NoError(t, err) rollbackMeta := *meta setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt) require.NoError(t, m.saveMetadata(meta)) - // The cleanup save fails, but the surviving on-disk record still points - // at the device, so retention must be reported as persisted. instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) require.NoError(t, os.Chmod(instanceDir, 0o555)) t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) @@ -603,12 +583,9 @@ func TestVGPUAssignmentClaimedByLiveInstanceGracesRecentDeadPIDClaim(t *testing. GPUAssignedAt: &assignedAt, }})) - // Same bounded grace as startup reconcile: a recent claim whose PID is - // dead fails closed instead of being treated as unclaimed. _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") require.Error(t, err) - // Past the grace period the dead claim no longer blocks the release. stale := assignedAt.Add(-2 * VGPUAssignmentStartupGracePeriod) meta, err := m.loadMetadata(claimantID) require.NoError(t, err) From 8e3d5a09443f631c3e45eb009a10920ed50ceb0e Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:32:09 +0000 Subject: [PATCH 64/76] Track vendor VFIO assignment age and owner for periodic sweeps Record the owning instance and assignment time for each vendor VFIO VF so a reconcile sweep can run while instances are being created: recently assigned VFs get a grace period before they are eligible (mirroring orphanedMdevGracePeriod), and owned VFs are destroyed with their recorded owner ID instead of failing the ownership check. --- lib/devices/vendor_vfio_linux.go | 34 +++++++++++++++++++------ lib/devices/vendor_vfio_linux_test.go | 36 ++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 2c576b4d2..74eae4948 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "log/slog" + "maps" "os" "path/filepath" "sort" @@ -14,6 +15,7 @@ import ( "strings" "sync" "syscall" + "time" "github.com/kernel/hypeman/lib/logger" ) @@ -21,13 +23,23 @@ import ( const ( pciDevicesPath = "/sys/bus/pci/devices" vfioDevicesPath = "/dev/vfio/devices" + + // vendorVFIOAssignmentGracePeriod protects assignments created by this + // process from the periodic sweep until their owning instance has had time + // to persist metadata and boot, mirroring orphanedMdevGracePeriod. + vendorVFIOAssignmentGracePeriod = 5 * time.Minute ) +type vendorVFIOOwner struct { + instanceID string + assignedAt time.Time +} + type vendorVFIOSysfs struct { pciDevicesPath string procPath string vfioDevicesPath string - owners map[string]string + owners map[string]vendorVFIOOwner framebufferByType map[string]int openVFIOPathsFunc func() (map[string]struct{}, error) } @@ -37,7 +49,7 @@ var ( pciDevicesPath: pciDevicesPath, procPath: procPath, vfioDevicesPath: vfioDevicesPath, - owners: make(map[string]string), + owners: make(map[string]vendorVFIOOwner), framebufferByType: make(map[string]int), } vendorVFIOMu sync.Mutex @@ -196,7 +208,7 @@ func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID str verifyErr := fmt.Errorf("verify vGPU on VF %s: type is %s, want %s", targetVF, currentType, requested.TypeName) return nil, s.rollbackCreate(currentTypePath, targetVF, instanceID, device, verifyErr) } - s.owners[targetVF] = instanceID + s.owners[targetVF] = vendorVFIOOwner{instanceID: instanceID, assignedAt: time.Now()} logger.FromContext(ctx).InfoContext(ctx, "created vendor VFIO vGPU", "profile", profileName, @@ -229,10 +241,10 @@ func (s vendorVFIOSysfs) destroy(ctx context.Context, vfAddress, instanceID stri if instanceID == "" { return fmt.Errorf("cannot release vendor VFIO vGPU on VF %s without instance ID", vfAddress) } - if owner != instanceID { + if owner.instanceID != instanceID { log.WarnContext(ctx, "skipping vendor VFIO vGPU release owned by another instance", "vf", vfAddress, - "owner_instance_id", owner, + "owner_instance_id", owner.instanceID, "requesting_instance_id", instanceID, ) return nil @@ -264,6 +276,9 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map if err != nil { return err } + vendorVFIOMu.Lock() + owners := maps.Clone(s.owners) + vendorVFIOMu.Unlock() log := logger.FromContext(ctx) protectedVFs := make(map[string]struct{}, len(protectedDevicePaths)) for path := range protectedDevicePaths { @@ -278,6 +293,11 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map log.DebugContext(ctx, "skipping vendor VFIO vGPU held by a live instance", "vf", vf.PCIAddress) continue } + owner := owners[vf.PCIAddress] + if !owner.assignedAt.IsZero() && time.Since(owner.assignedAt) < vendorVFIOAssignmentGracePeriod { + log.DebugContext(ctx, "skipping recently assigned vendor VFIO vGPU during grace period", "vf", vf.PCIAddress) + continue + } if openPaths == nil { if openPaths, err = s.openVFIOPaths(); err != nil { return fmt.Errorf("scan open VFIO handles: %w", err) @@ -292,7 +312,7 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map log.WarnContext(ctx, "preserving vendor VFIO vGPU held open without a live instance claim", "vf", vf.PCIAddress) continue } - if err := s.destroy(ctx, vf.PCIAddress, ""); err != nil { + if err := s.destroy(ctx, vf.PCIAddress, owner.instanceID); err != nil { log.WarnContext(ctx, "failed to destroy orphaned vendor VFIO vGPU", "vf", vf.PCIAddress, "error", err) } } @@ -514,7 +534,7 @@ func framebufferFromProfileName(name string) int { func (s vendorVFIOSysfs) rollbackCreate(currentTypePath, vfAddress, instanceID string, device VGPUDevice, verifyErr error) error { if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { - s.owners[vfAddress] = instanceID + s.owners[vfAddress] = vendorVFIOOwner{instanceID: instanceID, assignedAt: time.Now()} return &VGPUCreateCleanupPendingError{ Device: device, Err: errors.Join(verifyErr, fmt.Errorf("roll back vGPU on VF %s: %w", vfAddress, err)), diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index bd7293922..f7b0d6c3f 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -372,6 +373,32 @@ func TestVendorVFIOReconcile(t *testing.T) { assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "1148") } +func TestVendorVFIOReconcileSkipsRecentlyAssignedVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.owners["0000:82:00.4"] = vendorVFIOOwner{instanceID: "mid-create", assignedAt: time.Now()} + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcileDestroysOwnedVFPastGracePeriod(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.owners["0000:82:00.4"] = vendorVFIOOwner{ + instanceID: "deleted-instance", + assignedAt: time.Now().Add(-vendorVFIOAssignmentGracePeriod - time.Minute), + } + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "0") + assert.Empty(t, sysfs.owners) +} + func TestVendorVFIOReconcileRechecksOpenHandlesBeforeDestroy(t *testing.T) { t.Parallel() @@ -518,7 +545,7 @@ func TestRollbackVendorVFIOCreate(t *testing.T) { t.Run("preserves verification error", func(t *testing.T) { currentTypePath := filepath.Join(t.TempDir(), "current_vgpu_type") require.NoError(t, os.WriteFile(currentTypePath, []byte("1148"), 0644)) - sysfs := vendorVFIOSysfs{owners: make(map[string]string)} + sysfs := vendorVFIOSysfs{owners: make(map[string]vendorVFIOOwner)} err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) require.ErrorIs(t, err, verifyErr) @@ -528,7 +555,7 @@ func TestRollbackVendorVFIOCreate(t *testing.T) { t.Run("retains assignment when rollback fails", func(t *testing.T) { currentTypePath := filepath.Join(t.TempDir(), "missing", "current_vgpu_type") - sysfs := vendorVFIOSysfs{owners: make(map[string]string)} + sysfs := vendorVFIOSysfs{owners: make(map[string]vendorVFIOOwner)} err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) require.ErrorIs(t, err, verifyErr) @@ -536,7 +563,8 @@ func TestRollbackVendorVFIOCreate(t *testing.T) { var pending *VGPUCreateCleanupPendingError require.ErrorAs(t, err, &pending) assert.Equal(t, device, pending.Device) - assert.Equal(t, "instance-1", sysfs.owners[device.VFAddress]) + assert.Equal(t, "instance-1", sysfs.owners[device.VFAddress].instanceID) + assert.False(t, sysfs.owners[device.VFAddress].assignedAt.IsZero()) }) } @@ -566,7 +594,7 @@ func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { pciDevicesPath: pci, procPath: proc, vfioDevicesPath: vfio, - owners: make(map[string]string), + owners: make(map[string]vendorVFIOOwner), framebufferByType: make(map[string]int), }} } From 3545b1f40803bfbb3e905474b95e0e8e84412fab Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:32:16 +0000 Subject: [PATCH 65/76] Replace vGPU release retry machinery with a periodic reconciler Run the fail-closed vGPU reconcile once at startup and every minute after, skipping hosts without a vGPU framework. Each pass retries releases for assignments whose owner is no longer live (re-verified under the instance lock) and then sweeps device-level leftovers with no live metadata claim. This deletes the per-path orphan retry goroutines - whose path-keyed dedup could drop cleanup for a newer assignment reusing the same VF - the CAS/timer retry in ReconcileVGPUs, the stopped-instance release special case in StopInstance, the retention fallbacks that scheduled background retries, and the orphan-abandoned metric. --- cmd/api/main.go | 4 +- lib/devices/GPU.md | 2 +- lib/instances/delete.go | 3 +- lib/instances/lifecycle_noop_test.go | 44 ++---- lib/instances/manager.go | 17 +- lib/instances/metrics.go | 17 -- lib/instances/start.go | 3 +- lib/instances/vgpu.go | 30 +--- lib/instances/vgpu_orphan.go | 59 ------- lib/instances/vgpu_orphan_test.go | 130 --------------- lib/instances/vgpu_reconcile.go | 142 +++++++++++------ lib/instances/vgpu_reconcile_test.go | 226 +++++++++++++++++++++++---- lib/instances/vgpu_retention.go | 3 +- lib/instances/vgpu_test.go | 12 +- 14 files changed, 321 insertions(+), 371 deletions(-) delete mode 100644 lib/instances/vgpu_orphan.go delete mode 100644 lib/instances/vgpu_orphan_test.go diff --git a/cmd/api/main.go b/cmd/api/main.go index 3bd0fff39..faa9ac15a 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -385,8 +385,8 @@ func run() error { } logger.Info("Reconciling vGPU devices...") - if r, ok := app.InstanceManager.(interface{ ReconcileVGPUs(context.Context) }); ok { - r.ReconcileVGPUs(ctx) + if r, ok := app.InstanceManager.(interface{ StartVGPUReconciler(context.Context) }); ok { + r.StartVGPUReconciler(ctx) } // Wire up resource validator for aggregate limit checking diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index ff11c5cde..d04dcf599 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -97,7 +97,7 @@ Instance Create → Assign profile to VF → Attach VF to VM → Instance Runnin Instance Stop/Delete → Release profile → VF available again ``` -Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. A release that fails during delete (typically because a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait) is retried in the background for up to ten minutes, so a completed delete does not strand the VF until the next restart. +Hypeman reconciles orphaned assignments with a periodic fail-closed pass: once at startup and every minute afterward (skipped entirely on hosts without GPUs). Each pass releases assignments whose owning instance is no longer live and clears their metadata, then sweeps device-level leftovers with no live metadata claim. Devices held open by a running VMM and assignments younger than five minutes are preserved, so a release that fails during stop or delete (typically because a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait) is simply retried on later passes until the device is free. ### Hypervisor Support diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 9c5f6b055..0474138d2 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -148,8 +148,7 @@ func (m *manager) deleteInstanceWithOptions( log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } if err := m.releaseStoredVGPU(ctx, stored); err != nil { - log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) - m.scheduleOrphanedVGPURelease(ctx, *stored) + log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup; the periodic vGPU reconcile releases it once free", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) } else if hadVGPUAssignment { if err := m.saveMetadata(meta); err != nil { log.WarnContext(ctx, "failed to save metadata after vGPU release", "instance_id", id, "error", err) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 90f2e7aa1..1843485e3 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -152,7 +152,6 @@ func TestLifecycleNoopStandbyWithOptionsStillRejectsStandbyInstance(t *testing.T func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - m.orphanedVGPURetryDelay = time.Millisecond meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" @@ -162,9 +161,8 @@ func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { // A failed release is logged and the delete continues, matching the // pre-refactor contract; the leaked assignment is recovered by the - // background retry or startup reconciliation. + // periodic vGPU reconcile. require.NoError(t, m.DeleteInstance(context.Background(), id)) - waitForOrphanQueueEmpty(t, m) _, err = m.loadMetadata(id) require.Error(t, err, "instance data must be deleted despite the failed release") @@ -280,7 +278,6 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - m.orphanedVGPURetryDelay = time.Millisecond deviceManager := &recordingDeviceManager{} m.deviceManager = deviceManager meta, err := m.loadMetadata(id) @@ -294,7 +291,6 @@ func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { // The failed release must not block the rest of the teardown: devices // are detached and the instance is fully deleted. require.NoError(t, m.DeleteInstance(context.Background(), id)) - waitForOrphanQueueEmpty(t, m) assert.Equal(t, []string{"dev-1"}, deviceManager.detached) _, err = m.loadMetadata(id) @@ -343,8 +339,9 @@ func TestStartRejectsVGPURetentionRecord(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) } -func TestStopStoppedInstanceReleasesRetainedVGPU(t *testing.T) { +func TestReconcileReleasesRetainedVGPUOnStoppedInstance(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + m.reconcileVGPUDevices = func(context.Context, map[string]struct{}, bool) error { return nil } meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" @@ -352,38 +349,26 @@ func TestStopStoppedInstanceReleasesRetainedVGPU(t *testing.T) { meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" require.NoError(t, m.saveMetadata(meta)) + // Stop on an already-stopped instance is a no-op for the assignment; the + // periodic reconcile retries the release. inst, err := m.StopInstance(context.Background(), id) require.NoError(t, err) require.NotNil(t, inst) assert.Equal(t, StateStopped, inst.State) - stored, err := m.loadMetadata(id) require.NoError(t, err) - assert.Empty(t, stored.GPUDevicePath) -} - -func TestStopStoppedInstanceLeavesRetentionStubForDelete(t *testing.T) { - m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFrameworkNone - meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" - meta.GPURetainedForCleanup = true - require.NoError(t, m.saveMetadata(meta)) - - inst, err := m.StopInstance(context.Background(), id) - require.NoError(t, err) - require.NotNil(t, inst) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) - stored, err := m.loadMetadata(id) + m.ReconcileVGPUs(context.Background()) + stored, err = m.loadMetadata(id) require.NoError(t, err) - assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) - assert.True(t, stored.GPURetainedForCleanup) + assert.Empty(t, stored.GPUDevicePath) + assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") } -func TestStopStoppedInstanceVGPUReleaseFailureRemainsNoop(t *testing.T) { +func TestReconcileVGPUReleaseFailureKeepsStoppedInstanceUsable(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + m.reconcileVGPUDevices = func(context.Context, map[string]struct{}, bool) error { return nil } meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" @@ -391,10 +376,7 @@ func TestStopStoppedInstanceVGPUReleaseFailureRemainsNoop(t *testing.T) { meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" require.NoError(t, m.saveMetadata(meta)) - inst, err := m.StopInstance(context.Background(), id) - require.NoError(t, err) - require.NotNil(t, inst) - assert.Equal(t, StateStopped, inst.State) + m.ReconcileVGPUs(context.Background()) stored, err := m.loadMetadata(id) require.NoError(t, err) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index a37049f05..956e69d43 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -8,7 +8,6 @@ import ( "path/filepath" "strings" "sync" - "sync/atomic" "time" "github.com/kernel/hypeman/lib/devices" @@ -214,12 +213,10 @@ type manager struct { // Periodic TAP garbage collection reconciler. tapGCOnce sync.Once - orphanedVGPUMu sync.Mutex - orphanedVGPUs map[string]struct{} - orphanedVGPURetryDelay time.Duration - - vgpuReconcileRetryPending atomic.Bool - vgpuReconcileRetryDelay time.Duration + // Periodic vGPU reconciler. + vgpuReconcileOnce sync.Once + vgpuReconcileInterval time.Duration + discoverVGPU func() (devices.VGPUFramework, []devices.VirtualFunction, error) vgpuInitTermGrace time.Duration @@ -653,12 +650,6 @@ func (m *manager) StopInstance(ctx context.Context, id string) (*Instance, error if err := m.markRestartManualStopLocked(ctx, id); err != nil { return nil, err } - // A stopped instance can retain a vGPU assignment when the release - // failed during the original stop. Retry it here so the vGPU slot is - // not held until the next start, delete, or hypeman restart. A failed - // retry only logs, keeping stop's no-op contract for already-stopped - // instances. - m.releaseRetainedVGPULocked(ctx, id) updated, err := m.currentInstanceWithoutHydration(ctx, id) if err != nil { return nil, err diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index de71c2686..1ada5ac1e 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -94,7 +94,6 @@ type Metrics struct { lifecycleEventsDroppedTotal metric.Int64Counter forkMemFileShareFallbacksTotal metric.Int64Counter ttlReaperDeletionsTotal metric.Int64Counter - vgpuOrphanReleasesAbandonedTotal metric.Int64Counter tracer trace.Tracer } @@ -271,14 +270,6 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } - vgpuOrphanReleasesAbandonedTotal, err := meter.Int64Counter( - "hypeman_instances_vgpu_orphan_releases_abandoned_total", - metric.WithDescription("Total orphaned vGPU release retries that gave up, leaving the VF allocated until startup reconciliation or manual remediation"), - ) - if err != nil { - return nil, err - } - // Register observable gauge for instance counts by state instancesTotal, err := meter.Int64ObservableGauge( "hypeman_instances_total", @@ -473,7 +464,6 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M lifecycleEventsDroppedTotal: lifecycleEventsDroppedTotal, forkMemFileShareFallbacksTotal: forkMemFileShareFallbacksTotal, ttlReaperDeletionsTotal: ttlReaperDeletionsTotal, - vgpuOrphanReleasesAbandonedTotal: vgpuOrphanReleasesAbandonedTotal, tracer: tracer, }, nil } @@ -573,13 +563,6 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } -func (m *manager) recordVGPUOrphanReleaseAbandoned(ctx context.Context) { - if m.metrics == nil { - return - } - m.metrics.vgpuOrphanReleasesAbandonedTotal.Add(ctx, 1) -} - // recordStateTransition records a state transition with hypervisor label. func (m *manager) recordStateTransition(ctx context.Context, fromState, toState string, hvType hypervisor.Type) { if m.metrics == nil { diff --git a/lib/instances/start.go b/lib/instances/start.go index 8986d7fb3..19bb64697 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -55,7 +55,7 @@ func (m *manager) startInstance( // Release any assignment retained by an earlier failed release and // persist the cleared fields immediately, so a failure later in start // cannot leave on-disk metadata pointing at a device that is already - // gone (matching releaseRetainedVGPULocked). + // gone. if storedVGPUDevicePath(stored) != "" { if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.ErrorContext(ctx, "failed to release stale vGPU before start", "instance_id", id, "error", err) @@ -184,7 +184,6 @@ func (m *manager) startInstance( wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) - m.scheduleOrphanedVGPURelease(ctx, retentionMeta.StoredMetadata) return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} } return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 0e09f2f34..3fb26778a 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -26,7 +26,7 @@ func (e *VGPUCleanupPendingError) Error() string { if e.Retained { return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) } - return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the release is retried in the background and by the next startup reconcile", e.Err, e.InstanceID) + return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the periodic vGPU reconcile retries the release", e.Err, e.InstanceID) } func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } @@ -118,7 +118,6 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { return true, true } - m.scheduleOrphanedVGPURelease(ctx, cleanupMeta.StoredMetadata) return true, false } return retained, retained @@ -220,33 +219,6 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu return false, nil } -// releaseRetainedVGPULocked releases a vGPU assignment retained on a stopped -// instance after a failed release during the original stop. It is a no-op -// when no assignment is retained, and a failed retry only logs so the -// metadata stays for the next retry. The caller must hold the instance lock. -func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) { - log := logger.FromContext(ctx) - meta, err := m.loadMetadata(id) - if err != nil { - log.WarnContext(ctx, "failed to load metadata for retained vGPU release", "instance_id", id, "error", err) - return - } - stored := &meta.StoredMetadata - if stored.GPURetainedForCleanup { - return - } - if storedVGPUDevicePath(stored) == "" { - return - } - if err := m.releaseStoredVGPU(ctx, stored); err != nil { - log.WarnContext(ctx, "failed to destroy retained vGPU; retaining assignment metadata", "instance_id", id, "error", err) - return - } - if err := m.saveMetadata(meta); err != nil { - log.WarnContext(ctx, "failed to save metadata after retained vGPU release", "instance_id", id, "error", err) - } -} - func storedVGPUDevicePath(stored *StoredMetadata) string { if stored.GPUDevicePath != "" { return stored.GPUDevicePath diff --git a/lib/instances/vgpu_orphan.go b/lib/instances/vgpu_orphan.go deleted file mode 100644 index 041402642..000000000 --- a/lib/instances/vgpu_orphan.go +++ /dev/null @@ -1,59 +0,0 @@ -package instances - -import ( - "context" - "time" - - "github.com/kernel/hypeman/lib/logger" -) - -const ( - orphanedVGPUReleaseMaxAttempts = 20 - defaultOrphanedVGPUReleaseRetryDelay = 30 * time.Second -) - -func (m *manager) scheduleOrphanedVGPURelease(ctx context.Context, stored StoredMetadata) { - path := storedVGPUDevicePath(&stored) - if path == "" { - return - } - m.orphanedVGPUMu.Lock() - if m.orphanedVGPUs == nil { - m.orphanedVGPUs = make(map[string]struct{}) - } - if _, pending := m.orphanedVGPUs[path]; pending { - m.orphanedVGPUMu.Unlock() - return - } - m.orphanedVGPUs[path] = struct{}{} - m.orphanedVGPUMu.Unlock() - - delay := m.orphanedVGPURetryDelay - if delay <= 0 { - delay = defaultOrphanedVGPUReleaseRetryDelay - } - go m.retryOrphanedVGPURelease(context.WithoutCancel(ctx), stored, path, delay) -} - -func (m *manager) retryOrphanedVGPURelease(ctx context.Context, stored StoredMetadata, path string, delay time.Duration) { - log := logger.FromContext(ctx) - defer func() { - m.orphanedVGPUMu.Lock() - delete(m.orphanedVGPUs, path) - m.orphanedVGPUMu.Unlock() - }() - for attempt := 1; attempt <= orphanedVGPUReleaseMaxAttempts; attempt++ { - time.Sleep(delay) - if err := m.releaseStoredVGPUExcluding(ctx, &stored, ""); err != nil { - log.WarnContext(ctx, "orphaned vGPU release retry failed", - "instance_id", stored.Id, "device_path", path, "attempt", attempt, "error", err) - continue - } - log.InfoContext(ctx, "released orphaned vGPU after delete", - "instance_id", stored.Id, "device_path", path, "attempt", attempt) - return - } - m.recordVGPUOrphanReleaseAbandoned(ctx) - log.ErrorContext(ctx, "giving up on orphaned vGPU release; VF stays allocated until startup reconciliation or manual remediation", - "instance_id", stored.Id, "device_path", path, "attempts", orphanedVGPUReleaseMaxAttempts) -} diff --git a/lib/instances/vgpu_orphan_test.go b/lib/instances/vgpu_orphan_test.go deleted file mode 100644 index ee8576a3b..000000000 --- a/lib/instances/vgpu_orphan_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package instances - -import ( - "context" - "errors" - "sync/atomic" - "testing" - "time" - - "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/paths" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func waitForOrphanQueueEmpty(t *testing.T, m *manager) { - t.Helper() - require.Eventually(t, func() bool { - m.orphanedVGPUMu.Lock() - defer m.orphanedVGPUMu.Unlock() - return len(m.orphanedVGPUs) == 0 - }, 5*time.Second, 5*time.Millisecond, "orphan retry should finish and clear its queue entry") -} - -func TestScheduleOrphanedVGPUReleaseRetriesUntilSuccess(t *testing.T) { - t.Parallel() - - var attempts atomic.Int32 - m := &manager{ - paths: paths.New(t.TempDir()), - orphanedVGPURetryDelay: time.Millisecond, - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { - if attempts.Add(1) < 3 { - return errors.New("operation not permitted") - } - return nil - }, - } - m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ - Id: "deleted-instance", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - }) - - waitForOrphanQueueEmpty(t, m) - assert.Equal(t, int32(3), attempts.Load(), "release should succeed on the third attempt and stop retrying") -} - -func TestScheduleOrphanedVGPUReleaseGivesUpAfterMaxAttempts(t *testing.T) { - t.Parallel() - - var attempts atomic.Int32 - m := &manager{ - paths: paths.New(t.TempDir()), - orphanedVGPURetryDelay: time.Millisecond, - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { - attempts.Add(1) - return errors.New("vGPU destroy failed: 0xffffffff") - }, - } - m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ - Id: "deleted-instance", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - }) - - waitForOrphanQueueEmpty(t, m) - assert.Equal(t, int32(orphanedVGPUReleaseMaxAttempts), attempts.Load(), - "a wedged VF should get exactly the bounded number of attempts") -} - -func TestScheduleOrphanedVGPUReleaseDeduplicatesByDevicePath(t *testing.T) { - t.Parallel() - - var attempts atomic.Int32 - release := make(chan struct{}) - m := &manager{ - paths: paths.New(t.TempDir()), - orphanedVGPURetryDelay: 20 * time.Millisecond, - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { - attempts.Add(1) - <-release - return nil - }, - } - stored := StoredMetadata{ - Id: "deleted-instance", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - } - m.scheduleOrphanedVGPURelease(context.Background(), stored) - m.scheduleOrphanedVGPURelease(context.Background(), stored) - - require.Eventually(t, func() bool { return attempts.Load() == 1 }, 5*time.Second, 5*time.Millisecond) - close(release) - waitForOrphanQueueEmpty(t, m) - assert.Equal(t, int32(1), attempts.Load(), "the second schedule for the same path must be dropped") -} - -func TestOrphanedVGPUReleaseReappliesClaimScan(t *testing.T) { - t.Parallel() - - var destroys atomic.Int32 - m := &manager{ - paths: paths.New(t.TempDir()), - orphanedVGPURetryDelay: time.Millisecond, - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { - destroys.Add(1) - return nil - }, - } - require.NoError(t, m.ensureDirectories("mid-boot-claimant")) - assignedAt := time.Now() - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "mid-boot-claimant", - Name: "mid-boot-claimant", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, - }})) - - m.scheduleOrphanedVGPURelease(context.Background(), StoredMetadata{ - Id: "deleted-instance", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - }) - - waitForOrphanQueueEmpty(t, m) - assert.Zero(t, destroys.Load(), "no destroy may fire while the claim scan cannot clear the path") -} diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index aab2da2a1..966bf1118 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -2,51 +2,62 @@ package instances import ( "context" + "errors" "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/logger" ) -const vgpuReconcileFailureRetryDelay = time.Minute +const defaultVGPUReconcileInterval = time.Minute -func (m *manager) liveVGPUReconcileProtection(ctx context.Context) (map[string]struct{}, time.Duration, error) { - allInstances, err := m.listInstancesForReconcile(ctx) +// StartVGPUReconciler runs one reconcile pass and then keeps reconciling +// periodically until ctx is cancelled. Hosts without a vGPU framework skip +// reconciliation entirely. A discovery failure starts the reconciler anyway: +// a transient sysfs error must not disable cleanup on a GPU host. +func (m *manager) StartVGPUReconciler(ctx context.Context) { + discover := m.discoverVGPU + if discover == nil { + discover = devices.DiscoverVGPU + } + framework, _, err := discover() + if err == nil && framework == devices.VGPUFrameworkNone { + return + } if err != nil { - return nil, 0, err + logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU framework; starting vGPU reconciler anyway", "error", err) } - protected := make(map[string]struct{}) - var retryAfter time.Duration - for i := range allInstances { - stored := &allInstances[i].StoredMetadata - if stored.GPUDevicePath == "" { - continue + m.ReconcileVGPUs(ctx) + m.vgpuReconcileOnce.Do(func() { + interval := m.vgpuReconcileInterval + if interval <= 0 { + interval = defaultVGPUReconcileInterval } - livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) - live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID) - if !live { - continue - } - protected[stored.GPUDevicePath] = struct{}{} - if remaining > 0 && (retryAfter == 0 || remaining < retryAfter) { - retryAfter = remaining - } - } - return protected, retryAfter, nil + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + m.ReconcileVGPUs(ctx) + } + } + }() + }) } -// ReconcileVGPUs releases orphaned vGPU assignments. +// ReconcileVGPUs runs one fail-closed reconcile pass: stale instance-held +// assignments are released, then device-level leftovers not claimed by a live +// instance are swept. Failures only log; the next periodic pass retries. func (m *manager) ReconcileVGPUs(ctx context.Context) { log := logger.FromContext(ctx) - protected, retryAfter, err := m.liveVGPUReconcileProtection(ctx) + protected, err := m.reconcileVGPUAssignments(ctx) sweepVendorVFIO := err == nil if err != nil { - log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconcile, mdev reconcile still runs", "error", err) + log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO sweep until the next pass, mdev reconcile still runs", "error", err) protected = make(map[string]struct{}) - retryAfter = vgpuReconcileFailureRetryDelay - if m.vgpuReconcileRetryDelay > 0 { - retryAfter = m.vgpuReconcileRetryDelay - } } reconcileDevices := m.reconcileVGPUDevices if reconcileDevices == nil { @@ -54,29 +65,68 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { } if err := reconcileDevices(ctx, protected, sweepVendorVFIO); err != nil { log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) - deviceRetryAfter := vgpuReconcileFailureRetryDelay - if m.vgpuReconcileRetryDelay > 0 { - deviceRetryAfter = m.vgpuReconcileRetryDelay + } +} + +// reconcileVGPUAssignments retries releases for assignments whose owner is no +// longer live and returns the device paths still protected by live instances. +// Listing fails closed: any unreadable metadata aborts the pass so the vendor +// VFIO sweep cannot clear a VF whose claim it could not read. +func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]struct{}, error) { + allInstances, err := m.listInstancesForReconcile(ctx) + if err != nil { + return nil, err + } + protected := make(map[string]struct{}) + for i := range allInstances { + stored := &allInstances[i].StoredMetadata + if storedVGPUDevicePath(stored) == "" { + continue } - if retryAfter <= 0 || deviceRetryAfter < retryAfter { - retryAfter = deviceRetryAfter + livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { + if stored.GPUDevicePath != "" { + protected[stored.GPUDevicePath] = struct{}{} + } + continue } + m.releaseStaleVGPUAssignment(ctx, stored.Id) } - if retryAfter <= 0 { + return protected, nil +} + +// releaseStaleVGPUAssignment retries a release that previously failed, under +// the instance lock. Liveness is re-verified after locking so a concurrent +// start or restore keeps its assignment. A failed release only logs and keeps +// the metadata for the next pass. +func (m *manager) releaseStaleVGPUAssignment(ctx context.Context, id string) { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + log := logger.FromContext(ctx) + meta, err := m.loadMetadata(id) + if err != nil { + if !errors.Is(err, ErrNotFound) { + log.WarnContext(ctx, "failed to load metadata for stale vGPU release", "instance_id", id, "error", err) + } return } - if !m.vgpuReconcileRetryPending.CompareAndSwap(false, true) { + stored := &meta.StoredMetadata + path := storedVGPUDevicePath(stored) + if path == "" { return } - go func() { - timer := time.NewTimer(retryAfter) - defer timer.Stop() - select { - case <-ctx.Done(): - m.vgpuReconcileRetryPending.Store(false) - case <-timer.C: - m.vgpuReconcileRetryPending.Store(false) - m.ReconcileVGPUs(ctx) - } - }() + livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { + return + } + if err := m.releaseStoredVGPU(ctx, stored); err != nil { + log.WarnContext(ctx, "failed to release stale vGPU assignment; retrying on the next reconcile pass", "instance_id", id, "device_path", path, "error", err) + return + } + if err := m.saveMetadata(meta); err != nil { + log.WarnContext(ctx, "failed to save metadata after stale vGPU release", "instance_id", id, "error", err) + return + } + log.InfoContext(ctx, "released stale vGPU assignment", "instance_id", id, "device_path", path) } diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 03851f40d..468591dee 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -10,12 +10,13 @@ import ( "testing" "time" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestLiveVGPUReconcileProtectionBoundsStartupProtection(t *testing.T) { +func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { dead := exec.Command("true") require.NoError(t, dead.Run()) deadPID := dead.Process.Pid @@ -23,7 +24,17 @@ func TestLiveVGPUReconcileProtectionBoundsStartupProtection(t *testing.T) { recent := now.Add(-time.Minute) stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) - m := &manager{paths: paths.New(t.TempDir()), now: func() time.Time { return now }} + var protected map[string]struct{} + m := &manager{ + paths: paths.New(t.TempDir()), + now: func() time.Time { return now }, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, sweepVendorVFIO bool) error { + protected = p + assert.True(t, sweepVendorVFIO) + return nil + }, + } instances := []StoredMetadata{ {Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}, {Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}, @@ -36,22 +47,38 @@ func TestLiveVGPUReconcileProtectionBoundsStartupProtection(t *testing.T) { require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: instances[i]})) } - protected, retryAfter, err := m.liveVGPUReconcileProtection(t.Context()) - require.NoError(t, err) - assert.Equal(t, VGPUAssignmentStartupGracePeriod-time.Minute, retryAfter) + m.ReconcileVGPUs(t.Context()) assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") + + for _, id := range []string{"orphaned", "legacy", "dead"} { + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "stale assignment on %s must be released", id) + } + for _, id := range []string{"booting", "stale-pid-booting"} { + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.NotEmpty(t, stored.GPUDevicePath, "live assignment on %s must be kept", id) + } } -func TestReconcileVGPUsRetriesAfterListingFailure(t *testing.T) { +func TestReconcileVGPUsSkipsVendorSweepWhenListingFails(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } - m := &manager{paths: paths.New(t.TempDir()), vgpuReconcileRetryDelay: 250 * time.Millisecond} + var sweeps []bool + m := &manager{ + paths: paths.New(t.TempDir()), + reconcileVGPUDevices: func(_ context.Context, _ map[string]struct{}, sweepVendorVFIO bool) error { + sweeps = append(sweeps, sweepVendorVFIO) + return nil + }, + } const id = "unreadable" require.NoError(t, m.ensureDirectories(id)) require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{Id: id}})) @@ -59,36 +86,181 @@ func TestReconcileVGPUsRetriesAfterListingFailure(t *testing.T) { require.NoError(t, os.Chmod(instanceDir, 0o000)) t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - m.ReconcileVGPUs(ctx) - require.True(t, m.vgpuReconcileRetryPending.Load(), - "a listing failure must schedule a retry instead of disabling the vendor sweep until restart") + m.ReconcileVGPUs(t.Context()) + require.Equal(t, []bool{false}, sweeps, + "a listing failure must skip the vendor sweep, not run it with an empty protection set") require.NoError(t, os.Chmod(instanceDir, 0o755)) - require.Eventually(t, func() bool { - return !m.vgpuReconcileRetryPending.Load() - }, 5*time.Second, 10*time.Millisecond) + m.ReconcileVGPUs(t.Context()) + assert.Equal(t, []bool{false, true}, sweeps, "the next pass retries the vendor sweep") +} + +func TestReconcileVGPUsReleasesStaleAssignment(t *testing.T) { + t.Parallel() + + var destroyed []devices.VGPUAssignment + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const id = "stopped-retained" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + m.ReconcileVGPUs(t.Context()) + + require.Len(t, destroyed, 1) + assert.Equal(t, devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: "/sys/bus/pci/devices/0000:82:00.4", + InstanceID: id, + }, destroyed[0]) + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath) + assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") +} + +func TestReconcileVGPUsKeepsAssignmentWhenReleaseFails(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + if attempts.Add(1) == 1 { + return errors.New("vGPU destroy failed: 0xffffffff") + } + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const id = "wedged" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + m.ReconcileVGPUs(t.Context()) + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath, + "a failed release must keep the assignment metadata for the next pass") + + m.ReconcileVGPUs(t.Context()) + stored, err = m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "the next pass retries the release") + assert.Equal(t, int32(2), attempts.Load()) +} + +func TestReconcileVGPUsDefersToUnconfirmedClaimant(t *testing.T) { + t.Parallel() + + var destroys atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + destroys.Add(1) + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const path = "/sys/bus/pci/devices/0000:82:00.4" + assignedAt := time.Now() + for _, stored := range []StoredMetadata{ + {Id: "stale", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: path}, + {Id: "mid-boot-claimant", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: path, GPUAssignedAt: &assignedAt}, + } { + require.NoError(t, m.ensureDirectories(stored.Id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: stored})) + } + + m.ReconcileVGPUs(t.Context()) + + assert.Zero(t, destroys.Load(), "no destroy may fire while the claim scan cannot clear the path") + stale, err := m.loadMetadata("stale") + require.NoError(t, err) + assert.Equal(t, path, stale.GPUDevicePath, "the stale release retries once the claimant's liveness is decidable") +} + +func TestReconcileVGPUsReleasesRetentionStubAssignment(t *testing.T) { + t.Parallel() + + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const id = "retention-stub" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: "failed-create", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPURetainedForCleanup: true, + }})) + + m.ReconcileVGPUs(t.Context()) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "the stub's wedged assignment is released once free") + assert.True(t, stored.GPURetainedForCleanup, "the stub stays a delete-only record of the failed create") } -func TestReconcileVGPUsRetriesAfterDeviceFailure(t *testing.T) { - var calls atomic.Int32 +func TestStartVGPUReconcilerSkipsHostsWithoutGPUs(t *testing.T) { + t.Parallel() + + var passes atomic.Int32 m := &manager{ - paths: paths.New(t.TempDir()), - vgpuReconcileRetryDelay: 10 * time.Millisecond, + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkNone, nil, nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { + passes.Add(1) + return nil + }, + vgpuReconcileInterval: time.Millisecond, + } + + m.StartVGPUReconciler(t.Context()) + time.Sleep(20 * time.Millisecond) + assert.Zero(t, passes.Load(), "a host without GPUs must not reconcile at all") +} + +func TestStartVGPUReconcilerRunsPeriodically(t *testing.T) { + t.Parallel() + + var passes atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, nil, nil + }, reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { - if calls.Add(1) == 1 { + if passes.Add(1) == 1 { return errors.New("transient device error") } return nil }, + vgpuReconcileInterval: time.Millisecond, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - m.ReconcileVGPUs(ctx) - require.True(t, m.vgpuReconcileRetryPending.Load()) - require.Eventually(t, func() bool { - return calls.Load() >= 2 && !m.vgpuReconcileRetryPending.Load() - }, 5*time.Second, 10*time.Millisecond) + m.StartVGPUReconciler(t.Context()) + require.Eventually(t, func() bool { return passes.Load() >= 3 }, 5*time.Second, time.Millisecond, + "periodic passes must keep running after a failed pass") } diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index feac72ed2..121c13cb0 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -57,6 +57,8 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten id := retention.instanceID retainedVGPU := retention.stub log := logger.FromContext(ctx) + // When the retention record is lost, the assignment has no metadata claim + // left; the periodic vGPU reconcile sweeps the device once it is free. retentionSurvives := func() bool { meta, err := m.loadMetadata(id) if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { @@ -65,7 +67,6 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) } - m.scheduleOrphanedVGPURelease(ctx, *retainedVGPU) return false } if err := m.deleteInstanceData(id); err != nil { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 4b31bf7c7..e5ed8d13f 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -106,12 +106,7 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { } assert.False(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) _, err := m.loadMetadata(id) - require.Error(t, err) - - m.orphanedVGPUMu.Lock() - _, queued := m.orphanedVGPUs[stored.GPUDevicePath] - m.orphanedVGPUMu.Unlock() - assert.True(t, queued, "unpersisted retention must queue a background release") + require.Error(t, err, "the lost retention leaves no metadata claim, so the periodic sweep releases the VF") } func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T) { @@ -332,11 +327,6 @@ func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded") - - m.orphanedVGPUMu.Lock() - _, queued := m.orphanedVGPUs[device.SysfsPath] - m.orphanedVGPUMu.Unlock() - assert.True(t, queued, "unpersisted retention must queue a background release") } func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { From 110328c5e11522db840167a26be2523b0670a4f2 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:21:01 +0000 Subject: [PATCH 66/76] Update vgpu_cleanup_pending assertions for the periodic reconcile message --- cmd/api/api/instances_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 408cedc65..273e95cd6 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -104,7 +104,7 @@ func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance( assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") assert.Contains(t, pending.Message, network.ErrNameExists.Error(), "the underlying create failure must survive the cleanup guidance") - assert.Contains(t, pending.Message, "startup reconcile") + assert.Contains(t, pending.Message, "periodic vGPU reconcile") assert.NotContains(t, pending.Message, "delete") require.NotNil(t, pending.InnerError) require.NotNil(t, pending.InnerError.Code) @@ -1081,7 +1081,7 @@ func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") - assert.Contains(t, pending.Message, "startup reconcile") + assert.Contains(t, pending.Message, "periodic vGPU reconcile") assert.NotContains(t, pending.Message, "delete") require.NotNil(t, pending.InnerError) require.NotNil(t, pending.InnerError.Code) From a35de81c65c9044952152388e290fc9c88cc902a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:21:01 +0000 Subject: [PATCH 67/76] Check socket ownership for vGPU liveness without a persisted PID An assignment could lose its persisted hypervisor PID while its VM stays alive - a post-boot metadata save failure or a hypeman crash before the save. The liveness checks gated the socket-ownership scan on a non-nil PID, so once the startup grace expired the reconciler considered such an assignment stale and could remove an mdev out from under the live VM (DestroyMdev has no in-use guard). Run the socket-ownership scan unconditionally: a live VMM always holds its control-socket listener, and a missing listener still resolves to not-alive, so genuinely stopped instances are released as before. The claim scan in releaseStoredVGPUExcluding gets the same treatment. --- lib/instances/vgpu.go | 15 +++--- lib/instances/vgpu_reconcile.go | 8 ++- lib/instances/vgpu_reconcile_linux_test.go | 62 ++++++++++++++++++++++ 3 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 lib/instances/vgpu_reconcile_linux_test.go diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 3fb26778a..bb230f3d1 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -13,7 +13,7 @@ import ( // VGPUAssignmentStartupGracePeriod bounds how long an assignment without a // persisted hypervisor PID is treated as potentially live. -const VGPUAssignmentStartupGracePeriod = 5 * time.Minute +const VGPUAssignmentStartupGracePeriod = devices.VGPUAssignmentGracePeriod // VGPUCleanupPendingError reports a failed rollback that left a vGPU assigned. type VGPUCleanupPendingError struct { @@ -50,7 +50,7 @@ func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { } func vgpuAssignmentLiveness(stored *StoredMetadata, now time.Time, livePID bool) (live bool, graceRemaining time.Duration) { - if stored.HypervisorPID != nil && livePID { + if livePID { return true, 0 } if stored.GPUAssignedAt == nil { @@ -198,12 +198,11 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if storedVGPUDevicePath(stored) != devicePath { continue } - pid := 0 - if stored.HypervisorPID != nil { - pid, err = resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) - if err != nil { - return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) - } + // Resolve even without a persisted PID: the socket-ownership scan can + // still prove a claimant whose post-boot metadata save failed is live. + pid, err := resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) + if err != nil { + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), pid > 0) if pid > 0 && live { diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 966bf1118..5cfb43d4c 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -83,7 +83,11 @@ func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]stru if storedVGPUDevicePath(stored) == "" { continue } - livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + // The socket-ownership check runs even without a persisted PID: a VMM + // whose post-boot metadata save failed still holds its control-socket + // listener, and releasing its device would tear the vGPU out from + // under a live VM. + livePID := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { if stored.GPUDevicePath != "" { protected[stored.GPUDevicePath] = struct{}{} @@ -116,7 +120,7 @@ func (m *manager) releaseStaleVGPUAssignment(ctx context.Context, id string) { if path == "" { return } - livePID := stored.HypervisorPID != nil && hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + livePID := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { return } diff --git a/lib/instances/vgpu_reconcile_linux_test.go b/lib/instances/vgpu_reconcile_linux_test.go new file mode 100644 index 000000000..8708a1dd7 --- /dev/null +++ b/lib/instances/vgpu_reconcile_linux_test.go @@ -0,0 +1,62 @@ +//go:build linux + +package instances + +import ( + "context" + "net" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A VMM whose post-boot metadata save failed has no persisted PID, but it +// still holds its control-socket listener. The reconciler must protect its +// assignment past the startup grace period instead of releasing the device +// out from under the live VM. +func TestReconcileVGPUsProtectsSocketOwnerWithoutPersistedPID(t *testing.T) { + t.Parallel() + + socketPath := filepath.Join(t.TempDir(), "test.sock") + listener, err := net.Listen("unix", socketPath) + require.NoError(t, err) + defer listener.Close() + + var destroyed []devices.VGPUAssignment + var protected map[string]struct{} + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, _ bool) error { + protected = p + return nil + }, + } + const id = "pid-save-failed" + stale := time.Now().UTC().Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + SocketPath: socketPath, + GPUFramework: devices.VGPUFrameworkMdev, + GPUDevicePath: "/sys/bus/mdev/devices/test-mdev", + GPUMdevUUID: "test-mdev", + GPUAssignedAt: &stale, + }})) + + m.ReconcileVGPUs(t.Context()) + + assert.Empty(t, destroyed, "a live socket owner must block the release even with a nil persisted PID") + assert.Contains(t, protected, "/sys/bus/mdev/devices/test-mdev") + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/mdev/devices/test-mdev", stored.GPUDevicePath) +} From be01bd6e2050a61a012802ff61708e384197b272 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:21:01 +0000 Subject: [PATCH 68/76] Share one vGPU assignment grace period constant The five-minute fresh-assignment protection existed three times: the instances startup grace, the mdev orphan grace, and the vendor VFIO sweep grace. Define it once in lib/devices and alias the instances constant to it. --- lib/devices/mdev_linux.go | 13 ++++++------- lib/devices/types.go | 5 +++++ lib/devices/vendor_vfio_linux.go | 7 +------ lib/devices/vendor_vfio_linux_test.go | 2 +- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 6ef9f6c60..0908e8765 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -23,10 +23,9 @@ import ( ) const ( - mdevBusPath = "/sys/class/mdev_bus" - mdevDevices = "/sys/bus/mdev/devices" - orphanedMdevGracePeriod = 5 * time.Minute - procPath = "/proc" + mdevBusPath = "/sys/class/mdev_bus" + mdevDevices = "/sys/bus/mdev/devices" + procPath = "/proc" ) // mdevMu protects mdev creation/destruction to prevent race conditions @@ -747,7 +746,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro log.InfoContext(ctx, "reconciling mdev devices", "total_mdevs", len(mdevs), "managed_vfs", len(managedVFs), - "grace_period", orphanedMdevGracePeriod.String(), + "grace_period", VGPUAssignmentGracePeriod.String(), ) groupInUseCache := make(map[int]bool) @@ -782,7 +781,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro continue } - pastGracePeriod, age, err := mdevPastGracePeriod(mdev.UUID, orphanedMdevGracePeriod) + pastGracePeriod, age, err := mdevPastGracePeriod(mdev.UUID, VGPUAssignmentGracePeriod) if err != nil { log.WarnContext(ctx, "failed to determine mdev age, skipping cleanup", "uuid", mdev.UUID, "error", err) skippedProbeError++ @@ -792,7 +791,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro log.DebugContext(ctx, "skipping recently created mdev during grace period", "uuid", mdev.UUID, "age", age.String(), - "grace_period", orphanedMdevGracePeriod.String(), + "grace_period", VGPUAssignmentGracePeriod.String(), ) skippedGrace++ continue diff --git a/lib/devices/types.go b/lib/devices/types.go index 31ebd90ef..369bb3268 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -60,6 +60,11 @@ func ValidateDeviceName(name string) bool { // GPUMode represents the host's GPU configuration mode type GPUMode string +// VGPUAssignmentGracePeriod protects a fresh vGPU assignment from cleanup +// until its VM has had time to boot and become identifiable — by a persisted +// hypervisor PID, a control-socket owner, or an open VFIO handle. +const VGPUAssignmentGracePeriod = 5 * time.Minute + type VGPUFramework string const ( diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 74eae4948..a1378a3c7 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -23,11 +23,6 @@ import ( const ( pciDevicesPath = "/sys/bus/pci/devices" vfioDevicesPath = "/dev/vfio/devices" - - // vendorVFIOAssignmentGracePeriod protects assignments created by this - // process from the periodic sweep until their owning instance has had time - // to persist metadata and boot, mirroring orphanedMdevGracePeriod. - vendorVFIOAssignmentGracePeriod = 5 * time.Minute ) type vendorVFIOOwner struct { @@ -294,7 +289,7 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map continue } owner := owners[vf.PCIAddress] - if !owner.assignedAt.IsZero() && time.Since(owner.assignedAt) < vendorVFIOAssignmentGracePeriod { + if !owner.assignedAt.IsZero() && time.Since(owner.assignedAt) < VGPUAssignmentGracePeriod { log.DebugContext(ctx, "skipping recently assigned vendor VFIO vGPU during grace period", "vf", vf.PCIAddress) continue } diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index f7b0d6c3f..098d016e3 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -391,7 +391,7 @@ func TestVendorVFIOReconcileDestroysOwnedVFPastGracePeriod(t *testing.T) { sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") sysfs.owners["0000:82:00.4"] = vendorVFIOOwner{ instanceID: "deleted-instance", - assignedAt: time.Now().Add(-vendorVFIOAssignmentGracePeriod - time.Minute), + assignedAt: time.Now().Add(-VGPUAssignmentGracePeriod - time.Minute), } require.NoError(t, sysfs.reconcile(context.Background(), nil)) From 703de473989f68094dce959b34d347b15b072a07 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:09:21 +0000 Subject: [PATCH 69/76] Preserve delete-only vGPU retention records --- lib/instances/vgpu_retention.go | 81 +++++++++++++++++++++++++-------- lib/instances/vgpu_test.go | 6 ++- 2 files changed, 66 insertions(+), 21 deletions(-) diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index 121c13cb0..e630484ae 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -2,9 +2,13 @@ package instances import ( "context" + "encoding/json" + "fmt" + "os" "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -62,7 +66,16 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten retentionSurvives := func() bool { meta, err := m.loadMetadata(id) if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { - return true + saveErr := m.saveVGPURetentionStub(retainedVGPU) + if saveErr == nil { + return true + } + log.ErrorContext(ctx, "failed to replace surviving instance metadata with vGPU retention stub", "instance_id", id, "error", saveErr) + overwriteErr := m.overwriteVGPURetentionStub(retainedVGPU) + if overwriteErr == nil { + return true + } + log.ErrorContext(ctx, "failed to overwrite surviving instance metadata with vGPU retention stub", "instance_id", id, "error", overwriteErr) } if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) @@ -79,28 +92,56 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten retention.persisted = retentionSurvives() return } - retained := StoredMetadata{ - Id: id, - Name: retainedVGPU.Name, - Image: retainedVGPU.Image, - ResolvedImage: retainedVGPU.ResolvedImage, - Platform: retainedVGPU.Platform, - CreatedAt: retainedVGPU.CreatedAt, - HypervisorType: retainedVGPU.HypervisorType, - HypervisorVersion: retainedVGPU.HypervisorVersion, - SocketPath: retainedVGPU.SocketPath, - DataDir: retainedVGPU.DataDir, - GPUProfile: retainedVGPU.GPUProfile, - GPUFramework: retainedVGPU.GPUFramework, - GPUDevicePath: retainedVGPU.GPUDevicePath, - GPUMdevUUID: retainedVGPU.GPUMdevUUID, - GPUAssignedAt: retainedVGPU.GPUAssignedAt, - GPURetainedForCleanup: true, - } - if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { + if err := m.saveVGPURetentionStub(retainedVGPU); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) retention.persisted = retentionSurvives() return } retention.persisted = true } + +func vgpuRetentionMetadata(source *StoredMetadata) *metadata { + return &metadata{StoredMetadata: StoredMetadata{ + Id: source.Id, + Name: source.Name, + Image: source.Image, + ResolvedImage: source.ResolvedImage, + Platform: source.Platform, + CreatedAt: source.CreatedAt, + HypervisorType: source.HypervisorType, + HypervisorVersion: source.HypervisorVersion, + SocketPath: source.SocketPath, + DataDir: source.DataDir, + GPUProfile: source.GPUProfile, + GPUFramework: source.GPUFramework, + GPUDevicePath: source.GPUDevicePath, + GPUMdevUUID: source.GPUMdevUUID, + GPUAssignedAt: source.GPUAssignedAt, + GPURetainedForCleanup: true, + }} +} + +func (m *manager) saveVGPURetentionStub(source *StoredMetadata) error { + return m.saveMetadata(vgpuRetentionMetadata(source)) +} + +// overwriteVGPURetentionStub handles a surviving metadata file when its +// directory cannot create the temporary file used by saveMetadata. +func (m *manager) overwriteVGPURetentionStub(source *StoredMetadata) error { + retained := vgpuRetentionMetadata(source) + data, err := json.MarshalIndent(retained, "", " ") + if err != nil { + return fmt.Errorf("marshal metadata: %w", err) + } + unlockAliasReaders := hypervisor.LockSnapshotSourceAliasReaders() + defer unlockAliasReaders() + writeFile := m.writeFile + if writeFile == nil { + writeFile = os.WriteFile + } + if err := writeFile(m.paths.InstanceMetadata(source.Id), data, 0644); err != nil { + return fmt.Errorf("write metadata: %w", err) + } + m.syncAdmissionAllocation(retained) + return nil +} diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index e5ed8d13f..b9f7124d2 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -109,7 +109,7 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { require.Error(t, err, "the lost retention leaves no metadata claim, so the periodic sweep releases the VF") } -func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T) { +func TestCleanupFailedCreateRewritesFullMetadataWhenDirectoryIsReadOnly(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } @@ -133,6 +133,10 @@ func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T require.NoError(t, err) assert.Equal(t, stored.GPUFramework, retained.GPUFramework) assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) + assert.True(t, retained.GPURetainedForCleanup) + + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) + assert.ErrorIs(t, err, errVGPURetentionStub) } func TestVGPURetentionWrapPending(t *testing.T) { From 8932ba62f02061a65c8c276cea826521509b0eeb Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:44:10 +0000 Subject: [PATCH 70/76] Trim redundant vGPU coverage --- cmd/api/api/instances_test.go | 164 +++++++++------------------ lib/instances/lifecycle_noop_test.go | 18 +-- lib/instances/vgpu.go | 10 +- lib/instances/vgpu_reconcile_test.go | 55 +++------ lib/instances/vgpu_test.go | 117 +++++++------------ 5 files changed, 116 insertions(+), 248 deletions(-) diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 273e95cd6..fe717abdb 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -48,69 +48,37 @@ func TestGetInstance_NotFound(t *testing.T) { require.Error(t, err) } -type createErrorInstanceManager struct { - instances.Manager - err error -} - -func (m createErrorInstanceManager) CreateInstance(context.Context, instances.CreateInstanceRequest) (*instances.Instance, error) { - return nil, m.err -} - -func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { +func TestVGPUCleanupPendingDetail(t *testing.T) { t.Parallel() - svc := newTestService(t) - svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ - InstanceID: "inst-1", - Retained: true, - Err: network.ErrNameExists, - }} - - resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ - Body: &oapi.CreateInstanceRequest{Image: "test-image"}, - }) - require.NoError(t, err) - - pending, ok := resp.(oapi.CreateInstance500JSONResponse) - require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) - assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) - assert.Contains(t, pending.Message, "inst-1") - assert.Contains(t, pending.Message, network.ErrNameExists.Error(), - "the underlying create failure must survive the cleanup guidance") - assert.Contains(t, pending.Message, "delete it to retry") - require.NotNil(t, pending.InnerError) - require.NotNil(t, pending.InnerError.Code) - assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code) - require.NotNil(t, pending.InnerError.Message) - assert.Equal(t, "inst-1", *pending.InnerError.Message) -} - -func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) { - t.Parallel() - svc := newTestService(t) - svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ - InstanceID: "inst-1", - Err: network.ErrNameExists, - }} - - resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ - Body: &oapi.CreateInstanceRequest{Image: "test-image"}, - }) - require.NoError(t, err) - - pending, ok := resp.(oapi.CreateInstance500JSONResponse) - require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) - assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) - assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") - assert.Contains(t, pending.Message, network.ErrNameExists.Error(), - "the underlying create failure must survive the cleanup guidance") - assert.Contains(t, pending.Message, "periodic vGPU reconcile") - assert.NotContains(t, pending.Message, "delete") - require.NotNil(t, pending.InnerError) - require.NotNil(t, pending.InnerError.Code) - assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code) - require.NotNil(t, pending.InnerError.Message) - assert.Equal(t, "inst-1", *pending.InnerError.Message) + for _, tt := range []struct { + name string + retained bool + code string + guidance string + exclude string + }{ + {name: "retained", retained: true, code: "vgpu_retained_instance", guidance: "delete it to retry"}, + {name: "unretained", code: "vgpu_unretained_instance", guidance: "periodic vGPU reconcile", exclude: "delete"}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + message, inner := vgpuCleanupPendingDetail(&instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: tt.retained, + Err: network.ErrNameExists, + }, "create", "delete it to retry") + assert.Contains(t, message, "inst-1") + assert.Contains(t, message, network.ErrNameExists.Error()) + assert.Contains(t, message, tt.guidance) + if tt.exclude != "" { + assert.NotContains(t, message, tt.exclude) + } + require.NotNil(t, inner.Code) + assert.Equal(t, tt.code, *inner.Code) + require.NotNil(t, inner.Message) + assert.Equal(t, "inst-1", *inner.Message) + }) + } } func TestCreateInstance_AutoPullImage(t *testing.T) { @@ -908,6 +876,16 @@ func TestCreateInstance_ErrorStatusMapping(t *testing.T) { wantCode string wantMessage string }{ + { + name: "vGPU cleanup pending beats wrapped name conflict -> 500", + err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: true, + Err: network.ErrNameExists, + }, + wantType: oapi.CreateInstance500JSONResponse{}, + wantCode: "vgpu_cleanup_pending", + }, { name: "platform not available -> 404", err: fmt.Errorf("resolve image: %w", images.ErrPlatformNotAvailable), @@ -1034,61 +1012,23 @@ func TestRestoreInstance_ErrorMapping(t *testing.T) { func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { t.Parallel() - + svc := newTestService(t) resolved := &instances.Instance{ StoredMetadata: instances.StoredMetadata{Id: "inst-1", Name: "inst-1"}, State: instances.StateStopped, } + svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ + InstanceID: resolved.Id, + Retained: true, + Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), + }} - t.Run("retained", func(t *testing.T) { - t.Parallel() - svc := newTestService(t) - svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ - InstanceID: "inst-1", - Retained: true, - Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), - }} - - resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) - require.NoError(t, rerr) - - pending, ok := resp.(oapi.StartInstance500JSONResponse) - require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) - assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) - assert.Contains(t, pending.Message, "inst-1") - assert.Contains(t, pending.Message, instances.ErrInsufficientResources.Error(), - "the underlying start failure must survive the cleanup guidance") - assert.Contains(t, pending.Message, "delete it or retry start") - require.NotNil(t, pending.InnerError) - require.NotNil(t, pending.InnerError.Code) - assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code) - require.NotNil(t, pending.InnerError.Message) - assert.Equal(t, "inst-1", *pending.InnerError.Message) - }) - - t.Run("unretained", func(t *testing.T) { - t.Parallel() - svc := newTestService(t) - svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ - InstanceID: "inst-1", - Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), - }} - - resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) - require.NoError(t, rerr) - - pending, ok := resp.(oapi.StartInstance500JSONResponse) - require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) - assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) - assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") - assert.Contains(t, pending.Message, "periodic vGPU reconcile") - assert.NotContains(t, pending.Message, "delete") - require.NotNil(t, pending.InnerError) - require.NotNil(t, pending.InnerError.Code) - assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code) - require.NotNil(t, pending.InnerError.Message) - assert.Equal(t, "inst-1", *pending.InnerError.Message) - }) + resp, err := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) + require.NoError(t, err) + pending, ok := resp.(oapi.StartInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "delete it or retry start") } func TestInstanceActions_ImageNotFoundMapsTo404(t *testing.T) { diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 1843485e3..85083766b 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -214,10 +214,11 @@ func TestDeleteReleasesRetainedCreateStub(t *testing.T) { require.NoError(t, m.ensureDirectories(id)) assignedAt := time.Now().UTC() require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: id, - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + GPURetainedForCleanup: true, }})) require.NoError(t, m.DeleteInstance(context.Background(), id)) @@ -339,9 +340,8 @@ func TestStartRejectsVGPURetentionRecord(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) } -func TestReconcileReleasesRetainedVGPUOnStoppedInstance(t *testing.T) { +func TestStopStoppedInstanceLeavesVGPUForReconcile(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - m.reconcileVGPUDevices = func(context.Context, map[string]struct{}, bool) error { return nil } meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" @@ -358,12 +358,6 @@ func TestReconcileReleasesRetainedVGPUOnStoppedInstance(t *testing.T) { stored, err := m.loadMetadata(id) require.NoError(t, err) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) - - m.ReconcileVGPUs(context.Background()) - stored, err = m.loadMetadata(id) - require.NoError(t, err) - assert.Empty(t, stored.GPUDevicePath) - assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") } func TestReconcileVGPUReleaseFailureKeepsStoppedInstanceUsable(t *testing.T) { diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index bb230f3d1..37127e921 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -143,17 +143,13 @@ func restoreStartMutatedFields(dst, src *StoredMetadata) { } func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { - return m.releaseStoredVGPUExcluding(ctx, stored, stored.Id) -} - -func (m *manager) releaseStoredVGPUExcluding(ctx context.Context, stored *StoredMetadata, excludeID string) error { path := storedVGPUDevicePath(stored) if path != "" { // Vendor VFIO VFs are reusable, so release fails closed on an incomplete inventory. claimed := false if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { var err error - claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, excludeID, path) + claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) if err != nil { return err } @@ -204,8 +200,8 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu if err != nil { return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) } - live, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), pid > 0) - if pid > 0 && live { + _, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), pid > 0) + if pid > 0 { return true, nil } if remaining > 0 { diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 468591dee..395010532 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -25,10 +25,14 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) var protected map[string]struct{} + var destroyed []devices.VGPUAssignment m := &manager{ - paths: paths.New(t.TempDir()), - now: func() time.Time { return now }, - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + paths: paths.New(t.TempDir()), + now: func() time.Time { return now }, + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, sweepVendorVFIO bool) error { protected = p assert.True(t, sweepVendorVFIO) @@ -37,7 +41,7 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { } instances := []StoredMetadata{ {Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}, - {Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}, + {Id: "orphaned", GPUProfile: "NVIDIA L40S-2Q", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}, {Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}, {Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}}, {Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}, @@ -59,6 +63,14 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { require.NoError(t, err) assert.Empty(t, stored.GPUDevicePath, "stale assignment on %s must be released", id) } + assert.Contains(t, destroyed, devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: "/sys/bus/pci/devices/0000:82:00.5", + InstanceID: "orphaned", + }) + orphaned, err := m.loadMetadata("orphaned") + require.NoError(t, err) + assert.Equal(t, "NVIDIA L40S-2Q", orphaned.GPUProfile) for _, id := range []string{"booting", "stale-pid-booting"} { stored, err := m.loadMetadata(id) require.NoError(t, err) @@ -95,41 +107,6 @@ func TestReconcileVGPUsSkipsVendorSweepWhenListingFails(t *testing.T) { assert.Equal(t, []bool{false, true}, sweeps, "the next pass retries the vendor sweep") } -func TestReconcileVGPUsReleasesStaleAssignment(t *testing.T) { - t.Parallel() - - var destroyed []devices.VGPUAssignment - m := &manager{ - paths: paths.New(t.TempDir()), - destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { - destroyed = append(destroyed, assignment) - return nil - }, - reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, - } - const id = "stopped-retained" - require.NoError(t, m.ensureDirectories(id)) - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: id, - GPUProfile: "NVIDIA L40S-2Q", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - }})) - - m.ReconcileVGPUs(t.Context()) - - require.Len(t, destroyed, 1) - assert.Equal(t, devices.VGPUAssignment{ - Framework: devices.VGPUFrameworkVendorVFIO, - DevicePath: "/sys/bus/pci/devices/0000:82:00.4", - InstanceID: id, - }, destroyed[0]) - stored, err := m.loadMetadata(id) - require.NoError(t, err) - assert.Empty(t, stored.GPUDevicePath) - assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") -} - func TestReconcileVGPUsKeepsAssignmentWhenReleaseFails(t *testing.T) { t.Parallel() diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index b9f7124d2..55f88f4d9 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -526,87 +526,48 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. assert.True(t, claimed) } -func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnRecentNilPIDClaim(t *testing.T) { +func TestVGPUAssignmentClaimedByLiveInstanceLiveness(t *testing.T) { t.Parallel() - m := &manager{paths: paths.New(t.TempDir())} - require.NoError(t, m.ensureDirectories("booting-claimant")) - assignedAt := time.Now().UTC() - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "booting-claimant", - Name: "booting-claimant", - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, - }})) - - _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") - require.Error(t, err) - assert.Contains(t, err.Error(), "booting-claimant") -} - -func TestVGPUAssignmentClaimedByLiveInstanceIgnoresStaleNilPIDClaim(t *testing.T) { - t.Parallel() - - m := &manager{paths: paths.New(t.TempDir())} - require.NoError(t, m.ensureDirectories("stale-claimant")) - assignedAt := time.Now().Add(-VGPUAssignmentStartupGracePeriod - time.Minute) - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "stale-claimant", - Name: "stale-claimant", - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, - }})) - - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") - require.NoError(t, err) - assert.False(t, claimed) -} - -func TestVGPUAssignmentClaimedByLiveInstanceGracesRecentDeadPIDClaim(t *testing.T) { - m := &manager{paths: paths.New(t.TempDir())} - claimantID := "claimant-dead-pid" - require.NoError(t, m.ensureDirectories(claimantID)) - deadPID := 1<<22 - 1 - require.False(t, ProcessExists(deadPID)) - assignedAt := time.Now().UTC() - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: claimantID, - HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUAssignedAt: &assignedAt, - }})) - - _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") - require.Error(t, err) - - stale := assignedAt.Add(-2 * VGPUAssignmentStartupGracePeriod) - meta, err := m.loadMetadata(claimantID) - require.NoError(t, err) - meta.GPUAssignedAt = &stale - require.NoError(t, m.saveMetadata(meta)) - - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") - require.NoError(t, err) - assert.False(t, claimed) -} - -func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { - t.Parallel() - - m := &manager{paths: paths.New(t.TempDir())} - require.NoError(t, m.ensureDirectories("dead-claimant")) + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" deadPID := 1 << 30 - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ - Id: "dead-claimant", - Name: "dead-claimant", - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, - }})) - - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") - require.NoError(t, err) - assert.False(t, claimed, "a claim whose hypervisor is gone must not block the release") + require.False(t, ProcessExists(deadPID)) + recent := time.Now().UTC() + stale := recent.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + tests := []struct { + name string + assignedAt *time.Time + pid *int + wantErr string + }{ + {name: "recent without PID", assignedAt: &recent, wantErr: "no persisted hypervisor PID"}, + {name: "stale without PID", assignedAt: &stale}, + {name: "recent dead PID", assignedAt: &recent, pid: &deadPID, wantErr: "recorded hypervisor is not running"}, + {name: "legacy dead PID", pid: &deadPID}, + } + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + m := &manager{paths: paths.New(t.TempDir())} + id := fmt.Sprintf("claimant-%d", i) + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + GPUAssignedAt: tt.assignedAt, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: tt.pid}, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", devicePath) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.False(t, claimed) + }) + } } func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) { From eddeaf196418c950142a6db980041cabdaf6467a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:40:31 +0000 Subject: [PATCH 71/76] Add counters for vGPU cleanup failure paths A wedged or leaked VF presents as reduced GPU capacity while /resources still reports full capacity, and reconcile, stale-release, and retention failures were visible only as log lines. Count failed reconcile stages, failed stale releases, and retained assignments (by operation and whether the retention record persisted) so sustained failure can alert. --- lib/instances/metrics.go | 81 +++++++++++++++++++++++++++++++++ lib/instances/metrics_test.go | 45 ++++++++++++++++++ lib/instances/start.go | 3 ++ lib/instances/vgpu_reconcile.go | 3 ++ lib/instances/vgpu_retention.go | 3 ++ 5 files changed, 135 insertions(+) diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 1ada5ac1e..a380ff09f 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -73,6 +73,20 @@ type lifecycleEventDropReason string const lifecycleEventDropReasonBufferFull lifecycleEventDropReason = "buffer_full" +type vgpuReconcileStage string + +const ( + vgpuReconcileStageListInstances vgpuReconcileStage = "list_instances" + vgpuReconcileStageReconcileDevices vgpuReconcileStage = "reconcile_devices" +) + +type vgpuRetentionOperation string + +const ( + vgpuRetentionOperationCreate vgpuRetentionOperation = "create" + vgpuRetentionOperationStart vgpuRetentionOperation = "start" +) + // Metrics holds the metrics instruments for instance operations. type Metrics struct { createDuration metric.Float64Histogram @@ -94,6 +108,9 @@ type Metrics struct { lifecycleEventsDroppedTotal metric.Int64Counter forkMemFileShareFallbacksTotal metric.Int64Counter ttlReaperDeletionsTotal metric.Int64Counter + vgpuReconcileFailuresTotal metric.Int64Counter + vgpuStaleReleaseFailuresTotal metric.Int64Counter + vgpuAssignmentsRetainedTotal metric.Int64Counter tracer trace.Tracer } @@ -270,6 +287,30 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + vgpuReconcileFailuresTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_reconcile_failures_total", + metric.WithDescription("Total number of vGPU reconcile pass stages that failed, leaving stale assignments or device leftovers allocated while /resources still advertises the capacity"), + ) + if err != nil { + return nil, err + } + + vgpuStaleReleaseFailuresTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_stale_release_failures_total", + metric.WithDescription("Total number of stale vGPU assignment releases that failed, keeping the VF allocated until a later reconcile pass succeeds"), + ) + if err != nil { + return nil, err + } + + vgpuAssignmentsRetainedTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_assignments_retained_total", + metric.WithDescription("Total number of failed rollbacks that left a vGPU assignment behind, by whether the retention record the periodic reconcile needs was persisted"), + ) + if err != nil { + return nil, err + } + // Register observable gauge for instance counts by state instancesTotal, err := meter.Int64ObservableGauge( "hypeman_instances_total", @@ -464,6 +505,9 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M lifecycleEventsDroppedTotal: lifecycleEventsDroppedTotal, forkMemFileShareFallbacksTotal: forkMemFileShareFallbacksTotal, ttlReaperDeletionsTotal: ttlReaperDeletionsTotal, + vgpuReconcileFailuresTotal: vgpuReconcileFailuresTotal, + vgpuStaleReleaseFailuresTotal: vgpuStaleReleaseFailuresTotal, + vgpuAssignmentsRetainedTotal: vgpuAssignmentsRetainedTotal, tracer: tracer, }, nil } @@ -563,6 +607,43 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } +// recordVGPUReconcileFailure records a vGPU reconcile stage failing: stale +// assignments or device leftovers stay allocated (capacity silently reduced +// while /resources reports full) until a later pass succeeds, so sustained +// failure must be alertable beyond a log line. +func (m *manager) recordVGPUReconcileFailure(ctx context.Context, stage vgpuReconcileStage) { + if m.metrics == nil { + return + } + m.metrics.vgpuReconcileFailuresTotal.Add(ctx, 1, metric.WithAttributes( + attribute.String("stage", string(stage)), + )) +} + +// recordVGPUStaleReleaseFailure records a stale vGPU release failing: the VF +// stays allocated while /resources still advertises it, so a wedged release +// that fails every pass must be alertable beyond a log line. +func (m *manager) recordVGPUStaleReleaseFailure(ctx context.Context) { + if m.metrics == nil { + return + } + m.metrics.vgpuStaleReleaseFailuresTotal.Add(ctx, 1) +} + +// recordVGPURetainedAssignment records a failed rollback leaving a vGPU +// assignment behind. An unpersisted retention record has no metadata claim, +// so the device is only recovered by the periodic reconcile sweep; either way +// the VF is unavailable while /resources still advertises it. +func (m *manager) recordVGPURetainedAssignment(ctx context.Context, operation vgpuRetentionOperation, persisted bool) { + if m.metrics == nil { + return + } + m.metrics.vgpuAssignmentsRetainedTotal.Add(ctx, 1, metric.WithAttributes( + attribute.String("operation", string(operation)), + attribute.String("persisted", strconv.FormatBool(persisted)), + )) +} + // recordStateTransition records a state transition with hypervisor label. func (m *manager) recordStateTransition(ctx context.Context, fromState, toState string, hvType hypervisor.Type) { if m.metrics == nil { diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go index 811571e36..525002cbb 100644 --- a/lib/instances/metrics_test.go +++ b/lib/instances/metrics_test.go @@ -2,6 +2,7 @@ package instances import ( "context" + "errors" "os" "path/filepath" "testing" @@ -556,6 +557,50 @@ func TestEnsureSnapshotMemoryReadySkipsPendingCompressionWithoutPreemptionMetric assert.False(t, metricExists(rm, "hypeman_snapshot_compression_preemptions_total"), "pending-delay cancellation should not record a preemption") } +func TestVGPUReconcileFailureMetric_RecordStages(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + + m := &manager{ + paths: paths.New(t.TempDir()), + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { + return errors.New("sweep failed") + }, + } + metrics, err := newInstanceMetrics(provider.Meter("test"), nil, m) + require.NoError(t, err) + m.metrics = metrics + + const id = "unreadable" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{Id: id}})) + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o000)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + m.ReconcileVGPUs(t.Context()) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + + failuresMetric := findMetric(t, rm, "hypeman_instances_vgpu_reconcile_failures_total") + failures, ok := failuresMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, failures.DataPoints, 2) + for _, point := range failures.DataPoints { + switch metricLabel(t, point.Attributes, "stage") { + case "list_instances", "reconcile_devices": + assert.Equal(t, int64(1), point.Value) + default: + t.Fatalf("unexpected reconcile failure stage datapoint: %s", metricLabel(t, point.Attributes, "stage")) + } + } +} + func assertMetricNames(t *testing.T, rm metricdata.ResourceMetrics, expected []string) { t.Helper() diff --git a/lib/instances/start.go b/lib/instances/start.go index 19bb64697..ea8c2a6f4 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -183,9 +183,11 @@ func (m *manager) startInstance( setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, assignedAt) wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, false) log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} } + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, true) return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} } return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) @@ -198,6 +200,7 @@ func (m *manager) startInstance( retained, persisted := m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) if retained { retention.markRetained(persisted) + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, persisted) } }) if err := m.saveMetadata(meta); err != nil { diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 5cfb43d4c..9ca8cef72 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -56,6 +56,7 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { protected, err := m.reconcileVGPUAssignments(ctx) sweepVendorVFIO := err == nil if err != nil { + m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageListInstances) log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO sweep until the next pass, mdev reconcile still runs", "error", err) protected = make(map[string]struct{}) } @@ -64,6 +65,7 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { reconcileDevices = devices.ReconcileVGPUs } if err := reconcileDevices(ctx, protected, sweepVendorVFIO); err != nil { + m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageReconcileDevices) log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) } } @@ -125,6 +127,7 @@ func (m *manager) releaseStaleVGPUAssignment(ctx context.Context, id string) { return } if err := m.releaseStoredVGPU(ctx, stored); err != nil { + m.recordVGPUStaleReleaseFailure(ctx) log.WarnContext(ctx, "failed to release stale vGPU assignment; retrying on the next reconcile pass", "instance_id", id, "device_path", path, "error", err) return } diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index e630484ae..691ae5eaa 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -61,6 +61,9 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten id := retention.instanceID retainedVGPU := retention.stub log := logger.FromContext(ctx) + defer func() { + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationCreate, retention.persisted) + }() // When the retention record is lost, the assignment has no metadata claim // left; the periodic vGPU reconcile sweeps the device once it is free. retentionSurvives := func() bool { From 409adb727d245e311a808209870bfc4623d9473d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:28:01 +0000 Subject: [PATCH 72/76] Simplify vGPU lifecycle cleanup --- lib/devices/vgpu_linux.go | 28 +-- lib/devices/vgpu_linux_test.go | 16 -- lib/instances/lifecycle_noop_test.go | 36 --- lib/instances/manager.go | 10 +- lib/instances/metrics.go | 11 - lib/instances/process_identity.go | 8 +- lib/instances/process_identity_linux_test.go | 4 +- lib/instances/query_test.go | 24 +- lib/instances/vgpu.go | 96 ++------ lib/instances/vgpu_reconcile.go | 18 +- lib/instances/vgpu_reconcile_linux_test.go | 6 +- lib/instances/vgpu_reconcile_test.go | 2 +- lib/instances/vgpu_retention.go | 42 +--- lib/instances/vgpu_test.go | 233 +++++-------------- 14 files changed, 113 insertions(+), 421 deletions(-) diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index 72827f7b2..36bf86c9e 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -111,33 +111,13 @@ func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{ return err } - return reconcileDiscoveredVGPUs( - ctx, - framework, - protectedDevicePaths, - sweepVendorVFIO, - func(ctx context.Context) error { return ReconcileMdevs(ctx, nil) }, - hostVendorVFIO.reconcile, - ) -} - -func reconcileDiscoveredVGPUs( - ctx context.Context, - framework VGPUFramework, - protectedDevicePaths map[string]struct{}, - sweepVendorVFIO bool, - reconcileMdev func(context.Context) error, - reconcileVendorVFIO func(context.Context, map[string]struct{}) error, -) error { switch framework { case VGPUFrameworkMdev: - return reconcileMdev(ctx) + return ReconcileMdevs(ctx, nil) case VGPUFrameworkVendorVFIO: - if !sweepVendorVFIO { - return nil + if sweepVendorVFIO { + return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) } - return reconcileVendorVFIO(ctx, protectedDevicePaths) - default: - return nil } + return nil } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go index 32f9a30ea..7b03d9c26 100644 --- a/lib/devices/vgpu_linux_test.go +++ b/lib/devices/vgpu_linux_test.go @@ -3,7 +3,6 @@ package devices import ( - "context" "errors" "os" "path/filepath" @@ -13,21 +12,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestReconcileDiscoveredVGPUsControlsVendorSweep(t *testing.T) { - protected := make(map[string]struct{}) - vendorCalls := 0 - reconcileVendor := func(context.Context, map[string]struct{}) error { - vendorCalls++ - return nil - } - - require.NoError(t, reconcileDiscoveredVGPUs(context.Background(), VGPUFrameworkVendorVFIO, protected, false, nil, reconcileVendor)) - assert.Zero(t, vendorCalls) - - require.NoError(t, reconcileDiscoveredVGPUs(context.Background(), VGPUFrameworkVendorVFIO, protected, true, nil, reconcileVendor)) - assert.Equal(t, 1, vendorCalls) -} - func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { t.Parallel() diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 85083766b..77d15d733 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -150,24 +150,6 @@ func TestLifecycleNoopStandbyWithOptionsStillRejectsStandbyInstance(t *testing.T assertNoLifecycleEvent(t, events) } -func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { - m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFramework("future-framework") - meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" - require.NoError(t, m.saveMetadata(meta)) - - // A failed release is logged and the delete continues, matching the - // pre-refactor contract; the leaked assignment is recovered by the - // periodic vGPU reconcile. - require.NoError(t, m.DeleteInstance(context.Background(), id)) - - _, err = m.loadMetadata(id) - require.Error(t, err, "instance data must be deleted despite the failed release") -} - func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) var persisted *metadata @@ -360,24 +342,6 @@ func TestStopStoppedInstanceLeavesVGPUForReconcile(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) } -func TestReconcileVGPUReleaseFailureKeepsStoppedInstanceUsable(t *testing.T) { - m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - m.reconcileVGPUDevices = func(context.Context, map[string]struct{}, bool) error { return nil } - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFramework("future-framework") - meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" - require.NoError(t, m.saveMetadata(meta)) - - m.ReconcileVGPUs(context.Background()) - - stored, err := m.loadMetadata(id) - require.NoError(t, err) - assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) - assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) -} - // recordingDeviceManager is a devices.Manager stub that records passthrough // teardown calls. Only the methods delete exercises are implemented. type recordingDeviceManager struct { diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 956e69d43..5f68d8f9a 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -218,7 +218,7 @@ type manager struct { vgpuReconcileInterval time.Duration discoverVGPU func() (devices.VGPUFramework, []devices.VirtualFunction, error) - vgpuInitTermGrace time.Duration + vfioTermGrace time.Duration // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter @@ -292,8 +292,6 @@ func NewManagerWithConfigE(p *paths.Paths, imageManager images.Manager, systemMa defaultHypervisor: defaultHypervisor, now: time.Now, writeFile: os.WriteFile, - createVGPU: devices.CreateVGPU, - destroyVGPU: devices.DestroyVGPU, meter: meter, tracer: tracer, guestMemoryPolicy: policy, @@ -728,12 +726,12 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } -func (m *manager) listInstancesForReconcile(ctx context.Context) ([]Instance, error) { +func (m *manager) listMetadataForReconcile() ([]StoredMetadata, error) { files, err := m.listMetadataFilesStrict() if err != nil { return nil, err } - result := make([]Instance, 0, len(files)) + result := make([]StoredMetadata, 0, len(files)) for _, file := range files { id := filepath.Base(filepath.Dir(file)) meta, err := m.loadMetadata(id) @@ -743,7 +741,7 @@ func (m *manager) listInstancesForReconcile(ctx context.Context) ([]Instance, er } return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) } - result = append(result, Instance{StoredMetadata: meta.StoredMetadata}) + result = append(result, meta.StoredMetadata) } return result, nil } diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index a380ff09f..e14deb49b 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -607,10 +607,6 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } -// recordVGPUReconcileFailure records a vGPU reconcile stage failing: stale -// assignments or device leftovers stay allocated (capacity silently reduced -// while /resources reports full) until a later pass succeeds, so sustained -// failure must be alertable beyond a log line. func (m *manager) recordVGPUReconcileFailure(ctx context.Context, stage vgpuReconcileStage) { if m.metrics == nil { return @@ -620,9 +616,6 @@ func (m *manager) recordVGPUReconcileFailure(ctx context.Context, stage vgpuReco )) } -// recordVGPUStaleReleaseFailure records a stale vGPU release failing: the VF -// stays allocated while /resources still advertises it, so a wedged release -// that fails every pass must be alertable beyond a log line. func (m *manager) recordVGPUStaleReleaseFailure(ctx context.Context) { if m.metrics == nil { return @@ -630,10 +623,6 @@ func (m *manager) recordVGPUStaleReleaseFailure(ctx context.Context) { m.metrics.vgpuStaleReleaseFailuresTotal.Add(ctx, 1) } -// recordVGPURetainedAssignment records a failed rollback leaving a vGPU -// assignment behind. An unpersisted retention record has no metadata claim, -// so the device is only recovered by the periodic reconcile sweep; either way -// the VF is unavailable while /resources still advertises it. func (m *manager) recordVGPURetainedAssignment(ctx context.Context, operation vgpuRetentionOperation, persisted bool) { if m.metrics == nil { return diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 02f43fcda..7e8500402 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -27,9 +27,9 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second -func (m *manager) vgpuTermGrace() time.Duration { - if m.vgpuInitTermGrace > 0 { - return m.vgpuInitTermGrace +func (m *manager) vfioTerminationGrace() time.Duration { + if m.vfioTermGrace > 0 { + return m.vfioTermGrace } return hypervisor.VFIOTermGrace } @@ -37,7 +37,7 @@ func (m *manager) vgpuTermGrace() time.Duration { // SIGKILL during guest driver init can wedge a VF until the parent GPU is reset. func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { if inst.GPUProfile != "" || len(inst.Devices) > 0 { - if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vgpuTermGrace()) { + if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vfioTerminationGrace()) { return nil } logger.FromContext(ctx).WarnContext(ctx, "hypervisor with VFIO devices did not exit on SIGTERM; hard-killing, device may wedge if the guest driver was initializing", diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index b208a6ef3..2ec201450 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -546,7 +546,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) SocketPath: socketPath, }})) - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", devicePath) + claimed, err := m.vgpuAssignmentClaimedByLiveInstance("other-instance", devicePath) require.NoError(t, err) assert.True(t, claimed) } @@ -740,7 +740,7 @@ func TestKillHypervisorEscalatesToSIGKILLWhenSIGTERMIgnored(t *testing.T) { pid, identity := startTrapProcess(t, "") socketPath := filepath.Join(t.TempDir(), "missing.sock") - m := &manager{vgpuInitTermGrace: 50 * time.Millisecond} + m := &manager{vfioTermGrace: 50 * time.Millisecond} require.NoError(t, m.killHypervisor(context.Background(), &Instance{ State: StateInitializing, StoredMetadata: StoredMetadata{ diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 795b00baf..222e85a30 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { +func TestListMetadataForReconcileFailsOnInvalidMetadata(t *testing.T) { m := &manager{paths: paths.New(t.TempDir())} require.NoError(t, m.ensureDirectories("valid")) @@ -31,18 +31,18 @@ func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { require.NoError(t, err) require.Len(t, listed, 1) - _, err = m.listInstancesForReconcile(context.Background()) + _, err = m.listMetadataForReconcile() require.Error(t, err) assert.ErrorContains(t, err, "load metadata for instance invalid") require.NoError(t, os.Remove(m.paths.InstanceMetadata("invalid"))) - listed, err = m.listInstancesForReconcile(context.Background()) + metadata, err := m.listMetadataForReconcile() require.NoError(t, err) - require.Len(t, listed, 1) - assert.Equal(t, "valid", listed[0].Id) + require.Len(t, metadata, 1) + assert.Equal(t, "valid", metadata[0].Id) } -func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T) { +func TestListMetadataForReconcileSkipsInstanceDeletedDuringListing(t *testing.T) { m := &manager{paths: paths.New(t.TempDir())} for _, id := range []string{"aaa-ghost", "zzz-live"} { @@ -57,13 +57,13 @@ func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T unlock := hypervisor.LockSnapshotSourceAliasMutation() type result struct { - listed []Instance - err error + metadata []StoredMetadata + err error } done := make(chan result, 1) go func() { - listed, err := m.listInstancesForReconcile(context.Background()) - done <- result{listed, err} + metadata, err := m.listMetadataForReconcile() + done <- result{metadata, err} }() time.Sleep(100 * time.Millisecond) require.NoError(t, os.Remove(m.paths.InstanceMetadata("aaa-ghost"))) @@ -71,8 +71,8 @@ func TestListInstancesForReconcileSkipsInstanceDeletedDuringListing(t *testing.T res := <-done require.NoError(t, res.err) - require.Len(t, res.listed, 1) - assert.Equal(t, "zzz-live", res.listed[0].Id) + require.Len(t, res.metadata, 1) + assert.Equal(t, "zzz-live", res.metadata[0].Id) } func TestParseExitSentinelLine(t *testing.T) { diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 37127e921..7b70e807c 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -11,10 +11,6 @@ import ( "github.com/kernel/hypeman/lib/logger" ) -// VGPUAssignmentStartupGracePeriod bounds how long an assignment without a -// persisted hypervisor PID is treated as potentially live. -const VGPUAssignmentStartupGracePeriod = devices.VGPUAssignmentGracePeriod - // VGPUCleanupPendingError reports a failed rollback that left a vGPU assigned. type VGPUCleanupPendingError struct { InstanceID string @@ -49,18 +45,9 @@ func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { return &pending.Device, true } -func vgpuAssignmentLiveness(stored *StoredMetadata, now time.Time, livePID bool) (live bool, graceRemaining time.Duration) { - if livePID { - return true, 0 - } - if stored.GPUAssignedAt == nil { - return false, 0 - } - remaining := VGPUAssignmentStartupGracePeriod - now.Sub(*stored.GPUAssignedAt) - if remaining <= 0 { - return false, 0 - } - return true, remaining +func vgpuAssignmentMayBeLive(stored *StoredMetadata, now time.Time, hypervisorLive bool) bool { + return hypervisorLive || + stored.GPUAssignedAt != nil && now.Sub(*stored.GPUAssignedAt) < devices.VGPUAssignmentGracePeriod } func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { @@ -87,61 +74,32 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) { logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID) - assignment := devices.VGPUAssignment{ + releaseErr := m.destroyVGPUAssignment(ctx, devices.VGPUAssignment{ Framework: device.Framework, DevicePath: device.SysfsPath, MdevUUID: device.MdevUUID, InstanceID: instanceID, - } - cleanupMeta, err := m.loadMetadata(instanceID) - if err != nil { - logger.FromContext(ctx).WarnContext(ctx, "failed to load current metadata for vGPU cleanup; restoring rollback snapshot", "instance_id", instanceID, "error", err) - cleanupMeta = &rollbackMeta - } else { - restoreStartMutatedFields(&cleanupMeta.StoredMetadata, &rollbackMeta.StoredMetadata) - } - releaseErr := m.destroyVGPUAssignment(ctx, assignment) + }) if releaseErr != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID, "error", releaseErr) - setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) + setStoredVGPUDevice(&rollbackMeta.StoredMetadata, device, assignedAt) retained = true } - if err := m.saveMetadata(cleanupMeta); err != nil { + if err := m.saveMetadata(&rollbackMeta); err != nil { message := "failed to save metadata after vGPU cleanup" - if releaseErr != nil { + if retained { message = "failed to retain vGPU assignment metadata after cleanup failure" } logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err) if !retained { return false, false } - if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { - return true, true - } - return true, false + meta, loadErr := m.loadMetadata(instanceID) + return true, loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath } return retained, retained } -func restoreStartMutatedFields(dst, src *StoredMetadata) { - dst.HypervisorPID = src.HypervisorPID - dst.HypervisorStartTime = src.HypervisorStartTime - dst.HypervisorBootID = src.HypervisorBootID - dst.ExitCode = src.ExitCode - dst.ExitMessage = src.ExitMessage - dst.ProgramStartedAt = src.ProgramStartedAt - dst.GuestAgentReadyAt = src.GuestAgentReadyAt - dst.Entrypoint = src.Entrypoint - dst.Cmd = src.Cmd - dst.IP = src.IP - dst.MAC = src.MAC - dst.GPUFramework = src.GPUFramework - dst.GPUDevicePath = src.GPUDevicePath - dst.GPUMdevUUID = src.GPUMdevUUID - dst.GPUAssignedAt = src.GPUAssignedAt - dst.StartedAt = src.StartedAt -} - func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { @@ -149,7 +107,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) claimed := false if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { var err error - claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) + claimed, err = m.vgpuAssignmentClaimedByLiveInstance(stored.Id, path) if err != nil { return err } @@ -173,42 +131,28 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) return nil } -func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { - files, err := m.listMetadataFilesStrict() +func (m *manager) vgpuAssignmentClaimedByLiveInstance(excludeID, devicePath string) (bool, error) { + allMetadata, err := m.listMetadataForReconcile() if err != nil { return false, fmt.Errorf("list instances for vGPU release check: %w", err) } - for _, file := range files { - id := filepath.Base(filepath.Dir(file)) - if id == excludeID { - continue - } - meta, err := m.loadMetadata(id) - if err != nil { - if errors.Is(err, ErrNotFound) { - continue - } - return false, fmt.Errorf("load metadata for vGPU release check: instance %s: %w", id, err) - } - stored := &meta.StoredMetadata - if storedVGPUDevicePath(stored) != devicePath { + for i := range allMetadata { + stored := &allMetadata[i] + if stored.Id == excludeID || storedVGPUDevicePath(stored) != devicePath { continue } - // Resolve even without a persisted PID: the socket-ownership scan can - // still prove a claimant whose post-boot metadata save failed is live. pid, err := resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) if err != nil { - return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", stored.Id, devicePath, err) } - _, remaining := vgpuAssignmentLiveness(stored, m.nowUTC(), pid > 0) if pid > 0 { return true, nil } - if remaining > 0 { + if vgpuAssignmentMayBeLive(stored, m.nowUTC(), false) { if stored.HypervisorPID == nil { - return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", stored.Id, devicePath) } - return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", id, devicePath) + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", stored.Id, devicePath) } } return false, nil diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 9ca8cef72..066c9bdee 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -75,22 +75,18 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { // Listing fails closed: any unreadable metadata aborts the pass so the vendor // VFIO sweep cannot clear a VF whose claim it could not read. func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]struct{}, error) { - allInstances, err := m.listInstancesForReconcile(ctx) + allMetadata, err := m.listMetadataForReconcile() if err != nil { return nil, err } protected := make(map[string]struct{}) - for i := range allInstances { - stored := &allInstances[i].StoredMetadata + for i := range allMetadata { + stored := &allMetadata[i] if storedVGPUDevicePath(stored) == "" { continue } - // The socket-ownership check runs even without a persisted PID: a VMM - // whose post-boot metadata save failed still holds its control-socket - // listener, and releasing its device would tear the vGPU out from - // under a live VM. - livePID := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) - if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { + hypervisorLive := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if vgpuAssignmentMayBeLive(stored, m.nowUTC(), hypervisorLive) { if stored.GPUDevicePath != "" { protected[stored.GPUDevicePath] = struct{}{} } @@ -122,8 +118,8 @@ func (m *manager) releaseStaleVGPUAssignment(ctx context.Context, id string) { if path == "" { return } - livePID := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) - if live, _ := vgpuAssignmentLiveness(stored, m.nowUTC(), livePID); live { + hypervisorLive := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if vgpuAssignmentMayBeLive(stored, m.nowUTC(), hypervisorLive) { return } if err := m.releaseStoredVGPU(ctx, stored); err != nil { diff --git a/lib/instances/vgpu_reconcile_linux_test.go b/lib/instances/vgpu_reconcile_linux_test.go index 8708a1dd7..553c64c15 100644 --- a/lib/instances/vgpu_reconcile_linux_test.go +++ b/lib/instances/vgpu_reconcile_linux_test.go @@ -15,10 +15,6 @@ import ( "github.com/stretchr/testify/require" ) -// A VMM whose post-boot metadata save failed has no persisted PID, but it -// still holds its control-socket listener. The reconciler must protect its -// assignment past the startup grace period instead of releasing the device -// out from under the live VM. func TestReconcileVGPUsProtectsSocketOwnerWithoutPersistedPID(t *testing.T) { t.Parallel() @@ -41,7 +37,7 @@ func TestReconcileVGPUsProtectsSocketOwnerWithoutPersistedPID(t *testing.T) { }, } const id = "pid-save-failed" - stale := time.Now().UTC().Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + stale := time.Now().UTC().Add(-devices.VGPUAssignmentGracePeriod - time.Minute) require.NoError(t, m.ensureDirectories(id)) require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ Id: id, diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 395010532..7a9b66731 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -22,7 +22,7 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { deadPID := dead.Process.Pid now := time.Now().UTC() recent := now.Add(-time.Minute) - stale := now.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + stale := now.Add(-devices.VGPUAssignmentGracePeriod - time.Minute) var protected map[string]struct{} var destroyed []devices.VGPUAssignment diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index 691ae5eaa..2804e8abf 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -2,13 +2,9 @@ package instances import ( "context" - "encoding/json" - "fmt" - "os" "time" "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) @@ -46,8 +42,7 @@ func (r *vgpuRetention) wrapPending(err error) error { return &VGPUCleanupPendingError{InstanceID: r.instanceID, Retained: r.persisted, Err: err} } -// deferWrapPending must be deferred before cleanup so it observes retention -// state recorded by rollback. +// Defer before cleanup so rollback records retention before this wraps the error. func (r *vgpuRetention) deferWrapPending(retErr *error) { *retErr = r.wrapPending(*retErr) } @@ -64,21 +59,13 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten defer func() { m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationCreate, retention.persisted) }() - // When the retention record is lost, the assignment has no metadata claim - // left; the periodic vGPU reconcile sweeps the device once it is free. retentionSurvives := func() bool { meta, err := m.loadMetadata(id) if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { - saveErr := m.saveVGPURetentionStub(retainedVGPU) - if saveErr == nil { - return true + if err := m.saveVGPURetentionStub(retainedVGPU); err != nil { + log.ErrorContext(ctx, "failed to replace surviving instance metadata with vGPU retention stub; preserving existing assignment claim", "instance_id", id, "error", err) } - log.ErrorContext(ctx, "failed to replace surviving instance metadata with vGPU retention stub", "instance_id", id, "error", saveErr) - overwriteErr := m.overwriteVGPURetentionStub(retainedVGPU) - if overwriteErr == nil { - return true - } - log.ErrorContext(ctx, "failed to overwrite surviving instance metadata with vGPU retention stub", "instance_id", id, "error", overwriteErr) + return true } if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) @@ -127,24 +114,3 @@ func vgpuRetentionMetadata(source *StoredMetadata) *metadata { func (m *manager) saveVGPURetentionStub(source *StoredMetadata) error { return m.saveMetadata(vgpuRetentionMetadata(source)) } - -// overwriteVGPURetentionStub handles a surviving metadata file when its -// directory cannot create the temporary file used by saveMetadata. -func (m *manager) overwriteVGPURetentionStub(source *StoredMetadata) error { - retained := vgpuRetentionMetadata(source) - data, err := json.MarshalIndent(retained, "", " ") - if err != nil { - return fmt.Errorf("marshal metadata: %w", err) - } - unlockAliasReaders := hypervisor.LockSnapshotSourceAliasReaders() - defer unlockAliasReaders() - writeFile := m.writeFile - if writeFile == nil { - writeFile = os.WriteFile - } - if err := writeFile(m.paths.InstanceMetadata(source.Id), data, 0644); err != nil { - return fmt.Errorf("write metadata: %w", err) - } - m.syncAdmissionAllocation(retained) - return nil -} diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 55f88f4d9..b575c7546 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -24,53 +24,51 @@ func persistTestVGPURetention(m *manager, ctx context.Context, id string, stub * return retention.persisted } -func retainedVGPUFromCreateErrorForTest(stub StoredMetadata, assignedAt time.Time, err error) *StoredMetadata { - retention := vgpuRetention{} - retention.retainFromCreateError(stub, assignedAt, err) - return retention.stub -} - func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { t.Parallel() m := &manager{paths: paths.New(t.TempDir())} - assignedAt := time.Now().UTC() - stored := &StoredMetadata{ - Id: "failed-create", - Name: "failed-create", - GPUProfile: "NVIDIA L40S-2Q", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - GPUMdevUUID: "mdev-uuid", - GPUAssignedAt: &assignedAt, + const id = "failed-create" + stub := StoredMetadata{ + Id: id, + Name: id, NetworkEnabled: true, IP: "192.0.2.1", Volumes: []VolumeAttachment{{VolumeID: "volume"}}, HypervisorType: "qemu", - DataDir: m.paths.InstanceDir("failed-create"), + DataDir: m.paths.InstanceDir(id), } - require.NoError(t, m.ensureDirectories(stored.Id)) - require.NoError(t, os.WriteFile(m.paths.InstanceOverlay(stored.Id), []byte("overlay"), 0o644)) - require.NoError(t, os.WriteFile(m.paths.InstanceConfigDisk(stored.Id), []byte("config"), 0o644)) - require.NoError(t, os.MkdirAll(m.paths.InstanceVolumeOverlaysDir(stored.Id), 0o755)) - require.NoError(t, os.WriteFile(m.paths.InstanceVolumeOverlay(stored.Id, "volume"), []byte("volume overlay"), 0o644)) - - assert.True(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) - assert.NoFileExists(t, m.paths.InstanceOverlay(stored.Id)) - assert.NoFileExists(t, m.paths.InstanceConfigDisk(stored.Id)) - assert.NoDirExists(t, m.paths.InstanceVolumeOverlaysDir(stored.Id)) - - retained, err := m.loadMetadata(stored.Id) + device := devices.VGPUDevice{ + ProfileName: "NVIDIA L40S-2Q", + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + assignedAt := time.Now().UTC() + retention := vgpuRetention{instanceID: id} + retention.retainFromCreateError(stub, assignedAt, &devices.VGPUCreateCleanupPendingError{Device: device, Err: errors.New("rollback failed")}) + + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, os.WriteFile(m.paths.InstanceOverlay(id), []byte("overlay"), 0o644)) + require.NoError(t, os.WriteFile(m.paths.InstanceConfigDisk(id), []byte("config"), 0o644)) + require.NoError(t, os.MkdirAll(m.paths.InstanceVolumeOverlaysDir(id), 0o755)) + require.NoError(t, os.WriteFile(m.paths.InstanceVolumeOverlay(id, "volume"), []byte("volume overlay"), 0o644)) + + m.persistVGPURetention(context.Background(), &retention) + assert.True(t, retention.persisted) + assert.NoFileExists(t, m.paths.InstanceOverlay(id)) + assert.NoFileExists(t, m.paths.InstanceConfigDisk(id)) + assert.NoDirExists(t, m.paths.InstanceVolumeOverlaysDir(id)) + + retained, err := m.loadMetadata(id) require.NoError(t, err) - assert.Equal(t, stored.Id, retained.Id) - assert.Equal(t, stored.GPUFramework, retained.GPUFramework) - assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) - assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) - assert.Equal(t, stored.GPUAssignedAt, retained.GPUAssignedAt) - assert.Equal(t, stored.Name, retained.Name) - assert.Equal(t, stored.GPUProfile, retained.GPUProfile) - assert.Equal(t, stored.HypervisorType, retained.HypervisorType) - assert.Equal(t, stored.DataDir, retained.DataDir) + assert.Equal(t, id, retained.Id) + assert.Equal(t, device.Framework, retained.GPUFramework) + assert.Equal(t, device.SysfsPath, retained.GPUDevicePath) + assert.Equal(t, assignedAt, *retained.GPUAssignedAt) + assert.Equal(t, stub.Name, retained.Name) + assert.Equal(t, device.ProfileName, retained.GPUProfile) + assert.Equal(t, stub.HypervisorType, retained.HypervisorType) + assert.Equal(t, stub.DataDir, retained.DataDir) assert.False(t, retained.NetworkEnabled) assert.Empty(t, retained.IP) assert.Empty(t, retained.Volumes) @@ -109,7 +107,7 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { require.Error(t, err, "the lost retention leaves no metadata claim, so the periodic sweep releases the VF") } -func TestCleanupFailedCreateRewritesFullMetadataWhenDirectoryIsReadOnly(t *testing.T) { +func TestCleanupFailedCreatePreservesClaimWhenStubSaveFails(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } @@ -119,6 +117,7 @@ func TestCleanupFailedCreateRewritesFullMetadataWhenDirectoryIsReadOnly(t *testi require.NoError(t, m.ensureDirectories(id)) stored := &StoredMetadata{ Id: id, + Name: "surviving metadata", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } @@ -131,61 +130,10 @@ func TestCleanupFailedCreateRewritesFullMetadataWhenDirectoryIsReadOnly(t *testi assert.True(t, persistTestVGPURetention(m, context.Background(), id, stored)) retained, err := m.loadMetadata(id) require.NoError(t, err) + assert.Equal(t, stored.Name, retained.Name) assert.Equal(t, stored.GPUFramework, retained.GPUFramework) assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) - assert.True(t, retained.GPURetainedForCleanup) - - _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) - assert.ErrorIs(t, err, errVGPURetentionStub) -} - -func TestVGPURetentionWrapPending(t *testing.T) { - cause := errors.New("boot failed") - - retention := vgpuRetention{instanceID: "inst-1"} - assert.Same(t, cause, retention.wrapPending(cause)) - - retention.retained = true - pending := retention.wrapPending(cause) - var cleanupPending *VGPUCleanupPendingError - require.ErrorAs(t, pending, &cleanupPending) - assert.False(t, cleanupPending.Retained) - - retention.persisted = true - pending = retention.wrapPending(cause) - require.ErrorAs(t, pending, &cleanupPending) - assert.True(t, cleanupPending.Retained) -} - -func TestVGPUDevicePendingCleanup(t *testing.T) { - t.Parallel() - - device := devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - } - cause := errors.New("rollback failed") - pending := &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} - - wrapped := fmt.Errorf("create failed: %w", pending) - actual, ok := vgpuDevicePendingCleanup(wrapped) - require.True(t, ok) - assert.Equal(t, device, *actual) - - assignedAt := time.Now().UTC() - retained := retainedVGPUFromCreateErrorForTest(StoredMetadata{Id: "inst-1", Name: "named", Image: "img"}, assignedAt, wrapped) - require.NotNil(t, retained) - assert.Equal(t, "inst-1", retained.Id) - assert.Equal(t, "named", retained.Name, "identity fields must survive into the retention stub") - assert.Equal(t, "img", retained.Image) - assert.Equal(t, device.Framework, retained.GPUFramework) - assert.Equal(t, device.SysfsPath, retained.GPUDevicePath) - assert.Equal(t, assignedAt, *retained.GPUAssignedAt) - - actual, ok = vgpuDevicePendingCleanup(cause) - assert.False(t, ok) - assert.Nil(t, actual) - assert.Nil(t, retainedVGPUFromCreateErrorForTest(StoredMetadata{Id: "inst-1"}, assignedAt, cause)) + assert.False(t, retained.GPURetainedForCleanup) } type startRetentionNetworkManager struct { @@ -333,24 +281,6 @@ func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded") } -func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { - m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { - return nil - }) - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.HypervisorType = hypervisor.TypeCloudHypervisor - require.NoError(t, m.saveMetadata(meta)) - - cause := errors.New("create failed") - m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { - return nil, cause - } - - _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) - assert.ErrorIs(t, err, cause) -} - func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { var destroyed []devices.VGPUAssignment m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { @@ -439,37 +369,24 @@ func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { m := &manager{ - paths: paths.New(t.TempDir()), - destroyVGPU: func(context.Context, devices.VGPUAssignment) error { - return nil - }, + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, } const id = "failed-start" require.NoError(t, m.ensureDirectories(id)) previousStart := time.Now().Add(-time.Hour).UTC() - previousProgramStart := previousStart.Add(time.Second) exitCode := 1 rollbackMeta := metadata{StoredMetadata: StoredMetadata{ - Id: id, - Name: "original name", - GPUProfile: "NVIDIA L40S-2Q", - Entrypoint: []string{"old-entrypoint"}, - Cmd: []string{"old-command"}, - StartedAt: &previousStart, - ProgramStartedAt: &previousProgramStart, - ExitCode: &exitCode, - ExitMessage: "previous exit", + Id: id, + Name: "original name", + GPUProfile: "NVIDIA L40S-2Q", + Entrypoint: []string{"old-entrypoint"}, + StartedAt: &previousStart, + ExitCode: &exitCode, + ExitMessage: "previous exit", }} - - partial := rollbackMeta - partial.Name = "concurrent update" - partial.Entrypoint = []string{"new-entrypoint"} - partial.Cmd = []string{"new-command"} - partial.StartedAt = ptr(time.Now().UTC()) - partial.ProgramStartedAt = nil - partial.ExitCode = nil - partial.ExitMessage = "" + partial := metadata{StoredMetadata: StoredMetadata{Id: id, Name: "partial start"}} assignedAt := time.Now().UTC() device := &devices.VGPUDevice{ Framework: devices.VGPUFrameworkVendorVFIO, @@ -482,15 +399,7 @@ func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { stored, err := m.loadMetadata(id) require.NoError(t, err) - assert.Equal(t, "concurrent update", stored.Name) - assert.Equal(t, rollbackMeta.Entrypoint, stored.Entrypoint) - assert.Equal(t, rollbackMeta.Cmd, stored.Cmd) - assert.Equal(t, rollbackMeta.StartedAt, stored.StartedAt) - assert.Equal(t, rollbackMeta.ProgramStartedAt, stored.ProgramStartedAt) - assert.Equal(t, rollbackMeta.ExitCode, stored.ExitCode) - assert.Equal(t, rollbackMeta.ExitMessage, stored.ExitMessage) - assert.Empty(t, stored.GPUDevicePath) - assert.Nil(t, stored.GPUAssignedAt) + assert.Equal(t, rollbackMeta.StoredMetadata, stored.StoredMetadata) } func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { @@ -500,7 +409,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) require.NoError(t, m.ensureDirectories("invalid-instance")) require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) - _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + _, err := m.vgpuAssignmentClaimedByLiveInstance("other-instance", "/sys/bus/pci/devices/0000:82:00.4") require.Error(t, err) } @@ -521,7 +430,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing. }, }})) - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/mdev/devices/legacy-uuid") + claimed, err := m.vgpuAssignmentClaimedByLiveInstance("other-instance", "/sys/bus/mdev/devices/legacy-uuid") require.NoError(t, err) assert.True(t, claimed) } @@ -533,7 +442,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceLiveness(t *testing.T) { deadPID := 1 << 30 require.False(t, ProcessExists(deadPID)) recent := time.Now().UTC() - stale := recent.Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + stale := recent.Add(-devices.VGPUAssignmentGracePeriod - time.Minute) tests := []struct { name string assignedAt *time.Time @@ -559,7 +468,7 @@ func TestVGPUAssignmentClaimedByLiveInstanceLiveness(t *testing.T) { HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: tt.pid}, }})) - claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", devicePath) + claimed, err := m.vgpuAssignmentClaimedByLiveInstance("requester", devicePath) if tt.wantErr != "" { require.ErrorContains(t, err, tt.wantErr) return @@ -635,37 +544,3 @@ func TestStoredVGPUDevicePath(t *testing.T) { })) assert.Empty(t, storedVGPUDevicePath(&StoredMetadata{})) } - -func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) { - t.Parallel() - - m := &manager{paths: paths.New(t.TempDir())} - stored := &StoredMetadata{ - GPUFramework: devices.VGPUFramework("future-framework"), - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - } - err := m.releaseStoredVGPU(context.Background(), stored) - assert.Error(t, err) - assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) - assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) -} - -func TestSetAndClearStoredVGPUDevice(t *testing.T) { - t.Parallel() - - assignedAt := time.Now().UTC() - stored := &StoredMetadata{} - setStoredVGPUDevice(stored, &devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - }, assignedAt) - assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) - assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) - assert.Equal(t, assignedAt, *stored.GPUAssignedAt) - - clearStoredVGPUDevice(stored) - assert.Empty(t, stored.GPUFramework) - assert.Empty(t, stored.GPUDevicePath) - assert.Empty(t, stored.GPUMdevUUID) - assert.Nil(t, stored.GPUAssignedAt) -} From 84ec4383e2eb147d58f17c228e875f84c7d74115 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:11:57 +0000 Subject: [PATCH 73/76] Simplify vGPU retention persistence to a single attempt --- lib/instances/vgpu.go | 5 +++-- lib/instances/vgpu_retention.go | 19 +++---------------- lib/instances/vgpu_test.go | 33 ++------------------------------- 3 files changed, 8 insertions(+), 49 deletions(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 7b70e807c..6d45d82f6 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -94,8 +94,7 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if !retained { return false, false } - meta, loadErr := m.loadMetadata(instanceID) - return true, loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath + return true, false } return retained, retained } @@ -132,6 +131,8 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) } func (m *manager) vgpuAssignmentClaimedByLiveInstance(excludeID, devicePath string) (bool, error) { + // Each vendor VFIO release lists and loads every instance's metadata. This is + // acceptable at GPU-host scale (tens of VFs and instances); revisit at hundreds. allMetadata, err := m.listMetadataForReconcile() if err != nil { return false, fmt.Errorf("list instances for vGPU release check: %w", err) diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index 2804e8abf..a0e048bdc 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -59,32 +59,19 @@ func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuReten defer func() { m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationCreate, retention.persisted) }() - retentionSurvives := func() bool { - meta, err := m.loadMetadata(id) - if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { - if err := m.saveVGPURetentionStub(retainedVGPU); err != nil { - log.ErrorContext(ctx, "failed to replace surviving instance metadata with vGPU retention stub; preserving existing assignment claim", "instance_id", id, "error", err) - } - return true - } - if err := m.deleteInstanceData(id); err != nil { - log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) - } - return false - } + + // An unpersisted retention leaves no metadata claim. The periodic reconciler releases + // the VF after its grace period once no open VFIO handles remain. if err := m.deleteInstanceData(id); err != nil { log.ErrorContext(ctx, "failed to clean instance data before retaining vGPU assignment", "instance_id", id, "error", err) - retention.persisted = retentionSurvives() return } if err := m.ensureDirectories(id); err != nil { log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) - retention.persisted = retentionSurvives() return } if err := m.saveVGPURetentionStub(retainedVGPU); err != nil { log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) - retention.persisted = retentionSurvives() return } retention.persisted = true diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index b575c7546..d8814e17e 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -107,35 +107,6 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { require.Error(t, err, "the lost retention leaves no metadata claim, so the periodic sweep releases the VF") } -func TestCleanupFailedCreatePreservesClaimWhenStubSaveFails(t *testing.T) { - if os.Geteuid() == 0 { - t.Skip("root bypasses directory permissions") - } - - m := &manager{paths: paths.New(t.TempDir())} - const id = "failed-create" - require.NoError(t, m.ensureDirectories(id)) - stored := &StoredMetadata{ - Id: id, - Name: "surviving metadata", - GPUFramework: devices.VGPUFrameworkVendorVFIO, - GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", - } - require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: *stored})) - - instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) - require.NoError(t, os.Chmod(instanceDir, 0o555)) - t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) - - assert.True(t, persistTestVGPURetention(m, context.Background(), id, stored)) - retained, err := m.loadMetadata(id) - require.NoError(t, err) - assert.Equal(t, stored.Name, retained.Name) - assert.Equal(t, stored.GPUFramework, retained.GPUFramework) - assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) - assert.False(t, retained.GPURetainedForCleanup) -} - type startRetentionNetworkManager struct { network.Manager config network.NetworkConfig @@ -338,7 +309,7 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { assert.Empty(t, stored.Entrypoint) } -func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { +func TestCleanupStartVGPUReportsUnpersistedRetentionWhenRollbackSaveFails(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } @@ -364,7 +335,7 @@ func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta) assert.True(t, retained) - assert.True(t, persisted, "a surviving mid-start save keeps the assignment recoverable via delete") + assert.False(t, persisted) } func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { From 0736feeda571925304516b63db476975ebe45bfc Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:52:33 +0000 Subject: [PATCH 74/76] Report vGPU retention as persisted when the assignment claim survives When rollback release fails and the subsequent retention save also fails, the assignment save from earlier in start may still hold the claim on disk. Check for that surviving claim before reporting the retention as unpersisted, so the API does not emit vgpu_unretained_instance for an assignment that is still durably attributed. --- lib/instances/vgpu.go | 7 +++++ lib/instances/vgpu_test.go | 52 ++++++++++++++++++++++++-------------- 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 6d45d82f6..00e32ba01 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -94,6 +94,13 @@ func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, devic if !retained { return false, false } + // The pre-cleanup assignment save may already hold this claim on disk, in + // which case the retention is durable despite the failed rollback save. + if onDisk, loadErr := m.loadMetadata(instanceID); loadErr == nil && + onDisk.GPUDevicePath == device.SysfsPath && + onDisk.GPUAssignedAt != nil && onDisk.GPUAssignedAt.Equal(assignedAt) { + return true, true + } return true, false } return retained, retained diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d8814e17e..e82773115 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -309,33 +309,47 @@ func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { assert.Empty(t, stored.Entrypoint) } -func TestCleanupStartVGPUReportsUnpersistedRetentionWhenRollbackSaveFails(t *testing.T) { +func TestCleanupStartVGPUReportsRetentionWhenRollbackSaveFails(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } - m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { - return errors.New("destroy failed") - }) - device := devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + tests := []struct { + name string + assignmentSaved bool + wantPersisted bool + }{ + {name: "assignment save survived", assignmentSaved: true, wantPersisted: true}, + {name: "assignment never saved", assignmentSaved: false, wantPersisted: false}, } - assignedAt := time.Now().UTC() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + assignedAt := time.Now().UTC() - meta, err := m.loadMetadata(id) - require.NoError(t, err) - rollbackMeta := *meta - setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt) - require.NoError(t, m.saveMetadata(meta)) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + rollbackMeta := *meta + if tt.assignmentSaved { + setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt) + require.NoError(t, m.saveMetadata(meta)) + } - instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) - require.NoError(t, os.Chmod(instanceDir, 0o555)) - t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) - retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta) - assert.True(t, retained) - assert.False(t, persisted) + retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta) + assert.True(t, retained) + assert.Equal(t, tt.wantPersisted, persisted) + }) + } } func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { From 727aa4f15d5f1872b7e8294e6d926f4cddec861d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:49:27 +0000 Subject: [PATCH 75/76] Protect claimed mdevs during reconciliation --- lib/devices/mdev_darwin.go | 2 +- lib/devices/mdev_linux.go | 22 +++++++++++++++++++--- lib/devices/vgpu_linux.go | 22 +++++++++++++++++----- lib/devices/vgpu_linux_test.go | 24 ++++++++++++++++++++++++ lib/instances/vgpu_reconcile.go | 13 ++++++------- lib/instances/vgpu_reconcile_test.go | 20 +++++++++++--------- 6 files changed, 78 insertions(+), 25 deletions(-) diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 4b726bb08..93d3adbf7 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -58,7 +58,7 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { return nil } -func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepVendorVFIO bool) error { +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepDevices bool) error { return nil } diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 0908e8765..423d8473d 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -707,17 +707,27 @@ func mdevPastGracePeriod(mdevUUID string, gracePeriod time.Duration) (bool, time return age >= gracePeriod, age, nil } +func protectedMdevUUIDs(instanceInfos []MdevReconcileInfo) map[string]struct{} { + protected := make(map[string]struct{}, len(instanceInfos)) + for _, info := range instanceInfos { + if info.MdevUUID != "" && info.IsRunning { + protected[info.MdevUUID] = struct{}{} + } + } + return protected +} + // ReconcileMdevs destroys orphaned mdevs on managed VFs. -// This is called on server startup to clean up stale mdevs from previous runs. // // Policy: // - Consider only mdevs whose parent VF is currently managed by hypeman (discoverable via /sys/class/mdev_bus) +// - Keep mdevs claimed by live instance metadata // - Keep mdevs whose VFIO group has an open file handle (/dev/vfio/) // - Keep mdevs younger than a short grace period to avoid racing very recent state transitions // - Delete all remaining mdevs func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error { log := logger.FromContext(ctx) - _ = instanceInfos + protectedMdevs := protectedMdevUUIDs(instanceInfos) vfs, err := discoverMdevVFs() if err != nil { @@ -750,13 +760,18 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro ) groupInUseCache := make(map[int]bool) - var destroyed, failedDestroy, skippedUnmanagedVF, skippedInUse, skippedGrace, skippedProbeError int + var destroyed, failedDestroy, skippedUnmanagedVF, skippedClaimed, skippedInUse, skippedGrace, skippedProbeError int for _, mdev := range mdevs { if _, ok := managedVFs[mdev.VFAddress]; !ok { log.DebugContext(ctx, "skipping mdev on unmanaged VF", "uuid", mdev.UUID, "vf", mdev.VFAddress) skippedUnmanagedVF++ continue } + if _, ok := protectedMdevs[mdev.UUID]; ok { + log.DebugContext(ctx, "skipping mdev claimed by live instance", "uuid", mdev.UUID) + skippedClaimed++ + continue + } group, err := mdevIOMMUGroup(mdev.UUID) if err != nil { @@ -817,6 +832,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro "destroyed", destroyed, "failed_destroy", failedDestroy, "skipped_unmanaged_vf", skippedUnmanagedVF, + "skipped_claimed", skippedClaimed, "skipped_in_use", skippedInUse, "skipped_grace", skippedGrace, "skipped_probe_error", skippedProbeError, diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index 36bf86c9e..354217fd5 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -104,20 +104,32 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { } } +func mdevReconcileInfos(protectedDevicePaths map[string]struct{}) []MdevReconcileInfo { + instanceInfos := make([]MdevReconcileInfo, 0, len(protectedDevicePaths)) + for devicePath := range protectedDevicePaths { + instanceInfos = append(instanceInfos, MdevReconcileInfo{ + MdevUUID: filepath.Base(devicePath), + IsRunning: true, + }) + } + return instanceInfos +} + // ReconcileVGPUs releases orphaned vGPU assignments. -func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepVendorVFIO bool) error { +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepDevices bool) error { framework, _, err := DiscoverVGPU() if err != nil { return err } + if !sweepDevices { + return nil + } switch framework { case VGPUFrameworkMdev: - return ReconcileMdevs(ctx, nil) + return ReconcileMdevs(ctx, mdevReconcileInfos(protectedDevicePaths)) case VGPUFrameworkVendorVFIO: - if sweepVendorVFIO { - return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) - } + return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) } return nil } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go index 7b03d9c26..5bb52da1d 100644 --- a/lib/devices/vgpu_linux_test.go +++ b/lib/devices/vgpu_linux_test.go @@ -12,6 +12,30 @@ import ( "github.com/stretchr/testify/require" ) +func TestMdevReconcileInfosProtectClaimedDevicePath(t *testing.T) { + t.Parallel() + + infos := mdevReconcileInfos(map[string]struct{}{ + "/sys/bus/mdev/devices/claimed": {}, + }) + + assert.Equal(t, []MdevReconcileInfo{{MdevUUID: "claimed", IsRunning: true}}, infos) +} + +func TestProtectedMdevUUIDs(t *testing.T) { + t.Parallel() + + protected := protectedMdevUUIDs([]MdevReconcileInfo{ + {MdevUUID: "claimed", IsRunning: true}, + {MdevUUID: "stale"}, + {IsRunning: true}, + }) + + assert.Contains(t, protected, "claimed") + assert.NotContains(t, protected, "stale") + assert.Len(t, protected, 1) +} + func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { t.Parallel() diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 066c9bdee..27ecb9e01 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -54,17 +54,17 @@ func (m *manager) StartVGPUReconciler(ctx context.Context) { func (m *manager) ReconcileVGPUs(ctx context.Context) { log := logger.FromContext(ctx) protected, err := m.reconcileVGPUAssignments(ctx) - sweepVendorVFIO := err == nil + sweepDevices := err == nil if err != nil { m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageListInstances) - log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping vendor VFIO sweep until the next pass, mdev reconcile still runs", "error", err) + log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping device sweep until the next pass", "error", err) protected = make(map[string]struct{}) } reconcileDevices := m.reconcileVGPUDevices if reconcileDevices == nil { reconcileDevices = devices.ReconcileVGPUs } - if err := reconcileDevices(ctx, protected, sweepVendorVFIO); err != nil { + if err := reconcileDevices(ctx, protected, sweepDevices); err != nil { m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageReconcileDevices) log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) } @@ -82,14 +82,13 @@ func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]stru protected := make(map[string]struct{}) for i := range allMetadata { stored := &allMetadata[i] - if storedVGPUDevicePath(stored) == "" { + devicePath := storedVGPUDevicePath(stored) + if devicePath == "" { continue } hypervisorLive := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) if vgpuAssignmentMayBeLive(stored, m.nowUTC(), hypervisorLive) { - if stored.GPUDevicePath != "" { - protected[stored.GPUDevicePath] = struct{}{} - } + protected[devicePath] = struct{}{} continue } m.releaseStaleVGPUAssignment(ctx, stored.Id) diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 7a9b66731..8b8d71b66 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -33,9 +33,9 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { destroyed = append(destroyed, assignment) return nil }, - reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, sweepVendorVFIO bool) error { + reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, sweepDevices bool) error { protected = p - assert.True(t, sweepVendorVFIO) + assert.True(t, sweepDevices) return nil }, } @@ -45,6 +45,7 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { {Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}, {Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}}, {Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}, + {Id: "legacy-mdev-booting", GPUMdevUUID: "test-mdev", GPUAssignedAt: &recent}, } for i := range instances { require.NoError(t, m.ensureDirectories(instances[i].Id)) @@ -57,6 +58,7 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") + assert.Contains(t, protected, "/sys/bus/mdev/devices/test-mdev") for _, id := range []string{"orphaned", "legacy", "dead"} { stored, err := m.loadMetadata(id) @@ -71,14 +73,14 @@ func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { orphaned, err := m.loadMetadata("orphaned") require.NoError(t, err) assert.Equal(t, "NVIDIA L40S-2Q", orphaned.GPUProfile) - for _, id := range []string{"booting", "stale-pid-booting"} { + for _, id := range []string{"booting", "stale-pid-booting", "legacy-mdev-booting"} { stored, err := m.loadMetadata(id) require.NoError(t, err) - assert.NotEmpty(t, stored.GPUDevicePath, "live assignment on %s must be kept", id) + assert.NotEmpty(t, storedVGPUDevicePath(&stored.StoredMetadata), "live assignment on %s must be kept", id) } } -func TestReconcileVGPUsSkipsVendorSweepWhenListingFails(t *testing.T) { +func TestReconcileVGPUsSkipsDeviceSweepWhenListingFails(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root bypasses directory permissions") } @@ -86,8 +88,8 @@ func TestReconcileVGPUsSkipsVendorSweepWhenListingFails(t *testing.T) { var sweeps []bool m := &manager{ paths: paths.New(t.TempDir()), - reconcileVGPUDevices: func(_ context.Context, _ map[string]struct{}, sweepVendorVFIO bool) error { - sweeps = append(sweeps, sweepVendorVFIO) + reconcileVGPUDevices: func(_ context.Context, _ map[string]struct{}, sweepDevices bool) error { + sweeps = append(sweeps, sweepDevices) return nil }, } @@ -100,11 +102,11 @@ func TestReconcileVGPUsSkipsVendorSweepWhenListingFails(t *testing.T) { m.ReconcileVGPUs(t.Context()) require.Equal(t, []bool{false}, sweeps, - "a listing failure must skip the vendor sweep, not run it with an empty protection set") + "a listing failure must skip the device sweep, not run it with an empty protection set") require.NoError(t, os.Chmod(instanceDir, 0o755)) m.ReconcileVGPUs(t.Context()) - assert.Equal(t, []bool{false, true}, sweeps, "the next pass retries the vendor sweep") + assert.Equal(t, []bool{false, true}, sweeps, "the next pass retries the device sweep") } func TestReconcileVGPUsKeepsAssignmentWhenReleaseFails(t *testing.T) { From 081d9f755c98b57e5de5aa9fbb29910eb7046418 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:56:07 +0000 Subject: [PATCH 76/76] Unify vGPU retention wrapping and dedupe test fixtures --- lib/instances/create.go | 3 ++- lib/instances/start.go | 19 +++++++++--------- lib/instances/vgpu_retention.go | 5 ----- lib/instances/vgpu_test.go | 35 +++++++++++++-------------------- 4 files changed, 26 insertions(+), 36 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index 525f2c262..cd50deb5c 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -280,7 +280,8 @@ func (m *manager) createInstance( var gpuAssignedAt *time.Time retention := vgpuRetention{instanceID: id} - defer retention.deferWrapPending(&retErr) + // Deferred before cu.Clean so rollback records retention before this wraps the error. + defer func() { retErr = retention.wrapPending(retErr) }() cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) m.persistVGPURetention(ctx, &retention) diff --git a/lib/instances/start.go b/lib/instances/start.go index ea8c2a6f4..37b836427 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -121,7 +121,8 @@ func (m *manager) startInstance( // Setup cleanup stack for automatic rollback on errors retention := vgpuRetention{instanceID: id} - defer retention.deferWrapPending(&retErr) + // Deferred before cu.Clean so rollback records retention before this wraps the error. + defer func() { retErr = retention.wrapPending(retErr) }() cu := cleanup.Make(func() {}) defer cu.Clean() @@ -177,20 +178,20 @@ func (m *manager) startInstance( device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id) if err != nil { log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) + wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok { - assignedAt := m.nowUTC() retentionMeta := rollbackMeta - setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, assignedAt) - wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) + setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, m.nowUTC()) + persisted := true if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { - m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, false) log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) - return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} + wrapped = fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr) + persisted = false } - m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, true) - return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} + retention.markRetained(persisted) + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, persisted) } - return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) + return nil, wrapped } assignedAt := m.nowUTC() setStoredVGPUDevice(stored, device, assignedAt) diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go index a0e048bdc..1bcba2c42 100644 --- a/lib/instances/vgpu_retention.go +++ b/lib/instances/vgpu_retention.go @@ -42,11 +42,6 @@ func (r *vgpuRetention) wrapPending(err error) error { return &VGPUCleanupPendingError{InstanceID: r.instanceID, Retained: r.persisted, Err: err} } -// Defer before cleanup so rollback records retention before this wraps the error. -func (r *vgpuRetention) deferWrapPending(retErr *error) { - *retErr = r.wrapPending(*retErr) -} - func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuRetention) { if retention.stub == nil { m.deleteInstanceData(retention.instanceID) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index e82773115..07cdcb8e5 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -18,6 +18,16 @@ import ( "github.com/stretchr/testify/require" ) +func testVendorVFIODevice(profileName string) devices.VGPUDevice { + return devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: profileName, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } +} + func persistTestVGPURetention(m *manager, ctx context.Context, id string, stub *StoredMetadata) bool { retention := vgpuRetention{instanceID: id, stub: stub, retained: stub != nil} m.persistVGPURetention(ctx, &retention) @@ -131,13 +141,8 @@ func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, dev instanceLocks: sync.Map{}, bootMarkerScans: sync.Map{}, createVGPU: func(_ context.Context, profileName, _ string) (*devices.VGPUDevice, error) { - return &devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - VFAddress: "0000:82:00.4", - ProfileType: "1148", - ProfileName: profileName, - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - }, nil + device := testVendorVFIODevice(profileName) + return &device, nil }, destroyVGPU: destroy, } @@ -180,13 +185,7 @@ func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { meta.ExitMessage = "previous exit" require.NoError(t, m.saveMetadata(meta)) - device := devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - VFAddress: "0000:82:00.4", - ProfileType: "1148", - ProfileName: "NVIDIA L40S-2Q", - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - } + device := testVendorVFIODevice("NVIDIA L40S-2Q") cause := errors.New("create verification and rollback failed") m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} @@ -225,13 +224,7 @@ func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { return nil }) - device := devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - VFAddress: "0000:82:00.4", - ProfileType: "1148", - ProfileName: "NVIDIA L40S-2Q", - SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", - } + device := testVendorVFIODevice("NVIDIA L40S-2Q") cause := errors.New("create verification and rollback failed") m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { instanceDir := filepath.Dir(m.paths.InstanceMetadata(id))