diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 945ad8314..2d7f1bbb1 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -362,7 +362,16 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst inst, err := s.InstanceManager.CreateInstance(ctx, domainReq) if err != nil { + var vgpuPending *instances.VGPUCleanupPendingError switch { + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) + message, inner := vgpuCleanupPendingDetail(vgpuPending, "create", "delete it to retry") + return oapi.CreateInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: inner, + }, nil case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ Code: "image_not_ready", @@ -424,6 +433,19 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst return oapi.CreateInstance201JSONResponse(instanceToOAPI(*inst)), nil } +func vgpuCleanupPendingDetail(pending *instances.VGPUCleanupPendingError, action, retainedGuidance string) (string, *oapi.ErrorDetail) { + message := fmt.Sprintf("failed to %s instance: %v", action, pending) + innerCode := "vgpu_unretained_instance" + if pending.Retained { + message += "; " + retainedGuidance + innerCode = "vgpu_retained_instance" + } + return message, &oapi.ErrorDetail{ + Code: lo.ToPtr(innerCode), + Message: lo.ToPtr(pending.InstanceID), + } +} + // GetInstance gets instance details // The id parameter can be an instance ID, name, or ID prefix // Note: Resolution is handled by ResolveResource middleware @@ -829,7 +851,16 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan result, err := s.InstanceManager.StartInstance(ctx, inst.Id, startReq) if err != nil { + var vgpuPending *instances.VGPUCleanupPendingError switch { + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to start instance", "error", err) + message, inner := vgpuCleanupPendingDetail(vgpuPending, "start", "delete it or retry start to release it") + return oapi.StartInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: inner, + }, nil case errors.Is(err, instances.ErrInvalidState): return oapi.StartInstance409JSONResponse{ Code: "invalid_state", @@ -1264,6 +1295,9 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { if inst.GPUMdevUUID != "" { gpu.MdevUuid = lo.ToPtr(inst.GPUMdevUUID) } + if inst.GPUDevicePath != "" { + gpu.DevicePath = lo.ToPtr(inst.GPUDevicePath) + } oapiInst.Gpu = gpu } diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index fce9ee10a..fe717abdb 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -17,6 +17,7 @@ import ( "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/instances/phasetracking" mw "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/paths" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" @@ -47,6 +48,39 @@ func TestGetInstance_NotFound(t *testing.T) { require.Error(t, err) } +func TestVGPUCleanupPendingDetail(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + name string + retained bool + code string + guidance string + exclude string + }{ + {name: "retained", retained: true, code: "vgpu_retained_instance", guidance: "delete it to retry"}, + {name: "unretained", code: "vgpu_unretained_instance", guidance: "periodic vGPU reconcile", exclude: "delete"}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + message, inner := vgpuCleanupPendingDetail(&instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: tt.retained, + Err: network.ErrNameExists, + }, "create", "delete it to retry") + assert.Contains(t, message, "inst-1") + assert.Contains(t, message, network.ErrNameExists.Error()) + assert.Contains(t, message, tt.guidance) + if tt.exclude != "" { + assert.NotContains(t, message, tt.exclude) + } + require.NotNil(t, inner.Code) + assert.Equal(t, tt.code, *inner.Code) + require.NotNil(t, inner.Message) + assert.Equal(t, "inst-1", *inner.Message) + }) + } +} + func TestCreateInstance_AutoPullImage(t *testing.T) { t.Parallel() if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { @@ -842,6 +876,16 @@ func TestCreateInstance_ErrorStatusMapping(t *testing.T) { wantCode string wantMessage string }{ + { + name: "vGPU cleanup pending beats wrapped name conflict -> 500", + err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: true, + Err: network.ErrNameExists, + }, + wantType: oapi.CreateInstance500JSONResponse{}, + wantCode: "vgpu_cleanup_pending", + }, { name: "platform not available -> 404", err: fmt.Errorf("resolve image: %w", images.ErrPlatformNotAvailable), @@ -966,6 +1010,27 @@ func TestRestoreInstance_ErrorMapping(t *testing.T) { } } +func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { + t.Parallel() + svc := newTestService(t) + resolved := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{Id: "inst-1", Name: "inst-1"}, + State: instances.StateStopped, + } + svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ + InstanceID: resolved.Id, + Retained: true, + Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), + }} + + resp, err := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) + require.NoError(t, err) + pending, ok := resp.(oapi.StartInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "delete it or retry start") +} + func TestInstanceActions_ImageNotFoundMapsTo404(t *testing.T) { t.Parallel() diff --git a/cmd/api/main.go b/cmd/api/main.go index f1bbfcf3d..faa9ac15a 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -384,11 +384,9 @@ func run() error { return fmt.Errorf("reconcile device state: %w", err) } - // Reconcile mdev devices (clears orphaned vGPUs from previous runs) - logger.Info("Reconciling mdev devices...") - if err := devices.ReconcileMdevs(app.Ctx, nil); err != nil { - // Log but don't fail - mdev cleanup is best-effort - logger.Warn("failed to reconcile mdev devices", "error", err) + logger.Info("Reconciling vGPU devices...") + if r, ok := app.InstanceManager.(interface{ StartVGPUReconciler(context.Context) }); ok { + r.StartVGPUReconciler(ctx) } // Wire up resource validator for aggregate limit checking diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 7873aa35c..1f1a82776 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -324,11 +324,6 @@ func checkVGPUTestPrerequisites() (string, string) { if framework == devices.VGPUFrameworkNone { return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", "" } - if framework == devices.VGPUFrameworkVendorVFIO { - // CreateVGPU rejects vendor VFIO until the instance lifecycle - // integration lands. - return "vGPU test requires the vendor VFIO instance lifecycle integration", "" - } // Check for available profiles profiles, err := devices.ListGPUProfiles() diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 3a1504428..d04dcf599 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -97,7 +97,7 @@ Instance Create → Assign profile to VF → Attach VF to VM → Instance Runnin Instance Stop/Delete → Release profile → VF available again ``` -Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. +Hypeman reconciles orphaned assignments with a periodic fail-closed pass: once at startup and every minute afterward (skipped entirely on hosts without GPUs). Each pass releases assignments whose owning instance is no longer live and clears their metadata, then sweeps device-level leftovers with no live metadata claim. Devices held open by a running VMM and assignments younger than five minutes are preserved, so a release that fails during stop or delete (typically because a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait) is simply retried on later passes until the device is free. ### Hypervisor Support @@ -288,10 +288,17 @@ every request, so one wedged VF presents as all vGPU instances failing while `/resources` reports full capacity. The wedge itself leaves no host-side log: no kernel error, no XID, no plugin -crash. In the observed case it followed a period of heavy attach/teardown -churn on the VF, including QEMU processes that exited within seconds of -opening the VFIO device (failed start attempts that were then retried), so -suspect any workload that repeatedly kills the VMM mid-device-init. +crash. The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is +still initializing the VF (roughly the first seconds after process start): +a single hard kill in that window wedges the VF near-deterministically, +while QEMU processes that exit voluntarily — error exits, QMP quit, SIGTERM — +run their VFIO teardown and never wedge, and hard kills of fully-initialized +vGPU VMs are also safe. Hypeman therefore SIGTERMs a vGPU QEMU first and only +escalates to SIGKILL after a grace period, both in start-failure cleanup and +when force-killing any vGPU instance (the instance reports Running seconds +before driver init completes, so no state reliably marks the window); a hard +kill after an ignored SIGTERM logs `VF may wedge` with the device path. +External SIGKILLs (OOM killer, manual `kill -9`) can still trigger it. Confirm by assigning the same profile on a different VF: if that guest initializes, the VF is wedged, not the driver stack. Remediate by cycling diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 4274063ed..93d3adbf7 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -58,7 +58,7 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { return nil } -func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error { +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepDevices bool) error { return nil } diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 6ef9f6c60..423d8473d 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -23,10 +23,9 @@ import ( ) const ( - mdevBusPath = "/sys/class/mdev_bus" - mdevDevices = "/sys/bus/mdev/devices" - orphanedMdevGracePeriod = 5 * time.Minute - procPath = "/proc" + mdevBusPath = "/sys/class/mdev_bus" + mdevDevices = "/sys/bus/mdev/devices" + procPath = "/proc" ) // mdevMu protects mdev creation/destruction to prevent race conditions @@ -708,17 +707,27 @@ func mdevPastGracePeriod(mdevUUID string, gracePeriod time.Duration) (bool, time return age >= gracePeriod, age, nil } +func protectedMdevUUIDs(instanceInfos []MdevReconcileInfo) map[string]struct{} { + protected := make(map[string]struct{}, len(instanceInfos)) + for _, info := range instanceInfos { + if info.MdevUUID != "" && info.IsRunning { + protected[info.MdevUUID] = struct{}{} + } + } + return protected +} + // ReconcileMdevs destroys orphaned mdevs on managed VFs. -// This is called on server startup to clean up stale mdevs from previous runs. // // Policy: // - Consider only mdevs whose parent VF is currently managed by hypeman (discoverable via /sys/class/mdev_bus) +// - Keep mdevs claimed by live instance metadata // - Keep mdevs whose VFIO group has an open file handle (/dev/vfio/) // - Keep mdevs younger than a short grace period to avoid racing very recent state transitions // - Delete all remaining mdevs func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error { log := logger.FromContext(ctx) - _ = instanceInfos + protectedMdevs := protectedMdevUUIDs(instanceInfos) vfs, err := discoverMdevVFs() if err != nil { @@ -747,17 +756,22 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro log.InfoContext(ctx, "reconciling mdev devices", "total_mdevs", len(mdevs), "managed_vfs", len(managedVFs), - "grace_period", orphanedMdevGracePeriod.String(), + "grace_period", VGPUAssignmentGracePeriod.String(), ) groupInUseCache := make(map[int]bool) - var destroyed, failedDestroy, skippedUnmanagedVF, skippedInUse, skippedGrace, skippedProbeError int + var destroyed, failedDestroy, skippedUnmanagedVF, skippedClaimed, skippedInUse, skippedGrace, skippedProbeError int for _, mdev := range mdevs { if _, ok := managedVFs[mdev.VFAddress]; !ok { log.DebugContext(ctx, "skipping mdev on unmanaged VF", "uuid", mdev.UUID, "vf", mdev.VFAddress) skippedUnmanagedVF++ continue } + if _, ok := protectedMdevs[mdev.UUID]; ok { + log.DebugContext(ctx, "skipping mdev claimed by live instance", "uuid", mdev.UUID) + skippedClaimed++ + continue + } group, err := mdevIOMMUGroup(mdev.UUID) if err != nil { @@ -782,7 +796,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro continue } - pastGracePeriod, age, err := mdevPastGracePeriod(mdev.UUID, orphanedMdevGracePeriod) + pastGracePeriod, age, err := mdevPastGracePeriod(mdev.UUID, VGPUAssignmentGracePeriod) if err != nil { log.WarnContext(ctx, "failed to determine mdev age, skipping cleanup", "uuid", mdev.UUID, "error", err) skippedProbeError++ @@ -792,7 +806,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro log.DebugContext(ctx, "skipping recently created mdev during grace period", "uuid", mdev.UUID, "age", age.String(), - "grace_period", orphanedMdevGracePeriod.String(), + "grace_period", VGPUAssignmentGracePeriod.String(), ) skippedGrace++ continue @@ -818,6 +832,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro "destroyed", destroyed, "failed_destroy", failedDestroy, "skipped_unmanaged_vf", skippedUnmanagedVF, + "skipped_claimed", skippedClaimed, "skipped_in_use", skippedInUse, "skipped_grace", skippedGrace, "skipped_probe_error", skippedProbeError, diff --git a/lib/devices/types.go b/lib/devices/types.go index 31ebd90ef..369bb3268 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -60,6 +60,11 @@ func ValidateDeviceName(name string) bool { // GPUMode represents the host's GPU configuration mode type GPUMode string +// VGPUAssignmentGracePeriod protects a fresh vGPU assignment from cleanup +// until its VM has had time to boot and become identifiable — by a persisted +// hypervisor PID, a control-socket owner, or an open VFIO handle. +const VGPUAssignmentGracePeriod = 5 * time.Minute + type VGPUFramework string const ( diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 2c576b4d2..a1378a3c7 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "log/slog" + "maps" "os" "path/filepath" "sort" @@ -14,6 +15,7 @@ import ( "strings" "sync" "syscall" + "time" "github.com/kernel/hypeman/lib/logger" ) @@ -23,11 +25,16 @@ const ( vfioDevicesPath = "/dev/vfio/devices" ) +type vendorVFIOOwner struct { + instanceID string + assignedAt time.Time +} + type vendorVFIOSysfs struct { pciDevicesPath string procPath string vfioDevicesPath string - owners map[string]string + owners map[string]vendorVFIOOwner framebufferByType map[string]int openVFIOPathsFunc func() (map[string]struct{}, error) } @@ -37,7 +44,7 @@ var ( pciDevicesPath: pciDevicesPath, procPath: procPath, vfioDevicesPath: vfioDevicesPath, - owners: make(map[string]string), + owners: make(map[string]vendorVFIOOwner), framebufferByType: make(map[string]int), } vendorVFIOMu sync.Mutex @@ -196,7 +203,7 @@ func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID str verifyErr := fmt.Errorf("verify vGPU on VF %s: type is %s, want %s", targetVF, currentType, requested.TypeName) return nil, s.rollbackCreate(currentTypePath, targetVF, instanceID, device, verifyErr) } - s.owners[targetVF] = instanceID + s.owners[targetVF] = vendorVFIOOwner{instanceID: instanceID, assignedAt: time.Now()} logger.FromContext(ctx).InfoContext(ctx, "created vendor VFIO vGPU", "profile", profileName, @@ -229,10 +236,10 @@ func (s vendorVFIOSysfs) destroy(ctx context.Context, vfAddress, instanceID stri if instanceID == "" { return fmt.Errorf("cannot release vendor VFIO vGPU on VF %s without instance ID", vfAddress) } - if owner != instanceID { + if owner.instanceID != instanceID { log.WarnContext(ctx, "skipping vendor VFIO vGPU release owned by another instance", "vf", vfAddress, - "owner_instance_id", owner, + "owner_instance_id", owner.instanceID, "requesting_instance_id", instanceID, ) return nil @@ -264,6 +271,9 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map if err != nil { return err } + vendorVFIOMu.Lock() + owners := maps.Clone(s.owners) + vendorVFIOMu.Unlock() log := logger.FromContext(ctx) protectedVFs := make(map[string]struct{}, len(protectedDevicePaths)) for path := range protectedDevicePaths { @@ -278,6 +288,11 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map log.DebugContext(ctx, "skipping vendor VFIO vGPU held by a live instance", "vf", vf.PCIAddress) continue } + owner := owners[vf.PCIAddress] + if !owner.assignedAt.IsZero() && time.Since(owner.assignedAt) < VGPUAssignmentGracePeriod { + log.DebugContext(ctx, "skipping recently assigned vendor VFIO vGPU during grace period", "vf", vf.PCIAddress) + continue + } if openPaths == nil { if openPaths, err = s.openVFIOPaths(); err != nil { return fmt.Errorf("scan open VFIO handles: %w", err) @@ -292,7 +307,7 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map log.WarnContext(ctx, "preserving vendor VFIO vGPU held open without a live instance claim", "vf", vf.PCIAddress) continue } - if err := s.destroy(ctx, vf.PCIAddress, ""); err != nil { + if err := s.destroy(ctx, vf.PCIAddress, owner.instanceID); err != nil { log.WarnContext(ctx, "failed to destroy orphaned vendor VFIO vGPU", "vf", vf.PCIAddress, "error", err) } } @@ -514,7 +529,7 @@ func framebufferFromProfileName(name string) int { func (s vendorVFIOSysfs) rollbackCreate(currentTypePath, vfAddress, instanceID string, device VGPUDevice, verifyErr error) error { if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { - s.owners[vfAddress] = instanceID + s.owners[vfAddress] = vendorVFIOOwner{instanceID: instanceID, assignedAt: time.Now()} return &VGPUCreateCleanupPendingError{ Device: device, Err: errors.Join(verifyErr, fmt.Errorf("roll back vGPU on VF %s: %w", vfAddress, err)), diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index bd7293922..098d016e3 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -372,6 +373,32 @@ func TestVendorVFIOReconcile(t *testing.T) { assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "1148") } +func TestVendorVFIOReconcileSkipsRecentlyAssignedVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.owners["0000:82:00.4"] = vendorVFIOOwner{instanceID: "mid-create", assignedAt: time.Now()} + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcileDestroysOwnedVFPastGracePeriod(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.owners["0000:82:00.4"] = vendorVFIOOwner{ + instanceID: "deleted-instance", + assignedAt: time.Now().Add(-VGPUAssignmentGracePeriod - time.Minute), + } + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "0") + assert.Empty(t, sysfs.owners) +} + func TestVendorVFIOReconcileRechecksOpenHandlesBeforeDestroy(t *testing.T) { t.Parallel() @@ -518,7 +545,7 @@ func TestRollbackVendorVFIOCreate(t *testing.T) { t.Run("preserves verification error", func(t *testing.T) { currentTypePath := filepath.Join(t.TempDir(), "current_vgpu_type") require.NoError(t, os.WriteFile(currentTypePath, []byte("1148"), 0644)) - sysfs := vendorVFIOSysfs{owners: make(map[string]string)} + sysfs := vendorVFIOSysfs{owners: make(map[string]vendorVFIOOwner)} err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) require.ErrorIs(t, err, verifyErr) @@ -528,7 +555,7 @@ func TestRollbackVendorVFIOCreate(t *testing.T) { t.Run("retains assignment when rollback fails", func(t *testing.T) { currentTypePath := filepath.Join(t.TempDir(), "missing", "current_vgpu_type") - sysfs := vendorVFIOSysfs{owners: make(map[string]string)} + sysfs := vendorVFIOSysfs{owners: make(map[string]vendorVFIOOwner)} err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) require.ErrorIs(t, err, verifyErr) @@ -536,7 +563,8 @@ func TestRollbackVendorVFIOCreate(t *testing.T) { var pending *VGPUCreateCleanupPendingError require.ErrorAs(t, err, &pending) assert.Equal(t, device, pending.Device) - assert.Equal(t, "instance-1", sysfs.owners[device.VFAddress]) + assert.Equal(t, "instance-1", sysfs.owners[device.VFAddress].instanceID) + assert.False(t, sysfs.owners[device.VFAddress].assignedAt.IsZero()) }) } @@ -566,7 +594,7 @@ func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { pciDevicesPath: pci, procPath: proc, vfioDevicesPath: vfio, - owners: make(map[string]string), + owners: make(map[string]vendorVFIOOwner), framebufferByType: make(map[string]int), }} } diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index b3c497899..354217fd5 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -73,10 +73,7 @@ func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevic MdevUUID: mdev.UUID, }, nil case VGPUFrameworkVendorVFIO: - // The instance lifecycle does not yet persist vendor VFIO assignments - // durably or guard their release against live claims, so keep the - // backend out of the create path until that integration lands. - return nil, fmt.Errorf("vendor VFIO vGPU support is not yet integrated with the instance lifecycle") + return hostVendorVFIO.create(ctx, profileName, instanceID) default: return nil, fmt.Errorf("vGPU framework not available") } @@ -107,22 +104,32 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { } } +func mdevReconcileInfos(protectedDevicePaths map[string]struct{}) []MdevReconcileInfo { + instanceInfos := make([]MdevReconcileInfo, 0, len(protectedDevicePaths)) + for devicePath := range protectedDevicePaths { + instanceInfos = append(instanceInfos, MdevReconcileInfo{ + MdevUUID: filepath.Base(devicePath), + IsRunning: true, + }) + } + return instanceInfos +} + // ReconcileVGPUs releases orphaned vGPU assignments. -func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error { +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}, sweepDevices bool) error { framework, _, err := DiscoverVGPU() if err != nil { return err } + if !sweepDevices { + return nil + } switch framework { case VGPUFrameworkMdev: - return ReconcileMdevs(ctx, nil) + return ReconcileMdevs(ctx, mdevReconcileInfos(protectedDevicePaths)) case VGPUFrameworkVendorVFIO: - if protectedDevicePaths == nil { - return nil - } return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) - default: - return nil } + return nil } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go index 7b03d9c26..5bb52da1d 100644 --- a/lib/devices/vgpu_linux_test.go +++ b/lib/devices/vgpu_linux_test.go @@ -12,6 +12,30 @@ import ( "github.com/stretchr/testify/require" ) +func TestMdevReconcileInfosProtectClaimedDevicePath(t *testing.T) { + t.Parallel() + + infos := mdevReconcileInfos(map[string]struct{}{ + "/sys/bus/mdev/devices/claimed": {}, + }) + + assert.Equal(t, []MdevReconcileInfo{{MdevUUID: "claimed", IsRunning: true}}, infos) +} + +func TestProtectedMdevUUIDs(t *testing.T) { + t.Parallel() + + protected := protectedMdevUUIDs([]MdevReconcileInfo{ + {MdevUUID: "claimed", IsRunning: true}, + {MdevUUID: "stale"}, + {IsRunning: true}, + }) + + assert.Contains(t, protected, "claimed") + assert.NotContains(t, protected, "stale") + assert.Len(t, protected, 1) +} + func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { t.Parallel() diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index c02c5beef..f16d606eb 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -227,6 +227,7 @@ func buildQMPArgs(socketPath string) []string { type startedProcess struct { pid int socketPath string + termGrace time.Duration waitDone chan error waitConsumed bool waitErr error @@ -277,18 +278,47 @@ func (p *startedProcess) wait() error { return err } +func (p *startedProcess) waitFor(d time.Duration) bool { + if _, exited := p.checkExited(); exited { + return true + } + select { + case err := <-p.waitDone: + p.waitConsumed = true + p.waitErr = err + return true + case <-time.After(d): + return false + } +} + func (p *startedProcess) cleanup() { if _, exited := p.checkExited(); !exited { - _ = syscall.Kill(p.pid, syscall.SIGKILL) - _ = p.wait() + terminated := false + if p.termGrace > 0 { + if syscall.Kill(p.pid, syscall.SIGTERM) == nil { + terminated = p.waitFor(p.termGrace) + } + } + if !terminated { + _ = syscall.Kill(p.pid, syscall.SIGKILL) + _ = p.wait() + } } _ = os.Remove(p.socketPath) } +func vfioTermGraceFor(cfg hypervisor.VMConfig) time.Duration { + if cfg.VGPUDevicePath != "" || len(cfg.PCIDevices) > 0 { + return hypervisor.VFIOTermGrace + } + return 0 +} + // startQEMUProcess handles the common QEMU process startup logic. // Returns the PID, hypervisor client, and a cleanup function. // The cleanup function must be called on error; call cleanup.Release() on success. -func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version string, socketPath string, args []string) (int, *QEMU, *cleanup.Cleanup, error) { +func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version string, socketPath string, args []string, termGrace time.Duration) (int, *QEMU, *cleanup.Cleanup, error) { log := logger.FromContext(ctx) processAttrs := hypervisor.TraceAttributesFromContext(ctx) processAttrs = append(processAttrs, @@ -358,6 +388,7 @@ func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version } pid := proc.pid + proc.termGrace = termGrace log.DebugContext(processCtx, "QEMU process started", "pid", pid, "duration_ms", time.Since(processStartTime).Milliseconds()) // Setup cleanup to kill, reap, and remove the socket if subsequent steps fail. @@ -474,7 +505,7 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s // Build command arguments: QMP socket + VM configuration args := buildQMPArgs(socketPath) args = append(args, buildArgs(attempt, machineType)...) - pid, hv, cu, err = s.startQEMUProcess(ctx, p, version, socketPath, args) + pid, hv, cu, err = s.startQEMUProcess(ctx, p, version, socketPath, args, vfioTermGraceFor(attempt)) if err == nil { booted = attempt started = true @@ -609,7 +640,7 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, incomingURI := "exec:cat < " + memoryFile args = append(args, "-incoming", incomingURI) - pid, hv, cu, err := s.startQEMUProcess(ctx, p, version, socketPath, args) + pid, hv, cu, err := s.startQEMUProcess(ctx, p, version, socketPath, args, vfioTermGraceFor(config)) if err != nil { return 0, nil, err } diff --git a/lib/hypervisor/qemu/process_test.go b/lib/hypervisor/qemu/process_test.go index e8be0dda2..ad08aea24 100644 --- a/lib/hypervisor/qemu/process_test.go +++ b/lib/hypervisor/qemu/process_test.go @@ -1,6 +1,7 @@ package qemu import ( + "bufio" "context" "errors" "os" @@ -409,3 +410,58 @@ func TestWaitForSocketOrExitReturnsEarlyWhenProcessDies(t *testing.T) { require.NotNil(t, cmd.ProcessState) assert.True(t, cmd.ProcessState.Exited()) } + +func TestVFIOTermGraceFor(t *testing.T) { + tests := []struct { + name string + cfg hypervisor.VMConfig + want time.Duration + }{ + {name: "vGPU", cfg: hypervisor.VMConfig{VGPUDevicePath: "/sys/bus/mdev/devices/test"}, want: hypervisor.VFIOTermGrace}, + {name: "PCI device", cfg: hypervisor.VMConfig{PCIDevices: []string{"0000:01:00.0"}}, want: hypervisor.VFIOTermGrace}, + {name: "no VFIO device", cfg: hypervisor.VMConfig{}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, vfioTermGraceFor(tt.cfg)) + }) + } +} + +func TestCleanupSIGTERMsProcessWithTermGrace(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "qemu.sock") + markerPath := filepath.Join(t.TempDir(), "terminated") + + cmd := exec.Command("sh", "-c", "trap 'touch "+markerPath+"; exit 0' TERM; echo ready; sleep 30 & wait") + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + proc, err := startManagedProcess(cmd, socketPath) + require.NoError(t, err) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + proc.termGrace = 5 * time.Second + + proc.cleanup() + + assert.FileExists(t, markerPath, "process must get SIGTERM, not SIGKILL, when termGrace is set") + require.NoFileExists(t, socketPath) + require.NotNil(t, cmd.ProcessState) +} + +func TestCleanupEscalatesToSIGKILLAfterTermGrace(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "qemu.sock") + + cmd := exec.Command("sh", "-c", "trap '' TERM; echo ready; sleep 30 & wait") + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + proc, err := startManagedProcess(cmd, socketPath) + require.NoError(t, err) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + proc.termGrace = 50 * time.Millisecond + + proc.cleanup() + + assert.ErrorIs(t, syscall.Kill(proc.pid, 0), syscall.ESRCH, "SIGTERM-ignoring process must still be hard-killed") + require.NoFileExists(t, socketPath) +} diff --git a/lib/hypervisor/vfio.go b/lib/hypervisor/vfio.go new file mode 100644 index 000000000..1744cfc84 --- /dev/null +++ b/lib/hypervisor/vfio.go @@ -0,0 +1,7 @@ +package hypervisor + +import "time" + +// VFIOTermGrace allows VFIO teardown to finish after SIGTERM before SIGKILL; +// SIGKILL during initialization can wedge the VF, while teardown takes 1-2s. +const VFIOTermGrace = 5 * time.Second diff --git a/lib/instances/create.go b/lib/instances/create.go index b4b0952da..cd50deb5c 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -56,7 +56,7 @@ func wrapCreateVGPUErr(profile string, err error) error { if errors.Is(err, devices.ErrVGPUNotSupportedOnMacOS) { return fmt.Errorf("%w: %w", ErrInvalidRequest, err) } - return fmt.Errorf("create vGPU mdev for profile %s: %w", profile, err) + return fmt.Errorf("create vGPU for profile %s: %w", profile, err) } // generateVsockCID converts first 8 chars of instance ID to a unique CID @@ -277,11 +277,14 @@ func (m *manager) createInstance( var gpuFramework devices.VGPUFramework var gpuDevicePath string var gpuMdevUUID string + var gpuAssignedAt *time.Time + retention := vgpuRetention{instanceID: id} - // Setup cleanup stack early so device attachment errors trigger cleanup + // Deferred before cu.Clean so rollback records retention before this wraps the error. + defer func() { retErr = retention.wrapPending(retErr) }() cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) - m.deleteInstanceData(id) + m.persistVGPURetention(ctx, &retention) }) defer cu.Clean() @@ -297,9 +300,24 @@ func (m *manager) createInstance( // Handle vGPU profile request if req.GPU != nil && req.GPU.Profile != "" { + retentionStub := func() StoredMetadata { + return StoredMetadata{ + Id: id, + Name: req.Name, + Image: req.Image, + ResolvedImage: resolvedImageRef, + Platform: imageInfo.Platform, + CreatedAt: m.nowUTC(), + HypervisorType: hvType, + HypervisorVersion: hvVersion, + SocketPath: m.paths.InstanceSocket(id, starter.SocketName()), + DataDir: m.paths.InstanceDir(id), + } + } log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) - gpuDevice, err = devices.CreateVGPU(ctx, req.GPU.Profile, id) + gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) if err != nil { + retention.retainFromCreateError(retentionStub(), m.nowUTC(), err) log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } @@ -307,6 +325,8 @@ func (m *manager) createInstance( gpuFramework = gpuDevice.Framework gpuDevicePath = gpuDevice.SysfsPath gpuMdevUUID = gpuDevice.MdevUUID + assignedAt := m.nowUTC() + gpuAssignedAt = &assignedAt log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", gpuProfile, "uuid", gpuMdevUUID) // Add vGPU cleanup to stack @@ -318,8 +338,9 @@ func (m *manager) createInstance( MdevUUID: gpuDevice.MdevUUID, InstanceID: id, } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) + retention.retainFromDevice(retentionStub(), gpuDevice, *gpuAssignedAt) } }) } @@ -396,6 +417,7 @@ func (m *manager) createInstance( GPUFramework: gpuFramework, GPUDevicePath: gpuDevicePath, GPUMdevUUID: gpuMdevUUID, + GPUAssignedAt: gpuAssignedAt, Entrypoint: req.Entrypoint, Cmd: req.Cmd, SkipKernelHeaders: req.SkipKernelHeaders, diff --git a/lib/instances/create_mdev_test.go b/lib/instances/create_mdev_test.go index e6e4f55c5..05b3e9d8c 100644 --- a/lib/instances/create_mdev_test.go +++ b/lib/instances/create_mdev_test.go @@ -63,7 +63,7 @@ func TestWrapCreateVGPUErr(t *testing.T) { { name: "other vGPU error", err: errors.New("boom"), - wantMessage: "create vGPU mdev for profile profile: boom", + wantMessage: "create vGPU for profile profile: boom", }, } { t.Run(tc.name, func(t *testing.T) { diff --git a/lib/instances/delete.go b/lib/instances/delete.go index ad4a192d1..0474138d2 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -142,19 +142,13 @@ func (m *manager) deleteInstanceWithOptions( } m.closeFirecrackerUFFDSession(ctx, stored) - // 5b. Release the vGPU assignment if present, before any network, device, - // or volume teardown. Release failure is logged and the delete continues, - // matching the pre-refactor contract: the VMM is already confirmed dead, - // the guards inside the release never destroy a device they cannot prove - // is unowned, and a skipped release is recovered by startup - // reconciliation. + // Release before deleting metadata so a failed release can be retried safely. hadVGPUAssignment := storedVGPUDevicePath(stored) != "" if hadVGPUAssignment { log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } - if err := releaseStoredVGPU(ctx, stored); err != nil { - // Log error but continue with cleanup. - log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) + if err := m.releaseStoredVGPU(ctx, stored); err != nil { + log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup; the periodic vGPU reconcile releases it once free", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) } else if hadVGPUAssignment { if err := m.saveMetadata(meta); err != nil { log.WarnContext(ctx, "failed to save metadata after vGPU release", "instance_id", id, "error", err) @@ -242,7 +236,7 @@ func (m *manager) killHypervisor(ctx context.Context, inst *Instance) error { "instance_id", inst.Id, "stored_pid", *inst.HypervisorPID, "owner_pid", pid) } log.DebugContext(ctx, "killing hypervisor process", "instance_id", inst.Id, "pid", pid) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } diff --git a/lib/instances/fork.go b/lib/instances/fork.go index e0c778860..08ce4014a 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -219,6 +219,9 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin default: return nil, false, fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby)", ErrInvalidState, source.State) } + if stored.GPURetainedForCleanup { + return nil, false, errVGPURetentionStub + } if !supportValidated { if err := m.validateForkSupport(ctx, stored.HypervisorType); err != nil { diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 26bc6cfcb..c0bed670e 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -63,6 +63,29 @@ func TestForkInstanceClearsVGPUAssignment(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", source.GPUDevicePath) } +func TestForkInstanceRejectsVGPURetentionRecord(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + hvType := hypervisor.Type("fork-vgpu-retention-test") + hypervisor.RegisterCapabilities(hvType, hypervisor.Capabilities{SupportsConcurrentForkPrepare: true}) + manager.vmStarters[hvType] = concurrentForkPrepareTestStarter{} + + sourceID := "fork-vgpu-retention-source" + createStoppedSnapshotSourceFixture(t, manager, sourceID, sourceID, hvType) + + meta, err := manager.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, manager.saveMetadata(meta)) + + _, err = manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fork-vgpu-retention-copy"}) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") +} + func TestForkInstance_VZStoppedSourceSupported(t *testing.T) { t.Parallel() manager, _ := setupTestManager(t) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index a9b918dfb..77d15d733 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -3,6 +3,7 @@ package instances import ( "context" "errors" + "net" "os" "path/filepath" "sync" @@ -149,24 +150,6 @@ func TestLifecycleNoopStandbyWithOptionsStillRejectsStandbyInstance(t *testing.T assertNoLifecycleEvent(t, events) } -func TestDeleteContinuesWhenVGPUReleaseFails(t *testing.T) { - m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFramework("future-framework") - meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" - require.NoError(t, m.saveMetadata(meta)) - - // A failed release is logged and the delete continues, matching the - // pre-refactor contract; the leaked assignment is recovered by startup - // reconciliation. - require.NoError(t, m.DeleteInstance(context.Background(), id)) - - _, err = m.loadMetadata(id) - require.Error(t, err, "instance data must be deleted despite the failed release") -} - func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) var persisted *metadata @@ -195,6 +178,87 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { assert.Equal(t, restartpolicy.BlockedReasonManualStop, persisted.RestartStatus.BlockedReason) } +func TestDeleteReleasesRetainedCreateStub(t *testing.T) { + p := paths.New(t.TempDir()) + var destroyed []devices.VGPUAssignment + m := &manager{ + paths: p, + instanceLocks: sync.Map{}, + bootMarkerScans: sync.Map{}, + now: time.Now, + lifecycleEvents: newLifecycleSubscribers(), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + } + const id = "retained-stub" + require.NoError(t, m.ensureDirectories(id)) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + GPURetainedForCleanup: true, + }})) + + require.NoError(t, m.DeleteInstance(context.Background(), id)) + + require.Len(t, destroyed, 1) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", destroyed[0].DevicePath) + assert.Equal(t, id, destroyed[0].InstanceID) + _, err := m.loadMetadata(id) + require.Error(t, err, "retained stub must be fully deleted") +} + +func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { + now := time.Now().UTC() + m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, now) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkVendorVFIO + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + require.NoError(t, m.saveMetadata(meta)) + + claimantID := "inst-live-claimant" + require.NoError(t, m.ensureDirectories(claimantID)) + pid := os.Getpid() + // Bind under /tmp: a t.TempDir()-derived path exceeds the macOS AF_UNIX + // path limit. + socketDir, err := os.MkdirTemp("/tmp", "hypeman-claimant-socket-") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(socketDir) + }) + socketPath := filepath.Join(socketDir, "noop.sock") + listener, err := net.Listen("unix", socketPath) + require.NoError(t, err) + defer listener.Close() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: claimantID, + Name: claimantID, + Image: "test-image", + CreatedAt: now, + HypervisorType: lifecycleNoopHypervisorType, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, + SocketPath: socketPath, + DataDir: m.paths.InstanceDir(claimantID), + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + require.NoError(t, m.DeleteInstance(context.Background(), id)) + + _, err = m.loadMetadata(id) + require.Error(t, err, "deleted instance metadata should be gone") + claimant, err := m.loadMetadata(claimantID) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", claimant.GPUDevicePath, "live claimant keeps its assignment") +} + func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) deviceManager := &recordingDeviceManager{} @@ -239,42 +303,42 @@ func TestStartPersistsStaleVGPUReleaseImmediately(t *testing.T) { assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") } -func TestStopStoppedInstanceReleasesRetainedVGPU(t *testing.T) { +func TestStartRejectsVGPURetentionRecord(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" meta.GPUFramework = devices.VGPUFrameworkNone meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true require.NoError(t, m.saveMetadata(meta)) - inst, err := m.StopInstance(context.Background(), id) - require.NoError(t, err) - require.NotNil(t, inst) - assert.Equal(t, StateStopped, inst.State) + _, err = m.StartInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") stored, err := m.loadMetadata(id) require.NoError(t, err) - assert.Empty(t, stored.GPUDevicePath) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) } -func TestStopStoppedInstanceVGPUReleaseFailureRemainsNoop(t *testing.T) { +func TestStopStoppedInstanceLeavesVGPUForReconcile(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) meta, err := m.loadMetadata(id) require.NoError(t, err) meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUFramework = devices.VGPUFrameworkNone meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" require.NoError(t, m.saveMetadata(meta)) + // Stop on an already-stopped instance is a no-op for the assignment; the + // periodic reconcile retries the release. inst, err := m.StopInstance(context.Background(), id) require.NoError(t, err) require.NotNil(t, inst) assert.Equal(t, StateStopped, inst.State) - stored, err := m.loadMetadata(id) require.NoError(t, err) - assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) } @@ -300,6 +364,20 @@ func (m *recordingDeviceManager) UnbindFromVFIO(ctx context.Context, id string) return nil } +func TestLifecycleNoopStandbyRejectsVendorVFIOVGPU(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateRunning, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkVendorVFIO + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + require.NoError(t, m.saveMetadata(meta)) + + _, err = m.StandbyInstance(context.Background(), id, StandbyInstanceRequest{}) + require.ErrorIs(t, err, ErrInvalidState) + assert.ErrorContains(t, err, "standby is not supported for instances with vGPU attached") +} + func newLifecycleNoopManagerWithInstance(t *testing.T, state State, now time.Time) (*manager, string) { t.Helper() diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 28ac78f61..5f68d8f9a 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -2,8 +2,10 @@ package instances import ( "context" + "errors" "fmt" "os" + "path/filepath" "strings" "sync" "time" @@ -180,6 +182,9 @@ type manager struct { now func() time.Time writeFile func(string, []byte, os.FileMode) error deleteInstanceFn func(context.Context, string) error + createVGPU func(context.Context, string, string) (*devices.VGPUDevice, error) + destroyVGPU func(context.Context, devices.VGPUAssignment) error + reconcileVGPUDevices func(context.Context, map[string]struct{}, bool) error deleteSnapshotFn func(context.Context, string) error ttlReaperDeleteTimeout time.Duration egressProxy *egressproxy.Service @@ -208,6 +213,13 @@ type manager struct { // Periodic TAP garbage collection reconciler. tapGCOnce sync.Once + // Periodic vGPU reconciler. + vgpuReconcileOnce sync.Once + vgpuReconcileInterval time.Duration + discoverVGPU func() (devices.VGPUFramework, []devices.VirtualFunction, error) + + vfioTermGrace time.Duration + // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter defaultHypervisor hypervisor.Type // Default hypervisor type when not specified in request @@ -636,12 +648,6 @@ func (m *manager) StopInstance(ctx context.Context, id string) (*Instance, error if err := m.markRestartManualStopLocked(ctx, id); err != nil { return nil, err } - // A stopped instance can retain a vGPU assignment when the release - // failed during the original stop. Retry it here so the vGPU slot is - // not held until the next start, delete, or hypeman restart. A failed - // retry only logs, keeping stop's no-op contract for already-stopped - // instances. - m.releaseRetainedVGPULocked(ctx, id) updated, err := m.currentInstanceWithoutHydration(ctx, id) if err != nil { return nil, err @@ -720,6 +726,26 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } +func (m *manager) listMetadataForReconcile() ([]StoredMetadata, error) { + files, err := m.listMetadataFilesStrict() + if err != nil { + return nil, err + } + result := make([]StoredMetadata, 0, len(files)) + for _, file := range files { + id := filepath.Base(filepath.Dir(file)) + meta, err := m.loadMetadata(id) + if err != nil { + if errors.Is(err, ErrNotFound) { + continue + } + return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) + } + result = append(result, meta.StoredMetadata) + } + return result, nil +} + // ListInstances returns instances, optionally filtered by the given criteria. // Pass nil to return all instances. func (m *manager) ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) { diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 1ada5ac1e..e14deb49b 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -73,6 +73,20 @@ type lifecycleEventDropReason string const lifecycleEventDropReasonBufferFull lifecycleEventDropReason = "buffer_full" +type vgpuReconcileStage string + +const ( + vgpuReconcileStageListInstances vgpuReconcileStage = "list_instances" + vgpuReconcileStageReconcileDevices vgpuReconcileStage = "reconcile_devices" +) + +type vgpuRetentionOperation string + +const ( + vgpuRetentionOperationCreate vgpuRetentionOperation = "create" + vgpuRetentionOperationStart vgpuRetentionOperation = "start" +) + // Metrics holds the metrics instruments for instance operations. type Metrics struct { createDuration metric.Float64Histogram @@ -94,6 +108,9 @@ type Metrics struct { lifecycleEventsDroppedTotal metric.Int64Counter forkMemFileShareFallbacksTotal metric.Int64Counter ttlReaperDeletionsTotal metric.Int64Counter + vgpuReconcileFailuresTotal metric.Int64Counter + vgpuStaleReleaseFailuresTotal metric.Int64Counter + vgpuAssignmentsRetainedTotal metric.Int64Counter tracer trace.Tracer } @@ -270,6 +287,30 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + vgpuReconcileFailuresTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_reconcile_failures_total", + metric.WithDescription("Total number of vGPU reconcile pass stages that failed, leaving stale assignments or device leftovers allocated while /resources still advertises the capacity"), + ) + if err != nil { + return nil, err + } + + vgpuStaleReleaseFailuresTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_stale_release_failures_total", + metric.WithDescription("Total number of stale vGPU assignment releases that failed, keeping the VF allocated until a later reconcile pass succeeds"), + ) + if err != nil { + return nil, err + } + + vgpuAssignmentsRetainedTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_assignments_retained_total", + metric.WithDescription("Total number of failed rollbacks that left a vGPU assignment behind, by whether the retention record the periodic reconcile needs was persisted"), + ) + if err != nil { + return nil, err + } + // Register observable gauge for instance counts by state instancesTotal, err := meter.Int64ObservableGauge( "hypeman_instances_total", @@ -464,6 +505,9 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M lifecycleEventsDroppedTotal: lifecycleEventsDroppedTotal, forkMemFileShareFallbacksTotal: forkMemFileShareFallbacksTotal, ttlReaperDeletionsTotal: ttlReaperDeletionsTotal, + vgpuReconcileFailuresTotal: vgpuReconcileFailuresTotal, + vgpuStaleReleaseFailuresTotal: vgpuStaleReleaseFailuresTotal, + vgpuAssignmentsRetainedTotal: vgpuAssignmentsRetainedTotal, tracer: tracer, }, nil } @@ -563,6 +607,32 @@ func (m *manager) recordTimeToRunning(ctx context.Context, stored *StoredMetadat m.metrics.timeToRunning.Record(ctx, duration, metric.WithAttributes(attrs...)) } +func (m *manager) recordVGPUReconcileFailure(ctx context.Context, stage vgpuReconcileStage) { + if m.metrics == nil { + return + } + m.metrics.vgpuReconcileFailuresTotal.Add(ctx, 1, metric.WithAttributes( + attribute.String("stage", string(stage)), + )) +} + +func (m *manager) recordVGPUStaleReleaseFailure(ctx context.Context) { + if m.metrics == nil { + return + } + m.metrics.vgpuStaleReleaseFailuresTotal.Add(ctx, 1) +} + +func (m *manager) recordVGPURetainedAssignment(ctx context.Context, operation vgpuRetentionOperation, persisted bool) { + if m.metrics == nil { + return + } + m.metrics.vgpuAssignmentsRetainedTotal.Add(ctx, 1, metric.WithAttributes( + attribute.String("operation", string(operation)), + attribute.String("persisted", strconv.FormatBool(persisted)), + )) +} + // recordStateTransition records a state transition with hypervisor label. func (m *manager) recordStateTransition(ctx context.Context, fromState, toState string, hvType hypervisor.Type) { if m.metrics == nil { diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go index 811571e36..525002cbb 100644 --- a/lib/instances/metrics_test.go +++ b/lib/instances/metrics_test.go @@ -2,6 +2,7 @@ package instances import ( "context" + "errors" "os" "path/filepath" "testing" @@ -556,6 +557,50 @@ func TestEnsureSnapshotMemoryReadySkipsPendingCompressionWithoutPreemptionMetric assert.False(t, metricExists(rm, "hypeman_snapshot_compression_preemptions_total"), "pending-delay cancellation should not record a preemption") } +func TestVGPUReconcileFailureMetric_RecordStages(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + + m := &manager{ + paths: paths.New(t.TempDir()), + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { + return errors.New("sweep failed") + }, + } + metrics, err := newInstanceMetrics(provider.Meter("test"), nil, m) + require.NoError(t, err) + m.metrics = metrics + + const id = "unreadable" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{Id: id}})) + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o000)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + m.ReconcileVGPUs(t.Context()) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + + failuresMetric := findMetric(t, rm, "hypeman_instances_vgpu_reconcile_failures_total") + failures, ok := failuresMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, failures.DataPoints, 2) + for _, point := range failures.DataPoints { + switch metricLabel(t, point.Attributes, "stage") { + case "list_instances", "reconcile_devices": + assert.Equal(t, int64(1), point.Value) + default: + t.Fatalf("unexpected reconcile failure stage datapoint: %s", metricLabel(t, point.Attributes, "stage")) + } + } +} + func assertMetricNames(t *testing.T, rm metricdata.ResourceMetrics, expected []string) { t.Helper() diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 1d885d0b5..7e8500402 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -1,6 +1,7 @@ package instances import ( + "context" "errors" "fmt" "os" @@ -13,6 +14,7 @@ import ( "time" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/logger" ) // linuxBootIDPath is the kernel-provided boot ID used to scope process @@ -25,6 +27,25 @@ const linuxBootIDPath = "/proc/sys/kernel/random/boot_id" // does not unstick it, so the wait is short to keep stop and delete fast. const hypervisorSIGKILLWaitTimeout = 2 * time.Second +func (m *manager) vfioTerminationGrace() time.Duration { + if m.vfioTermGrace > 0 { + return m.vfioTermGrace + } + return hypervisor.VFIOTermGrace +} + +// SIGKILL during guest driver init can wedge a VF until the parent GPU is reset. +func (m *manager) terminateThenKill(ctx context.Context, inst *Instance, pid int) error { + if inst.GPUProfile != "" || len(inst.Devices) > 0 { + if syscall.Kill(pid, syscall.SIGTERM) == nil && WaitForProcessExit(pid, m.vfioTerminationGrace()) { + return nil + } + logger.FromContext(ctx).WarnContext(ctx, "hypervisor with VFIO devices did not exit on SIGTERM; hard-killing, device may wedge if the guest driver was initializing", + "instance_id", inst.Id, "device_path", inst.GPUDevicePath) + } + return killProcessAndWait(pid) +} + // killProcessAndWait SIGKILLs pid and waits for it to exit. A process that // survives the first wait gets its process group killed too (the hypervisor // may have spawned children in its own group) and a short grace period. An @@ -180,6 +201,12 @@ func classifyResolvedHypervisorOwner(socketPath string, stored, resolved int, er return 0, fmt.Errorf("cannot confirm ownership of socket %s: %w", socketPath, err) } +// Ambiguous ownership is treated as live; this must not authorize teardown. +func hypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { + pid, err := resolveLiveHypervisorPID(id, socketPath) + return err != nil || pid > 0 +} + // ProcessExists reports whether pid belongs to a live, non-zombie process. func ProcessExists(pid int) bool { if pid <= 0 { diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 7450ed112..2ec201450 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -17,7 +17,9 @@ import ( "testing" "time" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -508,6 +510,47 @@ func TestRefreshHypervisorPIDResolvesSocketOwnerWhenStoredPIDIsDead(t *testing.T assert.Equal(t, hostBootID(), stored.HypervisorBootID) } +func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "test.sock") + owner := exec.Command(os.Args[0], "-test.run=^TestSocketListenerHelper$") + owner.Env = append(os.Environ(), "HYPERVISOR_SOCKET_HELPER=1", "HYPERVISOR_SOCKET_PATH="+socketPath) + stdin, err := owner.StdinPipe() + require.NoError(t, err) + stdout, err := owner.StdoutPipe() + require.NoError(t, err) + require.NoError(t, owner.Start()) + t.Cleanup(func() { + _ = stdin.Close() + _ = owner.Process.Kill() + _ = owner.Wait() + }) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + + stale := exec.Command("sleep", "30") + require.NoError(t, stale.Start()) + t.Cleanup(func() { + _ = stale.Process.Kill() + _ = stale.Wait() + }) + + m := &manager{paths: paths.New(t.TempDir())} + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" + stalePID := stale.Process.Pid + require.NoError(t, m.ensureDirectories("live-claimant")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "live-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &stalePID}, + SocketPath: socketPath, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance("other-instance", devicePath) + require.NoError(t, err) + assert.True(t, claimed) +} + func TestKillHypervisorSurvivesConcurrentReaper(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "test.sock") process := exec.Command(os.Args[0], "-test.run=^TestSocketListenerHelper$") @@ -648,3 +691,84 @@ func TestResolveRuntimeHypervisorPIDMintsIdentityOnlyWhenConfirmed(t *testing.T) assert.Empty(t, stored.HypervisorBootID, "a dead fallback must not mint the identity token") }) } + +func startTrapProcess(t *testing.T, trapAction string) (int, HypervisorProcessIdentity) { + t.Helper() + script := fmt.Sprintf("trap '%s' TERM; echo ready; sleep 30 & wait", trapAction) + process := exec.Command("sh", "-c", script) + stdout, err := process.StdoutPipe() + require.NoError(t, err) + require.NoError(t, process.Start()) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + waitDone := make(chan error, 1) + go func() { waitDone <- process.Wait() }() + t.Cleanup(func() { + _ = process.Process.Kill() + <-waitDone + }) + + pid := process.Process.Pid + startTime := processStartTime(pid) + require.NotZero(t, startTime) + return pid, HypervisorProcessIdentity{HypervisorPID: &pid, HypervisorStartTime: startTime, HypervisorBootID: hostBootID()} +} + +func TestKillHypervisorSIGTERMsVGPUHypervisor(t *testing.T) { + markerPath := filepath.Join(t.TempDir(), "terminated") + pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateInitializing, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + GPUProfile: "NVIDIA L40S-1Q", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + + require.Eventually(t, func() bool { + return syscall.Kill(pid, 0) == syscall.ESRCH + }, 5*time.Second, 10*time.Millisecond) + assert.FileExists(t, markerPath, "hypervisor must be given SIGTERM, not SIGKILL, during vGPU driver init") +} + +func TestKillHypervisorEscalatesToSIGKILLWhenSIGTERMIgnored(t *testing.T) { + pid, identity := startTrapProcess(t, "") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{vfioTermGrace: 50 * time.Millisecond} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateInitializing, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + GPUProfile: "NVIDIA L40S-1Q", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + + assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH, "SIGTERM-ignoring hypervisor must still be hard-killed") +} + +func TestKillHypervisorHardKillsNonVGPUHypervisor(t *testing.T) { + markerPath := filepath.Join(t.TempDir(), "terminated") + pid, identity := startTrapProcess(t, "touch "+markerPath+"; exit 0") + socketPath := filepath.Join(t.TempDir(), "missing.sock") + + m := &manager{} + require.NoError(t, m.killHypervisor(context.Background(), &Instance{ + State: StateRunning, + StoredMetadata: StoredMetadata{ + Id: "kill-test", + HypervisorProcessIdentity: identity, + SocketPath: socketPath, + }, + })) + + assert.ErrorIs(t, syscall.Kill(pid, 0), syscall.ESRCH) + assert.NoFileExists(t, markerPath, "non-vGPU hypervisors keep the direct SIGKILL path") +} diff --git a/lib/instances/query.go b/lib/instances/query.go index 8e3ed61f1..0621460da 100644 --- a/lib/instances/query.go +++ b/lib/instances/query.go @@ -784,7 +784,7 @@ func parseSentinelTimestamp(line, sentinelPrefix string) (time.Time, bool) { return time.Time{}, false } -// listInstances returns all instances +// listInstances returns all instances, skipping metadata files that cannot be loaded. func (m *manager) listInstances(ctx context.Context) ([]Instance, error) { ctx, span := m.tracerOrDefault().Start(ctx, "instances.list_metadata") defer span.End() diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 8bb0bb464..222e85a30 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -14,6 +14,67 @@ import ( "github.com/stretchr/testify/require" ) +func TestListMetadataForReconcileFailsOnInvalidMetadata(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + + require.NoError(t, m.ensureDirectories("valid")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "valid", + Name: "valid", + CreatedAt: time.Now(), + DataDir: m.paths.InstanceDir("valid"), + }})) + require.NoError(t, m.ensureDirectories("invalid")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid"), []byte("{"), 0644)) + + listed, err := m.ListInstances(context.Background(), nil) + require.NoError(t, err) + require.Len(t, listed, 1) + + _, err = m.listMetadataForReconcile() + require.Error(t, err) + assert.ErrorContains(t, err, "load metadata for instance invalid") + + require.NoError(t, os.Remove(m.paths.InstanceMetadata("invalid"))) + metadata, err := m.listMetadataForReconcile() + require.NoError(t, err) + require.Len(t, metadata, 1) + assert.Equal(t, "valid", metadata[0].Id) +} + +func TestListMetadataForReconcileSkipsInstanceDeletedDuringListing(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + + for _, id := range []string{"aaa-ghost", "zzz-live"} { + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + CreatedAt: time.Now(), + DataDir: m.paths.InstanceDir(id), + }})) + } + + unlock := hypervisor.LockSnapshotSourceAliasMutation() + type result struct { + metadata []StoredMetadata + err error + } + done := make(chan result, 1) + go func() { + metadata, err := m.listMetadataForReconcile() + done <- result{metadata, err} + }() + time.Sleep(100 * time.Millisecond) + require.NoError(t, os.Remove(m.paths.InstanceMetadata("aaa-ghost"))) + unlock() + + res := <-done + require.NoError(t, res.err) + require.Len(t, res.metadata, 1) + assert.Equal(t, "zzz-live", res.metadata[0].Id) +} + func TestParseExitSentinelLine(t *testing.T) { t.Parallel() tests := []struct { diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 2d2676c72..a87acb614 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -66,6 +66,9 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata + if stored.GPURetainedForCleanup { + return nil, errVGPURetentionStub + } if err := validateForkVolumeSafety(stored.Volumes); err != nil { return nil, fmt.Errorf("%w: snapshot requires readonly volume attachments: %v", ErrNotSupported, err) } @@ -263,6 +266,9 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str if sourceInst.State == StateRunning { return nil, fmt.Errorf("%w: cannot restore snapshot while source is %s", ErrInvalidState, sourceInst.State) } + if sourceMeta.GPURetainedForCleanup { + return nil, errVGPURetentionStub + } targetState, err := resolveSnapshotTargetState(rec.Snapshot.Kind, req.TargetState) if err != nil { @@ -312,6 +318,7 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str restored.GPUFramework = sourceMeta.GPUFramework restored.GPUDevicePath = sourceMeta.GPUDevicePath restored.GPUMdevUUID = sourceMeta.GPUMdevUUID + restored.GPUAssignedAt = sourceMeta.GPUAssignedAt restored.HypervisorType = targetHypervisor restored.HypervisorVersion = targetHypervisorVersion restored.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index b23e364ee..31f0a4343 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -52,6 +52,63 @@ func TestForkSnapshotClearsVGPUAssignment(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", source.GPUDevicePath) } +func TestCreateSnapshotRejectsVGPURetentionRecord(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "snapshot-vgpu-retention" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, mgr.defaultHypervisor) + + meta, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, mgr.saveMetadata(meta)) + + _, err = mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "snapshot-vgpu-retention", + }) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") +} + +func TestRestoreSnapshotRejectsVGPURetentionRecord(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "snapshot-vgpu-restore-retention" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, mgr.defaultHypervisor) + + snapshot, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "snapshot-vgpu-restore-retention", + }) + require.NoError(t, err) + + meta, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, mgr.saveMetadata(meta)) + + _, err = mgr.RestoreSnapshot(ctx, sourceID, snapshot.Id, RestoreSnapshotRequest{ + TargetState: StateStopped, + TargetHypervisor: mgr.defaultHypervisor, + }) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") + + stored, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + assert.True(t, stored.GPURetainedForCleanup) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) +} + func TestRestoreSnapshotDoesNotResurrectStaleVGPUAssignment(t *testing.T) { mgr, _ := setupTestManager(t) ctx := context.Background() @@ -113,6 +170,8 @@ func TestRestoreSnapshotKeepsCurrentVGPUAssignment(t *testing.T) { meta.GPUFramework = devices.VGPUFramework("future-framework") meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" meta.GPUMdevUUID = "retained-uuid" + assignedAt := time.Now().UTC().Truncate(time.Second) + meta.GPUAssignedAt = &assignedAt require.NoError(t, mgr.saveMetadata(meta)) _, err = mgr.RestoreSnapshot(ctx, sourceID, snapshot.Id, RestoreSnapshotRequest{ @@ -126,6 +185,8 @@ func TestRestoreSnapshotKeepsCurrentVGPUAssignment(t *testing.T) { assert.Equal(t, devices.VGPUFramework("future-framework"), restored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", restored.GPUDevicePath) assert.Equal(t, "retained-uuid", restored.GPUMdevUUID) + require.NotNil(t, restored.GPUAssignedAt) + assert.True(t, assignedAt.Equal(*restored.GPUAssignedAt)) } func TestStoppedSnapshotLifecycleAndForkAfterSourceDeletion(t *testing.T) { diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 4c6fd7d33..5c6e0173e 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -376,7 +376,7 @@ func (m *manager) shutdownHypervisor(ctx context.Context, inst *Instance) error // alive; teardown is committed, so kill it rather than report a // completed shutdown for a VMM that is still running. log.WarnContext(ctx, "could not connect to hypervisor, force killing resolved owner", "instance_id", inst.Id, "pid", pid, "error", err) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } @@ -405,13 +405,13 @@ func (m *manager) shutdownHypervisor(ctx context.Context, inst *Instance) error log.DebugContext(ctx, "hypervisor shutdown gracefully", "instance_id", inst.Id, "pid", pid) } else { log.WarnContext(ctx, "hypervisor did not exit gracefully in time, force killing process", "instance_id", inst.Id, "pid", pid) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } } else { log.DebugContext(ctx, "skipping graceful exit wait; force killing hypervisor process", "instance_id", inst.Id, "pid", pid) - if err := killProcessAndWait(pid); err != nil { + if err := m.terminateThenKill(ctx, inst, pid); err != nil { return err } } diff --git a/lib/instances/start.go b/lib/instances/start.go index a6c832450..37b836427 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/egressproxy" "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/logger" @@ -48,13 +47,17 @@ func (m *manager) startInstance( log.ErrorContext(ctx, "invalid state for start", "instance_id", id, "state", inst.State) return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } + if stored.GPURetainedForCleanup { + log.ErrorContext(ctx, "refusing to start vGPU retention record", "instance_id", id) + return nil, errVGPURetentionStub + } // Release any assignment retained by an earlier failed release and // persist the cleared fields immediately, so a failure later in start // cannot leave on-disk metadata pointing at a device that is already - // gone (matching releaseRetainedVGPULocked). + // gone. if storedVGPUDevicePath(stored) != "" { - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.ErrorContext(ctx, "failed to release stale vGPU before start", "instance_id", id, "error", err) return nil, fmt.Errorf("release stale vGPU before start: %w", err) } @@ -64,6 +67,12 @@ func (m *manager) startInstance( } } + stored.HypervisorPID = nil + stored.HypervisorStartTime = 0 + stored.HypervisorBootID = "" + rollbackMeta := *meta + rollbackMeta.Phases = meta.Phases.Clone() + // 2a. Clear stale exit info from previous run and apply command overrides stored.ExitCode = nil stored.ExitMessage = "" @@ -111,6 +120,9 @@ func (m *manager) startInstance( } // Setup cleanup stack for automatic rollback on errors + retention := vgpuRetention{instanceID: id} + // Deferred before cu.Clean so rollback records retention before this wraps the error. + defer func() { retErr = retention.wrapPending(retErr) }() cu := cleanup.Make(func() {}) defer cu.Clean() @@ -162,27 +174,40 @@ func (m *manager) startInstance( // 4b. Recreate the vGPU if this instance had a GPU profile // Note: GPU availability was already validated in step 2b if stored.GPUProfile != "" { - log.InfoContext(ctx, "creating vGPU mdev for start", "instance_id", id, "profile", stored.GPUProfile) - device, err := devices.CreateVGPU(ctx, stored.GPUProfile, id) + log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) + device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id) if err != nil { log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) - return nil, fmt.Errorf("create vGPU mdev for profile %s: %w", stored.GPUProfile, err) + wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) + if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok { + retentionMeta := rollbackMeta + setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, m.nowUTC()) + persisted := true + if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) + wrapped = fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr) + persisted = false + } + retention.markRetained(persisted) + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, persisted) + } + return nil, wrapped } - setStoredVGPUDevice(stored, device) + assignedAt := m.nowUTC() + setStoredVGPUDevice(stored, device, assignedAt) log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) // Add vGPU cleanup to stack cu.Add(func() { - log.DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID) - assignment := devices.VGPUAssignment{ - Framework: device.Framework, - DevicePath: device.SysfsPath, - MdevUUID: device.MdevUUID, - InstanceID: id, - } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { - log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) + retained, persisted := m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) + if retained { + retention.markRetained(persisted) + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationStart, persisted) } }) + if err := m.saveMetadata(meta); err != nil { + log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) + return nil, fmt.Errorf("save metadata after vGPU creation: %w", err) + } } // 5. Regenerate config disk with new network configuration diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 7eddeb034..ebcf58708 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -232,7 +232,7 @@ func (m *manager) stopInstance( // 7. Release the vGPU assignment if present (frees the vGPU slot for other VMs). if path := storedVGPUDevicePath(stored); path != "" { log.InfoContext(ctx, "destroying vGPU on stop", "instance_id", id, "device_path", path) - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on stop; retaining assignment metadata", "instance_id", id, "device_path", path, "error", err) } } diff --git a/lib/instances/storage.go b/lib/instances/storage.go index a293fc6e1..1a4d325b0 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -187,8 +187,15 @@ func removeAllWithRetry(path string, removeAll func(string) error, sleep func(ti } } -// listMetadataFiles returns paths to all instance metadata files func (m *manager) listMetadataFiles() ([]string, error) { + return m.walkMetadataFiles(false) +} + +func (m *manager) listMetadataFilesStrict() ([]string, error) { + return m.walkMetadataFiles(true) +} + +func (m *manager) walkMetadataFiles(failOnStatError bool) ([]string, error) { guestsDir := m.paths.GuestsDir() // Ensure guests directory exists @@ -210,6 +217,8 @@ func (m *manager) listMetadataFiles() ([]string, error) { metaPath := filepath.Join(guestsDir, entry.Name(), "metadata.json") if _, err := os.Stat(metaPath); err == nil { metaFiles = append(metaFiles, metaPath) + } else if failOnStatError && !os.IsNotExist(err) { + return nil, fmt.Errorf("stat metadata for instance %s: %w", entry.Name(), err) } } diff --git a/lib/instances/types.go b/lib/instances/types.go index 6aac15985..ed32f5b5d 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -151,10 +151,12 @@ type StoredMetadata struct { Devices []string // Device IDs attached to this instance // GPU configuration (vGPU mode) - GPUProfile string // vGPU profile name (e.g., "L40S-1Q") - GPUFramework devices.VGPUFramework - GPUDevicePath string - GPUMdevUUID string // populated for mdev-backed vGPUs + GPUProfile string // vGPU profile name (e.g., "L40S-1Q") + GPUFramework devices.VGPUFramework + GPUDevicePath string + GPUMdevUUID string // populated for mdev-backed vGPUs + GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection + GPURetainedForCleanup bool // delete-only stub holding a vGPU assignment // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index a8ca6aceb..00e32ba01 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -2,63 +2,168 @@ package instances import ( "context" + "errors" + "fmt" "path/filepath" + "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/logger" ) -func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice) { +// VGPUCleanupPendingError reports a failed rollback that left a vGPU assigned. +type VGPUCleanupPendingError struct { + InstanceID string + Retained bool + Err error +} + +func (e *VGPUCleanupPendingError) Error() string { + if e.Retained { + return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) + } + return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the periodic vGPU reconcile retries the release", e.Err, e.InstanceID) +} + +func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } + +var errVGPURetentionStub = fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + +func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID string) (*devices.VGPUDevice, error) { + create := m.createVGPU + if create == nil { + create = devices.CreateVGPU + } + return create(ctx, profileName, instanceID) +} + +func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { + var pending *devices.VGPUCreateCleanupPendingError + if !errors.As(err, &pending) { + return nil, false + } + return &pending.Device, true +} + +func vgpuAssignmentMayBeLive(stored *StoredMetadata, now time.Time, hypervisorLive bool) bool { + return hypervisorLive || + stored.GPUAssignedAt != nil && now.Sub(*stored.GPUAssignedAt) < devices.VGPUAssignmentGracePeriod +} + +func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { + destroy := m.destroyVGPU + if destroy == nil { + destroy = devices.DestroyVGPU + } + return destroy(ctx, assignment) +} + +func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) { stored.GPUFramework = device.Framework stored.GPUDevicePath = device.SysfsPath stored.GPUMdevUUID = device.MdevUUID + stored.GPUAssignedAt = &assignedAt } func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUFramework = devices.VGPUFrameworkNone stored.GPUDevicePath = "" stored.GPUMdevUUID = "" + stored.GPUAssignedAt = nil } -func releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { +func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) { + logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID) + releaseErr := m.destroyVGPUAssignment(ctx, devices.VGPUAssignment{ + Framework: device.Framework, + DevicePath: device.SysfsPath, + MdevUUID: device.MdevUUID, + InstanceID: instanceID, + }) + if releaseErr != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID, "error", releaseErr) + setStoredVGPUDevice(&rollbackMeta.StoredMetadata, device, assignedAt) + retained = true + } + if err := m.saveMetadata(&rollbackMeta); err != nil { + message := "failed to save metadata after vGPU cleanup" + if retained { + message = "failed to retain vGPU assignment metadata after cleanup failure" + } + logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err) + if !retained { + return false, false + } + // The pre-cleanup assignment save may already hold this claim on disk, in + // which case the retention is durable despite the failed rollback save. + if onDisk, loadErr := m.loadMetadata(instanceID); loadErr == nil && + onDisk.GPUDevicePath == device.SysfsPath && + onDisk.GPUAssignedAt != nil && onDisk.GPUAssignedAt.Equal(assignedAt) { + return true, true + } + return true, false + } + return retained, retained +} + +func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { - assignment := devices.VGPUAssignment{ - Framework: stored.GPUFramework, - DevicePath: path, - MdevUUID: stored.GPUMdevUUID, - InstanceID: stored.Id, + // Vendor VFIO VFs are reusable, so release fails closed on an incomplete inventory. + claimed := false + if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { + var err error + claimed, err = m.vgpuAssignmentClaimedByLiveInstance(stored.Id, path) + if err != nil { + return err + } } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { - return err + if claimed { + logger.FromContext(ctx).WarnContext(ctx, "dropping stale vGPU assignment claimed by another live instance", + "instance_id", stored.Id, "device_path", path) + } else { + assignment := devices.VGPUAssignment{ + Framework: stored.GPUFramework, + DevicePath: path, + MdevUUID: stored.GPUMdevUUID, + InstanceID: stored.Id, + } + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { + return err + } } } clearStoredVGPUDevice(stored) return nil } -// releaseRetainedVGPULocked releases a vGPU assignment retained on a stopped -// instance after a failed release during the original stop. It is a no-op -// when no assignment is retained, and a failed retry only logs so the -// metadata stays for the next retry. The caller must hold the instance lock. -func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) { - log := logger.FromContext(ctx) - meta, err := m.loadMetadata(id) +func (m *manager) vgpuAssignmentClaimedByLiveInstance(excludeID, devicePath string) (bool, error) { + // Each vendor VFIO release lists and loads every instance's metadata. This is + // acceptable at GPU-host scale (tens of VFs and instances); revisit at hundreds. + allMetadata, err := m.listMetadataForReconcile() if err != nil { - log.WarnContext(ctx, "failed to load metadata for retained vGPU release", "instance_id", id, "error", err) - return - } - stored := &meta.StoredMetadata - if storedVGPUDevicePath(stored) == "" { - return + return false, fmt.Errorf("list instances for vGPU release check: %w", err) } - if err := releaseStoredVGPU(ctx, stored); err != nil { - log.WarnContext(ctx, "failed to destroy retained vGPU; retaining assignment metadata", "instance_id", id, "error", err) - return - } - if err := m.saveMetadata(meta); err != nil { - log.WarnContext(ctx, "failed to save metadata after retained vGPU release", "instance_id", id, "error", err) + for i := range allMetadata { + stored := &allMetadata[i] + if stored.Id == excludeID || storedVGPUDevicePath(stored) != devicePath { + continue + } + pid, err := resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) + if err != nil { + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", stored.Id, devicePath, err) + } + if pid > 0 { + return true, nil + } + if vgpuAssignmentMayBeLive(stored, m.nowUTC(), false) { + if stored.HypervisorPID == nil { + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", stored.Id, devicePath) + } + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", stored.Id, devicePath) + } } + return false, nil } func storedVGPUDevicePath(stored *StoredMetadata) string { diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go new file mode 100644 index 000000000..27ecb9e01 --- /dev/null +++ b/lib/instances/vgpu_reconcile.go @@ -0,0 +1,134 @@ +package instances + +import ( + "context" + "errors" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" +) + +const defaultVGPUReconcileInterval = time.Minute + +// StartVGPUReconciler runs one reconcile pass and then keeps reconciling +// periodically until ctx is cancelled. Hosts without a vGPU framework skip +// reconciliation entirely. A discovery failure starts the reconciler anyway: +// a transient sysfs error must not disable cleanup on a GPU host. +func (m *manager) StartVGPUReconciler(ctx context.Context) { + discover := m.discoverVGPU + if discover == nil { + discover = devices.DiscoverVGPU + } + framework, _, err := discover() + if err == nil && framework == devices.VGPUFrameworkNone { + return + } + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU framework; starting vGPU reconciler anyway", "error", err) + } + m.ReconcileVGPUs(ctx) + m.vgpuReconcileOnce.Do(func() { + interval := m.vgpuReconcileInterval + if interval <= 0 { + interval = defaultVGPUReconcileInterval + } + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + m.ReconcileVGPUs(ctx) + } + } + }() + }) +} + +// ReconcileVGPUs runs one fail-closed reconcile pass: stale instance-held +// assignments are released, then device-level leftovers not claimed by a live +// instance are swept. Failures only log; the next periodic pass retries. +func (m *manager) ReconcileVGPUs(ctx context.Context) { + log := logger.FromContext(ctx) + protected, err := m.reconcileVGPUAssignments(ctx) + sweepDevices := err == nil + if err != nil { + m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageListInstances) + log.ErrorContext(ctx, "failed to list instances for vGPU reconcile protection; skipping device sweep until the next pass", "error", err) + protected = make(map[string]struct{}) + } + reconcileDevices := m.reconcileVGPUDevices + if reconcileDevices == nil { + reconcileDevices = devices.ReconcileVGPUs + } + if err := reconcileDevices(ctx, protected, sweepDevices); err != nil { + m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageReconcileDevices) + log.WarnContext(ctx, "failed to reconcile vGPU devices", "error", err) + } +} + +// reconcileVGPUAssignments retries releases for assignments whose owner is no +// longer live and returns the device paths still protected by live instances. +// Listing fails closed: any unreadable metadata aborts the pass so the vendor +// VFIO sweep cannot clear a VF whose claim it could not read. +func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]struct{}, error) { + allMetadata, err := m.listMetadataForReconcile() + if err != nil { + return nil, err + } + protected := make(map[string]struct{}) + for i := range allMetadata { + stored := &allMetadata[i] + devicePath := storedVGPUDevicePath(stored) + if devicePath == "" { + continue + } + hypervisorLive := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if vgpuAssignmentMayBeLive(stored, m.nowUTC(), hypervisorLive) { + protected[devicePath] = struct{}{} + continue + } + m.releaseStaleVGPUAssignment(ctx, stored.Id) + } + return protected, nil +} + +// releaseStaleVGPUAssignment retries a release that previously failed, under +// the instance lock. Liveness is re-verified after locking so a concurrent +// start or restore keeps its assignment. A failed release only logs and keeps +// the metadata for the next pass. +func (m *manager) releaseStaleVGPUAssignment(ctx context.Context, id string) { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + log := logger.FromContext(ctx) + meta, err := m.loadMetadata(id) + if err != nil { + if !errors.Is(err, ErrNotFound) { + log.WarnContext(ctx, "failed to load metadata for stale vGPU release", "instance_id", id, "error", err) + } + return + } + stored := &meta.StoredMetadata + path := storedVGPUDevicePath(stored) + if path == "" { + return + } + hypervisorLive := hypervisorMayBeAlive(stored.HypervisorProcessIdentity, stored.SocketPath) + if vgpuAssignmentMayBeLive(stored, m.nowUTC(), hypervisorLive) { + return + } + if err := m.releaseStoredVGPU(ctx, stored); err != nil { + m.recordVGPUStaleReleaseFailure(ctx) + log.WarnContext(ctx, "failed to release stale vGPU assignment; retrying on the next reconcile pass", "instance_id", id, "device_path", path, "error", err) + return + } + if err := m.saveMetadata(meta); err != nil { + log.WarnContext(ctx, "failed to save metadata after stale vGPU release", "instance_id", id, "error", err) + return + } + log.InfoContext(ctx, "released stale vGPU assignment", "instance_id", id, "device_path", path) +} diff --git a/lib/instances/vgpu_reconcile_linux_test.go b/lib/instances/vgpu_reconcile_linux_test.go new file mode 100644 index 000000000..553c64c15 --- /dev/null +++ b/lib/instances/vgpu_reconcile_linux_test.go @@ -0,0 +1,58 @@ +//go:build linux + +package instances + +import ( + "context" + "net" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReconcileVGPUsProtectsSocketOwnerWithoutPersistedPID(t *testing.T) { + t.Parallel() + + socketPath := filepath.Join(t.TempDir(), "test.sock") + listener, err := net.Listen("unix", socketPath) + require.NoError(t, err) + defer listener.Close() + + var destroyed []devices.VGPUAssignment + var protected map[string]struct{} + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, _ bool) error { + protected = p + return nil + }, + } + const id = "pid-save-failed" + stale := time.Now().UTC().Add(-devices.VGPUAssignmentGracePeriod - time.Minute) + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + SocketPath: socketPath, + GPUFramework: devices.VGPUFrameworkMdev, + GPUDevicePath: "/sys/bus/mdev/devices/test-mdev", + GPUMdevUUID: "test-mdev", + GPUAssignedAt: &stale, + }})) + + m.ReconcileVGPUs(t.Context()) + + assert.Empty(t, destroyed, "a live socket owner must block the release even with a nil persisted PID") + assert.Contains(t, protected, "/sys/bus/mdev/devices/test-mdev") + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/mdev/devices/test-mdev", stored.GPUDevicePath) +} diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go new file mode 100644 index 000000000..8b8d71b66 --- /dev/null +++ b/lib/instances/vgpu_reconcile_test.go @@ -0,0 +1,245 @@ +package instances + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReconcileVGPUsBoundsStartupProtection(t *testing.T) { + dead := exec.Command("true") + require.NoError(t, dead.Run()) + deadPID := dead.Process.Pid + now := time.Now().UTC() + recent := now.Add(-time.Minute) + stale := now.Add(-devices.VGPUAssignmentGracePeriod - time.Minute) + + var protected map[string]struct{} + var destroyed []devices.VGPUAssignment + m := &manager{ + paths: paths.New(t.TempDir()), + now: func() time.Time { return now }, + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + reconcileVGPUDevices: func(_ context.Context, p map[string]struct{}, sweepDevices bool) error { + protected = p + assert.True(t, sweepDevices) + return nil + }, + } + instances := []StoredMetadata{ + {Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}, + {Id: "orphaned", GPUProfile: "NVIDIA L40S-2Q", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}, + {Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}, + {Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}}, + {Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}, + {Id: "legacy-mdev-booting", GPUMdevUUID: "test-mdev", GPUAssignedAt: &recent}, + } + for i := range instances { + require.NoError(t, m.ensureDirectories(instances[i].Id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: instances[i]})) + } + + m.ReconcileVGPUs(t.Context()) + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") + assert.Contains(t, protected, "/sys/bus/mdev/devices/test-mdev") + + for _, id := range []string{"orphaned", "legacy", "dead"} { + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "stale assignment on %s must be released", id) + } + assert.Contains(t, destroyed, devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: "/sys/bus/pci/devices/0000:82:00.5", + InstanceID: "orphaned", + }) + orphaned, err := m.loadMetadata("orphaned") + require.NoError(t, err) + assert.Equal(t, "NVIDIA L40S-2Q", orphaned.GPUProfile) + for _, id := range []string{"booting", "stale-pid-booting", "legacy-mdev-booting"} { + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.NotEmpty(t, storedVGPUDevicePath(&stored.StoredMetadata), "live assignment on %s must be kept", id) + } +} + +func TestReconcileVGPUsSkipsDeviceSweepWhenListingFails(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + var sweeps []bool + m := &manager{ + paths: paths.New(t.TempDir()), + reconcileVGPUDevices: func(_ context.Context, _ map[string]struct{}, sweepDevices bool) error { + sweeps = append(sweeps, sweepDevices) + return nil + }, + } + const id = "unreadable" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{Id: id}})) + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o000)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + m.ReconcileVGPUs(t.Context()) + require.Equal(t, []bool{false}, sweeps, + "a listing failure must skip the device sweep, not run it with an empty protection set") + + require.NoError(t, os.Chmod(instanceDir, 0o755)) + m.ReconcileVGPUs(t.Context()) + assert.Equal(t, []bool{false, true}, sweeps, "the next pass retries the device sweep") +} + +func TestReconcileVGPUsKeepsAssignmentWhenReleaseFails(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + if attempts.Add(1) == 1 { + return errors.New("vGPU destroy failed: 0xffffffff") + } + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const id = "wedged" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + m.ReconcileVGPUs(t.Context()) + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath, + "a failed release must keep the assignment metadata for the next pass") + + m.ReconcileVGPUs(t.Context()) + stored, err = m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "the next pass retries the release") + assert.Equal(t, int32(2), attempts.Load()) +} + +func TestReconcileVGPUsDefersToUnconfirmedClaimant(t *testing.T) { + t.Parallel() + + var destroys atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + destroys.Add(1) + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const path = "/sys/bus/pci/devices/0000:82:00.4" + assignedAt := time.Now() + for _, stored := range []StoredMetadata{ + {Id: "stale", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: path}, + {Id: "mid-boot-claimant", GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: path, GPUAssignedAt: &assignedAt}, + } { + require.NoError(t, m.ensureDirectories(stored.Id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: stored})) + } + + m.ReconcileVGPUs(t.Context()) + + assert.Zero(t, destroys.Load(), "no destroy may fire while the claim scan cannot clear the path") + stale, err := m.loadMetadata("stale") + require.NoError(t, err) + assert.Equal(t, path, stale.GPUDevicePath, "the stale release retries once the claimant's liveness is decidable") +} + +func TestReconcileVGPUsReleasesRetentionStubAssignment(t *testing.T) { + t.Parallel() + + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { return nil }, + } + const id = "retention-stub" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: "failed-create", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPURetainedForCleanup: true, + }})) + + m.ReconcileVGPUs(t.Context()) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "the stub's wedged assignment is released once free") + assert.True(t, stored.GPURetainedForCleanup, "the stub stays a delete-only record of the failed create") +} + +func TestStartVGPUReconcilerSkipsHostsWithoutGPUs(t *testing.T) { + t.Parallel() + + var passes atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkNone, nil, nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { + passes.Add(1) + return nil + }, + vgpuReconcileInterval: time.Millisecond, + } + + m.StartVGPUReconciler(t.Context()) + time.Sleep(20 * time.Millisecond) + assert.Zero(t, passes.Load(), "a host without GPUs must not reconcile at all") +} + +func TestStartVGPUReconcilerRunsPeriodically(t *testing.T) { + t.Parallel() + + var passes atomic.Int32 + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, nil, nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}, bool) error { + if passes.Add(1) == 1 { + return errors.New("transient device error") + } + return nil + }, + vgpuReconcileInterval: time.Millisecond, + } + + m.StartVGPUReconciler(t.Context()) + require.Eventually(t, func() bool { return passes.Load() >= 3 }, 5*time.Second, time.Millisecond, + "periodic passes must keep running after a failed pass") +} diff --git a/lib/instances/vgpu_retention.go b/lib/instances/vgpu_retention.go new file mode 100644 index 000000000..1bcba2c42 --- /dev/null +++ b/lib/instances/vgpu_retention.go @@ -0,0 +1,98 @@ +package instances + +import ( + "context" + "time" + + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" +) + +type vgpuRetention struct { + instanceID string + stub *StoredMetadata + retained bool + persisted bool +} + +func (r *vgpuRetention) retainFromCreateError(stub StoredMetadata, assignedAt time.Time, err error) { + device, ok := vgpuDevicePendingCleanup(err) + if !ok { + return + } + r.retainFromDevice(stub, device, assignedAt) +} + +func (r *vgpuRetention) retainFromDevice(stub StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) { + stub.GPUProfile = device.ProfileName + setStoredVGPUDevice(&stub, device, assignedAt) + r.stub = &stub + r.retained = true +} + +func (r *vgpuRetention) markRetained(persisted bool) { + r.retained = true + r.persisted = persisted +} + +func (r *vgpuRetention) wrapPending(err error) error { + if err == nil || !r.retained { + return err + } + return &VGPUCleanupPendingError{InstanceID: r.instanceID, Retained: r.persisted, Err: err} +} + +func (m *manager) persistVGPURetention(ctx context.Context, retention *vgpuRetention) { + if retention.stub == nil { + m.deleteInstanceData(retention.instanceID) + return + } + + id := retention.instanceID + retainedVGPU := retention.stub + log := logger.FromContext(ctx) + defer func() { + m.recordVGPURetainedAssignment(ctx, vgpuRetentionOperationCreate, retention.persisted) + }() + + // An unpersisted retention leaves no metadata claim. The periodic reconciler releases + // the VF after its grace period once no open VFIO handles remain. + if err := m.deleteInstanceData(id); err != nil { + log.ErrorContext(ctx, "failed to clean instance data before retaining vGPU assignment", "instance_id", id, "error", err) + return + } + if err := m.ensureDirectories(id); err != nil { + log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) + return + } + if err := m.saveVGPURetentionStub(retainedVGPU); err != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) + return + } + retention.persisted = true +} + +func vgpuRetentionMetadata(source *StoredMetadata) *metadata { + return &metadata{StoredMetadata: StoredMetadata{ + Id: source.Id, + Name: source.Name, + Image: source.Image, + ResolvedImage: source.ResolvedImage, + Platform: source.Platform, + CreatedAt: source.CreatedAt, + HypervisorType: source.HypervisorType, + HypervisorVersion: source.HypervisorVersion, + SocketPath: source.SocketPath, + DataDir: source.DataDir, + GPUProfile: source.GPUProfile, + GPUFramework: source.GPUFramework, + GPUDevicePath: source.GPUDevicePath, + GPUMdevUUID: source.GPUMdevUUID, + GPUAssignedAt: source.GPUAssignedAt, + GPURetainedForCleanup: true, + }} +} + +func (m *manager) saveVGPURetentionStub(source *StoredMetadata) error { + return m.saveMetadata(vgpuRetentionMetadata(source)) +} diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 6f2c46819..07cdcb8e5 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -2,53 +2,523 @@ package instances import ( "context" + "errors" + "fmt" + "os" + "path/filepath" + "sync" "testing" + "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestStoredVGPUDevicePath(t *testing.T) { +func testVendorVFIODevice(profileName string) devices.VGPUDevice { + return devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: profileName, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } +} + +func persistTestVGPURetention(m *manager, ctx context.Context, id string, stub *StoredMetadata) bool { + retention := vgpuRetention{instanceID: id, stub: stub, retained: stub != nil} + m.persistVGPURetention(ctx, &retention) + return retention.persisted +} + +func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { t.Parallel() - assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", storedVGPUDevicePath(&StoredMetadata{ - GPUDevicePath: "/sys/bus/mdev/devices/new-uuid", - GPUMdevUUID: "legacy-uuid", - })) - assert.Equal(t, "/sys/bus/mdev/devices/legacy-uuid", storedVGPUDevicePath(&StoredMetadata{ - GPUMdevUUID: "legacy-uuid", - })) - assert.Empty(t, storedVGPUDevicePath(&StoredMetadata{})) + m := &manager{paths: paths.New(t.TempDir())} + const id = "failed-create" + stub := StoredMetadata{ + Id: id, + Name: id, + NetworkEnabled: true, + IP: "192.0.2.1", + Volumes: []VolumeAttachment{{VolumeID: "volume"}}, + HypervisorType: "qemu", + DataDir: m.paths.InstanceDir(id), + } + device := devices.VGPUDevice{ + ProfileName: "NVIDIA L40S-2Q", + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + assignedAt := time.Now().UTC() + retention := vgpuRetention{instanceID: id} + retention.retainFromCreateError(stub, assignedAt, &devices.VGPUCreateCleanupPendingError{Device: device, Err: errors.New("rollback failed")}) + + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, os.WriteFile(m.paths.InstanceOverlay(id), []byte("overlay"), 0o644)) + require.NoError(t, os.WriteFile(m.paths.InstanceConfigDisk(id), []byte("config"), 0o644)) + require.NoError(t, os.MkdirAll(m.paths.InstanceVolumeOverlaysDir(id), 0o755)) + require.NoError(t, os.WriteFile(m.paths.InstanceVolumeOverlay(id, "volume"), []byte("volume overlay"), 0o644)) + + m.persistVGPURetention(context.Background(), &retention) + assert.True(t, retention.persisted) + assert.NoFileExists(t, m.paths.InstanceOverlay(id)) + assert.NoFileExists(t, m.paths.InstanceConfigDisk(id)) + assert.NoDirExists(t, m.paths.InstanceVolumeOverlaysDir(id)) + + retained, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, id, retained.Id) + assert.Equal(t, device.Framework, retained.GPUFramework) + assert.Equal(t, device.SysfsPath, retained.GPUDevicePath) + assert.Equal(t, assignedAt, *retained.GPUAssignedAt) + assert.Equal(t, stub.Name, retained.Name) + assert.Equal(t, device.ProfileName, retained.GPUProfile) + assert.Equal(t, stub.HypervisorType, retained.HypervisorType) + assert.Equal(t, stub.DataDir, retained.DataDir) + assert.False(t, retained.NetworkEnabled) + assert.Empty(t, retained.IP) + assert.Empty(t, retained.Volumes) + assert.True(t, retained.GPURetainedForCleanup) } -func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) { +func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { t.Parallel() + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("failed-create")) + + assert.False(t, persistTestVGPURetention(m, context.Background(), "failed-create", nil)) + _, err := m.loadMetadata("failed-create") + require.Error(t, err) +} + +func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m := &manager{paths: paths.New(t.TempDir())} + const id = "failed-create" + require.NoError(t, os.MkdirAll(m.paths.GuestsDir(), 0o755)) + require.NoError(t, os.Chmod(m.paths.GuestsDir(), 0o555)) + t.Cleanup(func() { _ = os.Chmod(m.paths.GuestsDir(), 0o755) }) + stored := &StoredMetadata{ - GPUFramework: devices.VGPUFramework("future-framework"), + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } - err := releaseStoredVGPU(context.Background(), stored) - assert.Error(t, err) - assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) - assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.False(t, persistTestVGPURetention(m, context.Background(), stored.Id, stored)) + _, err := m.loadMetadata(id) + require.Error(t, err, "the lost retention leaves no metadata claim, so the periodic sweep releases the VF") } -func TestSetAndClearStoredVGPUDevice(t *testing.T) { - t.Parallel() +type startRetentionNetworkManager struct { + network.Manager + config network.NetworkConfig + releaseCalls int +} + +func (m *startRetentionNetworkManager) CreateAllocation(context.Context, network.AllocateRequest) (*network.NetworkConfig, error) { + config := m.config + return &config, nil +} - stored := &StoredMetadata{} - setStoredVGPUDevice(stored, &devices.VGPUDevice{ - Framework: devices.VGPUFrameworkMdev, - SysfsPath: "/sys/bus/mdev/devices/new-uuid", - MdevUUID: "new-uuid", +func (m *startRetentionNetworkManager) ReleaseAllocation(context.Context, *network.Allocation) error { + m.releaseCalls++ + return nil +} + +func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) { + t.Helper() + m := &manager{ + paths: paths.New(t.TempDir()), + imageManager: readyFixtureImageManager{name: "test-image"}, + instanceLocks: sync.Map{}, + bootMarkerScans: sync.Map{}, + createVGPU: func(_ context.Context, profileName, _ string) (*devices.VGPUDevice, error) { + device := testVendorVFIODevice(profileName) + return &device, nil + }, + destroyVGPU: destroy, + } + const id = "start-rollback" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + Image: "test-image", + GPUProfile: "NVIDIA L40S-2Q", + HypervisorType: hypervisor.TypeQEMU, + SocketPath: m.paths.InstanceSocket(id, "noop.sock"), + DataDir: m.paths.InstanceDir(id), + }})) + return m, id +} + +func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + networkManager := &startRetentionNetworkManager{config: network.NetworkConfig{ + IP: "192.0.2.20", + MAC: "02:00:00:00:00:20", + TAPDevice: "tap-new", + }} + m.networkManager = networkManager + + previousProgramStart := time.Now().Add(-time.Hour).UTC() + previousExitCode := 23 + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.NetworkEnabled = true + meta.IP = "192.0.2.10" + meta.MAC = "02:00:00:00:00:10" + meta.Entrypoint = []string{"old-entrypoint"} + meta.Cmd = []string{"old-command"} + meta.ProgramStartedAt = &previousProgramStart + meta.ExitCode = &previousExitCode + meta.ExitMessage = "previous exit" + require.NoError(t, m.saveMetadata(meta)) + + device := testVendorVFIODevice("NVIDIA L40S-2Q") + cause := errors.New("create verification and rollback failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + } + + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{ + Entrypoint: []string{"new-entrypoint"}, + Cmd: []string{"new-command"}, + }) + require.ErrorIs(t, err, cause) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, id, pending.InstanceID) + assert.True(t, pending.Retained) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, device.Framework, stored.GPUFramework) + assert.Equal(t, device.SysfsPath, stored.GPUDevicePath) + assert.NotNil(t, stored.GPUAssignedAt) + assert.Equal(t, []string{"old-entrypoint"}, stored.Entrypoint) + assert.Equal(t, []string{"old-command"}, stored.Cmd) + assert.Equal(t, previousProgramStart, *stored.ProgramStartedAt) + assert.Equal(t, previousExitCode, *stored.ExitCode) + assert.Equal(t, "previous exit", stored.ExitMessage) + assert.Equal(t, "192.0.2.10", stored.IP) + assert.Equal(t, "02:00:00:00:00:10", stored.MAC) + assert.Equal(t, 1, networkManager.releaseCalls) +} + +func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + device := testVendorVFIODevice("NVIDIA L40S-2Q") + cause := errors.New("create verification and rollback failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + } + + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, cause) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, id, pending.InstanceID) + assert.False(t, pending.Retained) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded") +} + +func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { + var destroyed []devices.VGPUAssignment + m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil }) - assert.Equal(t, devices.VGPUFrameworkMdev, stored.GPUFramework) - assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", stored.GPUDevicePath) - assert.Equal(t, "new-uuid", stored.GPUMdevUUID) - clearStoredVGPUDevice(stored) + t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.Error(t, err) + + require.Len(t, destroyed, 1) + assert.Equal(t, devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: "/sys/bus/pci/devices/0000:82:00.4", + InstanceID: id, + }, destroyed[0]) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile) assert.Empty(t, stored.GPUFramework) assert.Empty(t, stored.GPUDevicePath) assert.Empty(t, stored.GPUMdevUUID) } + +func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + stalePID := os.Getpid() + meta.HypervisorPID = &stalePID + meta.HypervisorStartTime = 1 + meta.HypervisorBootID = "previous-boot" + require.NoError(t, m.saveMetadata(meta)) + + t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}}) + require.Error(t, err) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending, "a retained rollback assignment must surface as vgpu_cleanup_pending") + assert.Equal(t, id, pending.InstanceID) + assert.True(t, pending.Retained) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.NotNil(t, stored.GPUAssignedAt) + assert.Nil(t, stored.HypervisorPID) + assert.Zero(t, stored.HypervisorStartTime) + assert.Empty(t, stored.HypervisorBootID) + assert.Empty(t, stored.Entrypoint) +} + +func TestCleanupStartVGPUReportsRetentionWhenRollbackSaveFails(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + tests := []struct { + name string + assignmentSaved bool + wantPersisted bool + }{ + {name: "assignment save survived", assignmentSaved: true, wantPersisted: true}, + {name: "assignment never saved", assignmentSaved: false, wantPersisted: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + assignedAt := time.Now().UTC() + + meta, err := m.loadMetadata(id) + require.NoError(t, err) + rollbackMeta := *meta + if tt.assignmentSaved { + setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt) + require.NoError(t, m.saveMetadata(meta)) + } + + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta) + assert.True(t, retained) + assert.Equal(t, tt.wantPersisted, persisted) + }) + } +} + +func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + } + const id = "failed-start" + require.NoError(t, m.ensureDirectories(id)) + + previousStart := time.Now().Add(-time.Hour).UTC() + exitCode := 1 + rollbackMeta := metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: "original name", + GPUProfile: "NVIDIA L40S-2Q", + Entrypoint: []string{"old-entrypoint"}, + StartedAt: &previousStart, + ExitCode: &exitCode, + ExitMessage: "previous exit", + }} + partial := metadata{StoredMetadata: StoredMetadata{Id: id, Name: "partial start"}} + assignedAt := time.Now().UTC() + device := &devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + setStoredVGPUDevice(&partial.StoredMetadata, device, assignedAt) + require.NoError(t, m.saveMetadata(&partial)) + + m.cleanupStartVGPU(context.Background(), id, device, assignedAt, rollbackMeta) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, rollbackMeta.StoredMetadata, stored.StoredMetadata) +} + +func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("invalid-instance")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) + + _, err := m.vgpuAssignmentClaimedByLiveInstance("other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) +} + +func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("legacy-claimant")) + pid := os.Getpid() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "legacy-claimant", + Name: "legacy-claimant", + GPUMdevUUID: "legacy-uuid", + HypervisorProcessIdentity: HypervisorProcessIdentity{ + HypervisorPID: &pid, + HypervisorStartTime: processStartTime(pid), + HypervisorBootID: hostBootID(), + }, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance("other-instance", "/sys/bus/mdev/devices/legacy-uuid") + require.NoError(t, err) + assert.True(t, claimed) +} + +func TestVGPUAssignmentClaimedByLiveInstanceLiveness(t *testing.T) { + t.Parallel() + + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" + deadPID := 1 << 30 + require.False(t, ProcessExists(deadPID)) + recent := time.Now().UTC() + stale := recent.Add(-devices.VGPUAssignmentGracePeriod - time.Minute) + tests := []struct { + name string + assignedAt *time.Time + pid *int + wantErr string + }{ + {name: "recent without PID", assignedAt: &recent, wantErr: "no persisted hypervisor PID"}, + {name: "stale without PID", assignedAt: &stale}, + {name: "recent dead PID", assignedAt: &recent, pid: &deadPID, wantErr: "recorded hypervisor is not running"}, + {name: "legacy dead PID", pid: &deadPID}, + } + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + m := &manager{paths: paths.New(t.TempDir())} + id := fmt.Sprintf("claimant-%d", i) + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + GPUAssignedAt: tt.assignedAt, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: tt.pid}, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance("requester", devicePath) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.False(t, claimed) + }) + } +} + +func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) { + t.Parallel() + + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + } + require.NoError(t, m.ensureDirectories("invalid-instance")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) + + stored := &StoredMetadata{ + Id: "mdev-instance", + GPUFramework: devices.VGPUFrameworkMdev, + GPUMdevUUID: "uuid-1", + GPUDevicePath: "/sys/bus/mdev/devices/uuid-1", + } + require.NoError(t, m.releaseStoredVGPU(context.Background(), stored), + "an unreadable metadata file must not block mdev releases") + assert.Empty(t, stored.GPUDevicePath) +} + +func TestReleaseStoredVGPURetainsRequesterOnAmbiguousClaim(t *testing.T) { + t.Parallel() + + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + t.Fatal("destroyVGPU must not be called for an ambiguous claim") + return nil + }, + } + require.NoError(t, m.ensureDirectories("ambiguous-claimant")) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "ambiguous-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + GPUAssignedAt: &assignedAt, + }})) + + stored := &StoredMetadata{ + Id: "requester", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + } + err := m.releaseStoredVGPU(context.Background(), stored) + require.Error(t, err) + assert.Contains(t, err.Error(), "ambiguous-claimant") + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, devicePath, stored.GPUDevicePath) +} + +func TestStoredVGPUDevicePath(t *testing.T) { + t.Parallel() + + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", storedVGPUDevicePath(&StoredMetadata{ + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUMdevUUID: "legacy-uuid", + })) + assert.Equal(t, "/sys/bus/mdev/devices/legacy-uuid", storedVGPUDevicePath(&StoredMetadata{ + GPUMdevUUID: "legacy-uuid", + })) + assert.Empty(t, storedVGPUDevicePath(&StoredMetadata{})) +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 76e7e0a64..ba5df2db0 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1354,7 +1354,10 @@ type InstanceHypervisor string // InstanceGPU GPU information attached to the instance type InstanceGPU struct { - // MdevUuid mdev device UUID + // DevicePath sysfs path of the assigned vGPU device + DevicePath *string `json:"device_path,omitempty"` + + // MdevUuid mdev device UUID (mdev hosts only) MdevUuid *string `json:"mdev_uuid,omitempty"` // Profile vGPU profile name @@ -19032,204 +19035,205 @@ var swaggerSpec = []string{ "1VhohilpTMSQ6dcCWKt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtlrDJ+QW0P2ZRhQFGUOQM5Swmoqj/", "TQw51EPkbug4/UwQYfd5O/HKfl7KVeGDuQ5zuB2Y8OciuMJQG/RziA69hXK+f3sRvVXOkZNfbbaR/Wp0", "kwgGgiKeJ7HW+Mb6tjMGORJbM6QkynBn8y6V6J2pvVmdug09Vhz9nhOxQO9PTythD4JMNA9oN3HgEg37", - "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJlgz", - "5cdkPsrzkFaiHzkclHfvTo4rlILxk+1ng2fPe8/G2096e/Fgu4e3d5/0dvbxYLIbPd3d3tldkXHSIm3t", - "9ploQdU0EDBchIePXJh6KHq4KUmgJgTYwOcrymJ+VblngpGofu82ynVd98sx7K2HEMx8SbBUxk7QwDJO", - "4TYlkW7bRH7b1MiiplHYovjk7WD7c80sMLgGZvxW5Mz4L00yf2GrT70B+5tVHefteCsMyGWYrFstv/P2", - "izY42H9+sP+5i+ayJNaNsU5O97i5TaFXDuy4lobhUgE9g42zBHas8GHM5zZro9PtFIkl8DfcurWg5eJx", - "q2yppgPbDbORVfy7IUv5pKIYQEiIAbuLD7RI4AR/KJ1Q5MJrWeMo4XmMPKOXwf4Ch9eJpyToZsD/ZG1h", - "BsvTZD1oZQLAo6FEA2WaEYOjTzdiU5oP0Et4Fx7h1OhPdhCmUIjv48LxwgSm6PPlujbazOohn1tFBr7R", - "Wg3S/4Jp62WwttHVTRgx6AD9yuGbQq1ivG5kNa+DPrP8et0gu2FxsR1EBXRmZboD9FMhxxWSoJX8NiSx", - "f44swyqRYTYr+fl2xzuaWsqd83LNux2zop1uxy0U5KQvZ6e/K6l+6fz5pBiK2CI4gbNcJuPmiiYWDxtm", - "QqWikbRZGnpzm+QLW8OIxCOjpTQFf5oMT6vJFB858eX9KdoAyMO/IGtB1v/aLAJFK3fdzvO950+e7jx/", - "0grYqBzgehn0CPKPlwe3ViCNsnxkjRBNUz86e2eMDJFR34sgk/enPo5EJrhmPXrmrkG/8+f95z6eU8zz", - "ceJ58Cz4m4GPhQ0LQpYVvKgh4PB3mszpZMJ+/xhd7vxd0HT7+oncGW834NSajsL2rRPfi79kDCbjnqlH", - "FIbcAYISshGV6g2RMAN0ThQC+ukhHIEeUaQNW5Jz2FV2xYOEtbe7u/vs6f5OK7qyo/MOzgisXYFL2Y7A", - "O2LwJtp4c36OtjyCM2068AaAEmdWxwyfM2SLCQ+qAml/e7AbopKGi7ukGtv2PG1c8vdWT7OTsosO2c+F", - "Drd0yoOrvbs7eLq3/2y/3TG2dtiRuF7NYVxukFkei3jv7/wGSJNvD88QZN5OcFQ1orhQrBuNSt1oVFCt", - "waCs32Bgz54+2d/b3dluB68Wiu6wwIGVA1vlXYFDFyCKwG4ElmKZ9XabbouQOGUI7A2JEkzTw8jlMtRu", - "H4OmPhLmtXIT2lwM1vS/dHG1+LaVFamwDZlMGCMacIFyVtTw6K/3fX4RF2Yz1zbXw3quHsp/YXr1LA6Q", - "qVV2i6XMBJlTnssv0BBXJjl1knAubvRtk8Lyhsg8UcbPSCV6f/od8BRNa0gqklV1KEuNK9CSbjm5G53n", - "ComEibxpsVrtRputXzXhbsOp7a5Crqhwg0aMslhzrpytj7I8wkmUQ9UaXOynnhWAbUHufZYlCxNEnySc", - "MxTNMANvhPCghdCMJ3E/GHKqn4wmwfAFfoUSbtCVLwnJbEEXMwj9mRZh6JygDb+UmSGlWoHR/dQwGVuy", - "o0qN+2m4UiKWoaywIudcrydW3AP+NZ9UTI4Jn0pQChWkB/TrePMZFibqHzNToGieGl0yENkcGGKNmYdu", - "VHOT8olVcK3IARndZiVxJLiUiCR0CsVw3p/WEoVXJJcV6cLrIyerg21BusZzGLjKDOxU6zpmofsxkDjz", - "OTck0DAk562ISXTGyRSzHEq8eIRsLd791nGHMy7VqACAuuFgpRpB3YZckBKWrkhvL+xB7p3gvehY222W", - "ywb43urrJaoKN9U0wGaeGlzR8Gp1CxoMkfEyBNZK1K0SxquO2XQTVLgS6J9KaJV6+GBoA5JLPLbkYb1t", - "tolGCausup8lbdWW4Xy1Nzhvi5+2Gi7tDKvZCZvwAMjGDVyUzhJtw0IzIlIKlUtQTBglsdMlC1+lNXVB", - "ZnYiCYpzYlfOyKcC2wXH5ngDUAZzNjLKpjVeX++wjXnYjGF1WQfo177YJq5IhjNX34oc1soEBkqEyxzW", - "VtGWVI7C7qzlhgWZ5gkWyCIfthmyXKQJZZdtWpeLdMwTGiH9Qd0BPeFJwq9G+pH8Aeay2Wp2+oNRUw2g", - "czM4m4BnNqTWbzmFH/QsN2vpv2CJ2TLfbwE0S5swrWBI9k80IRZG7x2j1x6hV3HP93YGTWnpDY1WEtKX", - "IRhvyrktyQZPfC4DSXwrpRxXvYjEFozeiD1ZLk0hlRa3kkM7dS7A23l0qhkan4cBcmT4dQ0BBI0JJNi4", - "qS1zjRZssc1UgjUccjlDf+fjqkG0bXxtoDLYBiuxKASZBAPpYUdXGqTNG0tr4u3uTcAegK3qicJHN8RQ", - "WFdDrQxkauInb5bKic2IXTLq5mhKi7UoleECLQqcANtre8CAeuG3QGAwwMFItYASqlC3ZuFVM5RozIUA", - "qGct4XDmZgP4Jlrm0WvtAKbQ2xlZIEFSTNmQUVYYSQG1jCBG5kR46ahcaCVrSuI++pun4gE4dpqphUVd", - "B+P5dxLxK1aMccj8QerGc6nbOWTGsihyqMpfvqSbBa1PEwqkB4MTTAkoVEjVDE0EkTN/7qHilFrGu+Ii", - "bqz6s0DuFSgmAz5WpPglYT4rK5oJqoamoZH5ajlczlSWhadW/0SVYq+oXsx1dX+5JCIsJBZTKl5pFbri", - "HRVPOTFoKwA9AkX87F+GxRdwIy3ARcrm/+qaLH86Kxqv/lZ7zQMQcXi+h8ZsGzTBRiZfphbsU/WkrQ1V", - "gXyzVbAxy74EtOFilV0llKok4FUkaXVPtkt5q0flu9FsSRJVe997tv/0ScuSMJ/lrDMwWV/aNTdPV7jk", - "GnbqtI3f59n+s+fPd/f2n+/cyMPiEjga9qcpicPfH7RBrpU+rMm//vHP96c1r88+BDsPbjQok8IRHlJD", - "Gkd1QO9P//WPf7pR3XpAIUazDMXd4LdvjNJJ/J10gQJVF147J9kK/f6wYiTABZtBG2QyIWAGHZl165WD", - "qeFttJOCcYYjqhYBRo6vTFh58UoNUrqNO6g62JDIa9q28KOac8l8XGZ3brjO0X8a33CNFp61riwl83GT", - "H/p1vVfjhS69Fn6MQ4sQA1kUNV82cBfzucKyEjmt/44gwcGlci2ntZg3VsPb1nMOIIrFFlDzQgFDsOg1", - "edJ+5G9/bTs9v2XFrFNf8Q8rzmHzEbyR1TdwIweMvtH6HNYaf7AX4O2+Go39mm8ri+pVCsSVt+7N+22R", - "prtckKC4wW7en5eZeJMP6+C7QI92DHbJy7a7FZJooCYv6SRgQOMJ6RWBejYjBcnceAT1mbd47oFUyeiS", - "TyZVUNn9ZhBywNeBrCrXC1ZKayZdRK6dzaKOYG3AdIadfTnsaBVg2NlOh52a2yqYp5ji65HtoAqiMliF", - "Cl7mmdcGKd0MxgmPLk05M6iS3UcDlBLMJMoZHP6aV217sNo71O1k3t4UGNzEhDgtsS0Y05jM8JxC6Qfr", - "U5lWAjHJNVUSAkahnQMUcwOrVKnlameoXzNZhAflpOHSwWxhG9YN6vc4cxGt5btg4JtABVn2kQjetagA", - "mmO/fn3aNQEMEHpoBlaJb3QTNSPQDLLoolbHoPw9HD88TsgIxl3HxU+X19FP/gbPqiCSKGmBsktyqBEB", - "injOVB0wP22nyFXzt5avpJxBsJ8N/wAANNu7IRAUkwhOpFw+i1VCvwVx1/IG7EqHEgd2QyQMhwJ8SWFf", - "8RvrEK4PwBgbvDLMph0/rtt4CUdScVu3qzjVI3IdERLXkTXDr7SNlbdfBmPlX2ELxlNUSLZvQ7zz8uz6", - "d5dJBWNtWm0/pp9x1gMYELelFrLDYPBZUJgqoVUwvj3siFEIxzT0QpvUZnK9eq1/JdcKgMjjPDHocmHS", - "tazKXkbrVvzWKYRNB5oLsrYO3h3UhzPx5reqEGdD1R+iSJx9604Kwy3tzjlR7t1zS0aNO1StqFJxabmA", - "f/dKNcbGkFIX2QsebaebNRLcm4WtIhb9tmUyJMMpGWWCTOj1CuIxLxjFuIofUh6kIoPBAHlupPga7T1F", - "0QwLWRs7o9OZShbVAJy9AGjRZ1VPFEQR5gyFbXa+3E334XK0m91Ov/WQcHzuYfAs1Q6xIuloFUD1Uelt", - "s9b5DC/AitPoJHy6uzcY7O4MboVQ7YZ1g+U6Kj+xtQar7TSl1HnfWUd/JUrVb6HIZl4uYHslKCRFF8sk", - "lSA4PYDEmwxHBCVkAmh0ReHw9Z7FeterB28FKov8UtC/2yi7b84HX61NU3Rlwb3dNDrOuVgF+/Gfr3GI", - "NrCZaAm7LpBzt9sbPHm7vXuw/+Rge/suUKWLRWrK9nj6cfvqabKDJ3vJs8XT37dnT6c76W5QD7ukpgRP", - "G1r9Rb/bGGVTXpJV0KAKS0Mbdg4ZEfXKxPWK3pIklJGeLDKk1qcpruAFxv++9vzfzM5vZrBSdjivTtIX", - "IbAqF6dCWQ8DdGUns9J3UZ/NyfHqWdwqA6k+kDC91YcC5NVuMFAKYrvzmRAIOWt5Db3zXmx9Ea3Milt3", - "FYU87HDSg7vcsOIh8q4hIHizXnWBL19yAdvplAuqZunq26J4rcDrhrjpj1LFVWClPjqZMihL7v9chMn5", - "SpT+uNPtJB/3qmfG/t4eYstC/RYEaLfalwpahJFB1fvVqwCvlIqHMJHsWlfXY/5hu7f9HOIQko97Pwx6", - "z6sRB12zWv7ybbu3K78O2qyhX2vP1Wjafn6jiGu3nqso6BcaqhRX3ssWBNjSeFkE2l0dLuG2ssHl46U9", - "rkHmNAqgnyvp2ctt5AtNMUnwIgQC7xlqZU179IkMjcmUMtnGbrs7KAy3++mw00eHFokbdNmy5H+leSj2", - "7tEJTVMSUy1jGtW/OYNhp6Utrq5L3KwIiPsqIK31w+La8/UQCesSrtZdk/3PyMf9LO23nca7Cr0D7GpO", - "RQWwLnixi+gEYVarBErZHCc0ton0kBgJ8WoHDhGtJFnLA2QpBzo7SRdNuUJlCn1Le1vOmu2CxfjJNdhb", - "V2BmGILY+SKAKAVSF13Fvk6OUSZ4nEdl/mgCgy4RP0Rew0JbIeSvD8m9S/sGJGZPuEDr7RtNBo129smm", - "/a7ZJjXBNm/19mD9Vt+JUaTbybN4PQ8zL7XjYDeCSF+Tghgw0VSXvSYJepP50IKjv/FXcFnnNbbkSItE", - "eeYcLJqmlikp4G4BF0MorveYJERfU8uNIJ7EZZYElSUXXc9St588mzW5OMEjtTyQXwjJtK4C+EfQX4rZ", - "IjgwV9+zuEs2Bg5WWxqHV8/UBbKrVR3c07WSWONW+SbcploFhsvXbN4GL+XSM38XYNq+aLaMgOIYfkVI", - "e9OMtW+/dGFvjfbjuzDLPaSQ9tq6Hmr4qA69t4Ahd/2XscBarKsS717IPR8ii7dWM27CfK1ngfpW58Pe", - "/xgrMxr1D7Z++Mv/3fvwn0Frc01vlkT0YjKBQKNLsuiZKj9aR+9XEU+hxIAWpqeWVAhOwYYEaOL2MPrj", - "3R8UTGPxK06XpgARWl6Jnu21E/rLfzTHN3nL+A745FqS/ewKHHdRqVRxdx1tpERMXSy5SyTb7A8Z5KZd", - "koVEXuEvK9I4Qv1OFp94EejowoiBfcLmF2hMoZKiHDKt1eIoIpnWJmwtGWrKgXPgPoLgxG/HFiBzid/W", - "IWniCQh6f7oEl/v63dsfX7/79Xj0+uzFr4cno19e/DeEeFz1TA9xT9Pe3v4TWwTcX8ntYCGKm9dT6KNT", - "G6ZvXf2THBRawOmSKM1VDkEh5DpKcknnzkGokttXTlhO1r19JYLPhNpVKglFJVhI6IROCPj14TqxQTVU", - "OmKkEqqnW+MGZWj5xjaEM+wAJ/WK34fqVuitCK92ubHVRX8ya8dCDQhp4LBDxiuUuQ9oL1QCXoWL/fBe", - "RhuQOeJKvLrE2c2bgaIeFg0GIw+/cCWfwfMvUW3z3crymnOe9LR601CSIGhNNmsRjJyHpkxGQqfJ6TAd", - "B2R4a9qd0ikO+BlC/oQvUhXTDWhtxtTS/jeWBwvnMRzX6zWYY2mWqlZfoGYkkKrXnOaQaql2VJb+rwbP", - "5MzmrlIvtq6aqJoytWWr14bwMmIOqOGrspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzN", - "iCDeRsAHJQ7+DZfM5uW0QGEx1f8yIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1XUOTvF10QO4UrBc8j/C", - "PMo6S9svfwRM+jeutiSduCZgGDXlLozAXqWiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLH", - "v2GqfuIC1MFmzJM7B3KHyz8mAjDg6jDtrTDOaUriEc/V6vNvS9fbK7+oP1rWr3WqLwYijirpvE28wKFy", - "lGNYXmm9HCTKBVWLc71eNpgb0iBd0VhYSOgIfi47hkKdnz6B0XgSSBh5SRgRNIIyqPo8ppiBxoTen3rV", - "8ExhxCW8VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT", - "3ILS9TZ/2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfA66/u1AP", - "zK3cNcve2mYKacaQ/UKy13ZzP4CgDGcHprkzGFhgc2WvX8jdMQkDW3+30aNlv62kOrtEAZj7JTXPyZbF", - "0n/qdvYG2zca06qhwNkNdfyOYZvES0A737/hQtyq0xNm0vJskrUNh/JPHBCSf9Z++6D3TOZpisXCLZi/", - "WhmXTYIxkQi7d40epySKNKuAYjx99JoR8xxhhbCJXBY5gxrG7kNNodVTYNp2m1yAFP3I48UXW8JKH85G", - "8anKzvRx+bREz1+OdgoyXt5I+8ghbBuqvQcC+hEXBbgf7KTsDZ7ffadHnE0SGinUKwjYxiNTCSE/CeCF", - "O+whLtDvOVcYFeH8j+hIW5l1XJBbt7yKtv6g8SdzvBMSMoOfEZFiZpIjzDtrDv3ScTYuifI4r7zVHOFD", - "aQ+4qRwIj7moQJCrHlH/2qoLg8vX0V4AgcH2aaYXPyDh793DCbeTLWqwPuSRg8qXKJfkMR0n62Ibl0JI", - "UJZ7SdTXQvOD+7yybBGBP+EpeiwE/JIUEl65W0uXwlYmcmYU4KAE+KZMWLTffVcV/t6WT7woGfBr6Kah", - "nIUyflUcL/rIralR+tUCIJYEgXnGy9fKmR7e13LCdu7jhMGMC0/Rt2vq2zW16pQbanFTgIPpnfIWNogb", - "WSD+fPaHG1sfvtke2tseWlkeGLmy1oW/83Ef2YjUiMcEyRnPkxiNCTJ4Ry72RGHRn35EWEQzOicAagdF", - "2vJE0QwLiCxJUYwVNj70RsPESrNE0dyWbq7n4hDLBa7jWEgyAhy+URP+ZBmBSBkjMdKfWOi+Ek5wqW63", - "OftBA3vRYHk1oqsZl6TA82PKu80hvVka7Ria7Q/ZWwv0qhcQgqkdr5EkAbjaFfYfzhAeMvvB946FuEAw", - "idOSc2EBmIHUIFOabVlObdMjHcmIh7B23hKGmerJjER0QiM7rUuysPGcwQZb1V3SA3bjfH9aJGygnc0w", - "XhvAM4bBeY+LZ8hSUtV/wyAIOkryuHRyOQghLMY4SYKFOaYJH+NkZNbnkgR8gi/hDbsofn1/501iPCam", - "Vnu2UDPOzN/5OGcqN3+PBb+SRAw7m/0hg0QMu9Yk7pYCIrqCQm5pxvU5Ezw1fW6ZIW79cUkWn/pDdhin", - "lDmKgE9wIjki1/Ad1LcCzAzDvRrowZymsB/8KJeKpz7yqaM7M0yeqyxXNqNEEtUNoX4OmeLoD4ft+Gnr", - "j7LHT+AsJjjWdOK9YqYEsnXTqOUI69mP4NWAu53AAgw7+iI1YR5TgZkysJ0FOCWa+lu6UVRH0Id0s77C", - "EWYo45mpLAFENcOa5CptAFYDThKk4Ci5b7XgDjvZMB8LvZeOG3H3DFBa7RhRhk5/9A7TYO9Z+DxJEgkS", - "iij5r/PXvyK4lfUemNfKcC2T0sG0wIDiHFynjqe9wNEMGUcVFBMcdmg87BTu3HgTxppLGy7T64FP8Qc9", - "tB9MN10a/9Dv66aMu/IA/faHaeVAn6UsNTigw86nLvIeTKma5ePi2YfwgjbBl51XGAHaMNfcJnASTAFp", - "xrvxzRWJWYy4vQWSBcKo5EB+4MqYMiwWqxIJA0tvV5BPTCSjtxh/DCFycdg5GLrYxWGnO+wQNoffbIDj", - "sPMpvALWa9lcuQ7us8K5WRDRk8Fgcz0Stl3fgM+yhWPgC+uAjVpRUXZT76CFYf1z+Qf+rfXPwvWDme68", - "hCYyir8zvj9CB4QnsfuaaMAFURO7MYtI4sTu9Yae+3ce6M2KSJLcN4E+FHkW7rECqf9RkSNsVnmMVprv", - "H5jiBvd1qVTM9g9Dv4/Ofh6wnlvbOZm7UOdwnRLAoLGqNDIvIyzROYypd66V7xfwa9/+1+l+gKl4kfDp", - "xYFR3VHCpyihzOYDeIHKWjywawkfGRia4juLSuOKxG0YSeJf//gnDIqy6b/+8U+L7f6vf/wTjvuWgVeD", - "GtMXM4KFGhOsLg7QL4RkPZzQOXGTgSqwZE7EAu0OrM0fHiGv1L2V0uSQDdkbonLBvLwJU69N2gatq0DP", - "h7KcSAvjo1+kE1tMxsQ2Buw27iybpbzXE90NwCHCDLwJ6FvR0QBgyVFTaNtqop2wydTMuWI0rYdpLgXr", - "recvilwrQ709M8AbMhhY4tC5gwd20mjj/PzFZh+BtmWoAgoGge5QNmPViP43nrSeJxmOUmUosMqGN0U4", - "w2OaUGdybKh2Yo5giqMZZaSMLy6wxl0TB26kmsccnp0gGwjZhVeH7PX5FphYFYlULkjXcgJhEUbLcmjc", - "5rlAD8C/qILosJ59d8gmBEOe0MmxYQIeCHeRD1g0zADIA2JcqapUXusOmUGStcjF+uClPCYJfAT9T7Ei", - "V3jRRUWtW1cdJcFKK8Syq18eMoP1ategB1AlyBtmH/iZGVLPRfLanC1BJolWjSEC35T9hr43JlwgG+Hs", - "Vfl33ZkkSzMsvWgpjl6f6/lNQRPkxh4ILb0+d7ux2UWSoyihQA0RZkM2hUAgB97LWWVXi4SyGRZxL+L6", - "EvDBnC4Zv0pIPG3isUc+kd2hJFPpJ3Ccfq6T62MTLmbLE9CH2ADUrfbcHdt32rnubIt/Jt+dLQR5A+ed", - "seASw2/M6n5z5LVw5IXXzTn1Qp61Y4fAeHcRv6aLBwr4dbS3vObmibdkD2HRQxsO2ga8Ilygs6MThONY", - "ECk3/73tfXqmhkpL+U/fj5oVP0ToiR0LFxb0z9pbqgTyWNjBGztqhN286vV1/fttq1J8p/GmK+rwlFfe", - "3d8etU5vco2UQm9Ja99ukrXBtlRGHMoMltTSA9EoIYX4UpxTn4rWWZVNGG9x5awUlyx7Pjl2B/L+7Mu2", - "65zV74Z7YIrHNYb4gIywmmrtV81+TNT8rthFhza9wvz8dZHm4P6koPs2RYfI/DGpi3Ft2TQXNEAnjRfo", - "S6IMvMld6um2h8DEz4lwp9oMdGFmXUzLfIoMTgtMCCwxq3XfE/NKO9XXtPdn0nxheW4isdgl/yaitFB2", - "y7VapeCe2BLQd6ffQg83Um+/XNiKJbDAIoMVdezcTmBZ3cBywaLNb5ErX5yiTVxjqcQKN28SF5Zsg6ZU", - "6Fn3JdcdMr/euJbprF5LGZokdDqzToCYTiBWT/n1u2GUO/cwyqJOtsCK2BDFx5j3e6YX2XqB50Qo9Pro", - "xKy/f6Vu/QFBq+tVJce8Vt6u79686hEW8bhwnjTLpPbJF1aYDP1Xcnnv/9Q9wnxW6sSDJoHxM/bfBJMj", - "E//ep/x/7fyU0LHAYvG/dn7CSUYZ+V+7hwlWRKrNOyOWwX3ddPetwDxi4tP6C60uGrAmNgXI2DUCf/FW", - "S5nfvf+nEvvNpG8k+Bfr+k32byP7+8u1Uvy3W3GnCoDp44E8XAWxhVYbHn2DtLkHo6mlSA/SpuJFKkFt", - "ZlwqePT48pttUDktKM6/Nlpa/8sDufL6cKR7ctyFhYSK0lDRwqYP3pMvwI3j3oVb2+/9OwIO0zGd5jyX", - "fmZiilU0I9Jm7SakyoAfm9hdXs+NgvdXTKWD+7w67l2u/kb3dyTx1zfUMG/j0Fsn87u32sr89n0t8xtE", - "U5vZbMtudF1Jps2GQGuHadqWjCvQr8sB4KFxhXQR9E4rKqW6gECDOBiy/631j98UwemHH1wKZT4Y7DyB", - "3wmbf/jBZVGyU0cqhClBbQW9w1+PwYs6hUBZKLJXJmzXx2FqdgPpubIC/3YKUulIbq8hOSr8piG10pC8", - "5VqtIdm9uFsVqVqa5N51JEdvoQW3mOJ/Ti3pT+4eqWhwMp9MaEQJgwIvkJgul+IBjSb3zTNyy4RkZv2R", - "XjBRRRJprUYWXGuNhF7WlL73QLKTsojWfWuPrnz140ys4pmtB2v1tVJaaFbYvjZ6GNzv7XX/itpjJjGj", - "ES0vXaaF7kBlIFMTKs1NFlrxZQkq1kdv375yGWda4heuzpTirriUq7s5ZH5xqT56UVbtMi+4FrRETmKb", - "oQp5eLZcU0xwnFBGIESXyFByWLUk3IMeiy8vVIbr3bUSKu/5WNoKpg8nVD4YK7gX8e2kUhialzq+Xwqv", - "OC1OhINT86j4lWVAAcYTEp+2cK54z+awbs24ATYLYzueJTgCaEf9mkEds7ABBmbQbwqwAARPEiIMmlyW", - "K1cDcciKwVHm1Xi3NW4udPOjnCmaXHRNhAxAgkiE2cJCKg1ZpTOsFEkzBam9kLYOIxQkMyOuFX/Ug6Y8", - "l/AWZOH6XSKcXOGFHDKbDGw+h0K5gkQGeDFJ+uhnDjgMCE8xZR7jNRUIv5NDdkHjhIwsjMIFohLJGReK", - "MBKjlM+JrPZLsEgoETCJI6xXTqIULwDPzEA7mvXhGTGYYRWwBq7/jVlMoZad7rmY8sGQYbQzGKCUYCZt", - "6rXEE7hwbBsIBlEZ0PcIo73Bc/tVbd8Ac9ct/4Y+TUKQOY/wOFkgoqkYwB/UJmxgamtLmhq9evsmVEiz", - "X4XJ0BYNq2wsla5UYtxFOSuTy8F8nrMiF1xvl8oFg3laxxqhorgGLZ7GmERYryfj1X4AyZBHUS5CF6Te", - "aq/I6b+j4OhN7xyWKpy6nYAWHpEY9pxxNYMzzeEobX7fQFUlUf05LprgIeECYeTRdWkkIFEOrHEDkP8u", - "yop9zFXgvdj83p0dfXwtI3DH32DvPZb7CYiITyaVA7j+ajIHeFXKxDIJ/1nP6ZEr1eqzuJjiKeNS0cgx", - "w3pl928KYWuFcPXKBql5wsWlL1tV6fcnLi7bamAWT5Q+LkXMn+FXaNvXwwPs5oc38YOB2SgrmmjuXUmr", - "01dxSkHookq62GGOEs6m+hSVhu57t8T7Wt2GwWHTl6kw/uMCNUcrISP7o6n2qidja2mC1T6yrT40L9K9", - "34N/51euEE2zhKQEqsH2DLHpzS4RlqByPpUeztDNeKU+VX46sNEFpXHpd504BHTlNmwDpPfl7Qoy1YRP", - "1+P4FZ070LoAkN+Qmcr+BF0Yb84FKniwFmgNaj66mtFoBqB+oLfq9g3mH86yiwLPePMAvYSD7MM6Q+cb", - "Bitf05rkCTFYffM0vThYrnf6/vQUPjJ4fqay6cUBcjVOi/tD6rd8kD49iwRLhX610IMbhTIOO3qhsNY3", - "i/ltWvi+Em96yEJQfoxc2QbpBF14qH4XDZBTjt++4tMHE8a6zVUCzFwUR1Z1BNokLO40xU3QJAzotz0Y", - "hMCrW4ILmmHcMbbg0mBe8WlRoaBCyjjL2pKvHSZQ8TxNV9Aw2vCQyqSKea7+IlVMhICPLXU3ETfawJGt", - "ToUvNaFaXDp3sDeB/ILRQQYyPLhUmql2uh3C8rRz8Jv91zxNO92OHY8HNX4D4X4NSGO9weUoFr0zHhLj", - "N7H8JhiLVWbvgSzWbg6rTjdL5G/MC396b6Gz2T0gGYJ8UDPifk0iqDfeqsGH8QIsEkb2/D5GBoiSKEq4", - "JBUHz+PBo7KGrprM2Gwocmvc08OLc1fAp01QyLn99Nx9+RXo3utiRdyYkZvuvQeNLI/gMefWyqXZTLio", - "gxitiyb56gnpy23J0lTbUMg32ry5lbEVYWo9YZlF2A9iU9AN54qnWNEIiglFM86lR/YF4rAp+2WNxwVl", - "gmnFaLk2KP9Ck+qFNUNfWDXiwJrMEPYf2T768LkN5Q9/4R6VX/zkWQUKjt91oj8A7kO1c0HJBGU4l0RL", - "dXlKULSINFc01aMIjmYowpnKBYHCeASllNE0T30oab1jcwywFxfb6UUXjXOFEiymoJWZhy7YJuJpSlhM", - "wD43ZDOC51SrlAIlWBEWLXqSQEHdOUFXXFwmHMdgYshiDJ4eKMgniKZAwOVOicIxVhgEnQt94kcmL+ii", - "qLFr1HpGrktqiIdM5Ox7UyRAN3vhBnqBCKBgUzkrajFGOCYsCqJDn3/dbOzL26LPiapP9IEig27FSx8y", - "VMi3ubrhfB1RRI8svJkLu41t2PwKoVc2q7DVhApHRv+eR9rM1c3xgRxMxRKvOsVfh2epILqvxrv08O4j", - "LlCcm+68Uwlk/mf1CRUMxQ+2gmRNs423dQwVReeKZb4Rz9v6w/15cgtb3lfCCbuNin1TeaNy0l8Dy7Wr", - "eiue+0BGTGtL8m1yD8eCXUTXg4lPXHhc7rEYWy3DNkez4Ns+d1ICg/bF2Te2XWfbNuDhtmzb2WaXXPoe", - "I6esBzGiYQ5uzbiNrNqaDv5Ns1Fqs/NY5oOzyNJzcW9s8aRghIY1ZniRcBz/GYKEV/iPIi6EQZQAjIrH", - "hGjqWQ399ACwzZV107ouW/P96elmE5cQaiWPEOoRcwgvJUd/lsbLBtzXcyIEjS3wJzo6PbbhulQikbM+", - "ep1ShRRHl4RkZUYLZBX29fwctsZyjfYKiEa3Q5gSi4xTptaOonz1bgbz6VaV3e+ZT1qI7G/u8NbucLDs", - "Pz52BlwGcjbMBFZrpgqrtaU7KZtwkRq5DI95rlvXPEgvk95PUwJzQhMiF1KR1EQlTvIEjhuUW7Alde13", - "Zpe7EJOrT45Jl8uISKmUlDM5ZDZXJCNC960/1+17AVZBh4DCBX89M0zy6wje04Mx8WpYNa0aoCBBqc7O", - "QWcLZ9lWjBVuCBCzw/uMIf0E0XhILtIxT2iEEsouJdpI6KVRT9BcokT/sbkynG8E333pgsG3P1l6pU/Y", - "hAfLsRmaLYj5T5XVZdmac0w+Orb2kviHxfEf2OgwW1tfklgQnPSgxK/DwkG5ogn9aFidboRKRSOTcoSL", - "tXt/WjDV/pCdEiX0OxhS25LEIBqAdrmVCR5tDfPBYDfKKACq7RIYHDC85scp9Hh09s6koZKUi0V3yPQ/", - "oOG3h2fGuzvB1prgDdTWIkYnW6/XBDifwzL9G0cImgmuRC8Ibvg3l+DNMUYaz5BsOKI8W6Uq8exPH8Jq", - "JbhvdoXHaVcAkKdiNhtTgSMQiuUsVzG/YmEbwpwnear/Yf44WQcVpnA0ew+vfjXSrhnO2m7cBB/FobRz", - "iokpF/kgTg+zYI81ZlUvnJsCCDGVaMDgLXCo/ozU/eXN9/46foXuTruirhTrV3O27vvms2NwCBv+ejyW", - "Y24ozc1E8dXWpytMm61PPyY8upQWisU3G2q9DSDL9Y8lxLR1EYKYAJmhyEIYGaAsIrtDVjNAGsQfiTBS", - "RKSU4WQL5mwaAbBsZ8XCc04hQTuCPJWepDFgJiWAiA3wd3o2YKhyDXgeXWmLVfnv+M5IxdGYRDwlDkB8", - "M6S6/Q1T9RMXVTTwr4UvvvXWHyABMQV7+xoA9OYePwsQ/RRfQ6h0nFuHshvRxkte/mhMQV0EezPs7A7k", - "sNNFw85OOuzoHTjCYELFCu2jlLJcEdlHx8a+BSm4TwZIkoizWDocc2fB2x3IpoRcQ5YN2Z1P4Lv7FHss", - "VcFSvrGdhNiDfg/p7yFpB234B86eybgLhy5GPFfG3G/PlX0rJgrMI5v37qv1zsg33b4NJ/+bPb4VHgW7", - "rNmlt/WGs2e5nJFmk9srUxsoV2PAx3b1OuUM/Z2PZRcxcmWs4UKq/hLf01+fmQ7uA7tfd3UT3H4792+g", - "/S1A+8u1CoM1mgBLfSU76jCIjeQ640IBiqPNtTc0BJoEIEfwCCfo9dHJkEWaFRloQUFSDtzJQoybW/jw", - "b+foxdGbLjqG2pHo53y82UevWbJwFbyNj2bIjCRmmFeEGRobqiVx6Ho2Ywfquctgcd3BAxVjNicj4Flx", - "e+WCxLudGcExSCR/dF5x01kAdfjNK32AAPjXfFlse2el8NF5Q5RY9A4niojlZk9tnhQrMDPsJe0g6Kzg", - "ZoAvdYfSIa+VfRrZwEBj7O50AkgZn77VUbj7mqP34yUzcSKmgt04B6RRBkkGOF48rlgmOUMFcwyxQP+6", - "LioRNGUJW162UsGALpsiv78ik/tK3lXBlv93PV0w00fraMoq+6SJuKhgstbT65KDZwYO2TqqIpzhiKpF", - "F+EksXeUvQmKiJReIf6OBcGXMb9i/SF7U9ROsQm96OjsXdc5alFM5aVpwfpi++j1nAiZj4vBIThoxmsM", - "a07iIVMcRTiJ8kSLG2QyIRHk4kJJFNngyy2G0rnDs1N2Eqzf4kW154+ubFyYJmD3SrKoU9yW2eotQaIE", - "07QZfNwKahBwCKEGY90oZ4iySWJDqiLBpUS2qR5J6JSOExsgJPvo7YwgiVMyZFmCGSMC5dJExeuh9zJB", - "pMxNgrduAEB6DUV1UQksmAmubGhCwrmQJppAU/j7UyQVyVaQ2RvT8inM+Y5kW9O47emBjNS1MTSbQuwr", - "SG+IoRSz4JqO8sQFMN5rKLoZ0ENLiY/l4L8VdDolQp8KbJisCcczx9otpzn0lYzlxhKS58Vb7UpIFq16", - "WYlext5KYLhRibUdd24W9Rfo/JI2YgfaRzfLIv5Ff9Sy72q2angQ9tFnzvLPUpn/3EsSbGvAKin8sZmT", - "vJFXjmol0XY9rFbrzNq7zHRtjZ/1YLBZjxktC1fSZ5sU3q+PEAb3i/Jw30XWHjdtVdCuKrppQ8r/ejT9", - "r4IC7wZG/4FRTm4Bo/9V5d0DzvnD4Z8ED+pD5dFXfM+ufu2fHgn/rtLnDRw+wLE1pc8brmeDV1cqSu/t", - "O+3UJNvin0mCt/GON5Df3bJ/0/pbqAzeYq1zQWuCJ2mmFi6gzfoqy6AzST+SfoMjuIhbvTtX8C1COr8c", - "eTg6bQzo/HOWm3+QmFFbOpBKdHIcqOP+yDAG/TNXuVi29K3TwyKa0TlpNrpXT7BdokyQXsYzcK7EZsHs", - "eri7TGHRn35EtnmLuWr/BbUnAaqfxCimgkQqWZg6oJojmD6+k0hwrQnAcy4WzVEi5oj8JHh6aGez5j60", - "Z8oaw8o4w3TRi7HCvbnjNitMaJ8R3eniKTXDQ5Shlz+iDXKthKlwgSZa80F0UiypKeAvgSY3/QFvDxos", - "m/QjGU3HbUa5olbJa1sLBkW5VDx1e39yjDag9tmUML0XWtSfgCSbCT6nMYkrY+zMeWJWdbthQW9qd9VC", - "RVG4zikXZnAPIsO0uZCmH2lWZQtFSMyYMgyDW1sVpHqmTBK/7g9T5gJw7B65UXy7wqzmt+GUHU2JUIfT", - "LqLi3EA8b3675h7zNecnQ7k7rXLbufCc1cbrdvlRLdOW7qLwQ5E7d79m6/dfT0oPlY8ym8eazueFQtpk", - "Nv+6SHBwf/fDfZvL3z/iFNCXxCnfnqkcGtAthgjmFcR0x2ROEp6lUA8d3u10O7lIOgedmVLZwdYWxH7P", - "uFQHe8+f7nY+ffj0/wcAAP//uZ+MGF3vAQA=", + "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJhhK", + "kRmVgZF+q3IhJxbf0YrS2MlDBm+6gH8onfpyIbfGudzKIrpl82+2AJvjGWBz7AWTp2MyH+V5SDXSjxwY", + "y7t3J8doA34BbFlIoawSMMZPtp8Nnj3vPRtvP+ntxYPtHt7efdLb2ceDyW70dHd7Z3dFIkyLbLrbJ8gF", + "NeZAHHMRtT5y0fOhoOam3IWabGLjsa8oi/lV5foLBsj6vdvg23XdL4fWtx5CMCEnwVIZ80UDJzuFS55E", + "um0TkG4zNotSS2FD55O3g+3Ptf7A4BruiLciZ8atajAGChdC6g3Y36zqOG/H8mFALvFl3Wr5nbdftMHB", + "/vOD/c9dNJe8sW6MdXK6x81tighzGMy17BCXoejZkZyBsmNlImPVt8kknW6nyHeBv0EYqMVSF49bJXE1", + "HdhumI2sulYakqdPKvoKRKoYDL74QEsqTh+Big5Fir4WgY4SnsfIs8UZSDLww514uotuBtxi1kRnIEZN", + "MobWcQDTGipHUKYZMfgfdSM20/oAvYR34RFOjVpnB2Hql/iuNxwvTLyMPl+ua6NkrR7yudWv4ButbCH9", + "L5i2XgZrsl3dhJHODtCvHL4ptD3G67Zf8zqoWcuv1+3EGxau2yFnQGdW1DxAPxXiZSGgWoF0QxL758gy", + "rBKwZrMCG2B3vKOppdw5LwW+2zEr2ul23EJBqvxy0vy7kuqXzp9PiqFAMoITOMtljnCuaGJhumEmVCoa", + "SZs8oje3SeyxpZVIPDLKU1NMqkk8tQpW8ZGTqt6fog1AYvwLsoZt/a/NIn61ctftPN97/uTpzvMnrfCW", + "ygGuF42PIC16eXBr5eQoy0fWNtI09aOzd8b2ERmrQhH78v7Uh7fIBNesR8/cNeh3/rz/3IeZink+TjzH", + "osWkM6i2sGFBJLWCFzXEQf5OkzmdTNjvH6PLnb8Lmm5fP5E74+0G+FzTUdjsduIHFyzZqMm4Z8okhZGA", + "gKCEbATLekMkzACdE4WAfnoIR6DeFNnMluQcpJZd8SBh7e3u7j57ur/Tiq7s6LyDMwIjXOBStiPwjhi8", + "iTbenJ+jLY/gTJsOUwIQzplVfcPnDNkax4OqQNrfHuyGqKTh4i6pxrY9TxuX/L1VH+2k7KJDUnahWi6d", + "8uBq7+4Onu7tP9tvd4yteXgkrldzGJeyZJbHAvH7O78B0uTbwzMECcETHFVtOy5C7EajUjcaFRSRMODv", + "NxjYs6dP9vd2d7bbob6Fgk4snmHlwFZ5V+DQBYgisBuBpVhmvd2m2yIkThkCe0OiBNP0MHIpFrXbx4C8", + "j4R5rdyENheD1cCXLq4W37YybhUmK5OgY0QDLlDOitIi/fUu2S/iWW3m2uZ6WM/VQ2k5TK+ehScyJdRu", + "sZSZIHPKc/kFGuLK5MxOEs7Fjb5tUljeEJknythsqETvT78DnqJpDUlFsqoOZalxBYjTLSd3o/NcIZEw", + "kTctVqvdaLP1qybcbTi13VWAGhVu0AidFmvOlbP1wZ9HOIlyKKaDi/3UswIMMIAEyLJkYWL7k4RzhqIZ", + "ZuAkER7iEZrxJO4HI2H1k9EkGFXBr1DCDejzJSGZrTNjBqE/0yIMnRO04VdYM6RUq3u6nxomYyuJVKlx", + "Pw0XcMQylKxWpMLr9cSKe3jE5pOKJTThUwlKoYKshX4dBj/DwiQjYGbqJs1To0sGAq4DQ6wx89CNam5S", + "PrEKrhU5INHcrCSOBJcSkYROoUbP+9Na/vKKnLcii3l9QGd1sC1I1zg0A1eZQcNqXV4tdD8G8nk+54YE", + "GoacwRWhks44mWKWQ+UZj5CtIb7fOhxyxqUaFbhUNxysVCMoJ5ELUqLlFVn3hT3IvRO8Fx1ru81y2bjj", + "W329RFXhppoG2MxTgysaXq1uQYMhMl5G5loJBlaii9WhpG4CVlfWH6ASWqUebBnagJwXjy15EHSbbYJk", + "wiqr7mdJW7XVQV/tDc7bwrqtRnE7w2p2wiY8gP1xA8+ps0TbaNWMiJRCQRUUE0ZJ7HTJwoVqTV2QMJ5I", + "guKc2JUz8qnAdsGxOd7gs2LORkbZtMbr6x22MQ+bMayuNgH92hfbhDvJcELtW5HDWpl4RYlwmVrbKgiU", + "ylHYnbXcsCDTPMECWUDGNkOWizSh7LJN63KRjnlCI6Q/qPvFJzxJ+NVIP5I/wFw2W81OfzBqKk10bgZn", + "8wLNhtT6Lafwg57lZi0rGSwxW+b7LXCMtokeC0aK/0QTYtH93jF67RF6FY59b2fQlC3f0GglT34ZGfKm", + "nNuSbPDE5zKQW7hSynFFlUhsMfKN2JPl0tR3aXErORBW5wK8nUenmjjyedAkR4Zf14BJ0JhA3o+b2jLX", + "aMEW20wlWFoilzP0dz6uGkTbhv0GCpZtsBIiQ5BJML4fdnSlQdq8sbQm3u7eBIMC2KqeKHx0Q2iHdaXd", + "yviqJn7yZqnK2YzYJaNujqbiWYsKHi7+o4AvsL22xzGo16MLxCsDSo1UC6jsCuV0Fl6RRYnGXAhAoNYS", + "DmduNgC7omUevdYO9wq9nZEFEiTFlA0ZZYWRFMDUCGJkToSXJcuFVrKmJO6jv3kqHmB2p5laWDB4MJ5/", + "JxG/YsUYh8wfpG48l7qdQ2YsiyLPVKVcpG4WtD5NKJC1DE4wJaB+IlUzNBFEzvy5h2pmahnviou4sRjR", + "ArlXoMYN+FiR4peE+aysaCaoGpqGRuar5Sg+U/AWnlr9E1Vq0KJ6jdnV/eWSiLCQWEypeKVV6Ip3VDzl", + "xIDAACIK1Ba0fxkWX6CgtMA8KZv/q2uy/OmsaLz6W+01D9fEwQwfGrNt0AQbmTSeWrBP1ZO2NlQF0uBW", + "odks+xLQhguhdgVaqpKAVyil1T3ZLhOvnizgRrMlSVTtfe/Z/tMnLSvVfJazzqB3fWnX3Dxd4ZJr2KnT", + "Nn6fZ/vPnj/f3dt/vnMjD4vLK2nYn6bcEn9/0Aa5VvqwJv/6xz/fn9a8PvsQgz240aBMZkl4SA3ZJdUB", + "vT/91z/+6UZ16wGFGM0yQniD374xSifxd9IFClRdeO2cZCv0+8OKkQAXbAZtkMmEgBl0ZNatVw6mBgPS", + "TgrGGY6oWgQYOb4y0e7FKzWk6zbuoOpgQyKvaduiomrOJfNxmXS64TpH/2l8wzVaeNa64JXMx01+6Nf1", + "Xo0XuvRa+DEOLUIMZFFrfdnAXcznCstKQLf+O4K8C5dhtpxtY95YjbpbT4WAKBZb180LBQyhtdfkSfuR", + "v/217fT8lhWzTn3FP6w4h81H8EZW38CNHDD6RutTa2v8wV6At/tqNPZL0a2s9VepW1feujfvt0X28HKd", + "hOIGu3l/XsLkTT6sYwIDPdox2CUv2+5WSKKBmrxcmIABjSekVwTq2UQZJHPjEdRn3sLMBzI4o0s+mVSx", + "bvebsdEB9geSvVwvWCmtmXQRuXY2izqwtsH4GXb25bCjVYBhZzsddmpuq2D6ZIqvR7aDKrbLYBVYeZn+", + "XhukdDMYJzy6NFXWoHh3Hw1QSjCTKGdw+Gtete3Bau9Qt5N5e1NAgxMT4rTEtmBMYzLDcwoVKaxPZVoJ", + "xCTXVEkIGIV2DlDMDdpTpcSsnaF+zSQ3HpSThksHs4VtWDeo3+PMRbSW74KBbwKFbdlHInjXghVojv36", + "9WnXBDBA6KEZWCW+0U3UjEAzyKKLWnmF8vdw/PA4ISMYdx2uP11eRz8nHTyrgkiipMXvLsmhRgQo4jlT", + "dRz/tJ0iV00rW76ScgbBfjb8A3DZbO+GQFBMIjiRcvksVgn9FsRdyxuwKx1KHNgNkTAcCvAlhX3Fb6xD", + "uD4AY2zwqkObdvy4buMlHEnFbTmx4lSPyHVESFwH/Ay/0jZW3n4ZjJV/hS1GUFG42b4N8c7Ls+vfXYIX", + "jLVptf2YfsZZD9BJ3JZaJBEDDWixaqqEVoEe9yAtRiF41dALbTKuyfXqtf6VXCvAR4/zxIDehUnXsip7", + "Ga1b8VtnNjYdaC7I2vJ8d1C2zsSb36pwnQ1Vf4jadfatO6lXt7Q750S5d88tGTXuULXQS8Wl5QL+3SvV", + "GBtDSl1kL3i0nW7WSHBvFraKWFDeljmaDKdklAkyodcriMe8YBTjKqxJeZCKDAaDL7qR4mu09xRFMyxk", + "beyMTmcqWVQDcPYCWEqfVdRREEWYMxS22flyN92Hy9Fudjv91kPC8bkHDbRU0sSKpKNVuNlHpbfNWucz", + "vAArTqOT8Onu3mCwuzO4FXC2G9YNluuo/MSWQKy205RS531nHf2VKFW/hSLJermu7pWgkKtdLJNUguD0", + "ABJvMhwRlJAJgOQVCa3rPYv1rlcP3gpUNou2oH+3UXbfnA++WjKn6MpijrtpdJxzsYpB5D9f4xBtYDPR", + "EqReIOdutzd48nZ792D/ycH29l2AXReL1JTt8fTj9tXTZAdP9pJni6e/b8+eTnfS3aAedklNZaA2tPqL", + "frcxyqa8JKtYRhWWhjbsHDIi6gWT64XGJUkoIz1ZZEitT1NcwQuM/33t+b+Znd/MYKXscF6dpC9CYFUu", + "ToWyHgZ/y05mpe+iPpuT49WzuFUGUn0gYXqrDwXIq91goELFduczkRly1vIaeue92PoiWpkVt+4qCnnY", + "4aQHd7lhxUPkXQNm8Ga96gJfvuQCttMpF1TN0tW3RfFaASMOcdMfpYqreE99dDJlUC3d/7kIk/OVKP1x", + "p9tJPu5Vz4z9vT3yl0UgLgjQbrUvFbQII4Ni/KtXAV4pFQ9hItm1rq7H/MN2b/s5xCEkH/d+GPSeVyMO", + "uma1/OXbdm9Xfh20WUO/BKArHbX9/EYR1249V1HQLzRUwK68ly02saXxsja1uzpcwm1lg8vHS3tcQ/Jp", + "FEA/V9Kzl9vIF5pikuBFCJveM9TKmvboExkakyllso3ddndQGG7302Gnjw4tQDjosooX/fjNQw16j05o", + "mpKYahnTqP7NGQw7LW1xdV3iZrVJ3FcBaa0fFteer4dIWJdwte6a7H9GPu5nab/tNN5V6B1gV3MqKmCI", + "wYtdRCcIs1qBUsrmOKGxTaSHxEiIVztwQG0lyVoeIEs50NlJumjKFSpT6Fva23LWbBcsxk+uwd66AjPD", + "EMTOFwFEKQDE6Cr2dXKMMsHjPCrzRxMYdIn4IfIaRNsKIX99SO5d2jcgMXvCBVpv32gyaLSzTzbtd802", + "qQm2eau3B+u3+k6MIt1OnsXreZh5qR0HuxFy+5oUxICJprrsNUnQm8yHFhz9jb+CyzqvsSVHWiTKM+dg", + "0TS1TEkBdwu4GEJxvcckIfqaWm4E8SQusySoLLnoepa6/eTZrMnFCR6p5YH8QkimdRXAP4L+UswWwYG5", + "sqPFXbIxcGjf0ji8eqZckV2t6uCerpXEGrfKN+E2lVAwXL5m8zZ4KZee+bvA+PZFs2UEFMfwK0Lam+YS", + "APZLF/bWaD++C7PcQwppr63roQbb6kCFC3R0138ZC6zFuirx7oXc8yGyeGs14yYo2noWqG91Puz9j7Ey", + "o1H/YOuHv/zfvQ//GbQ21/RmSUQvJhMINLoki54pPqR19H4ViBUqH2hhempJheAUbEgAcm4Poz/e/UHB", + "NBa/4nRpChCh5VUO2l47ob/8R3N8k7eM74BPriXZzy4MchcFVBV319FGSsTUxZK7RLLN/pBBbtolWUjk", + "1SOzIo0j1O9k8YkXgY4ujBjYJ2x+gcYUCjzKIdNaLY4ikmltwpa4oaZKOQfuIwhO/HZsXTSX+G0dkiae", + "gKD3p0sovq/fvf3x9btfj0evz178engy+uXFf0OIx1XP9BD3NO3t7T+xtcn9ldwO1se4eZmHPjq1YfrW", + "1T/JQaEFnC6J0lzlEBRCrqMkl3TuHIQquX1Bh+Vk3dsXSPhMBGClklBUgkWqTuiEgF8frhMbVEOlI0Yq", + "oai7NW5QhpZvbEM4ww5wUq8mf6icht6K8GqXG1td9CezdizUYKMGDjtkvEL1/YD2QiXgVbjYD+9ltAGZ", + "I67yrEuc3bwZVuth0WAw8vALFxgaPP8SRUDfraz6OedJT6s3DZUSgtZksxbByHloymQkdJqcDtNxQIa3", + "pt0pneKAnyHkT/gixTrdgNZmTC3tf2PVsnAew3G9jIQ5lmapamUPakYCqXrNaQ6plmobgHcBWdjkrlIv", + "tq6aqJoytWWL6obwMmIOYOarspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzNiCDeRsAH", + "JTz/DZfM5uW0QGExRQkzIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1eUXTvF10QO4UrBc8j/CPMryT9sv", + "fwSo/Deu5CWduCZgGDXlLgwMX6WiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLHv2GqfuIC", + "1MFmzJM7x5eHyz8mAjDg6ujxraDXaUriEc/V6vNvK+rbK78oi1qW1XWqLwYijirpvE28wKFylGNYXmm9", + "HCTKBVWLc71eNpgb0iBdLVtYSOgIfi47hvqhnz6B0XgSSBh5SRgRNILqrPo8ppiBxoTen3pF+ky9xiW8", + "VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT3IKK+jZ/", + "2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfQ7kBd6EemFu5a5a9", + "tc0U0owh+4Vkr+3mfgBBGc4OTHNnMLDA5spev5C7YxIGtv5uo0fLfltJdXaJAuj7S2qeky2Lpf/U7ewN", + "tm80plVDgbMb6vgdwzaJl4B2vn/DhbhVpyfMpOXZJGsbDuWfOCAk/6z99kHvmczTFIuFWzB/tTIumwRj", + "IhF27xo9TkkUaVYBNYL66DUj5jnCCmETuSxyBqWV3YeaQqunwLTtNrkAKfqRx4svtoSVPpyN4lOVnenj", + "8mmJnr8c7RRkvLyR9pFD2DZUew8E9CMu6oI/2EnZGzy/+06POJskNFKoVxCwjUemEkJ+EsALd9hDXKDf", + "c64wKsL5H9GRtjLruCC3bnkVbf1B40/meCckZAY/IyLFzCRHmHfWHPql42xcEuVxXnmrOcI/Oe7Ym8qB", + "8JiLCgS56hH1r626MLh8He0FEBhsn2Z68QMS/t49nHA72aI07EMeOSjIiXJJHtNxsi62cSmEBGW5l0R9", + "LTQ/uM8ryxYR+BOeosdCwC9JIeGVu7V0KWxlImdGAQ5KgG/KhEX73XdV4e9t+cSLkgG/hm4aylko41fF", + "8aKP3JoapV8tAGJJEJhnvHytnOnhfS0nbOc+ThjMuPAUfbumvl1Tq065oRY3BTiY3ilvYYO4kQXiz2d/", + "uLH14Zvtob3toZXlgZEra134Ox/3kY1IjXhMkJzxPInRmCCDd+RiTxQW/elHhEU0o3MCoHZQpC1PFM2w", + "gMiSFMVYYeNDbzRMrDRLFM1t6eZ6Lg6xXOA6joUkI8DhGzXhT5YRiJQxEiP9iYXuK+EEl8qJm7MfNLAX", + "DZZXI7qacUkKPD+mvNsc0pul0Y6h2f6QvbVAr3oBIZja8RpJEoCrXWH/4QzhIbMffO9YiAsEkzgtORcW", + "gBlIDTKl2Zbl1DY90pGMeAhr5y1hmKmezEhEJzSy07okCxvPGWywVd0lPWA3zvenRcIG2tkM47UBPGMY", + "nPe4eIYsJVX9NwyCoKMkj0snl4MQwmKMkyRYmGOa8DFORmZ9LknAJ/gS3rCLUjpcSm8S4zExJeSzhZpx", + "Zv7OxzlTufl7LPiVJGLY2ewPGSRi2LUmcbcUENEVFHJLM67PmeCp6XPLDHHrj0uy+NQfssM4pcxRBHyC", + "E8kRuYbvoL4VYGYY7tVAD+Y0hf3gR7lUPPWRTx3dmWHyXGW5shklkqhuCPVzyBRHfzhsx09bf5Q9fgJn", + "McGxphPvFTMlkK2bRi1HWM9+BK8G3O0EFmDY0RepCfOYCsyUge0swCnR1N/SjaI6AlRMra9whBnKeGYq", + "SwBRzbAmuUobgNWAkwQpOEruWy24w042zMdC76XjRtw9A5RWO0aUodMfvcM02HsWPk+SRIKEIkr+6/z1", + "rwhuZb0H5rUyXMukdDAtMKA4B9ep42kvcDRDxlEFxQSHHRoPO4U7N96EsebShsv0euBT/EEP7QfTTZfG", + "P/T7uinjrjxAv/1hWjnQZylLDQ7osPOpi7wHU6pm+bh49iG8oE3wZecVRoA2zDW3CZwEU0Ca8W58c0Vi", + "FiNub4FkgTAqOZAfuDKmDIvFqkTCwNLbFeQTE8noLcYfQ4hcHHYOhi52cdjpDjuEzeE3G+A47HwKr4D1", + "WjZXroP7rHBuFkT0ZDDYXI+Ebdc34LNs4Rj4wjpgo1ZUlN3UO2hhWP9c/oF/a/2zcP1gpjsvoYmM4u+M", + "74/QAeFJ7L4mGnBB1MRuzCKSOLF7vaHn/p0HerMikiT3TaAPRZ6Fe6xA6n9U5AibVR6jleb7B6a4wX1d", + "KhWz/cPQ76Oznwes59Z2TuYu1DlcpwQwaKwqjczLCEt0DmPqnWvl+wX82rf/dbofYCpeJHx6cWBUd5Tw", + "KUoos/kAXqCyFg/sWsJHBoam+M6i0rgicRtGkvjXP/4Jg6Js+q9//NNiu//rH/+E475l4NWgxvTFjGCh", + "xgSriwP0CyFZDyd0TtxkoAosmROxQLsDa/OHR8grdW+lNDlkQ/aGqFwwL2/C1GuTtkHrKtDzoSwn0sL4", + "6BfpxBaTMbGNAbuNO8tmKe/1RHcDcIgwA28C+lZ0NABYctQU2raaaCdsMjVzrhhN62GaS8F66/mLItfK", + "UG/PDPCGDAaWOHTu4IGdNNo4P3+x2UegbRmqgIJBoDuUzVg1ov+NJ63nSYajVBkKrLLhTRHO8Jgm1Jkc", + "G6qdmCOY4mhGGSnjiwuscdfEgRup5jGHZyfIBkJ24dUhe32+BSZWRSKVC9K1nEBYhNGyHBq3eS7QA/Av", + "qiA6rGffHbIJwZAndHJsmIAHwl3kAxYNMwDygBhXqiqV17pDZpBkLXKxPngpj0kCH0H/U6zIFV50UVHr", + "1lVHSbDSCrHs6peHzGC92jXoAVQJ8obZB35mhtRzkbw2Z0uQSaJVY4jAN2W/oe+NCRfIRjh7Vf5ddybJ", + "0gxLL1qKo9fnen5T0AS5sQdCS6/P3W5sdpHkKEooUEOE2ZBNIRDIgfdyVtnVIqFshkXci7i+BHwwp0vG", + "rxIST5t47JFPZHcoyVT6CRynn+vk+tiEi9nyBPQhNgB1qz13x/addq472+KfyXdnC0HewHlnLLjE8Buz", + "ut8ceS0ceeF1c069kGft2CEw3l3Er+nigQJ+He0tr7l54i3ZQ1j00IaDtgGvCBfo7OgE4TgWRMrNf297", + "n56podJS/tP3o2bFDxF6YsfChQX9s/aWKoE8Fnbwxo4aYTeven1d/37bqhTfabzpijo85ZV397dHrdOb", + "XCOl0FvS2rebZG2wLZURhzKDJbX0QDRKSCG+FOfUp6J1VmUTxltcOSvFJcueT47dgbw/+7LtOmf1u+Ee", + "mOJxjSE+ICOsplr7VbMfEzW/K3bRoU2vMD9/XaQ5uD8p6L5N0SEyf0zqYlxbNs0FDdBJ4wX6kigDb3KX", + "errtITDxcyLcqTYDXZhZF9MynyKD0wITAkvMat33xLzSTvU17f2ZNF9YnptILHbJv4koLZTdcq1WKbgn", + "tgT03em30MON1NsvF7ZiCSywyGBFHTu3E1hWN7BcsGjzW+TKF6doE9dYKrHCzZvEhSXboCkVetZ9yXWH", + "zK83rmU6q9dShiYJnc6sEyCmE4jVU379bhjlzj2MsqiTLbAiNkTxMeb9nulFtl7gOREKvT46MevvX6lb", + "f0DQ6npVyTGvlbfruzeveoRFPC6cJ80yqX3yhRUmQ/+VXN77P3WPMJ+VOvGgSWD8jP03weTIxL/3Kf9f", + "Oz8ldCywWPyvnZ9wklFG/tfuYYIVkWrzzohlcF833X0rMI+Y+LT+QquLBqyJTQEydo3AX7zVUuZ37/+p", + "xH4z6RsJ/sW6fpP928j+/nKtFP/tVtypAmD6eCAPV0FsodWGR98gbe7BaGop0oO0qXiRSlCbGZcKHj2+", + "/GYbVE4LivOvjZbW//JArrw+HOmeHHdhIaGiNFS0sOmD9+QLcOO4d+HW9nv/joDDdEynOc+ln5mYYhXN", + "iLRZuwmpMuDHJnaX13Oj4P0VU+ngPq+Oe5erv9H9HUn89Q01zNs49NbJ/O6ttjK/fV/L/AbR1GY227Ib", + "XVeSabMh0NphmrYl4wr063IAeGhcIV0EvdOKSqkuINAgDobsf2v94zdFcPrhB5dCmQ8GO0/gd8LmH35w", + "WZTs1JEKYUpQW0Hv8Ndj8KJOIVAWiuyVCdv1cZia3UB6rqzAv52CVDqS22tIjgq/aUitNCRvuVZrSHYv", + "7lZFqpYmuXcdydFbaMEtpvifU0v6k7tHKhqczCcTGlHCoMALJKbLpXhAo8l984zcMiGZWX+kF0xUkURa", + "q5EF11ojoZc1pe89kOykLKJ139qjK1/9OBOreGbrwVp9rZQWmhW2r40eBvd7e92/ovaYScxoRMtLl2mh", + "O1AZyNSESnOThVZ8WYKK9dHbt69cxpmW+IWrM6W4Ky7l6m4OmV9cqo9elFW7zAuuBS2Rk9hmqEIeni3X", + "FBMcJ5QRCNElMpQcVi0J96DH4ssLleF6d62Eyns+lraC6cMJlQ/GCu5FfDupFIbmpY7vl8IrTosT4eDU", + "PCp+ZRlQgPGExKctnCveszmsWzNugM3C2I5nCY4A2lG/ZlDHLGyAgRn0mwIsAMGThAiDJpflytVAHLJi", + "cJR5Nd5tjZsL3fwoZ4omF10TIQOQIBJhtrCQSkNW6QwrRdJMQWovpK3DCAXJzIhrxR/1oCnPJbwFWbh+", + "lwgnV3ghh8wmA5vPoVCuIJEBXkySPvqZAw4DwlNMmcd4TQXC7+SQXdA4ISMLo3CBqERyxoUijMQo5XMi", + "q/0SLBJKBEziCOuVkyjFC8AzM9COZn14RgxmWAWsget/YxZTqGWney6mfDBkGO0MBiglmEmbei3xBC4c", + "2waCQVQG9D3CaG/w3H5V2zfA3HXLv6FPkxBkziM8ThaIaCoG8Ae1CRuY2tqSpkav3r4JFdLsV2EytEXD", + "KhtLpSuVGHdRzsrkcjCf56zIBdfbpXLBYJ7WsUaoKK5Bi6cxJhHW68l4tR9AMuRRlIvQBam32ity+u8o", + "OHrTO4elCqduJ6CFRySGPWdczeBMczhKm983UFVJVH+OiyZ4SLhAGHl0XRoJSJQDa9wA5L+LsmIfcxV4", + "Lza/d2dHH1/LCNzxN9h7j+V+AiLik0nlAK6/mswBXpUysUzCf9ZzeuRKtfosLqZ4yrhUNHLMsF7Z/ZtC", + "2FohXL2yQWqecHHpy1ZV+v2Ji8u2GpjFE6WPSxHzZ/gV2vb18AC7+eFN/GBgNsqKJpp7V9Lq9FWcUhC6", + "qJIudpijhLOpPkWlofveLfG+VrdhcNj0ZSqM/7hAzdFKyMj+aKq96snYWppgtY9sqw/Ni3Tv9+Df+ZUr", + "RNMsISmBarA9Q2x6s0uEJaicT6WHM3QzXqlPlZ8ObHRBaVz6XScOAV25DdsA6X15u4JMNeHT9Th+RecO", + "tC4A5DdkprI/QRfGm3OBCh6sBVqDmo+uZjSaAagf6K26fYP5h7PsosAz3jxAL+Eg+7DO0PmGwcrXtCZ5", + "QgxW3zxNLw6W652+Pz2Fjwyen6lsenGAXI3T4v6Q+i0fpE/PIsFSoV8t9OBGoYzDjl4orPXNYn6bFr6v", + "xJseshCUHyNXtkE6QRceqt9FA+SU47ev+PTBhLFuc5UAMxfFkVUdgTYJiztNcRM0CQP6bQ8GIfDqluCC", + "Zhh3jC24NJhXfFpUKKiQMs6ytuRrhwlUPE/TFTSMNjykMqlinqu/SBUTIeBjS91NxI02cGSrU+FLTagW", + "l84d7E0gv2B0kIEMDy6VZqqdboewPO0c/Gb/NU/TTrdjx+NBjd9AuF8D0lhvcDmKRe+Mh8T4TSy/CcZi", + "ldl7IIu1m8Oq080S+Rvzwp/eW+hsdg9IhiAf1Iy4X5MI6o23avBhvACLhJE9v4+RAaIkihIuScXB83jw", + "qKyhqyYzNhuK3Br39PDi3BXwaRMUcm4/PXdffgW697pYETdm5KZ770EjyyN4zLm1cmk2Ey7qIEbrokm+", + "ekL6cluyNNU2FPKNNm9uZWxFmFpPWGYR9oPYFHTDueIpVjSCYkLRjHPpkX2BOGzKflnjcUGZYFoxWq4N", + "yr/QpHphzdAXVo04sCYzhP1Hto8+fG5D+cNfuEflFz95VoGC43ed6A+A+1DtXFAyQRnOJdFSXZ4SFC0i", + "zRVN9SiCoxmKcKZyQaAwHkEpZTTNUx9KWu/YHAPsxcV2etFF41yhBIspaGXmoQu2iXiaEhYTsM8N2Yzg", + "OdUqpUAJVoRFi54kUFB3TtAVF5cJxzGYGLIYg6cHCvIJoikQcLlTonCMFQZB50Kf+JHJC7ooauwatZ6R", + "65Ia4iETOfveFAnQzV64gV4gAijYVM6KWowRjgmLgujQ5183G/vytuhzouoTfaDIoFvx0ocMFfJtrm44", + "X0cU0SMLb+bCbmMbNr9C6JXNKmw1ocKR0b/nkTZzdXN8IAdTscSrTvHX4VkqiO6r8S49vPuICxTnpjvv", + "VAKZ/1l9QgVD8YOtIFnTbONtHUNF0blimW/E87b+cH+e3MKW95Vwwm6jYt9U3qic9NfAcu2q3ornPpAR", + "09qSfJvcw7FgF9H1YOITFx6XeyzGVsuwzdEs+LbPnZTAoH1x9o1t19m2DXi4Ldt2ttkll77HyCnrQYxo", + "mINbM24jq7amg3/TbJTa7DyW+eAssvRc3BtbPCkYoWGNGV4kHMd/hiDhFf6jiAthECUAo+IxIZp6VkM/", + "PQBsc2XdtK7L1nx/errZxCWEWskjhHrEHMJLydGfpfGyAff1nAhBYwv8iY5Oj224LpVI5KyPXqdUIcXR", + "JSFZmdECWYV9PT+HrbFco70CotHtEKbEIuOUqbWjKF+9m8F8ulVl93vmkxYi+5s7vLU7HCz7j4+dAZeB", + "nA0zgdWaqcJqbelOyiZcpEYuw2Oe69Y1D9LLpPfTlMCc0ITIhVQkNVGJkzyB4wblFmxJXfud2eUuxOTq", + "k2PS5TIiUiol5UwOmc0VyYjQfevPdftegFXQIaBwwV/PDJP8OoL39GBMvBpWTasGKEhQqrNz0NnCWbYV", + "Y4UbAsTs8D5jSD9BNB6Si3TMExqhhLJLiTYSemnUEzSXKNF/bK4M5xvBd1+6YPDtT5Ze6RM24cFybIZm", + "C2L+U2V1WbbmHJOPjq29JP5hcfwHNjrM1taXJBYEJz0o8euwcFCuaEI/GlanG6FS0cikHOFi7d6fFky1", + "P2SnRAn9DobUtiQxiAagXW5lgkdbw3ww2I0yCoBquwQGBwyv+XEKPR6dvTNpqCTlYtEdMv0PaPjt4Znx", + "7k6wtSZ4A7W1iNHJ1us1Ac7nsEz/xhGCZoIr0QuCG/7NJXhzjJHGMyQbjijPVqlKPPvTh7BaCe6bXeFx", + "2hUA5KmYzcZU4AiEYjnLVcyvWNiGMOdJnup/mD9O1kGFKRzN3sOrX420a4azths3wUdxKO2cYmLKRT6I", + "08Ms2GONWdUL56YAQkwlGjB4CxyqPyN1f3nzvb+OX6G7066oK8X61Zyt+7757Bgcwoa/Ho/lmBtKczNR", + "fLX16QrTZuvTjwmPLqWFYvHNhlpvA8hy/WMJMW1dhCAmQGYoshBGBiiLyO6Q1QyQBvFHIowUESllONmC", + "OZtGACzbWbHwnFNI0I4gT6UnaQyYSQkgYgP8nZ4NGKpcA55HV9piVf47vjNScTQmEU+JAxDfDKluf8NU", + "/cRFFQ38a+GLb731B0hATMHevgYAvbnHzwJEP8XXECod59ah7Ea08ZKXPxpTUBfB3gw7uwM57HTRsLOT", + "Djt6B44wmFCxQvsopSxXRPbRsbFvQQrukwGSJOIslg7H3FnwdgeyKSHXkGVDducT+O4+xR5LVbCUb2wn", + "Ifag30P6e0jaQRv+gbNnMu7CoYsRz5Ux99tzZd+KiQLzyOa9+2q9M/JNt2/Dyf9mj2+FR8Eua3bpbb3h", + "7FkuZ6TZ5PbK1AbK1RjwsV29TjlDf+dj2UWMXBlruJCqv8T39NdnpoP7wO7XXd0Et9/O/RtofwvQ/nKt", + "wmCNJsBSX8mOOgxiI7nOuFCA4mhz7Q0NgSYByBE8wgl6fXQyZJFmRQZaUJCUA3eyEOPmFj782zl6cfSm", + "i46hdiT6OR9v9tFrlixcBW/joxkyI4kZ5hVhhsaGakkcup7N2IF67jJYXHfwQMWYzckIeFbcXrkg8W5n", + "RnAMEskfnVfcdBZAHX7zSh8gAP41Xxbb3lkpfHTeECUWvcOJImK52VObJ8UKzAx7STsIOiu4GeBL3aF0", + "yGtln0Y2MNAYuzudAFLGp291FO6+5uj9eMlMnIipYDfOAWmUQZIBjhePK5ZJzlDBHEMs0L+ui0oETVnC", + "lpetVDCgy6bI76/I5L6Sd1Ww5f9dTxfM9NE6mrLKPmkiLiqYrPX0uuTgmYFDto6qCGc4omrRRThJ7B1l", + "b4IiIqVXiL9jQfBlzK9Yf8jeFLVTbEIvOjp713WOWhRTeWlasL7YPno9J0Lm42JwCA6a8RrDmpN4yBRH", + "EU6iPNHiBplMSAS5uFASRTb4couhdO7w7JSdBOu3eFHt+aMrGxemCdi9kizqFLdltnpLkCjBNG0GH7eC", + "GgQcQqjBWDfKGaJsktiQqkhwKZFtqkcSOqXjxAYIyT56OyNI4pQMWZZgxohAuTRR8XrovUwQKXOT4K0b", + "AJBeQ1FdVAILZoIrG5qQcC6kiSbQFP7+FElFshVk9sa0fApzviPZ1jRue3ogI3VtDM2mEPsK0htiKMUs", + "uKajPHEBjPcaim4G9NBS4mM5+G8FnU6J0KcCGyZrwvHMsXbLaQ59JWO5sYTkefFWuxKSRateVqKXsbcS", + "GG5UYm3HnZtF/QU6v6SN2IH20c2yiH/RH7Xsu5qtGh6EffSZs/yzVOY/95IE2xqwSgp/bOYkb+SVo1pJ", + "tF0Pq9U6s/YuM11b42c9GGzWY0bLwpX02SaF9+sjhMH9ojzcd5G1x01bFbSrim7akPK/Hk3/q6DAu4HR", + "f2CUk1vA6H9VefeAc/5w+CfBg/pQefQV37OrX/unR8K/q/R5A4cPcGxN6fOG69ng1ZWK0nv7Tjs1ybb4", + "Z5LgbbzjDeR3t+zftP4WKoO3WOtc0JrgSZqphQtos77KMuhM0o+k3+AILuJW784VfIuQzi9HHo5OGwM6", + "/5zl5h8kZtSWDqQSnRwH6rg/MoxB/8xVLpYtfev0sIhmdE6aje7VE2yXKBOkl/EMnCuxWTC7Hu4uU1j0", + "px+Rbd5irtp/Qe1JgOonMYqpIJFKFqYOqOYIpo/vJBJcawLwnItFc5SIOSI/CZ4e2tmsuQ/tmbLGsDLO", + "MF30Yqxwb+64zQoT2mdEd7p4Ss3wEGXo5Y9og1wrYSpcoInWfBCdFEtqCvhLoMlNf8DbgwbLJv1IRtNx", + "m1GuqFXy2taCQVEuFU/d3p8cow2ofTYlTO+FFvUnIMlmgs9pTOLKGDtznphV3W5Y0JvaXbVQURSuc8qF", + "GdyDyDBtLqTpR5pV2UIREjOmDMPg1lYFqZ4pk8Sv+8OUuQAcu0duFN+uMKv5bThlR1Mi1OG0i6g4NxDP", + "m9+uucd8zfnJUO5Oq9x2LjxntfG6XX5Uy7Sluyj8UOTO3a/Z+v3Xk9JD5aPM5rGm83mhkDaZzb8uEhzc", + "3/1w3+by9484BfQlccq3ZyqHBnSLIYJ5BTHdMZmThGcp1EOHdzvdTi6SzkFnplR2sLUFsd8zLtXB3vOn", + "u51PHz79/wEAAP//zu89IfTvAQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index 48174e105..002c9fcce 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1757,8 +1757,12 @@ components: example: "L40S-1Q" mdev_uuid: type: string - description: mdev device UUID + description: mdev device UUID (mdev hosts only) example: "aa618089-8b16-4d01-a136-25a0f3c73123" + device_path: + type: string + description: sysfs path of the assigned vGPU device + example: "/sys/bus/pci/devices/0000:82:00.4" GPUProfile: type: object