From 6560b3735c0c4c666dc0b7f1fe7da14aa62c210e Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:23:35 +0000 Subject: [PATCH 1/6] Quarantine unhealthy vGPU VFs via a persisted health store Add a VF health store persisted at /gpu/vf-health.json: init failures reported against a VF are tallied per instance assignment, and once failures accumulate from gpu.vf_quarantine_threshold distinct assignments (default 2) the VF is quarantined. Quarantined VFs are excluded from placement and advertised profile availability, cards with quarantined VFs are deprioritized, and selection among equivalent free VFs is randomized. An exact-assignment success report clears the match and older tallies and rescinds that assignment's quarantine. An unreadable or invalid state file fails closed: mutations are refused, placement and advertised availability are disabled, and loads are retried after repair. Writes fsync before and after the rename. GET /resources reports allocatable_slots and quarantined_slots, and GPU admission gates on the allocatable count. GPU.md documents the store semantics, draining the parent GPU, the SR-IOV recovery cycle, and clearing quarantine state. --- cmd/api/api/resources.go | 8 +- cmd/api/config/config.go | 9 +- cmd/api/config/config_test.go | 12 + cmd/api/main.go | 1 + config.example.yaml | 6 + lib/devices/GPU.md | 65 +++- lib/devices/manager.go | 4 + lib/devices/vendor_vfio_linux.go | 49 ++- lib/devices/vendor_vfio_linux_test.go | 66 ++++ lib/devices/vf_health.go | 416 +++++++++++++++++++++ lib/devices/vf_health_test.go | 408 +++++++++++++++++++++ lib/oapi/oapi.go | 497 +++++++++++++------------- lib/paths/paths.go | 5 + lib/resources/gpu.go | 53 +-- lib/resources/gpu_test.go | 88 +++++ lib/resources/monitoring_test.go | 4 +- lib/resources/resource.go | 17 +- openapi.yaml | 12 +- 18 files changed, 1417 insertions(+), 303 deletions(-) create mode 100644 lib/devices/vf_health.go create mode 100644 lib/devices/vf_health_test.go create mode 100644 lib/resources/gpu_test.go diff --git a/cmd/api/api/resources.go b/cmd/api/api/resources.go index ebb6ad951..dec9f35eb 100644 --- a/cmd/api/api/resources.go +++ b/cmd/api/api/resources.go @@ -87,9 +87,11 @@ func convertResourceStatus(rs resources.ResourceStatus) oapi.ResourceStatus { func convertGPUResourceStatus(gs *resources.GPUResourceStatus) oapi.GPUResourceStatus { result := oapi.GPUResourceStatus{ - Mode: oapi.GPUResourceStatusMode(gs.Mode), - TotalSlots: gs.TotalSlots, - UsedSlots: gs.UsedSlots, + Mode: oapi.GPUResourceStatusMode(gs.Mode), + TotalSlots: gs.TotalSlots, + UsedSlots: gs.UsedSlots, + AllocatableSlots: gs.AllocatableSlots, + QuarantinedSlots: gs.QuarantinedSlots, } // Convert profiles (vGPU mode) diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index b46f4f2e5..c250a3621 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -269,7 +269,8 @@ type SnapshotConfig struct { // GPUConfig holds GPU-related settings. type GPUConfig struct { - ProfileCacheTTL string `koanf:"profile_cache_ttl"` + ProfileCacheTTL string `koanf:"profile_cache_ttl"` + VFQuarantineThreshold int `koanf:"vf_quarantine_threshold"` } // Config is the top-level Hypeman server configuration. @@ -494,7 +495,8 @@ func defaultConfig() *Config { }, GPU: GPUConfig{ - ProfileCacheTTL: "30m", + ProfileCacheTTL: "30m", + VFQuarantineThreshold: 2, }, } } @@ -647,6 +649,9 @@ func (c *Config) Validate() error { if c.Build.MaxConcurrentSourceBuilds <= 0 { return fmt.Errorf("build.max_concurrent_source_builds must be positive, got %d", c.Build.MaxConcurrentSourceBuilds) } + if c.GPU.VFQuarantineThreshold < 1 { + return fmt.Errorf("gpu.vf_quarantine_threshold must be >= 1, got %d", c.GPU.VFQuarantineThreshold) + } if c.Limits.MaxConcurrentPushes <= 0 { return fmt.Errorf("limits.max_concurrent_pushes must be positive, got %d", c.Limits.MaxConcurrentPushes) } diff --git a/cmd/api/config/config_test.go b/cmd/api/config/config_test.go index 5660d878e..efd52f208 100644 --- a/cmd/api/config/config_test.go +++ b/cmd/api/config/config_test.go @@ -250,6 +250,18 @@ func TestValidateRejectsInvalidMetricsPort(t *testing.T) { } } +func TestValidateRejectsInvalidVFQuarantineThreshold(t *testing.T) { + for _, threshold := range []int{0, -1} { + cfg := defaultConfig() + cfg.GPU.VFQuarantineThreshold = threshold + + err := cfg.Validate() + if err == nil { + t.Fatalf("expected validation error for vf_quarantine_threshold %d", threshold) + } + } +} + func TestValidateRejectsInvalidMetricExportInterval(t *testing.T) { cfg := defaultConfig() cfg.Otel.MetricExportInterval = "not-a-duration" diff --git a/cmd/api/main.go b/cmd/api/main.go index faa9ac15a..d6fba1b43 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -204,6 +204,7 @@ func run() error { // Configure GPU profile cache TTL devices.SetGPUProfileCacheTTL(cfg.GPU.ProfileCacheTTL) + devices.SetVFQuarantineThreshold(cfg.GPU.VFQuarantineThreshold) // Initialize OpenTelemetry (before wire initialization) otelCfg := otel.Config{ diff --git a/config.example.yaml b/config.example.yaml index ebef41257..70d55fa68 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -170,6 +170,12 @@ data_dir: /var/lib/hypeman # idle_ttl: "" # delete builders idle this long (e.g. "24h"); # # destructive, empty = disabled +# gpu: +# profile_cache_ttl: 30m # vGPU profile metadata cache TTL +# vf_quarantine_threshold: 2 # distinct instance assignments that must report +# # a guest driver init failure before the VF is +# # quarantined (must be >= 1) + # ============================================================================= # Resource Limits # ============================================================================= diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index d04dcf599..17c387723 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -49,8 +49,10 @@ curl -s http://localhost:4973/resources | jq .gpu "mode": "vgpu", "total_slots": 64, "used_slots": 5, + "allocatable_slots": 57, + "quarantined_slots": 2, "profiles": [ - {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 59}, + {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 57}, {"name": "L40S-2Q", "framebuffer_mb": 2048, "available": 30}, {"name": "L40S-4Q", "framebuffer_mb": 4096, "available": 16} ] @@ -121,6 +123,8 @@ curl -s http://localhost:4973/resources | jq .gpu "mode": "passthrough", "total_slots": 4, "used_slots": 2, + "allocatable_slots": 2, + "quarantined_slots": 0, "devices": [ {"name": "NVIDIA L40S", "available": true}, {"name": "NVIDIA L40S", "available": false} @@ -185,8 +189,10 @@ Returns GPU status along with other resources: "mode": "vgpu", "total_slots": 64, "used_slots": 5, + "allocatable_slots": 57, + "quarantined_slots": 2, "profiles": [ - {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 59} + {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 57} ] } } @@ -282,10 +288,25 @@ NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884) ``` (0x65 = timeout; the guest's init requests are never answered, and -`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). Because -placement is deterministic least-loaded, an idle host re-picks the same VF for -every request, so one wedged VF presents as all vGPU instances failing while -`/resources` reports full capacity. +`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). + +Hypeman tracks these failures in `/gpu/vf-health.json` (it survives +restarts): each reported init failure is tallied per instance assignment, and +once failures accumulate from `gpu.vf_quarantine_threshold` distinct +assignments (default 2), the VF is quarantined: excluded from placement and +from advertised profile availability, and its parent GPU becomes +overflow-only — deprioritized for new placements. Selection among a card's +equivalent free VFs is randomized so a wedged VF cannot capture every +placement. A reported init success clears failures only when that exact +assignment has a recorded failure, removing the match and older tallies; if +that assignment crossed the threshold, its later success also rescinds the +quarantine. If the state file exists but cannot be loaded, placement and +advertised availability fail closed until it is repaired or removed. + +`used_slots` includes quarantined VFs still held by running instances, so it +can overlap `quarantined_slots`; use `allocatable_slots` for admission. + +Quarantine only removes capacity — it never touches a running instance. The wedge itself leaves no host-side log: no kernel error, no XID, no plugin crash. The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is @@ -303,18 +324,44 @@ External SIGKILLs (OOM killer, manual `kill -9`) can still trigger it. Confirm by assigning the same profile on a different VF: if that guest initializes, the VF is wedged, not the driver stack. Remediate by cycling SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it -requires no vGPU assignments on that GPU): +requires no vGPU assignments on that GPU). The DCGM quiesce is not optional: +with `nv-hostengine`/`dcgm-exporter` holding the GPUs open, `sriov-manage -d` +fails with `Cannot obtain unbindLock` on first contact. + +Any manual edit to `vf-health.json` needs an immediate hypeman restart: the +store loads only at startup, and a failure report landing first re-persists +the in-memory set over your edit. The restart does not disturb running VMs — +startup reconciliation protects live VFs. + +**Draining the parent GPU.** Overflow-only is a preference, not a cordon: +under capacity pressure new placements still land on the card's healthy VFs +and refill it. To drain the card, quarantine all of its VFs by hand — add +records to the versioned `vf-health.json` (`{"version": 1, "records": +[{"vf_address": "...", "quarantined_at": "..."}]}`) and restart. Running +instances are untouched and +drain through their normal lifecycle: standby is blocked for vGPU instances, +so only a running VM pins a VF, and each stop or delete frees one for good. +Monitor by listing instances whose `gpu.device_path` sits under the parent +GPU; once none remain, run the cycle below. ```bash +# 1. Quiesce the services holding the GPU (required for the unbind lock). +systemctl stop nvidia-dcgm-exporter nvidia-dcgm + +# 2. Cycle SR-IOV on the parent GPU. /usr/lib/nvidia/sriov-manage -d /usr/lib/nvidia/sriov-manage -e + +# 3. Restart the quiesced services. +systemctl start nvidia-dcgm nvidia-dcgm-exporter ``` +After the cycle, remove the card's entries from `vf-health.json`, restart, +and boot a GPU instance to verify recovery. + Do not unbind/rebind the VF from the nvidia driver — it breaks the nvidia-vgpu-vfio core-device registration (`vfio_pci_core_device not found`) and the VF stops accepting assignments entirely until the SR-IOV cycle. -Services holding the GPU (DCGM, persistenced) must be stopped for the cycle -to obtain the unbind lock. ### vGPU assignment fails diff --git a/lib/devices/manager.go b/lib/devices/manager.go index 30763c04d..6b9f6340a 100644 --- a/lib/devices/manager.go +++ b/lib/devices/manager.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "os" "runtime" "strings" @@ -85,6 +86,9 @@ type manager struct { // NewManager creates a new device manager. // Use SetLivenessChecker after construction to enable accurate orphan detection. func NewManager(p *paths.Paths) Manager { + if err := initVFHealth(p.VFHealthState()); err != nil { + slog.Default().Error("failed to load VF health state; vGPU placement is disabled until the state file is repaired or removed", "error", err) + } return &manager{ paths: p, vfioBinder: NewVFIOBinder(), diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index a1378a3c7..b923439a3 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -8,12 +8,12 @@ import ( "fmt" "log/slog" "maps" + "math/rand/v2" "os" "path/filepath" "sort" "strconv" "strings" - "sync" "syscall" "time" @@ -37,18 +37,16 @@ type vendorVFIOSysfs struct { owners map[string]vendorVFIOOwner framebufferByType map[string]int openVFIOPathsFunc func() (map[string]struct{}, error) + pickVFIndex func(n int) int } -var ( - hostVendorVFIO = vendorVFIOSysfs{ - pciDevicesPath: pciDevicesPath, - procPath: procPath, - vfioDevicesPath: vfioDevicesPath, - owners: make(map[string]vendorVFIOOwner), - framebufferByType: make(map[string]int), - } - vendorVFIOMu sync.Mutex -) +var hostVendorVFIO = vendorVFIOSysfs{ + pciDevicesPath: pciDevicesPath, + procPath: procPath, + vfioDevicesPath: vfioDevicesPath, + owners: make(map[string]vendorVFIOOwner), + framebufferByType: make(map[string]int), +} func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { entries, err := os.ReadDir(s.pciDevicesPath) @@ -110,6 +108,10 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { // available_instances. This is a best-effort snapshot because creating on one // VF may revoke the type from siblings that share its GPU framebuffer. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { + quarantined, err := vfHealth.checkedAddresses() + if err != nil { + return nil, err + } profilesByType := make(map[string]profileMetadata) creatableVFs := make(map[string]int) for _, vf := range vfs { @@ -121,9 +123,10 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) continue } + _, bad := quarantined[vf.PCIAddress] for _, profile := range creatable { profilesByType[profile.TypeName] = profile - if !vf.Allocated { + if !vf.Allocated && !bad { creatableVFs[profile.TypeName]++ } } @@ -315,10 +318,22 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map } func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) { + quarantined, err := vfHealth.checkedAddresses() + if err != nil { + return "", err + } usageByGPU := make(map[string]int) unknownUsageByGPU := make(map[string]bool) + quarantinedByGPU := make(map[string]int) freeByGPU := make(map[string][]VirtualFunction) for _, vf := range vfs { + _, bad := quarantined[vf.PCIAddress] + if bad { + quarantinedByGPU[vf.ParentGPU]++ + if !vf.Allocated { + continue + } + } if vf.Allocated { // framebufferByType only covers currently creatable profiles, so // after a restart an allocated type can be missing when its @@ -352,6 +367,9 @@ func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType gpus = append(gpus, gpu) } sort.Slice(gpus, func(i, j int) bool { + if quarantinedByGPU[gpus[i]] != quarantinedByGPU[gpus[j]] { + return quarantinedByGPU[gpus[i]] < quarantinedByGPU[gpus[j]] + } if unknownUsageByGPU[gpus[i]] != unknownUsageByGPU[gpus[j]] { return !unknownUsageByGPU[gpus[i]] } @@ -363,7 +381,12 @@ func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType if len(gpus) == 0 { return "", nil } - return freeByGPU[gpus[0]][0].PCIAddress, nil + candidates := freeByGPU[gpus[0]] + pick := s.pickVFIndex + if pick == nil { + pick = rand.IntN + } + return candidates[pick(len(candidates))].PCIAddress, nil } func (s vendorVFIOSysfs) profileMetadata(vfs []VirtualFunction) ([]profileMetadata, error) { diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 098d016e3..a65fd0a88 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -621,3 +621,69 @@ func assertFileValue(t *testing.T, path, expected string) { require.NoError(t, err) assert.Equal(t, expected, string(value)) } + +func TestVendorVFIOSkipsQuarantinedVF(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.pickVFIndex = func(int) int { return 0 } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + +func TestVendorVFIONoVFWhenAllQuarantined(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + _, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.ErrorContains(t, err, "no available VF") +} + +func TestVendorVFIOCardBiasAvoidsGPUWithQuarantinedVF(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.pickVFIndex = func(int) int { return 0 } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) +} + +func TestVendorVFIOSelectUsesTiebreakAmongFreeVFs(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.pickVFIndex = func(n int) int { return n - 1 } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + +func TestVendorVFIOListProfilesExcludesQuarantinedFromAvailability(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-1Q")) +} diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go new file mode 100644 index 000000000..b47ff11a5 --- /dev/null +++ b/lib/devices/vf_health.go @@ -0,0 +1,416 @@ +package devices + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "path/filepath" + "regexp" + "sort" + "sync" + "time" +) + +const ( + vfHealthFileVersion = 1 + defaultVFQuarantineThreshold = 2 +) + +type vfInitFailure struct { + InstanceID string `json:"instance_id,omitempty"` + AssignedAt string `json:"assigned_at,omitempty"` + ReportedAt time.Time `json:"reported_at"` +} + +type vfHealthRecord struct { + VFAddress string `json:"vf_address"` + Failures []vfInitFailure `json:"failures,omitempty"` + QuarantinedAt *time.Time `json:"quarantined_at,omitempty"` +} + +type vfHealthFile struct { + Version int `json:"version"` + Records []vfHealthRecord `json:"records"` +} + +// VFInitFailureReport describes one guest-reported driver init failure. +type VFInitFailureReport struct { + VFAddress string + InstanceID string + AssignedAt string +} + +// VFInitSuccessReport identifies the assignment that successfully initialized. +type VFInitSuccessReport struct { + VFAddress string + InstanceID string + AssignedAt string +} + +// VFReportOutcome describes how a failure report changed a VF's health state. +type VFReportOutcome int + +const ( + // VFReportUnchanged means the VF was already quarantined or this + // assignment was already recorded. + VFReportUnchanged VFReportOutcome = iota + // VFReportRecorded means the failure was tallied below the quarantine threshold. + VFReportRecorded + // VFReportQuarantined means this report crossed the threshold and quarantined the VF. + VFReportQuarantined +) + +// VFReportResult is the outcome of recording a driver init failure. +type VFReportResult struct { + Outcome VFReportOutcome + Failures int + Threshold int +} + +// VFSuccessResult describes how a successful init changed a VF's health state. +type VFSuccessResult struct { + Cleared int + Rescinded bool +} + +type vfHealthStore struct { + mu sync.Mutex + path string + records map[string]vfHealthRecord + threshold int + loadErr error +} + +var vfHealthAddressPattern = regexp.MustCompile(`^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) + +var ( + vfHealth = &vfHealthStore{records: make(map[string]vfHealthRecord), threshold: defaultVFQuarantineThreshold} + vendorVFIOMu sync.Mutex +) + +func initVFHealth(path string) error { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.path = path + return vfHealth.loadLocked() +} + +// SetVFQuarantineThreshold configures the number of failed assignments +// required to quarantine a VF. +func SetVFQuarantineThreshold(n int) { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.threshold = n +} + +func (s *vfHealthStore) loadLocked() error { + s.records = make(map[string]vfHealthRecord) + s.loadErr = nil + + data, err := os.ReadFile(s.path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + s.loadErr = fmt.Errorf("read VF health state: %w", err) + return s.loadErr + } + var state vfHealthFile + if err := json.Unmarshal(data, &state); err != nil { + s.loadErr = fmt.Errorf("unmarshal VF health state: %w", err) + return s.loadErr + } + if state.Version != vfHealthFileVersion { + s.loadErr = fmt.Errorf("validate VF health state: unsupported version %d", state.Version) + return s.loadErr + } + if state.Records == nil { + s.loadErr = fmt.Errorf("validate VF health state: expected a records array") + return s.loadErr + } + loaded := make(map[string]vfHealthRecord, len(state.Records)) + for i, record := range state.Records { + if !vfHealthAddressPattern.MatchString(record.VFAddress) { + s.loadErr = fmt.Errorf("validate VF health state record %d: invalid VF address %q", i, record.VFAddress) + return s.loadErr + } + if record.QuarantinedAt != nil && record.QuarantinedAt.IsZero() { + s.loadErr = fmt.Errorf("validate VF health state record %d: missing quarantine timestamp", i) + return s.loadErr + } + if record.QuarantinedAt == nil && len(record.Failures) == 0 { + s.loadErr = fmt.Errorf("validate VF health state record %d: neither quarantined nor any recorded failures", i) + return s.loadErr + } + assignments := make(map[string]struct{}, len(record.Failures)) + for j, failure := range record.Failures { + if failure.ReportedAt.IsZero() { + s.loadErr = fmt.Errorf("validate VF health state record %d failure %d: missing report timestamp", i, j) + return s.loadErr + } + key := failure.InstanceID + "\x00" + failure.AssignedAt + if _, exists := assignments[key]; exists { + s.loadErr = fmt.Errorf("validate VF health state record %d: duplicate failure for assignment %q", i, failure.InstanceID) + return s.loadErr + } + assignments[key] = struct{}{} + } + if _, exists := loaded[record.VFAddress]; exists { + s.loadErr = fmt.Errorf("validate VF health state record %d: duplicate VF address %q", i, record.VFAddress) + return s.loadErr + } + loaded[record.VFAddress] = record + } + s.records = loaded + return nil +} + +func (s *vfHealthStore) ensureLoadedLocked() error { + if s.loadErr == nil { + return nil + } + return s.loadLocked() +} + +func (s *vfHealthStore) checkedAddresses() (map[string]struct{}, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return nil, fmt.Errorf("VF health state unavailable: %w", err) + } + addresses := make(map[string]struct{}, len(s.records)) + for address, record := range s.records { + if record.QuarantinedAt != nil { + addresses[address] = struct{}{} + } + } + return addresses, nil +} + +// VGPUAvailability returns free allocatable and quarantined VF counts. +func VGPUAvailability(framework VGPUFramework, vfs []VirtualFunction) (allocatable, quarantined int, err error) { + if framework != VGPUFrameworkVendorVFIO { + return countFreeVFs(vfs, nil), 0, nil + } + addresses, err := vfHealth.checkedAddresses() + if err != nil { + return 0, 0, err + } + for _, vf := range vfs { + if _, ok := addresses[vf.PCIAddress]; ok { + quarantined++ + } + } + return countFreeVFs(vfs, addresses), quarantined, nil +} + +func countFreeVFs(vfs []VirtualFunction, quarantined map[string]struct{}) int { + available := 0 + for _, vf := range vfs { + if vf.Allocated { + continue + } + if _, ok := quarantined[vf.PCIAddress]; !ok { + available++ + } + } + return available +} + +// ReportVFInitFailure records a guest-reported driver init failure and +// quarantines the VF once failures from enough distinct assignments accumulate. +func ReportVFInitFailure(report VFInitFailureReport) (VFReportResult, error) { + // Lock order is vendorVFIOMu before vfHealth.mu. This serializes quarantine + // mutations with vendor-VFIO create, destroy, and reconciliation so placement + // cannot select a VF while it is being quarantined. + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + return vfHealth.reportFailure(report) +} + +// ReportVFInitSuccess clears failures through an exactly matched successful +// assignment. A quarantine is rescinded only when that assignment triggered it. +func ReportVFInitSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { + // Lock order is vendorVFIOMu before vfHealth.mu. This serializes quarantine + // mutations with vendor-VFIO create, destroy, and reconciliation so placement + // cannot select a VF while it is being quarantined. + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + return vfHealth.reportSuccess(report) +} + +// VFHealthStoreUnavailable reports whether persisted state failed to load. +func VFHealthStoreUnavailable() bool { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + return vfHealth.loadErr != nil +} + +// TotalQuarantinedVFs returns the number of quarantined VFs in persisted state. +func TotalQuarantinedVFs() int { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + count := 0 + for _, record := range vfHealth.records { + if record.QuarantinedAt != nil { + count++ + } + } + return count +} + +func (s *vfHealthStore) sortedRecordsLocked() []vfHealthRecord { + records := make([]vfHealthRecord, 0, len(s.records)) + for _, record := range s.records { + records = append(records, record) + } + sort.Slice(records, func(i, j int) bool { return records[i].VFAddress < records[j].VFAddress }) + return records +} + +func (s *vfHealthStore) reportFailure(report VFInitFailureReport) (VFReportResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return VFReportResult{}, err + } + if !vfHealthAddressPattern.MatchString(report.VFAddress) { + return VFReportResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) + } + + previous, existed := s.records[report.VFAddress] + result := VFReportResult{Failures: len(previous.Failures), Threshold: s.threshold} + if previous.QuarantinedAt != nil { + return result, nil + } + for _, failure := range previous.Failures { + if sameVFAssignment(failure, report.InstanceID, report.AssignedAt) { + return result, nil + } + } + + record := vfHealthRecord{ + VFAddress: report.VFAddress, + Failures: append(append([]vfInitFailure(nil), previous.Failures...), vfInitFailure{ + InstanceID: report.InstanceID, + AssignedAt: report.AssignedAt, + ReportedAt: time.Now().UTC(), + }), + } + result.Failures = len(record.Failures) + result.Outcome = VFReportRecorded + if result.Failures >= s.threshold { + now := time.Now().UTC() + record.QuarantinedAt = &now + result.Outcome = VFReportQuarantined + } + s.records[report.VFAddress] = record + if err := s.persistLocked(); err != nil { + if existed { + s.records[report.VFAddress] = previous + } else { + delete(s.records, report.VFAddress) + } + return VFReportResult{}, err + } + return result, nil +} + +func sameVFAssignment(failure vfInitFailure, instanceID, assignedAt string) bool { + return failure.InstanceID == instanceID && failure.AssignedAt == assignedAt +} + +func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return VFSuccessResult{}, err + } + if !vfHealthAddressPattern.MatchString(report.VFAddress) { + return VFSuccessResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) + } + previous, ok := s.records[report.VFAddress] + if !ok || len(previous.Failures) == 0 { + return VFSuccessResult{}, nil + } + + match := -1 + for i, failure := range previous.Failures { + if sameVFAssignment(failure, report.InstanceID, report.AssignedAt) { + match = i + break + } + } + if match < 0 || (previous.QuarantinedAt != nil && match != len(previous.Failures)-1) { + return VFSuccessResult{}, nil + } + + remaining := append([]vfInitFailure(nil), previous.Failures[match+1:]...) + result := VFSuccessResult{ + Cleared: len(previous.Failures) - len(remaining), + Rescinded: previous.QuarantinedAt != nil, + } + if len(remaining) == 0 { + delete(s.records, report.VFAddress) + } else { + record := previous + record.Failures = remaining + s.records[report.VFAddress] = record + } + if err := s.persistLocked(); err != nil { + s.records[report.VFAddress] = previous + return VFSuccessResult{}, err + } + return result, nil +} + +func (s *vfHealthStore) persistLocked() error { + if s.path == "" { + return nil + } + data, err := json.MarshalIndent(vfHealthFile{ + Version: vfHealthFileVersion, + Records: s.sortedRecordsLocked(), + }, "", " ") + if err != nil { + return fmt.Errorf("marshal VF health state: %w", err) + } + if err := os.MkdirAll(filepath.Dir(s.path), 0755); err != nil { + return fmt.Errorf("create VF health state dir: %w", err) + } + tmp := s.path + ".tmp" + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("create VF health state: %w", err) + } + if _, err := f.Write(data); err != nil { + f.Close() + os.Remove(tmp) + return fmt.Errorf("write VF health state: %w", err) + } + if err := f.Sync(); err != nil { + f.Close() + os.Remove(tmp) + return fmt.Errorf("sync VF health state: %w", err) + } + if err := f.Close(); err != nil { + os.Remove(tmp) + return fmt.Errorf("close VF health state: %w", err) + } + if err := os.Rename(tmp, s.path); err != nil { + os.Remove(tmp) + return fmt.Errorf("rename VF health state: %w", err) + } + dirPath := filepath.Dir(s.path) + dir, err := os.Open(dirPath) + if err != nil { + slog.Default().Warn("failed to open VF health state directory for sync", "path", dirPath, "error", err) + return nil + } + if err := dir.Sync(); err != nil { + slog.Default().Warn("failed to sync VF health state directory", "path", dirPath, "error", err) + } + _ = dir.Close() + return nil +} diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go new file mode 100644 index 000000000..dd286c296 --- /dev/null +++ b/lib/devices/vf_health_test.go @@ -0,0 +1,408 @@ +package devices + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func resetVFHealthStore(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "vf-health.json") + require.NoError(t, initVFHealth(path)) + t.Cleanup(func() { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.path = "" + vfHealth.records = make(map[string]vfHealthRecord) + vfHealth.threshold = defaultVFQuarantineThreshold + vfHealth.loadErr = nil + }) + return path +} + +func quarantinedVFs() []vfHealthRecord { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + records := vfHealth.sortedRecordsLocked() + result := records[:0] + for _, record := range records { + if record.QuarantinedAt != nil { + result = append(result, record) + } + } + return result +} + +func quarantineVF(t *testing.T, address string) { + t.Helper() + SetVFQuarantineThreshold(1) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: address, InstanceID: "quarantine-helper"}) + require.NoError(t, err) + require.Equal(t, VFReportQuarantined, result.Outcome) + SetVFQuarantineThreshold(defaultVFQuarantineThreshold) +} + +func TestVGPUAvailability(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + vfs := []VirtualFunction{ + {PCIAddress: "0000:82:00.4"}, + {PCIAddress: "0000:82:00.5", Allocated: true}, + {PCIAddress: "0000:82:00.6"}, + } + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, vfs) + require.NoError(t, err) + assert.Equal(t, 1, available) + assert.Equal(t, 1, quarantined) + + available, quarantined, err = VGPUAvailability(VGPUFrameworkMdev, vfs) + require.NoError(t, err) + assert.Equal(t, 2, available) + assert.Zero(t, quarantined) +} + +func TestVGPUAvailabilityExcludesOnlyQuarantinedVFs(t *testing.T) { + resetVFHealthStore(t) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:82:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err) + assert.Equal(t, 1, available, "a below-threshold failure tally must not remove the VF from placement") + assert.Zero(t, quarantined) +} + +func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, os.WriteFile(path, []byte("not json"), 0o644)) + require.Error(t, initVFHealth(path)) + + _, _, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.ErrorContains(t, err, "VF health state unavailable") + + available, quarantined, err := VGPUAvailability(VGPUFrameworkMdev, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err) + assert.Equal(t, 1, available) + assert.Zero(t, quarantined) +} + +func TestReportVFInitFailureQuarantinesAtThreshold(t *testing.T) { + path := resetVFHealthStore(t) + + result, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + }) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 1, result.Failures) + assert.Equal(t, defaultVFQuarantineThreshold, result.Threshold) + assert.Empty(t, quarantinedVFs(), "one failure must not quarantine at the default threshold") + + result, err = ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-2", + AssignedAt: "2026-08-20T16:00:00Z", + }) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) + assert.Equal(t, 2, result.Failures) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, 1, TotalQuarantinedVFs()) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) + require.NotNil(t, records[0].QuarantinedAt) + require.Len(t, records[0].Failures, 2) + assert.Equal(t, "instance-1", records[0].Failures[0].InstanceID) + + result, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + + require.NoError(t, initVFHealth(path)) + reloaded := quarantinedVFs() + require.Len(t, reloaded, 1) + assert.Equal(t, "0000:e3:00.4", reloaded[0].VFAddress) + require.Len(t, reloaded[0].Failures, 2) +} + +func TestReportVFInitFailureDeduplicatesAssignments(t *testing.T) { + resetVFHealthStore(t) + + report := VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + } + result, err := ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + + result, err = ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + assert.Equal(t, 1, result.Failures) + assert.Empty(t, quarantinedVFs(), "a rescanned assignment must not count toward the threshold twice") +} + +func TestReportVFInitFailureRespectsConfiguredThreshold(t *testing.T) { + resetVFHealthStore(t) + SetVFQuarantineThreshold(3) + + for i, instance := range []string{"instance-1", "instance-2"} { + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, i+1, result.Failures) + assert.Equal(t, 3, result.Threshold) + } + + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) +} + +func TestReportVFInitSuccessClearsFailureTally(t *testing.T) { + path := resetVFHealthStore(t) + report := VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + } + + _, err := ReportVFInitFailure(report) + require.NoError(t, err) + + success := VFInitSuccessReport{ + VFAddress: report.VFAddress, + InstanceID: report.InstanceID, + AssignedAt: report.AssignedAt, + } + successResult, err := ReportVFInitSuccess(success) + require.NoError(t, err) + assert.Equal(t, 1, successResult.Cleared) + assert.False(t, successResult.Rescinded) + + successResult, err = ReportVFInitSuccess(success) + require.NoError(t, err) + assert.Zero(t, successResult.Cleared) + + require.NoError(t, initVFHealth(path)) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: report.VFAddress, InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 1, result.Failures) +} + +func TestReportVFInitSuccessRescindsQuarantineTriggeredByAssignment(t *testing.T) { + resetVFHealthStore(t) + vf := "0000:e3:00.4" + _, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-1", + AssignedAt: "2026-08-20T14:00:00Z", + }) + require.NoError(t, err) + trigger := VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-2", + AssignedAt: "2026-08-20T15:00:00Z", + } + result, err := ReportVFInitFailure(trigger) + require.NoError(t, err) + require.Equal(t, VFReportQuarantined, result.Outcome) + + success, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: trigger.VFAddress, + InstanceID: trigger.InstanceID, + AssignedAt: trigger.AssignedAt, + }) + require.NoError(t, err) + assert.Equal(t, 2, success.Cleared) + assert.True(t, success.Rescinded) + assert.Empty(t, quarantinedVFs()) +} + +func TestReportVFInitSuccessWithoutMatchingFailureClearsNothing(t *testing.T) { + resetVFHealthStore(t) + _, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + }) + require.NoError(t, err) + + result, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-2", + AssignedAt: "2026-08-20T16:00:00Z", + }) + require.NoError(t, err) + assert.Zero(t, result.Cleared) + assert.False(t, result.Rescinded) +} + +func TestReportVFInitSuccessNeverClearsAnotherAssignmentsQuarantine(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + result, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "another-instance", + }) + require.NoError(t, err) + assert.Zero(t, result.Cleared) + assert.False(t, result.Rescinded) + require.Len(t, quarantinedVFs(), 1) +} + +func TestReportVFInitFailureRejectsInvalidAddress(t *testing.T) { + resetVFHealthStore(t) + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "not-a-pci-address"}) + require.ErrorContains(t, err, "invalid VF address") + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "not-a-pci-address"}) + require.ErrorContains(t, err, "invalid VF address") + assert.Empty(t, quarantinedVFs()) +} + +func TestReportVFInitFailureRollsBackOnPersistFailure(t *testing.T) { + resetVFHealthStore(t) + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + + vfHealth.mu.Lock() + _, exists := vfHealth.records["0000:e3:00.4"] + vfHealth.mu.Unlock() + assert.False(t, exists, "a failure whose persist failed must be retried by the next report") +} + +func TestReportVFInitSuccessRollsBackOnPersistFailure(t *testing.T) { + resetVFHealthStore(t) + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + goodPath := vfHealth.path + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + vfHealth.path = goodPath + + vfHealth.mu.Lock() + record, exists := vfHealth.records["0000:e3:00.4"] + vfHealth.mu.Unlock() + require.True(t, exists, "a clear whose persist failed must be restored in memory") + assert.Len(t, record.Failures, 1) +} + +func TestCheckedAddressesFailsClosedOnUnloadedState(t *testing.T) { + path := resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) + require.Error(t, initVFHealth(path)) + + _, err := vfHealth.checkedAddresses() + require.Error(t, err) + + restored := `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0644)) + addresses, err := vfHealth.checkedAddresses() + require.NoError(t, err) + assert.Contains(t, addresses, "0000:e3:00.4") +} + +func TestCheckedAddressesFailsClosedOnInvalidRecord(t *testing.T) { + tests := []struct { + name string + state string + wantErr string + }{ + { + name: "unsupported version", + state: `{"version":2,"records":[]}`, + wantErr: "unsupported version 2", + }, + { + name: "missing records", + state: `{"version":1}`, + wantErr: "expected a records array", + }, + { + name: "invalid address", + state: `{"version":1,"records":[{"vf_address":"not-a-pci-address","quarantined_at":"2026-08-20T00:00:00Z"}]}`, + wantErr: "invalid VF address", + }, + { + name: "neither quarantined nor failed", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4"}]}`, + wantErr: "neither quarantined nor any recorded failures", + }, + { + name: "failure missing report timestamp", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[{"instance_id":"instance-1"}]}]}`, + wantErr: "missing report timestamp", + }, + { + name: "duplicate assignment", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[{"instance_id":"instance-1","assigned_at":"a","reported_at":"2026-08-20T00:00:00Z"},{"instance_id":"instance-1","assigned_at":"a","reported_at":"2026-08-21T00:00:00Z"}]}]}`, + wantErr: "duplicate failure for assignment", + }, + { + name: "duplicate address", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"},{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-21T00:00:00Z"}]}`, + wantErr: "duplicate VF address", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, os.WriteFile(path, []byte(tt.state), 0644)) + require.ErrorContains(t, initVFHealth(path), tt.wantErr) + assert.True(t, VFHealthStoreUnavailable()) + assert.Empty(t, quarantinedVFs()) + + _, err := vfHealth.checkedAddresses() + require.Error(t, err) + }) + } +} + +func TestReportVFInitFailureRefusesToClobberUnloadedState(t *testing.T) { + path := resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) + require.Error(t, initVFHealth(path)) + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5"}) + require.Error(t, err) + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "0000:e3:00.5"}) + require.Error(t, err) + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "not json", string(data), "a failed load must not be overwritten by later reports") + + restored := `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0644)) + quarantineVF(t, "0000:e3:00.5") + records := quarantinedVFs() + require.Len(t, records, 2, "reload must recover the previously persisted quarantine") + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index ba5df2db0..f2948267f 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1018,6 +1018,9 @@ type GPUProfile struct { // GPUResourceStatus GPU resource status. Null if no GPUs available. type GPUResourceStatus struct { + // AllocatableSlots Free slots eligible for placement, matching admission control (excludes quarantined VFs; 0 while VF health state is unavailable) + AllocatableSlots int `json:"allocatable_slots"` + // Devices Physical GPUs (only in passthrough mode) Devices *[]PassthroughDevice `json:"devices,omitempty"` @@ -1027,10 +1030,13 @@ type GPUResourceStatus struct { // Profiles Available vGPU profiles (only in vGPU mode) Profiles *[]GPUProfile `json:"profiles,omitempty"` + // QuarantinedSlots VFs quarantined after guest driver init failures (vGPU mode only). May overlap used_slots until the affected instance releases its VF. + QuarantinedSlots int `json:"quarantined_slots"` + // TotalSlots Total slots (VFs for vGPU, physical GPUs for passthrough) TotalSlots int `json:"total_slots"` - // UsedSlots Slots currently in use + // UsedSlots Slots currently in use. Includes quarantined VFs that are still assigned, so this can overlap quarantined_slots. UsedSlots int `json:"used_slots"` } @@ -18991,249 +18997,252 @@ var swaggerSpec = []string{ "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe+zHO0yyZmp++TZVLCtx2gR1wwe", "j+msj34kUvXIZMKFOrCBLOCQ8mq+4gUSJM41PfjQZJwhSccJnNGiVy3w61/0hAAIZJxPJkRUUQeeh2Rp", "7+1RGjDu/aSfI/NCAXtw+mNVntZye1ut/axCDaC2T3BE2XSz9XYHDIK1aazDFXx59u6NLRvUhLisl7Io", - "LWTAlvvo16Kwll5qWaIq9QOWwjpyX0Mi2dlsIWmEE9OiqYpBmW/gg9PQWkI/Kz+0ptCAnB6ux+ZOHtqY", - "T7Mczv35m97J6/dbaUzm3cqYIJBvxhOix73psae5QzMos94qXGneZGkxhCHbnlhvrQqW0XqRPAYRWB3F", - "FU5GMuGhmKG3+iGCh2jj/U8mHVmPoIuyylbq332oIZ++nwRPDCDYN3R7Dh3WTbaVAx7UXesQaJ3q9Cqd", - "ho6KSaValrGWiyHyy+pG88v1BfhMI839Hrlsr5pR3QqyyCSFGbxGFwaBzKfGam5NM5JkWGBFkkUtHrMK", - "qE6WPdrkmkQ3SDd7oV//ZIpo5IKM1EwQOeNJNQ5it7tciFVCzPGc2NpTZk6e4V9xlGJxCTexE+RRzswK", - "VEPWd9fhy8yUym4wqZ/fvj0z2r0iYo6TetKDXPLwH5MEL9CYqCtCmJsKlgj7Ma/1pFHZUP9HqFFGBOXV", - "NezsBvo9N3HQaCpwRJD5ypVxtlsiIb6o7VLaXgKIhVFEpGzY3+1V+2s/neRJuz0ODWt7bdXz6CYb/Pbo", - "zNWxKeoEu2XeWV7lMyJ65si5gsGrt3ZHrq6o5LpiBoh2SWAYEyixZDNN/ERQF98EJaT055XkS485SD9d", - "0PYDp8AsVdec8w+tpMv6cQ858lPM4lC9ZZNgYFLzp4DXBVHOIgfRj8Ym+MmgBRg9wM+TEATHlBEpa3nN", - "US6STrfTm9hZHWxtJTzCCUAW7u1uP9taHUa6Mn7YhkuNYrpKv3RBVSbsxmXRGrw3mHSVJLZwlrWwwJl1", - "XHM/AHtajleEGsv6bvMkPBfAPhgspWhf40i5qnBgkqu4XLF/bAFYuDIfaDA1hV60qP3cP5+DoA88w2pW", - "Jf+tJdqHuBiIadQ0YswdtXU0RP4xKFFxESr9xYWyGX9j4oIei/vQhRQ6CN2Kc27wzJ/lk/393f11fAiY", - "Te2Y23MXmKp5u5rTTLzltufXNgDhWVWZwx3p1dVL9bqsoSnNEZdIavWC8oywG63n/t7uzs3Ws+1ETlxY", - "WI0vhSBhjk6PjUwUcaYwZUSglCgcY4WrTAZsWZrLQG0ZTFJIC5p8v5q1NMRP+Bgvty2M9aW87w018t44", - "SOgUMzrRDNm+6fcsZ3hn/8mBqeQZk8ne/pN+v39T5IsXJdRFq63YMkF6HghGX84+bx/uAOCizVz+6Jwd", - "vv1ZM7JcCnNpbckxZQfev4t/lg/gD/PPMWVhYIw2xV/pZKnoazUeLbfIwyQ+QGV9byf3tIkPajBGQ3Qy", - "oPEEYeYqUZp3hydX0Ditlg24QW7wilxZLa68ZsmicW1uXce1rHCuvPqtfrZZi1qu9ONq/7ozd8E7tk8D", - "QF2UuV32rN+qULFcWctxqdRXRlhRvTFJzF8RZwDsGyrlWLki3bMWlcDgGrElv4ou/R+L3r0fj/yBeL+7", - "SmLeT7am44cbhsSsFEj/tiyHrudCThxdc5jDtsfiVmhbP9fi0AVjwR/4LrxN2Fi199fT//r9/8izp3/f", - "/v3V+/f/PX/5X8e/0v9+n5y9/iyMk9UQhA+KI/jFoANNTXgfP7AtKZ1iFQVsdFr9a1hh+8RYHFQ0g4qf", - "aEwOhqyHXlFFhKkfV0t+HHbQBgFNCb7S4i4UxzF5Z5v64zPj0dQf/+HE4E/1NmKblC7shhRYIzIfxzzF", - "lG0O2ZDZtpCbiAS9QP8VowhnpggcZUjrvws0FlCxz7qYys676A+cZZ82h8xW6Tdo2hmGomeTIuuLOQex", - "HZUJg7WvkwJ2wmQkDllxWxcYfMbP2C8B+ylJ6jlDDYuyWn+zmtOzQQitEPJZ9EZCURpQQQrK1mRUJNqg", - "Z4PNZX1ujY5R0NAK8rOOeJPweJiHzMVNSZLHJKYR8BWXJzizmaRFiqahNGvEywS/XsDevDHJazHCuZpp", - "XhTZxPqI80tKurClXXCHQeQHfGn8+TOe9caL3oxnBcgCFibaBRuPeFXJ/j89O9HeeyLoxPYUzJXXJBIQ", - "OuHI2JmZlMPCurA0sbemngfTos+c2NdNYRhpaoGYUG6VC+bKVRAoWgloHgX1kZBM/j2KEgpWJznjeRKj", - "GQDuKd1MCDOvMyhSZvA4ismk/u9qSMPO/hPQYN2/d3daZ6yapVtFZXkS0GlTx/pacGzDJmEARjwYOUP4", - "miAkfQNaPy7YKRSH/54j11B54gpGYrxTJodO2uoHifSy5zaDaU72GFjIkBG2p6nNfbR0CitZUy1aMNEJ", - "8FnSAqzkhcnWfPvqHCkiUpc/vxHp3YFTYpApelTK3BbiOjw6fbHZ7wSBliquKtiqlVlV1UEHsBZstEJT", - "EEZpo8Ep6aKTY8iWtddKqYtBesNPXKDE3IrlZXQAYB1Vcw82JfxOjq0AmizKkAcjtgw7m67FrH69HaA3", - "hQqIi6EUeY8lbbkmy8sEmrUBcCb3Yqn1Wpos+Mes+mfvY8i0gPqGhhcDWGTj/dXe5uigp/RFVbOQ3fhC", - "8qNQGu1f3t5/aVjlLy+j795MRrc+4FE2wzJE3TPfqwkvLe2770ausnvRHDNU6XckafBs/c1VefGuIUX0", - "PVf5PIQjut/b3n67vXdz891NEXGrUFgeTF4BitsezfYuUGEDGK9UjRqDy5F+bEPJnV3k/SmaYcm+U/Cw", - "Zh3Z3n3axigBvbYNy/YDsvnEDKngUg5XqwgnNghjlzRJjAAj6ZThBD1HG+cnL385efVqE/XQ69en9a1Y", - "9UVwf24Bjgu3AKyjyTALQCtVIAFQkTv49u0rOFwJgfQLI4df3h4yd61psQWErhvcy7N34PjHcuQCN5tz", - "FXGZ70uuqVRyGVWtVfzz50D2mk/bFfl3kzRtlLX+V+P+/lwBpg3C5G3eAWCvC15fWs4HwLJ9yCTBrw9H", - "dyXy7efC11o7wx2h1zZeaSHk1xoiwn7T7XZ7HNo7GU4FUCbEtnwJx2Vw3xr4tduhgezVQ6kvHhKjk7Oy", - "0lPpjHDN1+b0fKe//eQZFCvdHrRh7CmOVvR9enjUvvPBjrllDvD4IIoPQGG/rc/KErZRQXByhRdQ7M8s", - "7bBjLkxPu/WOrVUkW8XXLOPr3g5Oty7GNQDmgjjrApfkKF1Za6RFemIdNC3NLSRkSpOEShJxFsuqjDzD", - "EsnMIKGamhuFBD9kMMAuKkofg5SCcBSJvDQ9Wunayvt5Zuke6n5mnGkdAID/fyELiVIKTtCiewh9lKjI", - "gomHbEO4jKkiNQpKfsb6B8g/6NrI9lgPjSqoLaI/GDI5y5VmYpt9dMSZzFMirFUWjSl4jDaRzI1KC+OF", - "1VhohilpTMSQ6dcCWKt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtlrDJ+QW0P2ZRhQFGUOQM5Swmoqj/", - "TQw51EPkbug4/UwQYfd5O/HKfl7KVeGDuQ5zuB2Y8OciuMJQG/RziA69hXK+f3sRvVXOkZNfbbaR/Wp0", - "kwgGgiKeJ7HW+Mb6tjMGORJbM6QkynBn8y6V6J2pvVmdug09Vhz9nhOxQO9PTythD4JMNA9oN3HgEg37", - "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJhhK", - "kRmVgZF+q3IhJxbf0YrS2MlDBm+6gH8onfpyIbfGudzKIrpl82+2AJvjGWBz7AWTp2MyH+V5SDXSjxwY", - "y7t3J8doA34BbFlIoawSMMZPtp8Nnj3vPRtvP+ntxYPtHt7efdLb2ceDyW70dHd7Z3dFIkyLbLrbJ8gF", - "NeZAHHMRtT5y0fOhoOam3IWabGLjsa8oi/lV5foLBsj6vdvg23XdL4fWtx5CMCEnwVIZ80UDJzuFS55E", - "um0TkG4zNotSS2FD55O3g+3Ptf7A4BruiLciZ8atajAGChdC6g3Y36zqOG/H8mFALvFl3Wr5nbdftMHB", - "/vOD/c9dNJe8sW6MdXK6x81tighzGMy17BCXoejZkZyBsmNlImPVt8kknW6nyHeBv0EYqMVSF49bJXE1", - "HdhumI2sulYakqdPKvoKRKoYDL74QEsqTh+Big5Fir4WgY4SnsfIs8UZSDLww514uotuBtxi1kRnIEZN", - "MobWcQDTGipHUKYZMfgfdSM20/oAvYR34RFOjVpnB2Hql/iuNxwvTLyMPl+ua6NkrR7yudWv4ButbCH9", - "L5i2XgZrsl3dhJHODtCvHL4ptD3G67Zf8zqoWcuv1+3EGxau2yFnQGdW1DxAPxXiZSGgWoF0QxL758gy", - "rBKwZrMCG2B3vKOppdw5LwW+2zEr2ul23EJBqvxy0vy7kuqXzp9PiqFAMoITOMtljnCuaGJhumEmVCoa", - "SZs8oje3SeyxpZVIPDLKU1NMqkk8tQpW8ZGTqt6fog1AYvwLsoZt/a/NIn61ctftPN97/uTpzvMnrfCW", - "ygGuF42PIC16eXBr5eQoy0fWNtI09aOzd8b2ERmrQhH78v7Uh7fIBNesR8/cNeh3/rz/3IeZink+TjzH", - "osWkM6i2sGFBJLWCFzXEQf5OkzmdTNjvH6PLnb8Lmm5fP5E74+0G+FzTUdjsduIHFyzZqMm4Z8okhZGA", - "gKCEbATLekMkzACdE4WAfnoIR6DeFNnMluQcpJZd8SBh7e3u7j57ur/Tiq7s6LyDMwIjXOBStiPwjhi8", - "iTbenJ+jLY/gTJsOUwIQzplVfcPnDNkax4OqQNrfHuyGqKTh4i6pxrY9TxuX/L1VH+2k7KJDUnahWi6d", - "8uBq7+4Onu7tP9tvd4yteXgkrldzGJeyZJbHAvH7O78B0uTbwzMECcETHFVtOy5C7EajUjcaFRSRMODv", - "NxjYs6dP9vd2d7bbob6Fgk4snmHlwFZ5V+DQBYgisBuBpVhmvd2m2yIkThkCe0OiBNP0MHIpFrXbx4C8", - "j4R5rdyENheD1cCXLq4W37YybhUmK5OgY0QDLlDOitIi/fUu2S/iWW3m2uZ6WM/VQ2k5TK+ehScyJdRu", - "sZSZIHPKc/kFGuLK5MxOEs7Fjb5tUljeEJknythsqETvT78DnqJpDUlFsqoOZalxBYjTLSd3o/NcIZEw", - "kTctVqvdaLP1qybcbTi13VWAGhVu0AidFmvOlbP1wZ9HOIlyKKaDi/3UswIMMIAEyLJkYWL7k4RzhqIZ", - "ZuAkER7iEZrxJO4HI2H1k9EkGFXBr1DCDejzJSGZrTNjBqE/0yIMnRO04VdYM6RUq3u6nxomYyuJVKlx", - "Pw0XcMQylKxWpMLr9cSKe3jE5pOKJTThUwlKoYKshX4dBj/DwiQjYGbqJs1To0sGAq4DQ6wx89CNam5S", - "PrEKrhU5INHcrCSOBJcSkYROoUbP+9Na/vKKnLcii3l9QGd1sC1I1zg0A1eZQcNqXV4tdD8G8nk+54YE", - "GoacwRWhks44mWKWQ+UZj5CtIb7fOhxyxqUaFbhUNxysVCMoJ5ELUqLlFVn3hT3IvRO8Fx1ru81y2bjj", - "W329RFXhppoG2MxTgysaXq1uQYMhMl5G5loJBlaii9WhpG4CVlfWH6ASWqUebBnagJwXjy15EHSbbYJk", - "wiqr7mdJW7XVQV/tDc7bwrqtRnE7w2p2wiY8gP1xA8+ps0TbaNWMiJRCQRUUE0ZJ7HTJwoVqTV2QMJ5I", - "guKc2JUz8qnAdsGxOd7gs2LORkbZtMbr6x22MQ+bMayuNgH92hfbhDvJcELtW5HDWpl4RYlwmVrbKgiU", - "ylHYnbXcsCDTPMECWUDGNkOWizSh7LJN63KRjnlCI6Q/qPvFJzxJ+NVIP5I/wFw2W81OfzBqKk10bgZn", - "8wLNhtT6Lafwg57lZi0rGSwxW+b7LXCMtokeC0aK/0QTYtH93jF67RF6FY59b2fQlC3f0GglT34ZGfKm", - "nNuSbPDE5zKQW7hSynFFlUhsMfKN2JPl0tR3aXErORBW5wK8nUenmjjyedAkR4Zf14BJ0JhA3o+b2jLX", - "aMEW20wlWFoilzP0dz6uGkTbhv0GCpZtsBIiQ5BJML4fdnSlQdq8sbQm3u7eBIMC2KqeKHx0Q2iHdaXd", - "yviqJn7yZqnK2YzYJaNujqbiWYsKHi7+o4AvsL22xzGo16MLxCsDSo1UC6jsCuV0Fl6RRYnGXAhAoNYS", - "DmduNgC7omUevdYO9wq9nZEFEiTFlA0ZZYWRFMDUCGJkToSXJcuFVrKmJO6jv3kqHmB2p5laWDB4MJ5/", - "JxG/YsUYh8wfpG48l7qdQ2YsiyLPVKVcpG4WtD5NKJC1DE4wJaB+IlUzNBFEzvy5h2pmahnviou4sRjR", - "ArlXoMYN+FiR4peE+aysaCaoGpqGRuar5Sg+U/AWnlr9E1Vq0KJ6jdnV/eWSiLCQWEypeKVV6Ip3VDzl", - "xIDAACIK1Ba0fxkWX6CgtMA8KZv/q2uy/OmsaLz6W+01D9fEwQwfGrNt0AQbmTSeWrBP1ZO2NlQF0uBW", - "odks+xLQhguhdgVaqpKAVyil1T3ZLhOvnizgRrMlSVTtfe/Z/tMnLSvVfJazzqB3fWnX3Dxd4ZJr2KnT", - "Nn6fZ/vPnj/f3dt/vnMjD4vLK2nYn6bcEn9/0Aa5VvqwJv/6xz/fn9a8PvsQgz240aBMZkl4SA3ZJdUB", - "vT/91z/+6UZ16wGFGM0yQniD374xSifxd9IFClRdeO2cZCv0+8OKkQAXbAZtkMmEgBl0ZNatVw6mBgPS", - "TgrGGY6oWgQYOb4y0e7FKzWk6zbuoOpgQyKvaduiomrOJfNxmXS64TpH/2l8wzVaeNa64JXMx01+6Nf1", - "Xo0XuvRa+DEOLUIMZFFrfdnAXcznCstKQLf+O4K8C5dhtpxtY95YjbpbT4WAKBZb180LBQyhtdfkSfuR", - "v/217fT8lhWzTn3FP6w4h81H8EZW38CNHDD6RutTa2v8wV6At/tqNPZL0a2s9VepW1feujfvt0X28HKd", - "hOIGu3l/XsLkTT6sYwIDPdox2CUv2+5WSKKBmrxcmIABjSekVwTq2UQZJHPjEdRn3sLMBzI4o0s+mVSx", - "bvebsdEB9geSvVwvWCmtmXQRuXY2izqwtsH4GXb25bCjVYBhZzsddmpuq2D6ZIqvR7aDKrbLYBVYeZn+", - "XhukdDMYJzy6NFXWoHh3Hw1QSjCTKGdw+Gtete3Bau9Qt5N5e1NAgxMT4rTEtmBMYzLDcwoVKaxPZVoJ", - "xCTXVEkIGIV2DlDMDdpTpcSsnaF+zSQ3HpSThksHs4VtWDeo3+PMRbSW74KBbwKFbdlHInjXghVojv36", - "9WnXBDBA6KEZWCW+0U3UjEAzyKKLWnmF8vdw/PA4ISMYdx2uP11eRz8nHTyrgkiipMXvLsmhRgQo4jlT", - "dRz/tJ0iV00rW76ScgbBfjb8A3DZbO+GQFBMIjiRcvksVgn9FsRdyxuwKx1KHNgNkTAcCvAlhX3Fb6xD", - "uD4AY2zwqkObdvy4buMlHEnFbTmx4lSPyHVESFwH/Ay/0jZW3n4ZjJV/hS1GUFG42b4N8c7Ls+vfXYIX", - "jLVptf2YfsZZD9BJ3JZaJBEDDWixaqqEVoEe9yAtRiF41dALbTKuyfXqtf6VXCvAR4/zxIDehUnXsip7", - "Ga1b8VtnNjYdaC7I2vJ8d1C2zsSb36pwnQ1Vf4jadfatO6lXt7Q750S5d88tGTXuULXQS8Wl5QL+3SvV", - "GBtDSl1kL3i0nW7WSHBvFraKWFDeljmaDKdklAkyodcriMe8YBTjKqxJeZCKDAaDL7qR4mu09xRFMyxk", - "beyMTmcqWVQDcPYCWEqfVdRREEWYMxS22flyN92Hy9Fudjv91kPC8bkHDbRU0sSKpKNVuNlHpbfNWucz", - "vAArTqOT8Onu3mCwuzO4FXC2G9YNluuo/MSWQKy205RS531nHf2VKFW/hSLJermu7pWgkKtdLJNUguD0", - "ABJvMhwRlJAJgOQVCa3rPYv1rlcP3gpUNou2oH+3UXbfnA++WjKn6MpijrtpdJxzsYpB5D9f4xBtYDPR", - "EqReIOdutzd48nZ792D/ycH29l2AXReL1JTt8fTj9tXTZAdP9pJni6e/b8+eTnfS3aAedklNZaA2tPqL", - "frcxyqa8JKtYRhWWhjbsHDIi6gWT64XGJUkoIz1ZZEitT1NcwQuM/33t+b+Znd/MYKXscF6dpC9CYFUu", - "ToWyHgZ/y05mpe+iPpuT49WzuFUGUn0gYXqrDwXIq91goELFduczkRly1vIaeue92PoiWpkVt+4qCnnY", - "4aQHd7lhxUPkXQNm8Ga96gJfvuQCttMpF1TN0tW3RfFaASMOcdMfpYqreE99dDJlUC3d/7kIk/OVKP1x", - "p9tJPu5Vz4z9vT3yl0UgLgjQbrUvFbQII4Ni/KtXAV4pFQ9hItm1rq7H/MN2b/s5xCEkH/d+GPSeVyMO", - "uma1/OXbdm9Xfh20WUO/BKArHbX9/EYR1249V1HQLzRUwK68ly02saXxsja1uzpcwm1lg8vHS3tcQ/Jp", - "FEA/V9Kzl9vIF5pikuBFCJveM9TKmvboExkakyllso3ddndQGG7302Gnjw4tQDjosooX/fjNQw16j05o", - "mpKYahnTqP7NGQw7LW1xdV3iZrVJ3FcBaa0fFteer4dIWJdwte6a7H9GPu5nab/tNN5V6B1gV3MqKmCI", - "wYtdRCcIs1qBUsrmOKGxTaSHxEiIVztwQG0lyVoeIEs50NlJumjKFSpT6Fva23LWbBcsxk+uwd66AjPD", - "EMTOFwFEKQDE6Cr2dXKMMsHjPCrzRxMYdIn4IfIaRNsKIX99SO5d2jcgMXvCBVpv32gyaLSzTzbtd802", - "qQm2eau3B+u3+k6MIt1OnsXreZh5qR0HuxFy+5oUxICJprrsNUnQm8yHFhz9jb+CyzqvsSVHWiTKM+dg", - "0TS1TEkBdwu4GEJxvcckIfqaWm4E8SQusySoLLnoepa6/eTZrMnFCR6p5YH8QkimdRXAP4L+UswWwYG5", - "sqPFXbIxcGjf0ji8eqZckV2t6uCerpXEGrfKN+E2lVAwXL5m8zZ4KZee+bvA+PZFs2UEFMfwK0Lam+YS", - "APZLF/bWaD++C7PcQwppr63roQbb6kCFC3R0138ZC6zFuirx7oXc8yGyeGs14yYo2noWqG91Puz9j7Ey", - "o1H/YOuHv/zfvQ//GbQ21/RmSUQvJhMINLoki54pPqR19H4ViBUqH2hhempJheAUbEgAcm4Poz/e/UHB", - "NBa/4nRpChCh5VUO2l47ob/8R3N8k7eM74BPriXZzy4MchcFVBV319FGSsTUxZK7RLLN/pBBbtolWUjk", - "1SOzIo0j1O9k8YkXgY4ujBjYJ2x+gcYUCjzKIdNaLY4ikmltwpa4oaZKOQfuIwhO/HZsXTSX+G0dkiae", - "gKD3p0sovq/fvf3x9btfj0evz178engy+uXFf0OIx1XP9BD3NO3t7T+xtcn9ldwO1se4eZmHPjq1YfrW", - "1T/JQaEFnC6J0lzlEBRCrqMkl3TuHIQquX1Bh+Vk3dsXSPhMBGClklBUgkWqTuiEgF8frhMbVEOlI0Yq", - "oai7NW5QhpZvbEM4ww5wUq8mf6icht6K8GqXG1td9CezdizUYKMGDjtkvEL1/YD2QiXgVbjYD+9ltAGZ", - "I67yrEuc3bwZVuth0WAw8vALFxgaPP8SRUDfraz6OedJT6s3DZUSgtZksxbByHloymQkdJqcDtNxQIa3", - "pt0pneKAnyHkT/gixTrdgNZmTC3tf2PVsnAew3G9jIQ5lmapamUPakYCqXrNaQ6plmobgHcBWdjkrlIv", - "tq6aqJoytWWL6obwMmIOYOarspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzNiCDeRsAH", - "JTz/DZfM5uW0QGExRQkzIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1eUXTvF10QO4UrBc8j/CPMryT9sv", - "fwSo/Deu5CWduCZgGDXlLgwMX6WiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLHv2GqfuIC", - "1MFmzJM7x5eHyz8mAjDg6ujxraDXaUriEc/V6vNvK+rbK78oi1qW1XWqLwYijirpvE28wKFylGNYXmm9", - "HCTKBVWLc71eNpgb0iBdLVtYSOgIfi47hvqhnz6B0XgSSBh5SRgRNILqrPo8ppiBxoTen3pF+ky9xiW8", - "VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT3IKK+jZ/", - "2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfQ7kBd6EemFu5a5a9", - "tc0U0owh+4Vkr+3mfgBBGc4OTHNnMLDA5spev5C7YxIGtv5uo0fLfltJdXaJAuj7S2qeky2Lpf/U7ewN", - "tm80plVDgbMb6vgdwzaJl4B2vn/DhbhVpyfMpOXZJGsbDuWfOCAk/6z99kHvmczTFIuFWzB/tTIumwRj", - "IhF27xo9TkkUaVYBNYL66DUj5jnCCmETuSxyBqWV3YeaQqunwLTtNrkAKfqRx4svtoSVPpyN4lOVnenj", - "8mmJnr8c7RRkvLyR9pFD2DZUew8E9CMu6oI/2EnZGzy/+06POJskNFKoVxCwjUemEkJ+EsALd9hDXKDf", - "c64wKsL5H9GRtjLruCC3bnkVbf1B40/meCckZAY/IyLFzCRHmHfWHPql42xcEuVxXnmrOcI/Oe7Ym8qB", - "8JiLCgS56hH1r626MLh8He0FEBhsn2Z68QMS/t49nHA72aI07EMeOSjIiXJJHtNxsi62cSmEBGW5l0R9", - "LTQ/uM8ryxYR+BOeosdCwC9JIeGVu7V0KWxlImdGAQ5KgG/KhEX73XdV4e9t+cSLkgG/hm4aylko41fF", - "8aKP3JoapV8tAGJJEJhnvHytnOnhfS0nbOc+ThjMuPAUfbumvl1Tq065oRY3BTiY3ilvYYO4kQXiz2d/", - "uLH14Zvtob3toZXlgZEra134Ox/3kY1IjXhMkJzxPInRmCCDd+RiTxQW/elHhEU0o3MCoHZQpC1PFM2w", - "gMiSFMVYYeNDbzRMrDRLFM1t6eZ6Lg6xXOA6joUkI8DhGzXhT5YRiJQxEiP9iYXuK+EEl8qJm7MfNLAX", - "DZZXI7qacUkKPD+mvNsc0pul0Y6h2f6QvbVAr3oBIZja8RpJEoCrXWH/4QzhIbMffO9YiAsEkzgtORcW", - "gBlIDTKl2Zbl1DY90pGMeAhr5y1hmKmezEhEJzSy07okCxvPGWywVd0lPWA3zvenRcIG2tkM47UBPGMY", - "nPe4eIYsJVX9NwyCoKMkj0snl4MQwmKMkyRYmGOa8DFORmZ9LknAJ/gS3rCLUjpcSm8S4zExJeSzhZpx", - "Zv7OxzlTufl7LPiVJGLY2ewPGSRi2LUmcbcUENEVFHJLM67PmeCp6XPLDHHrj0uy+NQfssM4pcxRBHyC", - "E8kRuYbvoL4VYGYY7tVAD+Y0hf3gR7lUPPWRTx3dmWHyXGW5shklkqhuCPVzyBRHfzhsx09bf5Q9fgJn", - "McGxphPvFTMlkK2bRi1HWM9+BK8G3O0EFmDY0RepCfOYCsyUge0swCnR1N/SjaI6AlRMra9whBnKeGYq", - "SwBRzbAmuUobgNWAkwQpOEruWy24w042zMdC76XjRtw9A5RWO0aUodMfvcM02HsWPk+SRIKEIkr+6/z1", - "rwhuZb0H5rUyXMukdDAtMKA4B9ep42kvcDRDxlEFxQSHHRoPO4U7N96EsebShsv0euBT/EEP7QfTTZfG", - "P/T7uinjrjxAv/1hWjnQZylLDQ7osPOpi7wHU6pm+bh49iG8oE3wZecVRoA2zDW3CZwEU0Ca8W58c0Vi", - "FiNub4FkgTAqOZAfuDKmDIvFqkTCwNLbFeQTE8noLcYfQ4hcHHYOhi52cdjpDjuEzeE3G+A47HwKr4D1", - "WjZXroP7rHBuFkT0ZDDYXI+Ebdc34LNs4Rj4wjpgo1ZUlN3UO2hhWP9c/oF/a/2zcP1gpjsvoYmM4u+M", - "74/QAeFJ7L4mGnBB1MRuzCKSOLF7vaHn/p0HerMikiT3TaAPRZ6Fe6xA6n9U5AibVR6jleb7B6a4wX1d", - "KhWz/cPQ76Oznwes59Z2TuYu1DlcpwQwaKwqjczLCEt0DmPqnWvl+wX82rf/dbofYCpeJHx6cWBUd5Tw", - "KUoos/kAXqCyFg/sWsJHBoam+M6i0rgicRtGkvjXP/4Jg6Js+q9//NNiu//rH/+E475l4NWgxvTFjGCh", - "xgSriwP0CyFZDyd0TtxkoAosmROxQLsDa/OHR8grdW+lNDlkQ/aGqFwwL2/C1GuTtkHrKtDzoSwn0sL4", - "6BfpxBaTMbGNAbuNO8tmKe/1RHcDcIgwA28C+lZ0NABYctQU2raaaCdsMjVzrhhN62GaS8F66/mLItfK", - "UG/PDPCGDAaWOHTu4IGdNNo4P3+x2UegbRmqgIJBoDuUzVg1ov+NJ63nSYajVBkKrLLhTRHO8Jgm1Jkc", - "G6qdmCOY4mhGGSnjiwuscdfEgRup5jGHZyfIBkJ24dUhe32+BSZWRSKVC9K1nEBYhNGyHBq3eS7QA/Av", - "qiA6rGffHbIJwZAndHJsmIAHwl3kAxYNMwDygBhXqiqV17pDZpBkLXKxPngpj0kCH0H/U6zIFV50UVHr", - "1lVHSbDSCrHs6peHzGC92jXoAVQJ8obZB35mhtRzkbw2Z0uQSaJVY4jAN2W/oe+NCRfIRjh7Vf5ddybJ", - "0gxLL1qKo9fnen5T0AS5sQdCS6/P3W5sdpHkKEooUEOE2ZBNIRDIgfdyVtnVIqFshkXci7i+BHwwp0vG", - "rxIST5t47JFPZHcoyVT6CRynn+vk+tiEi9nyBPQhNgB1qz13x/addq472+KfyXdnC0HewHlnLLjE8Buz", - "ut8ceS0ceeF1c069kGft2CEw3l3Er+nigQJ+He0tr7l54i3ZQ1j00IaDtgGvCBfo7OgE4TgWRMrNf297", - "n56podJS/tP3o2bFDxF6YsfChQX9s/aWKoE8Fnbwxo4aYTeven1d/37bqhTfabzpijo85ZV397dHrdOb", - "XCOl0FvS2rebZG2wLZURhzKDJbX0QDRKSCG+FOfUp6J1VmUTxltcOSvFJcueT47dgbw/+7LtOmf1u+Ee", - "mOJxjSE+ICOsplr7VbMfEzW/K3bRoU2vMD9/XaQ5uD8p6L5N0SEyf0zqYlxbNs0FDdBJ4wX6kigDb3KX", - "errtITDxcyLcqTYDXZhZF9MynyKD0wITAkvMat33xLzSTvU17f2ZNF9YnptILHbJv4koLZTdcq1WKbgn", - "tgT03em30MON1NsvF7ZiCSywyGBFHTu3E1hWN7BcsGjzW+TKF6doE9dYKrHCzZvEhSXboCkVetZ9yXWH", - "zK83rmU6q9dShiYJnc6sEyCmE4jVU379bhjlzj2MsqiTLbAiNkTxMeb9nulFtl7gOREKvT46MevvX6lb", - "f0DQ6npVyTGvlbfruzeveoRFPC6cJ80yqX3yhRUmQ/+VXN77P3WPMJ+VOvGgSWD8jP03weTIxL/3Kf9f", - "Oz8ldCywWPyvnZ9wklFG/tfuYYIVkWrzzohlcF833X0rMI+Y+LT+QquLBqyJTQEydo3AX7zVUuZ37/+p", - "xH4z6RsJ/sW6fpP928j+/nKtFP/tVtypAmD6eCAPV0FsodWGR98gbe7BaGop0oO0qXiRSlCbGZcKHj2+", - "/GYbVE4LivOvjZbW//JArrw+HOmeHHdhIaGiNFS0sOmD9+QLcOO4d+HW9nv/joDDdEynOc+ln5mYYhXN", - "iLRZuwmpMuDHJnaX13Oj4P0VU+ngPq+Oe5erv9H9HUn89Q01zNs49NbJ/O6ttjK/fV/L/AbR1GY227Ib", - "XVeSabMh0NphmrYl4wr063IAeGhcIV0EvdOKSqkuINAgDobsf2v94zdFcPrhB5dCmQ8GO0/gd8LmH35w", - "WZTs1JEKYUpQW0Hv8Ndj8KJOIVAWiuyVCdv1cZia3UB6rqzAv52CVDqS22tIjgq/aUitNCRvuVZrSHYv", - "7lZFqpYmuXcdydFbaMEtpvifU0v6k7tHKhqczCcTGlHCoMALJKbLpXhAo8l984zcMiGZWX+kF0xUkURa", - "q5EF11ojoZc1pe89kOykLKJ139qjK1/9OBOreGbrwVp9rZQWmhW2r40eBvd7e92/ovaYScxoRMtLl2mh", - "O1AZyNSESnOThVZ8WYKK9dHbt69cxpmW+IWrM6W4Ky7l6m4OmV9cqo9elFW7zAuuBS2Rk9hmqEIeni3X", - "FBMcJ5QRCNElMpQcVi0J96DH4ssLleF6d62Eyns+lraC6cMJlQ/GCu5FfDupFIbmpY7vl8IrTosT4eDU", - "PCp+ZRlQgPGExKctnCveszmsWzNugM3C2I5nCY4A2lG/ZlDHLGyAgRn0mwIsAMGThAiDJpflytVAHLJi", - "cJR5Nd5tjZsL3fwoZ4omF10TIQOQIBJhtrCQSkNW6QwrRdJMQWovpK3DCAXJzIhrxR/1oCnPJbwFWbh+", - "lwgnV3ghh8wmA5vPoVCuIJEBXkySPvqZAw4DwlNMmcd4TQXC7+SQXdA4ISMLo3CBqERyxoUijMQo5XMi", - "q/0SLBJKBEziCOuVkyjFC8AzM9COZn14RgxmWAWsget/YxZTqGWney6mfDBkGO0MBiglmEmbei3xBC4c", - "2waCQVQG9D3CaG/w3H5V2zfA3HXLv6FPkxBkziM8ThaIaCoG8Ae1CRuY2tqSpkav3r4JFdLsV2EytEXD", - "KhtLpSuVGHdRzsrkcjCf56zIBdfbpXLBYJ7WsUaoKK5Bi6cxJhHW68l4tR9AMuRRlIvQBam32ity+u8o", - "OHrTO4elCqduJ6CFRySGPWdczeBMczhKm983UFVJVH+OiyZ4SLhAGHl0XRoJSJQDa9wA5L+LsmIfcxV4", - "Lza/d2dHH1/LCNzxN9h7j+V+AiLik0nlAK6/mswBXpUysUzCf9ZzeuRKtfosLqZ4yrhUNHLMsF7Z/ZtC", - "2FohXL2yQWqecHHpy1ZV+v2Ji8u2GpjFE6WPSxHzZ/gV2vb18AC7+eFN/GBgNsqKJpp7V9Lq9FWcUhC6", - "qJIudpijhLOpPkWlofveLfG+VrdhcNj0ZSqM/7hAzdFKyMj+aKq96snYWppgtY9sqw/Ni3Tv9+Df+ZUr", - "RNMsISmBarA9Q2x6s0uEJaicT6WHM3QzXqlPlZ8ObHRBaVz6XScOAV25DdsA6X15u4JMNeHT9Th+RecO", - "tC4A5DdkprI/QRfGm3OBCh6sBVqDmo+uZjSaAagf6K26fYP5h7PsosAz3jxAL+Eg+7DO0PmGwcrXtCZ5", - "QgxW3zxNLw6W652+Pz2Fjwyen6lsenGAXI3T4v6Q+i0fpE/PIsFSoV8t9OBGoYzDjl4orPXNYn6bFr6v", - "xJseshCUHyNXtkE6QRceqt9FA+SU47ev+PTBhLFuc5UAMxfFkVUdgTYJiztNcRM0CQP6bQ8GIfDqluCC", - "Zhh3jC24NJhXfFpUKKiQMs6ytuRrhwlUPE/TFTSMNjykMqlinqu/SBUTIeBjS91NxI02cGSrU+FLTagW", - "l84d7E0gv2B0kIEMDy6VZqqdboewPO0c/Gb/NU/TTrdjx+NBjd9AuF8D0lhvcDmKRe+Mh8T4TSy/CcZi", - "ldl7IIu1m8Oq080S+Rvzwp/eW+hsdg9IhiAf1Iy4X5MI6o23avBhvACLhJE9v4+RAaIkihIuScXB83jw", - "qKyhqyYzNhuK3Br39PDi3BXwaRMUcm4/PXdffgW697pYETdm5KZ770EjyyN4zLm1cmk2Ey7qIEbrokm+", - "ekL6cluyNNU2FPKNNm9uZWxFmFpPWGYR9oPYFHTDueIpVjSCYkLRjHPpkX2BOGzKflnjcUGZYFoxWq4N", - "yr/QpHphzdAXVo04sCYzhP1Hto8+fG5D+cNfuEflFz95VoGC43ed6A+A+1DtXFAyQRnOJdFSXZ4SFC0i", - "zRVN9SiCoxmKcKZyQaAwHkEpZTTNUx9KWu/YHAPsxcV2etFF41yhBIspaGXmoQu2iXiaEhYTsM8N2Yzg", - "OdUqpUAJVoRFi54kUFB3TtAVF5cJxzGYGLIYg6cHCvIJoikQcLlTonCMFQZB50Kf+JHJC7ooauwatZ6R", - "65Ia4iETOfveFAnQzV64gV4gAijYVM6KWowRjgmLgujQ5183G/vytuhzouoTfaDIoFvx0ocMFfJtrm44", - "X0cU0SMLb+bCbmMbNr9C6JXNKmw1ocKR0b/nkTZzdXN8IAdTscSrTvHX4VkqiO6r8S49vPuICxTnpjvv", - "VAKZ/1l9QgVD8YOtIFnTbONtHUNF0blimW/E87b+cH+e3MKW95Vwwm6jYt9U3qic9NfAcu2q3ornPpAR", - "09qSfJvcw7FgF9H1YOITFx6XeyzGVsuwzdEs+LbPnZTAoH1x9o1t19m2DXi4Ldt2ttkll77HyCnrQYxo", - "mINbM24jq7amg3/TbJTa7DyW+eAssvRc3BtbPCkYoWGNGV4kHMd/hiDhFf6jiAthECUAo+IxIZp6VkM/", - "PQBsc2XdtK7L1nx/errZxCWEWskjhHrEHMJLydGfpfGyAff1nAhBYwv8iY5Oj224LpVI5KyPXqdUIcXR", - "JSFZmdECWYV9PT+HrbFco70CotHtEKbEIuOUqbWjKF+9m8F8ulVl93vmkxYi+5s7vLU7HCz7j4+dAZeB", - "nA0zgdWaqcJqbelOyiZcpEYuw2Oe69Y1D9LLpPfTlMCc0ITIhVQkNVGJkzyB4wblFmxJXfud2eUuxOTq", - "k2PS5TIiUiol5UwOmc0VyYjQfevPdftegFXQIaBwwV/PDJP8OoL39GBMvBpWTasGKEhQqrNz0NnCWbYV", - "Y4UbAsTs8D5jSD9BNB6Si3TMExqhhLJLiTYSemnUEzSXKNF/bK4M5xvBd1+6YPDtT5Ze6RM24cFybIZm", - "C2L+U2V1WbbmHJOPjq29JP5hcfwHNjrM1taXJBYEJz0o8euwcFCuaEI/GlanG6FS0cikHOFi7d6fFky1", - "P2SnRAn9DobUtiQxiAagXW5lgkdbw3ww2I0yCoBquwQGBwyv+XEKPR6dvTNpqCTlYtEdMv0PaPjt4Znx", - "7k6wtSZ4A7W1iNHJ1us1Ac7nsEz/xhGCZoIr0QuCG/7NJXhzjJHGMyQbjijPVqlKPPvTh7BaCe6bXeFx", - "2hUA5KmYzcZU4AiEYjnLVcyvWNiGMOdJnup/mD9O1kGFKRzN3sOrX420a4azths3wUdxKO2cYmLKRT6I", - "08Ms2GONWdUL56YAQkwlGjB4CxyqPyN1f3nzvb+OX6G7066oK8X61Zyt+7757Bgcwoa/Ho/lmBtKczNR", - "fLX16QrTZuvTjwmPLqWFYvHNhlpvA8hy/WMJMW1dhCAmQGYoshBGBiiLyO6Q1QyQBvFHIowUESllONmC", - "OZtGACzbWbHwnFNI0I4gT6UnaQyYSQkgYgP8nZ4NGKpcA55HV9piVf47vjNScTQmEU+JAxDfDKluf8NU", - "/cRFFQ38a+GLb731B0hATMHevgYAvbnHzwJEP8XXECod59ah7Ea08ZKXPxpTUBfB3gw7uwM57HTRsLOT", - "Djt6B44wmFCxQvsopSxXRPbRsbFvQQrukwGSJOIslg7H3FnwdgeyKSHXkGVDducT+O4+xR5LVbCUb2wn", - "Ifag30P6e0jaQRv+gbNnMu7CoYsRz5Ux99tzZd+KiQLzyOa9+2q9M/JNt2/Dyf9mj2+FR8Eua3bpbb3h", - "7FkuZ6TZ5PbK1AbK1RjwsV29TjlDf+dj2UWMXBlruJCqv8T39NdnpoP7wO7XXd0Et9/O/RtofwvQ/nKt", - "wmCNJsBSX8mOOgxiI7nOuFCA4mhz7Q0NgSYByBE8wgl6fXQyZJFmRQZaUJCUA3eyEOPmFj782zl6cfSm", - "i46hdiT6OR9v9tFrlixcBW/joxkyI4kZ5hVhhsaGakkcup7N2IF67jJYXHfwQMWYzckIeFbcXrkg8W5n", - "RnAMEskfnVfcdBZAHX7zSh8gAP41Xxbb3lkpfHTeECUWvcOJImK52VObJ8UKzAx7STsIOiu4GeBL3aF0", - "yGtln0Y2MNAYuzudAFLGp291FO6+5uj9eMlMnIipYDfOAWmUQZIBjhePK5ZJzlDBHEMs0L+ui0oETVnC", - "lpetVDCgy6bI76/I5L6Sd1Ww5f9dTxfM9NE6mrLKPmkiLiqYrPX0uuTgmYFDto6qCGc4omrRRThJ7B1l", - "b4IiIqVXiL9jQfBlzK9Yf8jeFLVTbEIvOjp713WOWhRTeWlasL7YPno9J0Lm42JwCA6a8RrDmpN4yBRH", - "EU6iPNHiBplMSAS5uFASRTb4couhdO7w7JSdBOu3eFHt+aMrGxemCdi9kizqFLdltnpLkCjBNG0GH7eC", - "GgQcQqjBWDfKGaJsktiQqkhwKZFtqkcSOqXjxAYIyT56OyNI4pQMWZZgxohAuTRR8XrovUwQKXOT4K0b", - "AJBeQ1FdVAILZoIrG5qQcC6kiSbQFP7+FElFshVk9sa0fApzviPZ1jRue3ogI3VtDM2mEPsK0htiKMUs", - "uKajPHEBjPcaim4G9NBS4mM5+G8FnU6J0KcCGyZrwvHMsXbLaQ59JWO5sYTkefFWuxKSRateVqKXsbcS", - "GG5UYm3HnZtF/QU6v6SN2IH20c2yiH/RH7Xsu5qtGh6EffSZs/yzVOY/95IE2xqwSgp/bOYkb+SVo1pJ", - "tF0Pq9U6s/YuM11b42c9GGzWY0bLwpX02SaF9+sjhMH9ojzcd5G1x01bFbSrim7akPK/Hk3/q6DAu4HR", - "f2CUk1vA6H9VefeAc/5w+CfBg/pQefQV37OrX/unR8K/q/R5A4cPcGxN6fOG69ng1ZWK0nv7Tjs1ybb4", - "Z5LgbbzjDeR3t+zftP4WKoO3WOtc0JrgSZqphQtos77KMuhM0o+k3+AILuJW784VfIuQzi9HHo5OGwM6", - "/5zl5h8kZtSWDqQSnRwH6rg/MoxB/8xVLpYtfev0sIhmdE6aje7VE2yXKBOkl/EMnCuxWTC7Hu4uU1j0", - "px+Rbd5irtp/Qe1JgOonMYqpIJFKFqYOqOYIpo/vJBJcawLwnItFc5SIOSI/CZ4e2tmsuQ/tmbLGsDLO", - "MF30Yqxwb+64zQoT2mdEd7p4Ss3wEGXo5Y9og1wrYSpcoInWfBCdFEtqCvhLoMlNf8DbgwbLJv1IRtNx", - "m1GuqFXy2taCQVEuFU/d3p8cow2ofTYlTO+FFvUnIMlmgs9pTOLKGDtznphV3W5Y0JvaXbVQURSuc8qF", - "GdyDyDBtLqTpR5pV2UIREjOmDMPg1lYFqZ4pk8Sv+8OUuQAcu0duFN+uMKv5bThlR1Mi1OG0i6g4NxDP", - "m9+uucd8zfnJUO5Oq9x2LjxntfG6XX5Uy7Sluyj8UOTO3a/Z+v3Xk9JD5aPM5rGm83mhkDaZzb8uEhzc", - "3/1w3+by9484BfQlccq3ZyqHBnSLIYJ5BTHdMZmThGcp1EOHdzvdTi6SzkFnplR2sLUFsd8zLtXB3vOn", - "u51PHz79/wEAAP//zu89IfTvAQA=", + "LWTAlvvo16Kwll5qWaIq9QOWwho9mQBq/cJIJjwUjfKTIATBs7Kgpz574ME2GL1F1ACOAaOJM1dWEG1A", + "lFdMJPo9xwIzRRmJ0fuf5PdoYF3m739CJovGMiAq/cDGqqb4NLSljdlwZ7OFpBFOzLKY0h6U+VZKONKt", + "1Yyz8kNrzw0oG+Gico59oI35NMthAc/f9E5ev99KYzLvVsYE0YgznhA97k2Px84dJEOZuldhrfMmc5Gh", + "btmW7XhrVfC91ovkcbnA6nhU0ERw73+qEou5eEyqqjXXUkYVQG8B3vJGMUiDEN1Hp3hhtfoMOLjpykNA", + "xZOJAaAsOJsgCcESqlFL9P6n/towQcUVTprm8FY/tKdmQ09I76keZhdlFaKEk+QhP/ndPgkysHI+AacH", + "dFi3oPfRCQsfQhOLAthtBoZJSjplWtOQNhwlwqxYyaW9q3LyoJGijnXXqS5cZTrdADsKUUyId5rcumWh", + "e7k6Jr+sHhp+ub4io2mkud8jl/5X87JYzcbxN8gSdHExyHxq3CjWVidJhgVWJFnUAnSrCPtkOcSBXJPo", + "BvmHL/Trn0xVlVyQkZoJImc8qQbG7HaXK/NKCEKfE1uMzMzJ8wQpjlIsLuGUOc0O5cysQDWHYXcd4NBM", + "qewGk/r57dszY+5RRMxxUs+CkUshH8ckwQs0JuqKEOamgiXCfhB0PYtYNhSEEmqUEUF5dQ07u4F+z01g", + "PJoKHBFkvnJ1vQu2po9e26W0vQQgLKOISNmwv9ur9td+OsmTdnscGtb22jL40U02+O3RmStsVBSOdsu8", + "s7zKZ0T0zJFzFaRXb+2OXF1iy3XFDDLxkgQ5JlBzy6Ye+ZnBLuANaorpzyvZuB5zkH7+qO0HToFZqq45", + "5x9aqRv14x6K7Egxi0MFuE3GicFqmAKAG4S9ixx0ARqbaDhzJ3v3s02cEQTHlBEpa4nuUS6STrfTm9hZ", + "HWxtaX6fAIbl3u72s63VccUrA8pt/NwopqsMDi7KzsRhubRqAwAIk66SxBbOshYmWbOOa+4HYE/LAaxQ", + "dFvfbZ7I7zIaBoOlnP1rHClXJhBstBUfPPaPLSBNVwUZ3WBqKv9o3eu5fz4HwaCIDKtZlfy3lmgfAqUg", + "yFXTiLF/1dbREPnHoHTKRagWHBfKpoCOiYuCLe5DF2PqMJUr3trBM3+WT/b3d/fX8SFgNrVjbs9dYKrm", + "7WqSO/GW255f2wDE61VlDnekV5ez1euyhqY0R1wiqdULyjPCbrSe+3u7Ozdbz7YTOXFxgjW+FMIIOjo9", + "NjKR1iwxZUSglCgcY4WrTAaMm5rLQLEhTFLIE5t8v5q1NATU+KA/t62U9qXCMRqKJr5xGOEpZnQCSpJ5", + "0+9ZzvDO/pMDU9o1JpO9/Sf9fv+mUCgvSuyTVluxZaI2PVSUvpx93j7cAeJJm7n80Tk7fPuzZmS5FObS", + "2pJjyg68fxf/LB/AH+afY8rCSCltqgHTyVIV4GqAYm6hqEl8gMqC707uaRMw1uCdgHB1gGcK4g5Wwnbv", + "DmCwoHFarSNxg2TxFcnTWlx5zZJF49rcurBvWfJeeQV9fbtCi+K+9OPqgAtn/4R3bJ/G0lHUPV4OtbhV", + "5Wq5srjnUu23jLCinGeSmL8izgDpOVTbs3JFumctSsPBNWJrwBVd+j8WvXs/HvkD8X53peW8n2yRzw83", + "jJFaKZD+bVkOXc+FnDi65jCHjdHFrdC2oLIFJgwmBzzwXXibOMJq76+n//X7/5FnT/++/fur9+//e/7y", + "v45/pf/9Pjl7/VmgN6sxKR8UWPKLYUlC8FwFULItKZ1iFQVsdFr9a1hh+8RYHFQ0gxKwaEwOhqyHXlFF", + "hCkoWMuGHXbQBgFNCb7S4i5USzKJiJv64zPj4tYf/+HE4E/1NmKLUiDshhTgMzIfxzzFlG0O2ZDZtpCb", + "iAS9QP8VowhnpiogZUjrvws0FlDC0focy8676A+cZZ82hwyssuTawKtnGKrgTYo0QOYiBuyoTFy0fZ0U", + "OCQmRXXIitu6AGU0jud+WcGBkqSeRNawKKv1N6s5PRuE4CshwUlvJFQpAhWkoGxNRkXmFXo22FzW59bo", + "GAUNrSA/G5lhMmAP85C5uClr9pjENAK+4hJHZza1uMjZNZRmjXiZ4NcL2Js3JpsxRjhXM82LIou0EHF+", + "SUkXtrQL/lEIBYIvTYDHjGe98aI341mBuoGFCX/CJkSiqmT/n56daO89EXRiewqCJ2gSCQidcGTszEwO", + "amFdWJrYW1PghWnRZ07s66ZSkDTFYUxsv8oFc/VLCFQxBXiXgvpISCb/HkUJBauTnPE8idEMEBiVbiYE", + "otgZFDlUeBzFZFL/dzXGZWf/CWiw7t+7O61TmM3SraKyPAnotKljfS04tmGTMAAjHoycIXxNVJq+Aa1j", + "H+wUisN/z5FrqDxxBSMxnj6TVCltOYxEeumUm8G8N3sMLIbMCNvT1OY+WjqFlTS6Fi2YcBX4LGmBXvPC", + "pO++fXWOFBGpA1TYiPTuwCkxUCU9KmVuK7MdHp2+2Ox3gshbFZcWbNXKNLvqoAPgGzZ8pSkqp7TR4JR0", + "0ckxpE/ba6XUxSDf5ScuUGJuxfIyOgD0lqq5B5uajifHVgBNFmUMjBFbhp1N12JWv94O0JtCBcTFUIpE", + "2JK2XJPlZQLN2ohIk4yz1Hotbxr8Y1b9s/cxpN5AwUvDiwE9tPH+am9zdFhk+qKqWchufCH5YUmN9i9v", + "7780zvaXl9F3byajWy/0KJthGaLume/VhJeW9t13ZFfZvWgOIqv0O5I0eLb+5sr+eNeQIvqeq3weApbd", + "721vv93eu7n57qYQyVVsNA83sUBJbg9vfBcwwQHQX6pGjdkGSD+2uQXOLvL+FM2wZN8peFizjmzvPm1j", + "lIBe28bp+xH6fGKGVHApB7RWxJcbyLlLmiRGgJF0ynCCnqON85OXv5y8erWJeuj169P6Vqz6Irg/t0BL", + "hlsA1tGkHAawtioYEahIJn379hUcroRAPo6Rwy9vj6G81rTYAlPZDe7l2Ttw/GM5cpG8zcmruEwAJ9dU", + "KrkMs9cqIP5zMJzNp6VhrM0kTRs2xG8tEPTPFaTiIG7i5h0gOLtshqXlfABw44fMGv36gJVXQiF/Lp6x", + "tTPcEZxx45UWggKuQWTsN91utwcmvpPhVBCGQmzLl3BcSv+tkYC7HRpIZz60cXzo5Kws/VU6I1zztTk9", + "3+lvP3kG1Wu3B20Ye4qjFX2fHh6173ywY26ZAzw+iOIDUNhv67OyhG1UEJxc4QVUfzRLO+yYC9PTbr1j", + "axXJVvE1y4DLt8NXrotxDQjKIM66wCU5SlcWn2mRr1pH0UtzixGa0iShkkScxbIqI8+wRDIz0LimCEsh", + "wQ8ZDLCLilrYIKUgHEUiL02PVrq28n6eWbqHQrAZZ1oHgEoQv5CFRCkFJ2jRPYQ+SlSkRcVDtiFcCl2R", + "Kwc1YGP9AySkdG2qQ9yFqGEoNqM/GDI5y5VmYpt9dMSZzFMirFUWjSl4jDaRzI1KC+OF1VhohilpTMSQ", + "6dcC4Lt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtuLRJAgf4R2ZhpgFXUuQM5SwmoigITww51EPkbug4", + "/UxUafd5O/HKfl7KVeGDuQ6Euh269OdC+sJQG/RziA69hXK+f3sRvVUSmpNfbfqZ/Wp0kwgGgiKeJ7HW", + "+Mb6tjMGORJbM6QkynDnIhPknSnGWp26DT1WHP2eE7FA709PK2EPgkw0D2g3ceASDfvAsxttw84aG8na", + "0dzEvewBIN8H6HFdUvEkxC8Ocex7GF1WvKHQimGrojivsq9prTKYkUOZ2SdNNCsmWKvHazBEysBIv1W5", + "kBML+GlFaZdTYQHICzyQ0qkvF3JrnMutLKJbNpdpC8BangFYy14wmz4m81Geh1Qj/cih87x7d3KMNuAX", + "ABs2CTKV7jF+sv1s8Ox579l4+0lvLx5s9/D27pPezj4eTHajp7vbO7srkopapFfePmMyqDEH4piLqPWR", + "i54PBTU35S7UZBMbj31FWcyvKtdfMEDW790G367rfjm0vvUQgilBCZbKmC8aONkpXPIk0m2bgHSbwlvU", + "3gobOp+8HWx/rvUHBtdwR7wVOTNuVQM6UbgQUm/A/mZVx3k7lg8Dcokv61bL77z9og0O9p8f7H/uornk", + "jXVjrJPTPW5uU0SYA+WuZYe4lFXPjuQMlB0rExmrvk0m6XQ7Rb4L/A3CQC2WunjcKomr6cB2w2xk1bXS", + "kE1/UtFXIFLFgDLGB1pScfoIlPgoMBu0CHSU8DxGni3OYNSBH+7E0110M+AWsyY6k0ZrkjEgM5JKm89H", + "mWbE4H/UjdjU+wP0Et6FRzg1ap0dhClo47vecLww8TL6fLmujZK1esjnVr+Cb7SyhfS/YNp6GazJdnUT", + "Rjo7QL9y+KbQ9hiv237N66BmLb9etxNvWPx2B6UCnVlR8wD9VIiXhYBqBdINSeyfI8uwSgSjzQqOhN3x", + "jqaWcuc8TIRux6xop9txCwXYCcsoCu9Kql86fz4phgLJCE7gLJdJ47miicVth5lQqWgkbfKI3twmscfm", + "ZpJ4ZJSnpphUk/pqFaziIydVvT9FGwDN+RdkDdv6X5tF/Grlrtt5vvf8ydOd509aAXCVA1wvGh9Bnvzy", + "4NbKyVGWj6xtpGnqR2fvjO0jMlaFIvbl/amPd5IJrlmPnrlr0O/8ef+5jzsW83yceI5FC1JoYI5hw4LQ", + "egUvaoiD/J0mczqZsN8/Rpc7fxc03b5+InfG2w14yqajsNntxA8uWLJRk3HP1M0KQ0MBQQnZiJ72hkiY", + "ATonCgH99BCOQL0p8qktyTmMNbviQcLa293dffZ0f6cVXdnReQdnBEa4wKVsR+AdMXgTbbw5P0dbHsGZ", + "Nh3ICEDeM6v6hs8ZskWvB1WBtL892A1RScPFXVKNbXueNi75e6s+2knZRYfk7UK1XDrlwdXe3R083dt/", + "tt/uGFvz8Ehcr+YwLmXJLI+tzODv/AZIk28PzxAkBE9wVLXtuAixG41K3WhUUFXEVAO4wcCePX2yv7e7", + "s90OBjAUdGIBLisHtsq7AocuQBSB3QgsxTLr7TbdFiFxyhDYGxIlmKaHkUuxqN0+BvV/JMxr5Sa0uRis", + "Br50cbX4tpVxqzBZmQQdIxpwgXJW1Jrpr3fJfhHPajPXNtfDeq4eSsthevUsXpWpqXeLpcwEmVOeyy/Q", + "EFcmZ3aScC5u9G2TwvKGyDxRxmZDJXp/+h3wFE1rSCqSVXUoS40rUL1uObkbnecKiYSJvGmxWu1Gm61f", + "NeFuw6ntrgLUqHCDRiy9WHOunK0P/jzCSZRDdSVc7KeeFYDCASRAliULE9ufJJwzFM0wAyeJ8CCw0Iwn", + "cT8YCaufjCbBqAp+hRJuUMAvCcls4SEzCP2ZFmHonKANv+SeIaVaIdz91DAZW1qmSo37abiiJ5ahZLUi", + "FV6vJ1bcA6g2n1QsoQmfSlAKFWQt9Ot1ETIsTDICZqaQ1jw1umQg4DowxBozD92o5iblE6vgWpEDEs3N", + "SuJIcOmBU70/reUvr8h5K7KY1wd0VgfbgnSNQzNwlRl4tNb19kL3YyCf53NuSKBhyBlcESrpjJMpZjmU", + "IvII2Rri+63DIWdcqlEB8nXDwUo1gvoiuSAlfGKRdV/Yg9w7wXvRsbbbLJeNO77V10tUFW6qaYDNPDW4", + "ouHV6hY0GCLjZZSzlcBqJdxcHczqJuiFZUEKKqFV6uHYoQ3IefHYkodJuNkmSCassup+lrRVWy721d7g", + "vC3O32pYvzOsZidswgPYHzfwnDpLtI1WzYhwUHsxYZTETpcsXKjW1AUJ44kkKM6JXTkjnwpsFxyb4w0+", + "K+ZsZJRNa7y+3mEb87AZw+ryI9CvfbFNuJMMJ9S+FTmslYlXlAiXqbWtgkCpHIXdWcsNCzLNEyyQRehs", + "M2S5SBPKLtu0LhfpmCc0QvqDul98wpOEX430I/kDzGWz1ez0B6OmWlXnZnA2L9BsSK3fcgo/6Flu1rKS", + "wRKzZb7fAsdom+ixYKT4TzQhFinxHaPXHqFX8fn3dgZN2fINjVby5JehQm/KuS3JBk98LgO5hSulHFdl", + "i8S2aIIRe7JcmoI/LW4lh8rrXIC38+hUE0c+D5rkyPDrGjAJGhPI+3FTW+YaLdhim6kEa43kcob+zsdV", + "g2jbsN9ABbsNVkJkCDIJxvfDjq40SJs3ltbE292bYFAAW9UThY9uCO2wrtZfGV/VxE/eLJW9mxG7ZNTN", + "0ZTAa1HSxcV/FPAFttf2OAb1AoWBeGVAqZFqAaV+ob7Swqu6KdGYCwGQ5FrC4czNBmBXtMyj19rhXqG3", + "M7JAgqSYsiGjrDCSApgaQYzMifCyZLnQStaUxH30N0/FAxD3NFMLWx0AjOffScSvWDHGIfMHqRvPpW7n", + "kBnLosgzVakfqpsFrU8TCmQtgxNMCSioSdUMTQSRM3/uoSKqWsa74iJurE61QO4VKHoEPlak+CVhPisr", + "mgmqhqahkflqOYrPVECGp1b/RJWixKhedHh1f7kkIiwkFlMqXmkVuuIdFU85MSAwgIgCxSbtX4bFFygo", + "LTBPyub/6posfzorGq/+VnvNwzVxuNOHxmwbNMFGJo2nFuxT9aStDVWBNLhVaDbLvgS04UKoXcWeqiTg", + "Vc5pdU+2y8SrJwu40WxJElV733u2//RJy9JFn+WsM+hdX9o1N09XuOQaduq0jd/n2f6z58939/af79zI", + "w+LyShr2pym3xN8ftEGulT6syb/+8c/3pzWvzz7EYA9uNCiTWRIeUkN2SXVA70//9Y9/ulHdekAhRrMM", + "Gd/gt2+M0kn8nXSBAlUXXjsn2Qr9/rBiJMAFm0EbBKC46ZyMzLr1ysHUYEDaScE4wxFViwAjx1cm2r14", + "pYa13cYdVB1sSOQ1bVtUVM25ZD4uk043XOfoP41vuEYLz1pXQJP5uMkP/breq/FCl14LP8ahRYiBLIrv", + "Lxu4i/lcYVkJ6NZ/R5B34TLMlrNtzBurUXfrqRAQxWIL/XmhgCHk+5o8aT/yt7+2nZ7fsmLWqa/4hxXn", + "sPkI3sjqG7iRA0bfaH1qbY0/2Avwdl+Nxn5twpXFHyuFDMtb9+b9tsgeXi6cUdxgN+/PS5i8yYd1TGCg", + "RzsGu+Rl290KSTRQk5cLEzCg8YT0vOIFpvCTzI1HUJ95CzMfyOCMLvlkUsW63W/GRgfYH0j2cr1gpbRm", + "0kXk2tks6sDaBuNn2NmXw45WAYad7XTYqbmtgumTKb4e2Q6q2C6DVWDlZfp7bZDSzWCc8OjSlN2Dau59", + "NEApwUyinMHhr3nVtgervUPdTubtTQENTkyI0xLbgjGNyQzPKVT3sD6VaSUQk1xTJSFgFNo5QDE3aE+V", + "msN2hvo1k9x4UE4aLh3MFrZh3aB+jzMX0Vq+Cwa+CVQ6Zh+J4F0LVqA59uvXp10TwAChh2ZglfhGN1Ez", + "As0giy5q5RXK38Pxw+OEjGDcdbj+dHkd/Zx08KwKIomSFr+7JIcaEaCI50zVcfzTdopcNa1s+UrKGQT7", + "2fAPwGWzvRsCQTGJ4ETK5bNYJfRbEHctb8CudChxYDdEwnAowJcU9hW/sQ7h+gCMscErF27a8eO6jZdw", + "JBW39eWKUz0i1xEhcR3wM/xK21h5+2UwVv4VthhBRSVv+zbEOy/Prn93CV4w1qbV9mP6GWc9QCdxW2qR", + "RAw0oMWqqRJaBXrcg7QYheBVQy+0ybgm16vX+ldyrQAfPc4TA3oXJl3LquxltG7Fb53Z2HSguSBr6zXe", + "QR1DE29+q0qGNlT9IYoZ2rfupIDh0u6cE+XePbdk1LhD1UIvFZeWC/h3r1RjbAwpdZG94NF2ulkjwb1Z", + "2CpiQXlb5mgynJJRJsiEXq8gHvOCUYyrsCblQSoyGAy+6EaKr9HeUxTNsJC1sTM6nalkUQ3A2QtgKX1W", + "lU9BFGHOUNhm58vddB8uR7vZ7fRbDwnH5x400FJJEyuSjlbhZh+V3jZrnc/wAqw4jU7Cp7t7g8HuzuBW", + "wNluWDdYrqPyE1sTs9pOU0qd95119FeiVP0WiiTr5ULLV4JCrnaxTFIJgtMDSLzJcERQQiYAklcktK73", + "LNa7Xj14K1DZLNqC/t1G2X1zPvhqyZyiK4s57qbRcc7FKgaR/3yNQ7SBzURLkHqBnLvd3uDJ2+3dg/0n", + "B9vbdwF2XSxSU7bH04/bV0+THTzZS54tnv6+PXs63Ul3g3rYJTWVgdrQ6i/63cYom/KSrGIZVVga2rBz", + "yIioV9CuV56XJKGM9GSRIbU+TXEFLzD+97Xn/2Z2fjODlbLDeXWSvgiBVbk4Fcp6GPwtO5mVvov6bE6O", + "V8/iVhlI9YGE6a0+FCCvdoOBChXbnc9EZshZy2vonfdi64toZVbcuqso5GGHkx7c5YYVD5F3DZjBm/Wq", + "C3z5kgvYTqdcUDVLV98WxWsFjDjETX+UKq7iPfXRyZRB+Xz/5yJMzlei9Medbif5uFc9M/b39shfFoG4", + "IEC71b5U0CKMLCFzkqxeBXilVDyEiWTXuroe8w/bve3nEIeQfNz7YdB7Xo046JrV8pdv271d+XXQZg39", + "EoCudNT28xtFXLv1XEVBv9BQAbvyXrbYxJbGy2Ll7upwCbeVDS4fL+1xDcmnUQD9XEnPXm4jX2iKSYIX", + "IWx6z1Ara9qjT2RoTKaUyTZ2291BYbjdT4edPjq0AOGgyype9OM3r2nFpxOapiSmWsY0qn9zBsNOS1tc", + "XZe4WW0S91VAWuuHxbXn6yES1iVcrbsm+5+Rj/tZ2m87jXcVegfY1ZyKChhi8GIX0QnCrFaglLI5Tmhs", + "E+khMRLi1Q4cUFtJspYHyFIOdHaSLppyhcoU+pb2tpw12wWL8ZNrsLeuwMwwBLHzRQBRCgAxuop9nRyj", + "TPA4j8r80QQGXSJ+iLwG0bZCyF8fknuX9g1IzJ5wgdbbN5oMGu3sk037XbNNaoJt3urtwfqtvhOjSLeT", + "Z/F6HmZeasfBboTcviYFMWCiqS57TRL0JvOhBUd/46/gss5rbMmRFonyzDlYNE0tU1LA3QIuhlBc7zFJ", + "iL6mlhtBPInLLAkqSy66nqVuP3k2a3JxgkdqeSC/EJJpXQXwj6C/FLNFcGCu7Ghxl2wMHNq3NA6vnilX", + "ZFerOrinayWxxq3yTbhNJRQMl6/ZvA1eyqVn/i4wvn3RbBkBxTH8ipD2prkEgP3Shb012o/vwiz3kELa", + "a+t6qMG2OlDhAh3d9V/GAmuxrkq8eyH3fIgs3lrNuAmKtp4F6ludD3v/Y6zMaNQ/2PrhL/9378N/Bq3N", + "Nb1ZEtGLyQQCjS7JomeKD2kdvV8FYoXKB1qYnlpSITgFGxKAnNvD6I93f1AwjcWvOF2aAkRoeZWDttdO", + "6C//0Rzf5C3jO+CTa0n2swuD3EUBVcXddbSREjF1seQukWyzP2SQm3ZJFhJ59cisSOMI9TtZfOJFoKML", + "Iwb2CZtfoDGFAo9yyLRWi6OIZFqbsCVuqKlSzoH7CIITvx1bF80lfluHpIknIOj96RKK7+t3b398/e7X", + "49Hrsxe/Hp6Mfnnx3xDicdUzPcQ9TXt7+09sbXJ/JbeD9TFuXuahj05tmL519U9yUGgBp0uiNFc5BIWQ", + "6yjJJZ07B6FKbl/QYTlZ9/YFEj4TAVipJBSVYJGqEzoh4NeH68QG1VDpiJFKKOpujRuUoeUb2xDOsAOc", + "1KvJHyqnobcivNrlxlYX/cmsHQs12KiBww4Zr1B9P6C9UAl4FS72w3sZbUDmiKs86xJnN2+G1XpYNBiM", + "PPzCBYYGz79EEdB3K6t+znnS0+pNQ6WEoDXZrEUwch6aMhkJnSanw3QckOGtaXdKpzjgZwj5E75IsU43", + "oLUZU0v731i1LJzHcFwvI2GOpVmqWtmDmpFAql5zmkOqpdoG4F1AFja5q9SLrasmqqZMbdmiuiG8jJgD", + "mPmqbOXylDl0xB58tD4Jd6Ve5c3MG0nz3pw69aGm4KxYoDO9NFczIoi3EfBBCc9/wyWzeTktUFhMUcKM", + "iDJm1SX1aKkU3M0SbRSWH7cERbbxsjl8dfmFU3xd9ACuFCyX/I8wj7L80/bLHwEq/40reUknrgkYRk25", + "CwPDV6lo1Zo4qlreDJ+qludt3g8ePMurVnC/prNVI86yjwpphujxb5iqn7gAdbAZ8+TO8eXh8o+JAAy4", + "Onp8K+h1mpJ4xHO1+vzbivr2yi/KopZldZ3qi4GIo0o6bxMvcKgc5RiWV1ovB4lyQdXiXK+XDeaGNEhX", + "yxYWEjqCn8uOoX7op09gNJ4EEkZeEkYEjaA6qz6PKWagMaH3p16RPlOvcQmvFUSg10cn1tzgIH9BfaQK", + "SM/FXR6enXS6nTkRRuXuDPq7/QEc5owwnNHOQWe3v90fdECrmsEUt6Civs2ftvnGheJ6EltJ6Ef3kv5S", + "4JQo+OK3ABIAxB3a10EFwVNPicwwFVaLzBJAKDAEQ/XXUG7AXagH5lbummVvbTOFNGPIfiHZa7u5H0BQ", + "hrMD09wZDCywubLXL+TumISBrb/b6NGy31ZSnV2iAPr+kprnZMti6T91O3uD7RuNadVQ4OyGOn7HsE3i", + "JaCd799wIW7V6QkzaXk2ydqGQ/knDgjJP2u/fdB7JvM0xWLhFsxfrYzLJsGYSITdu0aPUxJFmlVAjaA+", + "es2IeY6wQthELoucQWll96Gm0OopMG27TS5Ain7k8eKLLWGlD2ej+FRlZ/q4fFqi5y9HOwUZL2+kfeQQ", + "tg3V3gMB/YiLuuAPdlL2Bs/vvtMjziYJjRTqFQRs45GphJCfBPDCHfYQF+j3nCuMinD+R3Skrcw6Lsit", + "W15FW3/Q+JM53gkJmcHPiEgxM8kR5p01h37pOBuXRHmcV95qjvBPjjv2pnIgPOaiAkGuekT9a6suDC5f", + "R3sBBAbbp5le/ICEv3cPJ9xOtigN+5BHDgpyolySx3ScrIttXAohQVnuJVFfC80P7vPKskUE/oSn6LEQ", + "8EtSSHjlbi1dCluZyJlRgIMS4JsyYdF+911V+HtbPvGiZMCvoZuGchbK+FVxvOgjt6ZG6VcLgFgSBOYZ", + "L18rZ3p4X8sJ27mPEwYzLjxF366pb9fUqlNuqMVNAQ6md8pb2CBuZIH489kfbmx9+GZ7aG97aGV5YOTK", + "Whf+zsd9ZCNSIx4TJGc8T2I0JsjgHbnYE4VFf/oRYRHN6JwAqB0UacsTRTMsILIkRTFW2PjQGw0TK80S", + "RXNburmei0MsF7iOYyHJCHD4Rk34k2UEImWMxEh/YqH7SjjBpXLi5uwHDexFg+XViK5mXJICz48p7zaH", + "9GZptGNotj9kby3Qq15ACKZ2vEaSBOBqV9h/OEN4yOwH3zsW4gLBJE5LzoUFYAZSg0xptmU5tU2PdCQj", + "HsLaeUsYZqonMxLRCY3stC7JwsZzBhtsVXdJD9iN8/1pkbCBdjbDeG0AzxgG5z0uniFLSVX/DYMg6CjJ", + "49LJ5SCEsBjjJAkW5pgmfIyTkVmfSxLwCb6EN+yilA6X0pvEeExMCflsoWacmb/zcc5Ubv4eC34liRh2", + "NvtDBokYdq1J3C0FRHQFhdzSjOtzJnhq+twyQ9z645IsPvWH7DBOKXMUAZ/gRHJEruE7qG8FmBmGezXQ", + "gzlNYT/4US4VT33kU0d3Zpg8V1mubEaJJKobQv0cMsXRHw7b8dPWH2WPn8BZTHCs6cR7xUwJZOumUcsR", + "1rMfwasBdzuBBRh29EVqwjymAjNlYDsLcEo09bd0o6iOABVT6yscYYYynpnKEkBUM6xJrtIGYDXgJEEK", + "jpL7VgvusJMN87HQe+m4EXfPAKXVjhFl6PRH7zAN9p6Fz5MkkSChiJL/On/9K4JbWe+Bea0M1zIpHUwL", + "DCjOwXXqeNoLHM2QcVRBMcFhh8bDTuHOjTdhrLm04TK9HvgUf9BD+8F006XxD/2+bsq4Kw/Qb3+YVg70", + "WcpSgwM67HzqIu/BlKpZPi6efQgvaBN82XmFEaANc81tAifBFJBmvBvfXJGYxYjbWyBZIIxKDuQHrowp", + "w2KxKpEwsPR2BfnERDJ6i/HHECIXh52DoYtdHHa6ww5hc/jNBjgOO5/CK2C9ls2V6+A+K5ybBRE9GQw2", + "1yNh2/UN+CxbOAa+sA7YqBUVZTf1DloY1j+Xf+DfWv8sXD+Y6c5LaCKj+Dvj+yN0QHgSu6+JBlwQNbEb", + "s4gkTuxeb+i5f+eB3qyIJMl9E+hDkWfhHiuQ+h8VOcJmlcdopfn+gSlucF+XSsVs/zD0++js5wHrubWd", + "k7kLdQ7XKQEMGqtKI/MywhKdw5h651r5fgG/9u1/ne4HmIoXCZ9eHBjVHSV8ihLKbD6AF6isxQO7lvCR", + "gaEpvrOoNK5I3IaRJP71j3/CoCib/usf/7TY7v/6xz/huG8ZeDWoMX0xI1ioMcHq4gD9QkjWwwmdEzcZ", + "qAJL5kQs0O7A2vzhEfJK3VspTQ7ZkL0hKhfMy5sw9dqkbdC6CvR8KMuJtDA++kU6scVkTGxjwG7jzrJZ", + "yns90d0AHCLMwJuAvhUdDQCWHDWFtq0m2gmbTM2cK0bTepjmUrDeev6iyLUy1NszA7whg4ElDp07eGAn", + "jTbOz19s9hFoW4YqoGAQ6A5lM1aN6H/jSet5kuEoVYYCq2x4U4QzPKYJdSbHhmon5gimOJpRRsr44gJr", + "3DVx4Eaqeczh2QmygZBdeHXIXp9vgYlVkUjlgnQtJxAWYbQsh8Ztngv0APyLKogO69l3h2xCMOQJnRwb", + "JuCBcBf5gEXDDIA8IMaVqkrlte6QGSRZi1ysD17KY5LAR9D/FCtyhRddVNS6ddVREqy0Qiy7+uUhM1iv", + "dg16AFWCvGH2gZ+ZIfVcJK/N2RJkkmjVGCLwTdlv6HtjwgWyEc5elX/XnUmyNMPSi5bi6PW5nt8UNEFu", + "7IHQ0utztxubXSQ5ihIK1BBhNmRTCARy4L2cVXa1SCibYRH3Iq4vAR/M6ZLxq4TE0yYee+QT2R1KMpV+", + "Asfp5zq5PjbhYrY8AX2IDUDdas/dsX2nnevOtvhn8t3ZQpA3cN4ZCy4x/Mas7jdHXgtHXnjdnFMv5Fk7", + "dgiMdxfxa7p4oIBfR3vLa26eeEv2EBY9tOGgbcArwgU6OzpBOI4FkXLz39vep2dqqLSU//T9qFnxQ4Se", + "2LFwYUH/rL2lSiCPhR28saNG2M2rXl/Xv9+2KsV3Gm+6og5PeeXd/e1R6/Qm10gp9Ja09u0mWRtsS2XE", + "ocxgSS09EI0SUogvxTn1qWidVdmE8RZXzkpxybLnk2N3IO/Pvmy7zln9brgHpnhcY4gPyAirqdZ+1ezH", + "RM3vil10aNMrzM9fF2kO7k8Kum9TdIjMH5O6GNeWTXNBA3TSeIG+JMrAm9ylnm57CEz8nAh3qs1AF2bW", + "xbTMp8jgtMCEwBKzWvc9Ma+0U31Ne38mzReW5yYSi13ybyJKC2W3XKtVCu6JLQF9d/ot9HAj9fbLha1Y", + "AgssMlhRx87tBJbVDSwXLNr8FrnyxSnaxDWWSqxw8yZxYck2aEqFnnVfct0h8+uNa5nO6rWUoUlCpzPr", + "BIjpBGL1lF+/G0a5cw+jLOpkC6yIDVF8jHm/Z3qRrRd4ToRCr49OzPr7V+rWHxC0ul5Vcsxr5e367s2r", + "HmERjwvnSbNMap98YYXJ0H8ll/f+T90jzGelTjxoEhg/Y/9NMDky8e99yv/Xzk8JHQssFv9r5yecZJSR", + "/7V7mGBFpNq8M2IZ3NdNd98KzCMmPq2/0OqiAWtiU4CMXSPwF2+1lPnd+38qsd9M+kaCf7Gu32T/NrK/", + "v1wrxX+7FXeqAJg+HsjDVRBbaLXh0TdIm3swmlqK9CBtKl6kEtRmxqWCR48vv9kGldOC4vxro6X1vzyQ", + "K68PR7onx11YSKgoDRUtbPrgPfkC3DjuXbi1/d6/I+AwHdNpznPpZyamWEUzIm3WbkKqDPixid3l9dwo", + "eH/FVDq4z6vj3uXqb3R/RxJ/fUMN8zYOvXUyv3urrcxv39cyv0E0tZnNtuxG15Vk2mwItHaYpm3JuAL9", + "uhwAHhpXSBdB77SiUqoLCDSIgyH731r/+E0RnH74waVQ5oPBzhP4nbD5hx9cFiU7daRCmBLUVtA7/PUY", + "vKhTCJSFIntlwnZ9HKZmN5CeKyvwb6cglY7k9hqSo8JvGlIrDclbrtUakt2Lu1WRqqVJ7l1HcvQWWnCL", + "Kf7n1JL+5O6RigYn88mERpQwKPACielyKR7QaHLfPCO3TEhm1h/pBRNVJJHWamTBtdZI6GVN6XsPJDsp", + "i2jdt/boylc/zsQqntl6sFZfK6WFZoXta6OHwf3eXvevqD1mEjMa0fLSZVroDlQGMjWh0txkoRVflqBi", + "ffT27SuXcaYlfuHqTCnuiku5uptD5heX6qMXZdUu84JrQUvkJLYZqpCHZ8s1xQTHCWUEQnSJDCWHVUvC", + "Peix+PJCZbjeXSuh8p6Ppa1g+nBC5YOxgnsR304qhaF5qeP7pfCK0+JEODg1j4pfWQYUYDwh8WkL54r3", + "bA7r1owbYLMwtuNZgiOAdtSvGdQxCxtgYAb9pgALQPAkIcKgyWW5cjUQh6wYHGVejXdb4+ZCNz/KmaLJ", + "RddEyAAkiESYLSyk0pBVOsNKkTRTkNoLaeswQkEyM+Ja8Uc9aMpzCW9BFq7fJcLJFV7IIbPJwOZzKJQr", + "SGSAF5Okj37mgMOA8BRT5jFeU4HwOzlkFzROyMjCKFwgKpGccaEIIzFK+ZzIar8Ei4QSAZM4wnrlJErx", + "AvDMDLSjWR+eEYMZVgFr4PrfmMUUatnpnospHwwZRjuDAUoJZtKmXks8gQvHtoFgEJUBfY8w2hs8t1/V", + "9g0wd93yb+jTJASZ8wiPkwUimooB/EFtwgamtrakqdGrt29ChTT7VZgMbdGwysZS6Uolxl2UszK5HMzn", + "OStywfV2qVwwmKd1rBEqimvQ4mmMSYT1ejJe7QeQDHkU5SJ0Qeqt9oqc/jsKjt70zmGpwqnbCWjhEYlh", + "zxlXMzjTHI7S5vcNVFUS1Z/jogkeEi4QRh5dl0YCEuXAGjcA+e+irNjHXAXei83v3dnRx9cyAnf8Dfbe", + "Y7mfgIj4ZFI5gOuvJnOAV6VMLJPwn/WcHrlSrT6LiymeMi4VjRwzrFd2/6YQtlYIV69skJonXFz6slWV", + "fn/i4rKtBmbxROnjUsT8GX6Ftn09PMBufngTPxiYjbKiiebelbQ6fRWnFIQuqqSLHeYo4WyqT1Fp6L53", + "S7yv1W0YHDZ9mQrjPy5Qc7QSMrI/mmqvejK2liZY7SPb6kPzIt37Pfh3fuUK0TRLSEqgGmzPEJve7BJh", + "CSrnU+nhDN2MV+pT5acDG11QGpd+14lDQFduwzZAel/eriBTTfh0PY5f0bkDrQsA+Q2ZqexP0IXx5lyg", + "ggdrgdag5qOrGY1mAOoHeqtu32D+4Sy7KPCMNw/QSzjIPqwzdL5hsPI1rUmeEIPVN0/Ti4PleqfvT0/h", + "I4PnZyqbXhwgV+O0uD+kfssH6dOzSLBU6FcLPbhRKOOwoxcKa32zmN+mhe8r8aaHLATlx8iVbZBO0IWH", + "6nfRADnl+O0rPn0wYazbXCXAzEVxZFVHoE3C4k5T3ARNwoB+24NBCLy6JbigGcYdYwsuDeYVnxYVCiqk", + "jLOsLfnaYQIVz9N0BQ2jDQ+pTKqY5+ovUsVECPjYUncTcaMNHNnqVPhSE6rFpXMHexPILxgdZCDDg0ul", + "mWqn2yEsTzsHv9l/zdO00+3Y8XhQ4zcQ7teANNYbXI5i0TvjITF+E8tvgrFYZfYeyGLt5rDqdLNE/sa8", + "8Kf3Fjqb3QOSIcgHNSPu1ySCeuOtGnwYL8AiYWTP72NkgCiJooRLUnHwPB48KmvoqsmMzYYit8Y9Pbw4", + "dwV82gSFnNtPz92XX4HuvS5WxI0Zuenee9DI8ggec26tXJrNhIs6iNG6aJKvnpC+3JYsTbUNhXyjzZtb", + "GVsRptYTllmE/SA2Bd1wrniKFY2gmFA041x6ZF8gDpuyX9Z4XFAmmFaMlmuD8i80qV5YM/SFVSMOrMkM", + "Yf+R7aMPn9tQ/vAX7lH5xU+eVaDg+F0n+gPgPlQ7F5RMUIZzSbRUl6cERYtIc0VTPYrgaIYinKlcECiM", + "R1BKGU3z1IeS1js2xwB7cbGdXnTROFcowWIKWpl56IJtIp6mhMUE7HNDNiN4TrVKKVCCFWHRoicJFNSd", + "E3TFxWXCcQwmhizG4OmBgnyCaAoEXO6UKBxjhUHQudAnfmTygi6KGrtGrWfkuqSGeMhEzr43RQJ0sxdu", + "oBeIAAo2lbOiFmOEY8KiIDr0+dfNxr68LfqcqPpEHygy6Fa89CFDhXybqxvO1xFF9MjCm7mw29iGza8Q", + "emWzCltNqHBk9O95pM1c3RwfyMFULPGqU/x1eJYKovtqvEsP7z7iAsW56c47lUDmf1afUMFQ/GArSNY0", + "23hbx1BRdK5Y5hvxvK0/3J8nt7DlfSWcsNuo2DeVNyon/TWwXLuqt+K5D2TEtLYk3yb3cCzYRXQ9mPjE", + "hcflHoux1TJsczQLvu1zJyUwaF+cfWPbdbZtAx5uy7adbXbJpe8xcsp6ECMa5uDWjNvIqq3p4N80G6U2", + "O49lPjiLLD0X98YWTwpGaFhjhhcJx/GfIUh4hf8o4kIYRAnAqHhMiKae1dBPDwDbXFk3reuyNd+fnm42", + "cQmhVvIIoR4xh/BScvRnabxswH09J0LQ2AJ/oqPTYxuuSyUSOeuj1ylVSHF0SUhWZrRAVmFfz89hayzX", + "aK+AaHQ7hCmxyDhlau0oylfvZjCfblXZ/Z75pIXI/uYOb+0OB8v+42NnwGUgZ8NMYLVmqrBaW7qTsgkX", + "qZHL8JjnunXNg/Qy6f00JTAnNCFyIRVJTVTiJE/guEG5BVtS135ndrkLMbn65Jh0uYyIlEpJOZNDZnNF", + "MiJ03/pz3b4XYBV0CChc8NczwyS/juA9PRgTr4ZV06oBChKU6uwcdLZwlm3FWOGGADE7vM8Y0k8QjYfk", + "Ih3zhEYooexSoo2EXhr1BM0lSvQfmyvD+Ubw3ZcuGHz7k6VX+oRNeLAcm6HZgpj/VFldlq05x+SjY2sv", + "iX9YHP+BjQ6ztfUliQXBSQ9K/DosHJQrmtCPhtXpRqhUNDIpR7hYu/enBVPtD9kpUUK/gyG1LUkMogFo", + "l1uZ4NHWMB8MdqOMAqDaLoHBAcNrfpxCj0dn70waKkm5WHSHTP8DGn57eGa8uxNsrQneQG0tYnSy9XpN", + "gPM5LNO/cYSgmeBK9ILghn9zCd4cY6TxDMmGI8qzVaoSz/70IaxWgvtmV3icdgUAeSpmszEVOAKhWM5y", + "FfMrFrYhzHmSp/of5o+TdVBhCkez9/DqVyPtmuGs7cZN8FEcSjunmJhykQ/i9DAL9lhjVvXCuSmAEFOJ", + "BgzeAofqz0jdX95876/jV+jutCvqSrF+NWfrvm8+OwaHsOGvx2M55obS3EwUX219usK02fr0Y8KjS2mh", + "WHyzodbbALJc/1hCTFsXIYgJkBmKLISRAcoisjtkNQOkQfyRCCNFREoZTrZgzqYRAMt2Viw85xQStCPI", + "U+lJGgNmUgKI2AB/p2cDhirXgOfRlbZYlf+O74xUHI1JxFPiAMQ3Q6rb3zBVP3FRRQP/WvjiW2/9ARIQ", + "U7C3rwFAb+7xswDRT/E1hErHuXUouxFtvOTlj8YU1EWwN8PO7kAOO1007Oykw47egSMMJlSs0D5KKcsV", + "kX10bOxbkIL7ZIAkiTiLpcMxdxa83YFsSsg1ZNmQ3fkEvrtPscdSFSzlG9tJiD3o95D+HpJ20IZ/4OyZ", + "jLtw6GLEc2XM/fZc2bdiosA8snnvvlrvjHzT7dtw8r/Z41vhUbDLml16W284e5bLGWk2ub0ytYFyNQZ8", + "bFevU87Q3/lYdhEjV8YaLqTqL/E9/fWZ6eA+sPt1VzfB7bdz/wba3wK0v1yrMFijCbDUV7KjDoPYSK4z", + "LhSgONpce0NDoEkAcgSPcIJeH50MWaRZkYEWFCTlwJ0sxLi5hQ//do5eHL3pomOoHYl+zsebffSaJQtX", + "wdv4aIbMSGKGeUWYobGhWhKHrmczdqCeuwwW1x08UDFmczICnhW3Vy5IvNuZERyDRPJH5xU3nQVQh9+8", + "0gcIgH/Nl8W2d1YKH503RIlF73CiiFhu9tTmSbECM8Ne0g6CzgpuBvhSdygd8lrZp5ENDDTG7k4ngJTx", + "6VsdhbuvOXo/XjITJ2Iq2I1zQBplkGSA48XjimWSM1QwxxAL9K/rohJBU5aw5WUrFQzosiny+ysyua/k", + "XRVs+X/X0wUzfbSOpqyyT5qIiwomaz29Ljl4ZuCQraMqwhmOqFp0EU4Se0fZm6CISOkV4u9YEHwZ8yvW", + "H7I3Re0Um9CLjs7edZ2jFsVUXpoWrC+2j17PiZD5uBgcgoNmvMaw5iQeMsVRhJMoT7S4QSYTEkEuLpRE", + "kQ2+3GIonTs8O2UnwfotXlR7/ujKxoVpAnavJIs6xW2Zrd4SJEowTZvBx62gBgGHEGow1o1yhiibJDak", + "KhJcSmSb6pGETuk4sQFCso/ezgiSOCVDliWYMSJQLk1UvB56LxNEytwkeOsGAKTXUFQXlcCCmeDKhiYk", + "nAtpogk0hb8/RVKRbAWZvTEtn8Kc70i2NY3bnh7ISF0bQ7MpxL6C9IYYSjELrukoT1wA472GopsBPbSU", + "+FgO/ltBp1Mi9KnAhsmacDxzrN1ymkNfyVhuLCF5XrzVroRk0aqXlehl7K0EhhuVWNtx52ZRf4HOL2kj", + "dqB9dLMs4l/0Ry37rmarhgdhH33mLP8slfnPvSTBtgasksIfmznJG3nlqFYSbdfDarXOrL3LTNfW+FkP", + "Bpv1mNGycCV9tknh/foIYXC/KA/3XWTtcdNWBe2qops2pPyvR9P/KijwbmD0Hxjl5BYw+l9V3j3gnD8c", + "/knwoD5UHn3F9+zq1/7pkfDvKn3ewOEDHFtT+rzhejZ4daWi9N6+005Nsi3+mSR4G+94A/ndLfs3rb+F", + "yuAt1joXtCZ4kmZq4QLarK+yDDqT9CPpNziCi7jVu3MF3yKk88uRh6PTxoDOP2e5+QeJGbWlA6lEJ8eB", + "Ou6PDGPQP3OVi2VL3zo9LKIZnZNmo3v1BNslygTpZTwD50psFsyuh7vLFBb96Udkm7eYq/ZfUHsSoPpJ", + "jGIqSKSShakDqjmC6eM7iQTXmgA852LRHCVijshPgqeHdjZr7kN7pqwxrIwzTBe9GCvcmztus8KE9hnR", + "nS6eUjM8RBl6+SPaINdKmAoXaKI1H0QnxZKaAv4SaHLTH/D2oMGyST+S0XTcZpQrapW8trVgUJRLxVO3", + "9yfHaANqn00J03uhRf0JSLKZ4HMak7gyxs6cJ2ZVtxsW9KZ2Vy1UFIXrnHJhBvcgMkybC2n6kWZVtlCE", + "xIwpwzC4tVVBqmfKJPHr/jBlLgDH7pEbxbcrzGp+G07Z0ZQIdTjtIirODcTz5rdr7jFfc34ylLvTKred", + "C89Zbbxulx/VMm3pLgo/FLlz92u2fv/1pPRQ+SizeazpfF4opE1m86+LBAf3dz/ct7n8/SNOAX1JnPLt", + "mcqhAd1iiGBeQUx3TOYk4VkK9dDh3U63k4ukc9CZKZUdbG1B7PeMS3Ww9/zpbufTh0//fwAAAP//5962", + "rgXyAQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/paths/paths.go b/lib/paths/paths.go index 814dc1432..833a7a911 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -343,6 +343,11 @@ func (p *Paths) DeviceMetadata(id string) string { return filepath.Join(p.DeviceDir(id), "metadata.json") } +// VFHealthState returns the path to the persisted vGPU VF health file. +func (p *Paths) VFHealthState() string { + return filepath.Join(p.dataDir, "gpu", "vf-health.json") +} + // Volume path methods // VolumesDir returns the root volumes directory. diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 054e3744e..2c7843532 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -10,16 +10,19 @@ import ( // GPUResourceStatus represents the GPU resource status for the API response. // Returns nil if no GPU is available on the host. type GPUResourceStatus struct { - Mode string `json:"mode"` // "vgpu" or "passthrough" - TotalSlots int `json:"total_slots"` // VFs for vGPU, physical GPUs for passthrough - UsedSlots int `json:"used_slots"` // Slots currently in use - Profiles []devices.GPUProfile `json:"profiles,omitempty"` // vGPU mode only - Devices []devices.PassthroughDevice `json:"devices,omitempty"` // passthrough mode only + Mode string `json:"mode"` // "vgpu" or "passthrough" + TotalSlots int `json:"total_slots"` // VFs for vGPU, physical GPUs for passthrough + UsedSlots int `json:"used_slots"` // Slots currently in use, including assigned quarantined VFs + AllocatableSlots int `json:"allocatable_slots"` // Healthy free slots used by admission control + QuarantinedSlots int `json:"quarantined_slots"` // Quarantined VFs; may overlap UsedSlots + Profiles []devices.GPUProfile `json:"profiles,omitempty"` // vGPU mode only + Devices []devices.PassthroughDevice `json:"devices,omitempty"` // passthrough mode only } -// GetGPUStatus returns the current GPU resource status. -// Returns nil if no GPU is available or the mode is "none". -func GetGPUStatus(ctx context.Context) *GPUResourceStatus { +// GetGPUStatus returns the current GPU resource status and any error that +// prevents determining allocatable vGPU capacity. It returns nil if no GPU is +// available or the mode is "none". +func GetGPUStatus(ctx context.Context) (*GPUResourceStatus, error) { framework, vfs, err := devices.DiscoverVGPU() if err != nil { // Only report passthrough once vGPU discovery confirms no vGPU @@ -27,16 +30,16 @@ func GetGPUStatus(ctx context.Context) *GPUResourceStatus { // expose the PFs/VFs as available passthrough slots while active vGPU // assignments exist. logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err) - return nil + return nil, nil } if framework != devices.VGPUFrameworkNone { return getVGPUStatus(ctx, framework, vfs) } - return getPassthroughStatus() + return getPassthroughStatus(), nil } // getVGPUStatus returns GPU status for vGPU mode (SR-IOV). -func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) *GPUResourceStatus { +func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) (*GPUResourceStatus, error) { usedSlots := 0 // Count used VFs (those with a vGPU assigned) for _, vf := range vfs { @@ -51,13 +54,20 @@ func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []d logger.FromContext(ctx).WarnContext(ctx, "failed to list vGPU profiles; reporting none", "framework", framework, "error", err) profiles = nil } - - return &GPUResourceStatus{ - Mode: string(devices.GPUModeVGPU), - TotalSlots: len(vfs), - UsedSlots: usedSlots, - Profiles: profiles, + allocatableSlots, quarantinedSlots, err := devices.VGPUAvailability(framework, vfs) + status := &GPUResourceStatus{ + Mode: string(devices.GPUModeVGPU), + TotalSlots: len(vfs), + UsedSlots: usedSlots, + AllocatableSlots: allocatableSlots, + QuarantinedSlots: quarantinedSlots, + Profiles: profiles, + } + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to count allocatable vGPU slots; reporting none", "framework", framework, "error", err) + status.AllocatableSlots = 0 } + return status, err } // getPassthroughStatus returns GPU status for whole-GPU passthrough mode. @@ -92,9 +102,10 @@ func getPassthroughStatus() *GPUResourceStatus { } return &GPUResourceStatus{ - Mode: string(devices.GPUModePassthrough), - TotalSlots: len(passthroughDevices), - UsedSlots: usedSlots, - Devices: passthroughDevices, + Mode: string(devices.GPUModePassthrough), + TotalSlots: len(passthroughDevices), + UsedSlots: usedSlots, + AllocatableSlots: len(passthroughDevices) - usedSlots, + Devices: passthroughDevices, } } diff --git a/lib/resources/gpu_test.go b/lib/resources/gpu_test.go new file mode 100644 index 000000000..825a20260 --- /dev/null +++ b/lib/resources/gpu_test.go @@ -0,0 +1,88 @@ +package resources + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func initVFHealthForTest(t *testing.T, state []byte) { + t.Helper() + dataDir := t.TempDir() + if state != nil { + path := paths.New(dataDir).VFHealthState() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, state, 0o644)) + } + devices.NewManager(paths.New(dataDir)) + resetDir := t.TempDir() + t.Cleanup(func() { devices.NewManager(paths.New(resetDir)) }) +} + +func TestGetVGPUStatusFailsClosedWhenVFHealthIsUnavailable(t *testing.T) { + initVFHealthForTest(t, []byte("not json")) + + status, err := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + assert.Zero(t, status.AllocatableSlots) + assert.Zero(t, status.QuarantinedSlots) + require.ErrorContains(t, err, "VF health state unavailable") +} + +func TestGetVGPUStatusReportsQuarantinedSlots(t *testing.T) { + initVFHealthForTest(t, nil) + for _, instance := range []string{"instance-1", "instance-2"} { + _, err := devices.ReportVFInitFailure(devices.VFInitFailureReport{VFAddress: "0000:82:00.4", InstanceID: instance}) + require.NoError(t, err) + } + + status, err := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4"}, + {PCIAddress: "0000:82:00.5", Allocated: true}, + {PCIAddress: "0000:82:00.6"}, + }) + require.NoError(t, err) + assert.Equal(t, 3, status.TotalSlots) + assert.Equal(t, 1, status.UsedSlots) + assert.Equal(t, 1, status.AllocatableSlots) + assert.Equal(t, 1, status.QuarantinedSlots) +} + +func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { + status := &GPUResourceStatus{ + Mode: string(devices.GPUModeVGPU), + TotalSlots: 4, + UsedSlots: 1, + AllocatableSlots: 0, + } + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + t.Cleanup(func() { setGPUStatusProvider(nil) }) + + mgr := NewManager(&config.Config{}, paths.New(t.TempDir())) + ctx := context.Background() + + err := mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "no allocatable vgpu slots") + + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { + return status, errors.New("VF health state unavailable: read failed") + }) + err = mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "vGPU placement is disabled: VF health state unavailable") + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + + status.AllocatableSlots = 1 + require.NoError(t, mgr.ReserveAllocation(ctx, "pending-a", 0, 0, 0, 0, 0, 0, true)) + err = mgr.ReserveAllocation(ctx, "pending-b", 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "no allocatable vgpu slots") + + mgr.FinishAllocation("pending-a") + require.NoError(t, mgr.ReserveAllocation(ctx, "pending-b", 0, 0, 0, 0, 0, 0, true)) +} diff --git a/lib/resources/monitoring_test.go b/lib/resources/monitoring_test.go index bef0740dc..39166856e 100644 --- a/lib/resources/monitoring_test.go +++ b/lib/resources/monitoring_test.go @@ -198,7 +198,7 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { mgr, _, _ := monitoringTestManager(t) originalProvider := currentGPUStatusProvider() - setGPUStatusProvider(func(context.Context) *GPUResourceStatus { + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return &GPUResourceStatus{ Mode: "vgpu", TotalSlots: 8, @@ -207,7 +207,7 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { {Name: "L40S-1Q", Available: 5}, {Name: "L40S-2Q", Available: 2}, }, - } + }, nil }) defer func() { setGPUStatusProvider(originalProvider) diff --git a/lib/resources/resource.go b/lib/resources/resource.go index 86f644bda..9c28fda0f 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -37,13 +37,13 @@ var ( gpuStatusProvider = GetGPUStatus ) -func currentGPUStatusProvider() func(context.Context) *GPUResourceStatus { +func currentGPUStatusProvider() func(context.Context) (*GPUResourceStatus, error) { gpuStatusProviderMu.RLock() defer gpuStatusProviderMu.RUnlock() return gpuStatusProvider } -func setGPUStatusProvider(fn func(context.Context) *GPUResourceStatus) { +func setGPUStatusProvider(fn func(context.Context) (*GPUResourceStatus, error)) { if fn == nil { fn = GetGPUStatus } @@ -427,7 +427,7 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error } // Get GPU status - gpuStatus := currentGPUStatusProvider()(ctx) + gpuStatus, _ := currentGPUStatusProvider()(ctx) return &FullResourceStatus{ CPU: *cpuStatus, @@ -691,15 +691,18 @@ func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string // Check GPU if needed if req.GPUSlots > 0 { - gpuStatus := currentGPUStatusProvider()(ctx) + gpuStatus, gpuStatusErr := currentGPUStatusProvider()(ctx) if gpuStatus == nil { return fmt.Errorf("insufficient GPU: no GPU available on this host") } - availableSlots := gpuStatus.TotalSlots - gpuStatus.UsedSlots - pending.GPUSlots + availableSlots := gpuStatus.AllocatableSlots - pending.GPUSlots if availableSlots < req.GPUSlots { + if gpuStatusErr != nil { + return fmt.Errorf("insufficient GPU: vGPU placement is disabled: %w", gpuStatusErr) + } if availableSlots <= 0 { - return fmt.Errorf("insufficient GPU: all %d %s slots are in use", - gpuStatus.TotalSlots, gpuStatus.Mode) + return fmt.Errorf("insufficient GPU: no allocatable %s slots available (%d total, %d in use)", + gpuStatus.Mode, gpuStatus.TotalSlots, gpuStatus.UsedSlots) } return fmt.Errorf("insufficient GPU: requested %d %s slot(s), but only %d available", req.GPUSlots, gpuStatus.Mode, availableSlots) diff --git a/openapi.yaml b/openapi.yaml index 002c9fcce..eb31e5863 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1800,7 +1800,7 @@ components: type: object description: GPU resource status. Null if no GPUs available. nullable: true - required: [mode, total_slots, used_slots] + required: [mode, total_slots, used_slots, allocatable_slots, quarantined_slots] properties: mode: type: string @@ -1813,8 +1813,16 @@ components: example: 64 used_slots: type: integer - description: Slots currently in use + description: Slots currently in use. Includes quarantined VFs that are still assigned, so this can overlap quarantined_slots. example: 5 + allocatable_slots: + type: integer + description: Free slots eligible for placement, matching admission control (excludes quarantined VFs; 0 while VF health state is unavailable) + example: 57 + quarantined_slots: + type: integer + description: VFs quarantined after guest driver init failures (vGPU mode only). May overlap used_slots until the affected instance releases its VF. + example: 2 profiles: type: array description: Available vGPU profiles (only in vGPU mode) From c784d9627fa1140e62bc2afc36e73a88c7f27726 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:56:12 +0000 Subject: [PATCH 2/6] Fail vGPU placement closed on VF health persist failures A failed state write previously rolled memory back and left the store reporting healthy, so a VF whose threshold-crossing failure could not be persisted stayed allocatable. Latch write failures and refuse placement until a later write succeeds; re-reported markers retry the write. Also make acknowledged reports crash-durable (fsync the parent when the state dir is first created, treat directory sync failures as persist failures instead of logging success), and re-evaluate persisted tallies against the configured threshold at load and on threshold changes so a lowered gpu.vf_quarantine_threshold applies to existing failures. --- lib/devices/GPU.md | 7 ++- lib/devices/vf_health.go | 85 +++++++++++++++++++++++++++-------- lib/devices/vf_health_test.go | 73 ++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 21 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 17c387723..678871562 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -300,8 +300,11 @@ equivalent free VFs is randomized so a wedged VF cannot capture every placement. A reported init success clears failures only when that exact assignment has a recorded failure, removing the match and older tallies; if that assignment crossed the threshold, its later success also rescinds the -quarantine. If the state file exists but cannot be loaded, placement and -advertised availability fail closed until it is repaired or removed. +quarantine. If the state file exists but cannot be loaded, or the last write +to it failed, placement and advertised availability fail closed until a load +or write succeeds. Recorded tallies are re-evaluated against the configured +threshold at load, so lowering `gpu.vf_quarantine_threshold` quarantines VFs +whose persisted failures already meet the new value. `used_slots` includes quarantined VFs still held by running instances, so it can overlap `quarantined_slots`; use `allocatable_slots` for admission. diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index b47ff11a5..b0870e212 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -75,11 +75,12 @@ type VFSuccessResult struct { } type vfHealthStore struct { - mu sync.Mutex - path string - records map[string]vfHealthRecord - threshold int - loadErr error + mu sync.Mutex + path string + records map[string]vfHealthRecord + threshold int + loadErr error + persistErr error } var vfHealthAddressPattern = regexp.MustCompile(`^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) @@ -97,16 +98,39 @@ func initVFHealth(path string) error { } // SetVFQuarantineThreshold configures the number of failed assignments -// required to quarantine a VF. +// required to quarantine a VF. Already-recorded tallies are re-evaluated so a +// lowered threshold applies to failures persisted before the change. func SetVFQuarantineThreshold(n int) { vfHealth.mu.Lock() defer vfHealth.mu.Unlock() vfHealth.threshold = n + vfHealth.requarantineLocked() +} + +// requarantineLocked quarantines records whose failure tallies meet the +// current threshold, so threshold changes and loaded state agree. +func (s *vfHealthStore) requarantineLocked() { + changed := false + for address, record := range s.records { + if record.QuarantinedAt != nil || len(record.Failures) < s.threshold { + continue + } + now := time.Now().UTC() + record.QuarantinedAt = &now + s.records[address] = record + changed = true + } + if changed { + if err := s.persistLocked(); err != nil { + slog.Default().Error("failed to persist re-evaluated VF quarantines; vGPU placement is disabled until a write succeeds", "error", err) + } + } } func (s *vfHealthStore) loadLocked() error { s.records = make(map[string]vfHealthRecord) s.loadErr = nil + s.persistErr = nil data, err := os.ReadFile(s.path) if err != nil { @@ -163,6 +187,7 @@ func (s *vfHealthStore) loadLocked() error { loaded[record.VFAddress] = record } s.records = loaded + s.requarantineLocked() return nil } @@ -179,6 +204,9 @@ func (s *vfHealthStore) checkedAddresses() (map[string]struct{}, error) { if err := s.ensureLoadedLocked(); err != nil { return nil, fmt.Errorf("VF health state unavailable: %w", err) } + if s.persistErr != nil { + return nil, fmt.Errorf("VF health state unavailable: last write failed: %w", s.persistErr) + } addresses := make(map[string]struct{}, len(s.records)) for address, record := range s.records { if record.QuarantinedAt != nil { @@ -240,11 +268,12 @@ func ReportVFInitSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { return vfHealth.reportSuccess(report) } -// VFHealthStoreUnavailable reports whether persisted state failed to load. +// VFHealthStoreUnavailable reports whether persisted state failed to load or +// the last write failed. func VFHealthStoreUnavailable() bool { vfHealth.mu.Lock() defer vfHealth.mu.Unlock() - return vfHealth.loadErr != nil + return vfHealth.loadErr != nil || vfHealth.persistErr != nil } // TotalQuarantinedVFs returns the number of quarantined VFs in persisted state. @@ -365,10 +394,19 @@ func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResu return result, nil } +// persistLocked writes the current records to disk. A failure is latched and +// fails placement closed until a later write succeeds, because in-memory +// rollback alone would leave a reported-unhealthy VF allocatable. func (s *vfHealthStore) persistLocked() error { if s.path == "" { return nil } + err := s.writeStateLocked() + s.persistErr = err + return err +} + +func (s *vfHealthStore) writeStateLocked() error { data, err := json.MarshalIndent(vfHealthFile{ Version: vfHealthFileVersion, Records: s.sortedRecordsLocked(), @@ -376,8 +414,15 @@ func (s *vfHealthStore) persistLocked() error { if err != nil { return fmt.Errorf("marshal VF health state: %w", err) } - if err := os.MkdirAll(filepath.Dir(s.path), 0755); err != nil { - return fmt.Errorf("create VF health state dir: %w", err) + dirPath := filepath.Dir(s.path) + if _, err := os.Stat(dirPath); os.IsNotExist(err) { + if err := os.MkdirAll(dirPath, 0755); err != nil { + return fmt.Errorf("create VF health state dir: %w", err) + } + // Make the new directory entry itself durable. + if err := syncDir(filepath.Dir(dirPath)); err != nil { + return fmt.Errorf("sync VF health state parent dir: %w", err) + } } tmp := s.path + ".tmp" f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) @@ -402,15 +447,17 @@ func (s *vfHealthStore) persistLocked() error { os.Remove(tmp) return fmt.Errorf("rename VF health state: %w", err) } - dirPath := filepath.Dir(s.path) - dir, err := os.Open(dirPath) - if err != nil { - slog.Default().Warn("failed to open VF health state directory for sync", "path", dirPath, "error", err) - return nil - } - if err := dir.Sync(); err != nil { - slog.Default().Warn("failed to sync VF health state directory", "path", dirPath, "error", err) + if err := syncDir(dirPath); err != nil { + return fmt.Errorf("sync VF health state dir: %w", err) } - _ = dir.Close() return nil } + +func syncDir(path string) error { + dir, err := os.Open(path) + if err != nil { + return err + } + defer dir.Close() + return dir.Sync() +} diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index dd286c296..75d8fbabf 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -1,6 +1,7 @@ package devices import ( + "encoding/json" "os" "path/filepath" "testing" @@ -20,6 +21,7 @@ func resetVFHealthStore(t *testing.T) string { vfHealth.records = make(map[string]vfHealthRecord) vfHealth.threshold = defaultVFQuarantineThreshold vfHealth.loadErr = nil + vfHealth.persistErr = nil }) return path } @@ -92,6 +94,77 @@ func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { assert.Zero(t, quarantined) } +func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { + resetVFHealthStore(t) + SetVFQuarantineThreshold(1) + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0o644)) + goodPath := vfHealth.path + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + assert.True(t, VFHealthStoreUnavailable()) + _, _, err = VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + require.ErrorContains(t, err, "last write failed") + + vfHealth.path = goodPath + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) + assert.False(t, VFHealthStoreUnavailable()) + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + require.NoError(t, err) + assert.Zero(t, available) + assert.Equal(t, 1, quarantined) +} + +func TestSetVFQuarantineThresholdReevaluatesRecordedFailures(t *testing.T) { + path := resetVFHealthStore(t) + SetVFQuarantineThreshold(3) + for _, instance := range []string{"instance-1", "instance-2"} { + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + } + + SetVFQuarantineThreshold(2) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt, "the re-evaluated quarantine must be persisted") +} + +func TestLoadReevaluatesTalliesAgainstConfiguredThreshold(t *testing.T) { + path := resetVFHealthStore(t) + SetVFQuarantineThreshold(3) + for _, instance := range []string{"instance-1", "instance-2"} { + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + } + + // Simulate a restart where the threshold is configured lower before the + // persisted tallies are loaded. + vfHealth.mu.Lock() + vfHealth.records = make(map[string]vfHealthRecord) + vfHealth.threshold = 2 + vfHealth.mu.Unlock() + require.NoError(t, initVFHealth(path)) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) +} + func TestReportVFInitFailureQuarantinesAtThreshold(t *testing.T) { path := resetVFHealthStore(t) From 52414cb472e1c82990ae905321e9e7f3c860745c Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:51:36 +0000 Subject: [PATCH 3/6] Preserve quarantines across sync failures --- lib/devices/vendor_vfio_linux.go | 119 ++++++++++++++++++------------- lib/devices/vf_health.go | 93 +++++++++++++++--------- lib/devices/vf_health_test.go | 75 +++++++++++++++++++ 3 files changed, 203 insertions(+), 84 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index b923439a3..0d8a27383 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -30,6 +30,26 @@ type vendorVFIOOwner struct { assignedAt time.Time } +type vendorVFIOGPUPlacement struct { + usage int + unknownUsage bool + quarantined int + freeVFs []VirtualFunction +} + +func (p *vendorVFIOGPUPlacement) preferredTo(other *vendorVFIOGPUPlacement, gpu, otherGPU string) bool { + if p.quarantined != other.quarantined { + return p.quarantined < other.quarantined + } + if p.unknownUsage != other.unknownUsage { + return !p.unknownUsage + } + if p.usage != other.usage { + return p.usage < other.usage + } + return gpu < otherGPU +} + type vendorVFIOSysfs struct { pciDevicesPath string procPath string @@ -317,71 +337,70 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map return nil } +func (s vendorVFIOSysfs) addVFToPlacement(placement *vendorVFIOGPUPlacement, vf VirtualFunction, profileType string, quarantined bool) { + if quarantined { + placement.quarantined++ + if !vf.Allocated { + return + } + } + if vf.Allocated { + // framebufferByType only covers currently creatable profiles, so + // after a restart an allocated type can be missing when its + // capacity is exhausted. Prefer GPUs whose load is fully known + // instead of rejecting placement outright; the kernel driver + // still enforces real capacity through creatable_vgpu_types. + framebuffer, ok := s.framebufferByType[vf.ProfileType] + if !ok { + placement.unknownUsage = true + return + } + placement.usage += framebuffer + return + } + profiles, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + // An unreadable free VF is just not a placement candidate. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + return + } + for _, profile := range profiles { + if profile.TypeName == profileType { + placement.freeVFs = append(placement.freeVFs, vf) + return + } + } +} + func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) { quarantined, err := vfHealth.checkedAddresses() if err != nil { return "", err } - usageByGPU := make(map[string]int) - unknownUsageByGPU := make(map[string]bool) - quarantinedByGPU := make(map[string]int) - freeByGPU := make(map[string][]VirtualFunction) + placementByGPU := make(map[string]*vendorVFIOGPUPlacement) for _, vf := range vfs { - _, bad := quarantined[vf.PCIAddress] - if bad { - quarantinedByGPU[vf.ParentGPU]++ - if !vf.Allocated { - continue - } - } - if vf.Allocated { - // framebufferByType only covers currently creatable profiles, so - // after a restart an allocated type can be missing when its - // capacity is exhausted. Prefer GPUs whose load is fully known - // instead of rejecting placement outright; the kernel driver - // still enforces real capacity through creatable_vgpu_types. - framebuffer, ok := s.framebufferByType[vf.ProfileType] - if !ok { - unknownUsageByGPU[vf.ParentGPU] = true - continue - } - usageByGPU[vf.ParentGPU] += framebuffer - continue - } - profiles, err := s.readCreatableProfiles(vf.PCIAddress) - if err != nil { - // An unreadable free VF is just not a placement candidate. - slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) - continue - } - for _, profile := range profiles { - if profile.TypeName == profileType { - freeByGPU[vf.ParentGPU] = append(freeByGPU[vf.ParentGPU], vf) - break - } + placement := placementByGPU[vf.ParentGPU] + if placement == nil { + placement = &vendorVFIOGPUPlacement{} + placementByGPU[vf.ParentGPU] = placement } + _, bad := quarantined[vf.PCIAddress] + s.addVFToPlacement(placement, vf, profileType, bad) } - gpus := make([]string, 0, len(freeByGPU)) - for gpu := range freeByGPU { - gpus = append(gpus, gpu) + gpus := make([]string, 0, len(placementByGPU)) + for gpu, placement := range placementByGPU { + if len(placement.freeVFs) > 0 { + gpus = append(gpus, gpu) + } } sort.Slice(gpus, func(i, j int) bool { - if quarantinedByGPU[gpus[i]] != quarantinedByGPU[gpus[j]] { - return quarantinedByGPU[gpus[i]] < quarantinedByGPU[gpus[j]] - } - if unknownUsageByGPU[gpus[i]] != unknownUsageByGPU[gpus[j]] { - return !unknownUsageByGPU[gpus[i]] - } - if usageByGPU[gpus[i]] == usageByGPU[gpus[j]] { - return gpus[i] < gpus[j] - } - return usageByGPU[gpus[i]] < usageByGPU[gpus[j]] + return placementByGPU[gpus[i]].preferredTo(placementByGPU[gpus[j]], gpus[i], gpus[j]) }) if len(gpus) == 0 { return "", nil } - candidates := freeByGPU[gpus[0]] + candidates := placementByGPU[gpus[0]].freeVFs pick := s.pickVFIndex if pick == nil { pick = rand.IntN diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index b0870e212..479696661 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -75,18 +75,23 @@ type VFSuccessResult struct { } type vfHealthStore struct { - mu sync.Mutex - path string - records map[string]vfHealthRecord - threshold int - loadErr error - persistErr error + mu sync.Mutex + path string + records map[string]vfHealthRecord + threshold int + loadErr error + persistErr error + syncDirFunc func(string) error } var vfHealthAddressPattern = regexp.MustCompile(`^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) var ( - vfHealth = &vfHealthStore{records: make(map[string]vfHealthRecord), threshold: defaultVFQuarantineThreshold} + vfHealth = &vfHealthStore{ + records: make(map[string]vfHealthRecord), + threshold: defaultVFQuarantineThreshold, + syncDirFunc: syncDir, + } vendorVFIOMu sync.Mutex ) @@ -121,7 +126,7 @@ func (s *vfHealthStore) requarantineLocked() { changed = true } if changed { - if err := s.persistLocked(); err != nil { + if _, err := s.persistLocked(); err != nil { slog.Default().Error("failed to persist re-evaluated VF quarantines; vGPU placement is disabled until a write succeeds", "error", err) } } @@ -307,6 +312,9 @@ func (s *vfHealthStore) reportFailure(report VFInitFailureReport) (VFReportResul if !vfHealthAddressPattern.MatchString(report.VFAddress) { return VFReportResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) } + if err := s.retryPersistLocked(); err != nil { + return VFReportResult{}, err + } previous, existed := s.records[report.VFAddress] result := VFReportResult{Failures: len(previous.Failures), Threshold: s.threshold} @@ -335,11 +343,14 @@ func (s *vfHealthStore) reportFailure(report VFInitFailureReport) (VFReportResul result.Outcome = VFReportQuarantined } s.records[report.VFAddress] = record - if err := s.persistLocked(); err != nil { - if existed { - s.records[report.VFAddress] = previous - } else { - delete(s.records, report.VFAddress) + renamed, err := s.persistLocked() + if err != nil { + if !renamed { + if existed { + s.records[report.VFAddress] = previous + } else { + delete(s.records, report.VFAddress) + } } return VFReportResult{}, err } @@ -359,6 +370,9 @@ func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResu if !vfHealthAddressPattern.MatchString(report.VFAddress) { return VFSuccessResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) } + if err := s.retryPersistLocked(); err != nil { + return VFSuccessResult{}, err + } previous, ok := s.records[report.VFAddress] if !ok || len(previous.Failures) == 0 { return VFSuccessResult{}, nil @@ -387,70 +401,81 @@ func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResu record.Failures = remaining s.records[report.VFAddress] = record } - if err := s.persistLocked(); err != nil { - s.records[report.VFAddress] = previous + renamed, err := s.persistLocked() + if err != nil { + if !renamed { + s.records[report.VFAddress] = previous + } return VFSuccessResult{}, err } return result, nil } +func (s *vfHealthStore) retryPersistLocked() error { + if s.persistErr == nil { + return nil + } + _, err := s.persistLocked() + return err +} + // persistLocked writes the current records to disk. A failure is latched and -// fails placement closed until a later write succeeds, because in-memory -// rollback alone would leave a reported-unhealthy VF allocatable. -func (s *vfHealthStore) persistLocked() error { +// fails placement closed until a later write succeeds. The returned boolean +// reports whether the rename made the new state visible. +func (s *vfHealthStore) persistLocked() (bool, error) { if s.path == "" { - return nil + return false, nil } - err := s.writeStateLocked() + renamed, err := s.writeStateLocked() s.persistErr = err - return err + return renamed, err } -func (s *vfHealthStore) writeStateLocked() error { +func (s *vfHealthStore) writeStateLocked() (bool, error) { data, err := json.MarshalIndent(vfHealthFile{ Version: vfHealthFileVersion, Records: s.sortedRecordsLocked(), }, "", " ") if err != nil { - return fmt.Errorf("marshal VF health state: %w", err) + return false, fmt.Errorf("marshal VF health state: %w", err) } dirPath := filepath.Dir(s.path) if _, err := os.Stat(dirPath); os.IsNotExist(err) { if err := os.MkdirAll(dirPath, 0755); err != nil { - return fmt.Errorf("create VF health state dir: %w", err) + return false, fmt.Errorf("create VF health state dir: %w", err) } // Make the new directory entry itself durable. - if err := syncDir(filepath.Dir(dirPath)); err != nil { - return fmt.Errorf("sync VF health state parent dir: %w", err) + if err := s.syncDirFunc(filepath.Dir(dirPath)); err != nil { + return false, fmt.Errorf("sync VF health state parent dir: %w", err) } } tmp := s.path + ".tmp" f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { - return fmt.Errorf("create VF health state: %w", err) + return false, fmt.Errorf("create VF health state: %w", err) } if _, err := f.Write(data); err != nil { f.Close() os.Remove(tmp) - return fmt.Errorf("write VF health state: %w", err) + return false, fmt.Errorf("write VF health state: %w", err) } if err := f.Sync(); err != nil { f.Close() os.Remove(tmp) - return fmt.Errorf("sync VF health state: %w", err) + return false, fmt.Errorf("sync VF health state: %w", err) } if err := f.Close(); err != nil { os.Remove(tmp) - return fmt.Errorf("close VF health state: %w", err) + return false, fmt.Errorf("close VF health state: %w", err) } if err := os.Rename(tmp, s.path); err != nil { os.Remove(tmp) - return fmt.Errorf("rename VF health state: %w", err) + return false, fmt.Errorf("rename VF health state: %w", err) } - if err := syncDir(dirPath); err != nil { - return fmt.Errorf("sync VF health state dir: %w", err) + if err := s.syncDirFunc(dirPath); err != nil { + return true, fmt.Errorf("sync VF health state dir: %w", err) } - return nil + return true, nil } func syncDir(path string) error { diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index 75d8fbabf..5166a76d2 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -2,6 +2,7 @@ package devices import ( "encoding/json" + "errors" "os" "path/filepath" "testing" @@ -22,6 +23,7 @@ func resetVFHealthStore(t *testing.T) string { vfHealth.threshold = defaultVFQuarantineThreshold vfHealth.loadErr = nil vfHealth.persistErr = nil + vfHealth.syncDirFunc = syncDir }) return path } @@ -362,6 +364,79 @@ func TestReportVFInitFailureRollsBackOnPersistFailure(t *testing.T) { assert.False(t, exists, "a failure whose persist failed must be retried by the next report") } +func TestReportVFInitFailureRetainsRenamedStateAfterSyncFailure(t *testing.T) { + path := resetVFHealthStore(t) + vf := "0000:e3:00.4" + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-1"}) + require.NoError(t, err) + + vfHealth.syncDirFunc = func(string) error { return errors.New("injected sync failure") } + _, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-2"}) + require.ErrorContains(t, err, "sync VF health state dir") + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt) + require.Len(t, quarantinedVFs(), 1, "memory must retain state already renamed into place") + assert.True(t, VFHealthStoreUnavailable()) + + vfHealth.syncDirFunc = syncDir + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5", InstanceID: "other-instance"}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.False(t, VFHealthStoreUnavailable()) + + data, err = os.ReadFile(path) + require.NoError(t, err) + state = vfHealthFile{} + require.NoError(t, json.Unmarshal(data, &state)) + found := false + for _, record := range state.Records { + if record.VFAddress == vf { + found = true + assert.NotNil(t, record.QuarantinedAt, "a later write must not erase the renamed quarantine") + } + } + require.True(t, found) + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: vf}}) + require.NoError(t, err) + assert.Zero(t, available) + assert.Equal(t, 1, quarantined) +} + +func TestReportRetriesFailedThresholdPersistence(t *testing.T) { + path := resetVFHealthStore(t) + vf := "0000:e3:00.4" + SetVFQuarantineThreshold(3) + for _, instance := range []string{"instance-1", "instance-2"} { + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: instance}) + require.NoError(t, err) + } + + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + vfHealth.path = filepath.Join(blocker, "vf-health.json") + SetVFQuarantineThreshold(2) + assert.True(t, VFHealthStoreUnavailable()) + + vfHealth.path = path + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + assert.False(t, VFHealthStoreUnavailable()) + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt) +} + func TestReportVFInitSuccessRollsBackOnPersistFailure(t *testing.T) { resetVFHealthStore(t) _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) From 493ab4206405bd4073209cc4adcfc1a2ceb4b857 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:41:07 +0000 Subject: [PATCH 4/6] Retry VF health parent directory sync --- lib/devices/vf_health.go | 13 +++++------ lib/devices/vf_health_test.go | 41 ++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index 479696661..843060ca9 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -440,14 +440,11 @@ func (s *vfHealthStore) writeStateLocked() (bool, error) { return false, fmt.Errorf("marshal VF health state: %w", err) } dirPath := filepath.Dir(s.path) - if _, err := os.Stat(dirPath); os.IsNotExist(err) { - if err := os.MkdirAll(dirPath, 0755); err != nil { - return false, fmt.Errorf("create VF health state dir: %w", err) - } - // Make the new directory entry itself durable. - if err := s.syncDirFunc(filepath.Dir(dirPath)); err != nil { - return false, fmt.Errorf("sync VF health state parent dir: %w", err) - } + if err := os.MkdirAll(dirPath, 0755); err != nil { + return false, fmt.Errorf("create VF health state dir: %w", err) + } + if err := s.syncDirFunc(filepath.Dir(dirPath)); err != nil { + return false, fmt.Errorf("sync VF health state parent dir: %w", err) } tmp := s.path + ".tmp" f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index 5166a76d2..260c5cc83 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -364,13 +364,52 @@ func TestReportVFInitFailureRollsBackOnPersistFailure(t *testing.T) { assert.False(t, exists, "a failure whose persist failed must be retried by the next report") } +func TestReportVFInitFailureRetriesParentSyncAfterFailure(t *testing.T) { + resetVFHealthStore(t) + parentDir := t.TempDir() + vfHealth.path = filepath.Join(parentDir, "gpu", "vf-health.json") + + parentSyncs := 0 + retrySawPersistErr := false + vfHealth.syncDirFunc = func(path string) error { + if path != parentDir { + return syncDir(path) + } + parentSyncs++ + if parentSyncs == 1 { + return errors.New("injected parent sync failure") + } + if parentSyncs == 2 { + retrySawPersistErr = vfHealth.persistErr != nil + } + return syncDir(path) + } + + report := VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"} + _, err := ReportVFInitFailure(report) + require.ErrorContains(t, err, "sync VF health state parent dir") + assert.True(t, VFHealthStoreUnavailable()) + + result, err := ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 3, parentSyncs) + assert.True(t, retrySawPersistErr, "retry must sync the parent before clearing the write failure") + assert.False(t, VFHealthStoreUnavailable()) +} + func TestReportVFInitFailureRetainsRenamedStateAfterSyncFailure(t *testing.T) { path := resetVFHealthStore(t) vf := "0000:e3:00.4" _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-1"}) require.NoError(t, err) - vfHealth.syncDirFunc = func(string) error { return errors.New("injected sync failure") } + vfHealth.syncDirFunc = func(path string) error { + if path == filepath.Dir(vfHealth.path) { + return errors.New("injected sync failure") + } + return syncDir(path) + } _, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-2"}) require.ErrorContains(t, err, "sync VF health state dir") From f9a27f43d194c6a104e6ca5eac599b1925720da0 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:44:17 +0000 Subject: [PATCH 5/6] Deduplicate VF health lock-order comment --- lib/devices/vf_health.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index 843060ca9..3ae8d02c0 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -92,6 +92,9 @@ var ( threshold: defaultVFQuarantineThreshold, syncDirFunc: syncDir, } + // vendorVFIOMu is acquired before vfHealth.mu. It serializes quarantine + // mutations with vendor-VFIO create, destroy, and reconciliation so + // placement cannot select a VF while it is being quarantined. vendorVFIOMu sync.Mutex ) @@ -254,9 +257,6 @@ func countFreeVFs(vfs []VirtualFunction, quarantined map[string]struct{}) int { // ReportVFInitFailure records a guest-reported driver init failure and // quarantines the VF once failures from enough distinct assignments accumulate. func ReportVFInitFailure(report VFInitFailureReport) (VFReportResult, error) { - // Lock order is vendorVFIOMu before vfHealth.mu. This serializes quarantine - // mutations with vendor-VFIO create, destroy, and reconciliation so placement - // cannot select a VF while it is being quarantined. vendorVFIOMu.Lock() defer vendorVFIOMu.Unlock() return vfHealth.reportFailure(report) @@ -265,9 +265,6 @@ func ReportVFInitFailure(report VFInitFailureReport) (VFReportResult, error) { // ReportVFInitSuccess clears failures through an exactly matched successful // assignment. A quarantine is rescinded only when that assignment triggered it. func ReportVFInitSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { - // Lock order is vendorVFIOMu before vfHealth.mu. This serializes quarantine - // mutations with vendor-VFIO create, destroy, and reconciliation so placement - // cannot select a VF while it is being quarantined. vendorVFIOMu.Lock() defer vendorVFIOMu.Unlock() return vfHealth.reportSuccess(report) From 5a8fcae3c72c478ecdda9733a7f1f06e70ffaa35 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:44:17 +0000 Subject: [PATCH 6/6] Collapse redundant VF health test cases Fold the below-threshold placement assertion into TestVGPUAvailability and the repaired-state recovery assertion into TestVGPUAvailabilityFailsWhenStoreUnavailable, exercising both through the public API. Drop TestReportVFInitFailureRespectsConfiguredThreshold and TestCheckedAddressesFailsClosedOnUnloadedState, whose remaining coverage is subsumed by the threshold re-evaluation and invalid-record tests. --- lib/devices/vf_health_test.go | 58 +++++++---------------------------- 1 file changed, 11 insertions(+), 47 deletions(-) diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index 260c5cc83..00f14dbf7 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -53,6 +53,9 @@ func quarantineVF(t *testing.T, address string) { func TestVGPUAvailability(t *testing.T) { resetVFHealthStore(t) quarantineVF(t, "0000:82:00.4") + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:82:00.6", InstanceID: "instance-1"}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) vfs := []VirtualFunction{ {PCIAddress: "0000:82:00.4"}, {PCIAddress: "0000:82:00.5", Allocated: true}, @@ -61,7 +64,7 @@ func TestVGPUAvailability(t *testing.T) { available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, vfs) require.NoError(t, err) - assert.Equal(t, 1, available) + assert.Equal(t, 1, available, "a below-threshold failure tally must not remove the VF from placement") assert.Equal(t, 1, quarantined) available, quarantined, err = VGPUAvailability(VGPUFrameworkMdev, vfs) @@ -70,18 +73,6 @@ func TestVGPUAvailability(t *testing.T) { assert.Zero(t, quarantined) } -func TestVGPUAvailabilityExcludesOnlyQuarantinedVFs(t *testing.T) { - resetVFHealthStore(t) - result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:82:00.4", InstanceID: "instance-1"}) - require.NoError(t, err) - require.Equal(t, VFReportRecorded, result.Outcome) - - available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) - require.NoError(t, err) - assert.Equal(t, 1, available, "a below-threshold failure tally must not remove the VF from placement") - assert.Zero(t, quarantined) -} - func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { path := resetVFHealthStore(t) require.NoError(t, os.WriteFile(path, []byte("not json"), 0o644)) @@ -94,6 +85,13 @@ func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, available) assert.Zero(t, quarantined) + + restored := `{"version":1,"records":[{"vf_address":"0000:82:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0o644)) + available, quarantined, err = VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err, "a repaired state file must re-enable placement without a new report") + assert.Zero(t, available) + assert.Equal(t, 1, quarantined) } func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { @@ -228,23 +226,6 @@ func TestReportVFInitFailureDeduplicatesAssignments(t *testing.T) { assert.Empty(t, quarantinedVFs(), "a rescanned assignment must not count toward the threshold twice") } -func TestReportVFInitFailureRespectsConfiguredThreshold(t *testing.T) { - resetVFHealthStore(t) - SetVFQuarantineThreshold(3) - - for i, instance := range []string{"instance-1", "instance-2"} { - result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) - require.NoError(t, err) - assert.Equal(t, VFReportRecorded, result.Outcome) - assert.Equal(t, i+1, result.Failures) - assert.Equal(t, 3, result.Threshold) - } - - result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-3"}) - require.NoError(t, err) - assert.Equal(t, VFReportQuarantined, result.Outcome) -} - func TestReportVFInitSuccessClearsFailureTally(t *testing.T) { path := resetVFHealthStore(t) report := VFInitFailureReport{ @@ -497,23 +478,6 @@ func TestReportVFInitSuccessRollsBackOnPersistFailure(t *testing.T) { assert.Len(t, record.Failures, 1) } -func TestCheckedAddressesFailsClosedOnUnloadedState(t *testing.T) { - path := resetVFHealthStore(t) - quarantineVF(t, "0000:e3:00.4") - - require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) - require.Error(t, initVFHealth(path)) - - _, err := vfHealth.checkedAddresses() - require.Error(t, err) - - restored := `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` - require.NoError(t, os.WriteFile(path, []byte(restored), 0644)) - addresses, err := vfHealth.checkedAddresses() - require.NoError(t, err) - assert.Contains(t, addresses, "0000:e3:00.4") -} - func TestCheckedAddressesFailsClosedOnInvalidRecord(t *testing.T) { tests := []struct { name string