diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 13a3ac159..6dc38ef54 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -26,8 +26,9 @@ file is canonical; `services/THIRD_PARTY_NOTICES.md` is only a pointer. Electron Chromium dependency notices ship separately with the Electron runtime as `LICENSES.chromium.html`; they are not reproduced here. Optional inference -engines (Ollama, LM Studio, oMLX) and the Phoenix container image are installed -separately, not bundled in the desktop application, and are governed by their +engines (Ollama, LM Studio, oMLX, llama.cpp), downloaded models and the Phoenix +container image are installed separately, not bundled in the desktop +application, and are governed by their upstream terms. Vendored Kaizen CSS attribution is outside the npm dependency graph and requires separate review, including preservation in built CSS. diff --git a/desktop/README.md b/desktop/README.md index a2c7dd7be..cb7501cda 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -70,6 +70,29 @@ devices remain read-only. Launch text is displayed only in its editor. The renderer consumes the node-owned settings protocol described in [`ENGINE_SETTINGS.md`](../services/nvpair-ui-broker/ENGINE_SETTINGS.md). +## llama.cpp models + +On macOS arm64, expand **llama.cpp → Models → Add model** to browse the +Go-owned catalog grouped by publisher or search public single-file Q4_K_M GGUFs. +Search bounds and download failures are shown as service messages. Download, +Cancel, Canceling, Cancelled, Interrupted and Retry reuse the existing model +manager's operation tracking. llama.cpp does not promise partial-file resume. +Installed models can be refreshed while stopped; deletion requires a stopped +engine and a PAIR-owned file. + +In **Settings → Startup model**, choose a downloaded catalog ID or **No model +(router mode)**. The selector writes a cache-relative file into the existing +arguments editor; Apply still previews, checks revisions and confirms restarts. +The build is not notarized, like PAIR. PAIR downloads the engine and models at +runtime; neither is bundled with the desktop app. + +Verified peers show installed llama.cpp cache files from authenticated discovery +inventory, including while stopped. Owned rows offer Delete; stop the peer's +engine first. Unrecorded files are visible but not deletable, and previews receive +no inventory. Download/cancel/delete reuse the existing remote controls. +The startup picker remains local; peer Settings arguments still accept a known +installed catalog ID. + ## Develop ```bash diff --git a/desktop/docs/services-api.md b/desktop/docs/services-api.md index 4bf199ba3..fd17773ce 100644 --- a/desktop/docs/services-api.md +++ b/desktop/docs/services-api.md @@ -320,10 +320,14 @@ Client surface: requests are sent to the broker; notifications are consumed from | `discovery:get-nodes` | request (TUI sends) | ✅ yes | | `discovery:subscribe` | request (TUI sends) | ✅ yes | | `engine:action` | request (TUI sends) | ✅ yes | +| `engine:apply-settings` | request (TUI sends) | ✅ yes | | `engine:cancel-pull` | request (TUI sends) | ✅ yes | | `engine:get-installed` | request (TUI sends) | ✅ yes | +| `engine:get-settings` | request (TUI sends) | ✅ yes | | `engine:install` | request (TUI sends) | ✅ yes | +| `engine:model-catalog` | request (TUI sends) | ✅ yes | | `engine:models` | request (TUI sends) | ✅ yes | +| `engine:preview-settings` | request (TUI sends) | ✅ yes | | `engine:remote-cancel-pull` | request (TUI sends) | ✅ yes | | `engine:remote-delete-model` | request (TUI sends) | ✅ yes | | `engine:remote-get-installed` | request (TUI sends) | ✅ yes | diff --git a/desktop/docs/services-backend.md b/desktop/docs/services-backend.md index c2d87d6f8..3fffd5f68 100644 --- a/desktop/docs/services-backend.md +++ b/desktop/docs/services-backend.md @@ -13,6 +13,13 @@ Personal AI Router does not reimplement backend behavior in TypeScript. Electron process startup, translates JSON-RPC into the stable frontend contract, and projects backend notifications into renderer state. +Model inventory carries the additive `downloadErrorsByEngine` map through verified +manual peers, scanner enrichment and broker discovery. The fixed llama.cpp code +`cache_unavailable` becomes an engine-scoped `EngineModels.downloadError`; only +llama.cpp's installed-models panel shows “Couldn't read llama.cpp's downloaded +models”. No filesystem errors are displayed, no other engine is marked down, +and the next successful cache read clears the message. + ## Runtime ownership `nvpair-ui-broker` is the only backend process Electron starts directly. The diff --git a/desktop/scripts/generate-licenses.ts b/desktop/scripts/generate-licenses.ts index ec5aa5ee5..079a0baf0 100644 --- a/desktop/scripts/generate-licenses.ts +++ b/desktop/scripts/generate-licenses.ts @@ -169,8 +169,9 @@ function renderMarkdown(entries: Entry[]): string { '', 'Electron Chromium dependency notices ship separately with the Electron runtime', 'as `LICENSES.chromium.html`; they are not reproduced here. Optional inference', - 'engines (Ollama, LM Studio, oMLX) and the Phoenix container image are installed', - 'separately, not bundled in the desktop application, and are governed by their', + 'engines (Ollama, LM Studio, oMLX, llama.cpp), downloaded models and the Phoenix', + 'container image are installed separately, not bundled in the desktop', + 'application, and are governed by their', 'upstream terms. Vendored Kaizen CSS attribution is outside the npm dependency', 'graph and requires separate review, including preservation in built CSS.', '', diff --git a/desktop/src/electron/service-bridge/empty-handlers.ts b/desktop/src/electron/service-bridge/empty-handlers.ts index 6d8d86859..b59bb2f81 100644 --- a/desktop/src/electron/service-bridge/empty-handlers.ts +++ b/desktop/src/electron/service-bridge/empty-handlers.ts @@ -6,6 +6,7 @@ import type { ClusterInitialSnapshot } from '@/shared/types/bootstrap' import type { ClusterNode, ClusterNodeIdentity, Invite } from '@/shared/types/cluster' import type { ManualNodeKind, ManualNodeStatus } from '@/shared/types/manual-nodes' import type { EngineType } from '@/shared/types/engines' +import type { DownloadedModel, EngineHubModel } from '@/shared/types/engine-api' import type { ServiceError } from '@/shared/types/errors' import type { NodeInferenceMetricsSnapshot } from '@/shared/types/metrics' import type { @@ -752,6 +753,20 @@ async function handleEngineCommand(payload?: WsInvokeRequest<'engine:command'>): const selfId = getModularBridgeState().getSelfId() if (selfId && payload.nodeId && payload.nodeId !== selfId) { if (supervisor.hasProcess('broker')) { + if ( + payload.engineType === 'llamacpp' && + payload.command === 'deleteModel' && + payload.model + ) { + await supervisor.modelActionRemote( + payload.nodeId, + 'llamacpp', + 'llamacpp', + payload.model, + 'deleteModel' + ) + return null + } routeRemoteEngineCommand(payload) return null } @@ -771,6 +786,16 @@ async function handleEngineCommand(payload?: WsInvokeRequest<'engine:command'>): // Lifecycle + model management is the local engine-manager's job, reached // through the broker's `engine:*` relay (the broker supervises it). if (supervisor.hasProcess('broker')) { + // The installed-cache UI needs the authoritative completion, not a + // fire-and-forget acknowledgement that could refresh before deletion. + if ( + payload.engineType === 'llamacpp' && + payload.command === 'deleteModel' && + payload.model + ) { + await supervisor.deleteModel('llamacpp', 'llamacpp', payload.model) + return null + } routeEngineManagerCommand(payload) return null } @@ -1384,8 +1409,64 @@ const EMPTY_SERVICE_BRIDGE_HANDLERS: BridgeHandlerMap = { ) ), 'engine:command': payload => handleEngineCommand(payload), - 'engine:search-hub': payload => - payload ? getEngineHubModels(payload.engineType) : { models: [] }, + 'engine:downloaded-models': async payload => { + if (!payload) throw new Error('A device is required.') + if (payload.nodeId !== getModularBridgeState().getSelfId()) + return getModularBridgeState().getPeerDownloadedModels(payload.nodeId) + const result = await getModularSupervisor().callProcess( + 'broker', + 'engine:action', + { + engine: 'llamacpp', + action: 'list_downloaded' + }, + 15_000 + ) + const rows = objectValue(result)?.models + if (!Array.isArray(rows)) throw new Error('Could not read installed models.') + return rows.flatMap(row => { + const model = objectValue(row) + if (!model || !stringValue(model.id) || !stringValue(model.file)) return [] + const value: DownloadedModel = { + id: stringValue(model.id), + file: stringValue(model.file), + size: numberValue(model.size), + sha256: stringValue(model.sha256) + } + return [value] + }) + }, + 'engine:search-hub': async payload => { + if (!payload) return { models: [] } + if (payload.engineType !== 'llamacpp') return getEngineHubModels(payload.engineType) + const result = await getModularSupervisor().callProcess( + 'broker', + 'engine:model-catalog', + { query: payload.query ?? '' }, + 30_000 + ) + const rows = objectValue(result)?.models + if (!Array.isArray(rows)) throw new Error('Could not read the model catalog.') + const models: EngineHubModel[] = rows.flatMap(row => { + const model = objectValue(row) + const id = model ? stringValue(model.id) : '' + if (!model || !id) return [] + return [ + { + id, + name: id, + author: id.split('/')[0], + url: '', + size: numberValue(model.size), + downloads: 0, + likes: 0, + updatedAt: '', + tags: [] + } + ] + }) + return { models } + }, 'engine:lookup-hub-model': payload => payload ? lookupEngineHubModel(payload.engineType, payload.name) : { model: null }, diff --git a/desktop/src/electron/service-bridge/modular-state.ts b/desktop/src/electron/service-bridge/modular-state.ts index 9df92a32c..aa9d1f91d 100644 --- a/desktop/src/electron/service-bridge/modular-state.ts +++ b/desktop/src/electron/service-bridge/modular-state.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { randomUUID } from 'node:crypto' -import type { EngineInitialState } from '@/shared/types/engine-api' +import type { DownloadedModel, EngineInitialState } from '@/shared/types/engine-api' import type { EngineModels, EngineProcessStatus, @@ -197,6 +197,11 @@ interface ModularNode extends NodeRecordIdentity { // empty object means no successful attribution (or a peer that predates // per-engine attribution) and falls back to {@link models}. Mirrors // noderec.DirectoryNode.EngineModels. + downloadError: string | undefined + downloadedByEngine: Record< + string, + { id: string; file: string; size: number; sha256: string; owned: boolean }[] + > modelsByEngine: Record // Per-engine set of models currently loaded in memory, keyed by // engine-manager engine name ("ollama", "lmstudio"), carried on @@ -1319,6 +1324,8 @@ function parseProxyNode(params: JsonValue | undefined, engine: ProxyEngine): Mod // proxy-sourced node contributes engine presence only, never models. models: [], modelsByEngine: {}, + downloadError: undefined, + downloadedByEngine: {}, // A proxy `node/*` event carries no loaded-model state either; the // broker-enriched value is preserved across the proxy merge (see mergeNode). loadedByEngine: {}, @@ -1342,6 +1349,29 @@ function parseProxyNode(params: JsonValue | undefined, engine: ProxyEngine): Mod } } +function parseDownloadedByEngine(value: JsonValue | undefined): ModularNode['downloadedByEngine'] { + const engines = objectValue(value) + const result: ModularNode['downloadedByEngine'] = {} + if (!engines) return result + for (const [engine, rows] of Object.entries(engines)) { + if (!Array.isArray(rows) || rows.length > 4096) continue + result[engine] = rows.flatMap(row => { + const model = objectValue(row) + if (!model || !stringValue(model.id) || !stringValue(model.file)) return [] + return [ + { + id: stringValue(model.id), + file: stringValue(model.file), + size: numberValue(model.size), + sha256: stringValue(model.sha256), + owned: booleanValue(model.owned) + } + ] + }) + } + return result +} + function parseBrokerNode(params: JsonValue | undefined): ModularNode | null { const obj = objectValue(params) if (!obj) return null @@ -1393,6 +1423,17 @@ function parseBrokerNode(params: JsonValue | undefined): ModularNode | null { // peer that predates per-engine attribution) drives the // {@link modelsForEngine} fallback to the flat union. modelsByEngine: parseModelsByEngine(obj.modelsByEngine), + downloadError: + (identity.identityVerification === 'verified' || + identity.identityVerification === 'local') && + objectValue(obj.downloadErrorsByEngine)?.llamacpp === 'cache_unavailable' + ? "Couldn't read llama.cpp's downloaded models" + : undefined, + downloadedByEngine: + identity.identityVerification === 'verified' || + identity.identityVerification === 'local' + ? parseDownloadedByEngine(obj.downloadedByEngine) + : {}, // Broker `AvailableNode.loadedByEngine`: the per-engine set of models // loaded in memory, enriched from the peer's engine-manager // `/v1/models`. `omitempty` on the wire — absent for a peer that reports @@ -1454,6 +1495,8 @@ function sameNode(left: ModularNode, right: ModularNode): boolean { sameTelemetry(left, right.gpus, right.cpu, right.memory, right.inferenceHardwareIds) && sameStringList(left.models, right.models) && sameModelsByEngine(left.modelsByEngine, right.modelsByEngine) && + left.downloadError === right.downloadError && + JSON.stringify(left.downloadedByEngine) === JSON.stringify(right.downloadedByEngine) && // A residency-only change (a model loaded/evicted with the installed set // unchanged) must still count as a node change so the loaded dot updates. sameModelsByEngine(left.loadedByEngine, right.loadedByEngine) && @@ -1787,6 +1830,12 @@ class ModularBridgeState { } } + getPeerDownloadedModels(nodeId: string): DownloadedModel[] { + const node = this.nodes.get(nodeId) + if (!node || !isPairPeer(node)) throw new Error('Installed models require a verified peer.') + return node.downloadedByEngine.llamacpp ?? [] + } + getAvailableNodes(): WsInvokeResponse<'discovery:get-nodes'> { return Array.from(this.nodes.values()).map(node => { const addresses = nodeAddresses(node) @@ -2671,9 +2720,17 @@ class ModularBridgeState { return { engineType: engine, nodeId: node.id, - models: this.modelsForEngine(node, engine).map(name => - modelItem(name, loaded.has(name)) - ) + ...(engine === 'llamacpp' ? { downloadError: node.downloadError } : {}), + models: + engine === 'llamacpp' + ? (node.downloadedByEngine.llamacpp ?? []).map(model => ({ + ...modelItem(model.id, false), + size: model.size, + digest: model.owned ? model.sha256 : '' + })) + : this.modelsForEngine(node, engine).map(name => + modelItem(name, loaded.has(name)) + ) } } @@ -3313,6 +3370,8 @@ class ModularBridgeState { // loaded set) we already merged. models: existing.models, modelsByEngine: existing.modelsByEngine, + downloadedByEngine: existing.downloadedByEngine, + downloadError: existing.downloadError, loadedByEngine: existing.loadedByEngine, // A proxy event carries no engine-native view; keep the // broker-enriched one we already merged. diff --git a/desktop/src/shared/types/engine-api.ts b/desktop/src/shared/types/engine-api.ts index e3433dbcb..eeb5cc95b 100644 --- a/desktop/src/shared/types/engine-api.ts +++ b/desktop/src/shared/types/engine-api.ts @@ -4,7 +4,9 @@ /** * Engine command and initial state types for the frontend API. * - * Commands are fire-and-forget (void). Errors arrive via errors:update push events. + * Commands generally fire-and-forget. Local llama.cpp delete waits for its + * acknowledgement so the installed cache can refresh. Errors arrive via + * errors:update push events. * All state flows via push events, not command responses. */ import type { EngineStatusData, EngineType } from '@/shared/types/engines' @@ -46,6 +48,13 @@ export interface EngineHubLookupResponse { model: EngineHubModel | null } +export interface DownloadedModel { + id: string + file: string + size: number + sha256: string +} + /** Engine command discriminator. */ export type EngineCommandType = | 'toggle' diff --git a/desktop/src/shared/types/engines.ts b/desktop/src/shared/types/engines.ts index 7d8003a78..e715a16f6 100644 --- a/desktop/src/shared/types/engines.ts +++ b/desktop/src/shared/types/engines.ts @@ -123,6 +123,8 @@ export interface EngineUpdateAvailable { } export interface EngineModels { + /** Engine-scoped, non-sensitive downloaded inventory diagnostic. */ + downloadError?: string engineType: EngineType nodeId: string models: ModelItem[] diff --git a/desktop/src/shared/types/ws-channels.ts b/desktop/src/shared/types/ws-channels.ts index 09f51318c..8ccabb0b1 100644 --- a/desktop/src/shared/types/ws-channels.ts +++ b/desktop/src/shared/types/ws-channels.ts @@ -145,7 +145,14 @@ export interface WsInvokeChannelMap { 'engines:preview-settings': { request: EngineSettingsRequest; response: EngineSettingsPreview } 'engines:apply-settings': { request: EngineSettingsRequest; response: EngineSettingsReceipt } 'engine:command': { request: EngineCommandPayload; response: null } - 'engine:search-hub': { request: { engineType: EngineType }; response: EngineHubSearchResponse } + 'engine:search-hub': { + request: { engineType: EngineType; query?: string } + response: EngineHubSearchResponse + } + 'engine:downloaded-models': { + request: { nodeId: string } + response: import('@/shared/types/engine-api').DownloadedModel[] + } // Exact-name lookup, used only when a search matches nothing, so the // catalog the app ships with cannot hide a model the engine could pull. 'engine:lookup-hub-model': { diff --git a/desktop/src/ui/api/engine-api.ts b/desktop/src/ui/api/engine-api.ts index fa843b2e9..70543c5ee 100644 --- a/desktop/src/ui/api/engine-api.ts +++ b/desktop/src/ui/api/engine-api.ts @@ -70,11 +70,12 @@ export interface IEngineApi { /** Unload a model from memory on a node. */ unloadModel(engineType: EngineType, nodeId: string, model: string): void /** Delete a downloaded model from a node. */ - deleteModel(engineType: EngineType, nodeId: string, model: string): void + deleteModel(engineType: EngineType, nodeId: string, model: string): Promise /** Set the model keep-alive expiry duration on a node. */ setModelExpiry(engineType: EngineType, nodeId: string, model: string, expiry: string): void /** Search the model registry/hub for available models. */ - searchHub(engineType: EngineType): Promise + searchHub(engineType: EngineType, query?: string): Promise + downloadedModels(nodeId: string): Promise /** Resolve one exact model name the hub catalog does not list. */ lookupHubModel(engineType: EngineType, name: string): Promise @@ -132,8 +133,24 @@ export function createEngineApi(transport: ServiceTransport): IEngineApi { fireCommand(transport, { command: 'loadModel', engineType, nodeId, model }), unloadModel: (engineType, nodeId, model) => fireCommand(transport, { command: 'unloadModel', engineType, nodeId, model }), - deleteModel: (engineType, nodeId, model) => - fireCommand(transport, { command: 'deleteModel', engineType, nodeId, model }), + deleteModel: async (engineType, nodeId, model) => { + if (engineType !== 'llamacpp') { + fireCommand(transport, { command: 'deleteModel', engineType, nodeId, model }) + return + } + const payload: EngineCommandPayload = { + command: 'deleteModel', + engineType, + nodeId, + model + } + usePendingActionsStore.getState().begin(payload) + try { + await transport.invoke('engine:command', payload) + } finally { + usePendingActionsStore.getState().completeModelAction(nodeId, engineType, model) + } + }, setModelExpiry: (engineType, nodeId, model, expiry) => fireCommand(transport, { command: 'setModelExpiry', @@ -142,7 +159,9 @@ export function createEngineApi(transport: ServiceTransport): IEngineApi { model, expiry }), - searchHub: engineType => transport.invoke('engine:search-hub', { engineType }), + searchHub: (engineType, query) => + transport.invoke('engine:search-hub', { engineType, query }), + downloadedModels: nodeId => transport.invoke('engine:downloaded-models', { nodeId }), lookupHubModel: (engineType, name) => transport.invoke('engine:lookup-hub-model', { engineType, name }), diff --git a/desktop/src/ui/components/BackendRow/BackendRow.tsx b/desktop/src/ui/components/BackendRow/BackendRow.tsx index 1396676c1..7e051b82f 100644 --- a/desktop/src/ui/components/BackendRow/BackendRow.tsx +++ b/desktop/src/ui/components/BackendRow/BackendRow.tsx @@ -188,7 +188,7 @@ export function BackendRow({ onUpdate={handleUpdate} /> - {canShowAccordions && backend.type !== 'llamacpp' && ( + {canShowAccordions && ( )} @@ -228,7 +228,10 @@ export function BackendRow({ onInstall={handleInstall} /> {backend.type === 'llamacpp' && ( - Not notarized, like PAIR itself. + + macOS arm64 only. Not notarized, like PAIR itself. Catalog models are + single-file Q4_K_M GGUF. + )} {expanded && } {content} diff --git a/desktop/src/ui/components/BackendRow/EngineSettingsSection.tsx b/desktop/src/ui/components/BackendRow/EngineSettingsSection.tsx index e09f1fea9..1979f3688 100644 --- a/desktop/src/ui/components/BackendRow/EngineSettingsSection.tsx +++ b/desktop/src/ui/components/BackendRow/EngineSettingsSection.tsx @@ -26,6 +26,8 @@ import { } from '@/ui/stores/engine-settings.store' import { useConnectionStore } from '@/ui/stores/connection.store' import { useNodesStore } from '@/ui/stores/nodes.store' +import { useLlamaInstalledModels } from '@/ui/hooks/useLlamaInstalledModels' +import { selectLlamaModel, selectedLlamaModel } from '@/ui/utils/llamacpp-models' import { ConfirmModal } from '@/ui/components/ConfirmModal' interface Draft { @@ -94,6 +96,9 @@ function SettingsEditor({ target, disabled }: { target: EngineSettingsTarget; di snapshot.format !== 'pair-arguments-v1' const locked = readOnly || disabled || applying const isOmlx = engine === 'omlx' + const local = useConnectionStore(state => state.selfId === nodeId) + const installed = useLlamaInstalledModels(nodeId, engine === 'llamacpp' && local) + const selected = selectedLlamaModel(draft.launchText) useEffect(() => watchEngineSettings(), []) useEffect(() => { @@ -303,6 +308,48 @@ function SettingsEditor({ target, disabled }: { target: EngineSettingsTarget; di } + {engine === 'llamacpp' && ( + + + {installed.error && {installed.error}} + {!local && ( + + Paired-device cache listing is not available yet. Engine arguments + can select a known installed catalog ID. + + )} + + )} {isOmlx ? ( {"oMLX's server settings are managed in the oMLX app"} diff --git a/desktop/src/ui/components/ModelHub/EngineHubListRow.tsx b/desktop/src/ui/components/ModelHub/EngineHubListRow.tsx index 17bba0431..20a1a1d8e 100644 --- a/desktop/src/ui/components/ModelHub/EngineHubListRow.tsx +++ b/desktop/src/ui/components/ModelHub/EngineHubListRow.tsx @@ -28,8 +28,8 @@ const EngineHubListRow = memo(function EngineHubListRow({ }, [disabled, model.id, onToggleSelectModel]) const displayName = useMemo( - () => formatModelName(model.name, model.author), - [model.name, model.author] + () => model.displayName ?? formatModelName(model.name, model.author), + [model.name, model.author, model.displayName] ) const openExternal = useCallback((evt: React.MouseEvent, url: string) => { diff --git a/desktop/src/ui/components/ModelHub/ModelHubContent.tsx b/desktop/src/ui/components/ModelHub/ModelHubContent.tsx index ed6f7d641..e810302fd 100644 --- a/desktop/src/ui/components/ModelHub/ModelHubContent.tsx +++ b/desktop/src/ui/components/ModelHub/ModelHubContent.tsx @@ -2,7 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 import { startTransition, useCallback, useEffect, useMemo, useRef, useState } from 'react' -import { Stack } from '@nvidia/foundations-react-core' +import { Stack, Text } from '@nvidia/foundations-react-core' +import { groupCatalog } from '@/ui/utils/llamacpp-models' import { ModelHubSearchBar } from './ModelHubSearchBar' import { ModelHubActions } from './ModelHubActions' import { InlineErrorBanner } from '@/ui/components/InlineErrorBanner' @@ -69,23 +70,24 @@ export const ModelHubContent = ({ const lookupGenRef = useRef(0) const loadModels = useCallback(async (backend: EngineType) => { - const cached = cacheRef.current.get(backend) + const gen = ++fetchGenRef.current + // Go owns llama.cpp's bounded cache and TTL; do not shadow it for a whole UI session. + const cached = backend === 'llamacpp' ? undefined : cacheRef.current.get(backend) if (cached) { setAllModels(cached) setLoading(false) return } - const gen = ++fetchGenRef.current setLoading(true) try { const result = await searchEngineHub(backend) if (fetchGenRef.current !== gen) return - cacheRef.current.set(backend, result) + if (backend !== 'llamacpp') cacheRef.current.set(backend, result) setAllModels(result) setLoading(false) } catch (error) { if (fetchGenRef.current !== gen) return - console.warn('Model hub load error', error) + // Display service errors without logging upstream content. setErrors(prev => [ ...prev, { id: performance.now().toString(), message: getErrorString(error) } @@ -105,18 +107,51 @@ export const ModelHubContent = ({ setQuery('') setSelectedModels(new Set()) setSort(resolveStoredSort(engine, SORT_DEFAULT_STATE)) - void loadModels(engine) + if (engine !== 'llamacpp') void loadModels(engine) return () => { // Invalidate any in-flight fetch for the previous engine. fetchGenRef.current += 1 } }, [engine, loadModels]) + useEffect(() => { + if (engine !== 'llamacpp' || !query.trim()) return + const generation = ++fetchGenRef.current + setLoading(true) + setErrors([]) + const timer = setTimeout(() => { + void searchEngineHub('llamacpp', query) + .then(result => { + if (fetchGenRef.current !== generation) return + setAllModels(result) + setLoading(false) + }) + .catch(error => { + if (fetchGenRef.current !== generation) return + setErrors([{ id: 'search', message: getErrorString(error) }]) + setAllModels([]) + setLoading(false) + }) + }, LOOKUP_DEBOUNCE_MS) + return () => { + clearTimeout(timer) + fetchGenRef.current++ + } + }, [engine, query]) + + useEffect(() => { + if (engine === 'llamacpp' && !query.trim()) { + setErrors([]) + void loadModels(engine) + } + }, [engine, query, loadModels]) + const queryFiltered = useMemo(() => { + if (engine === 'llamacpp') return allModels const q = query.toLowerCase().trim() if (!q) return allModels return allModels.filter(m => m.name.toLowerCase().includes(q)) - }, [allModels, query]) + }, [allModels, query, engine]) const visibleModels = useMemo(() => { if (!engine || !downloadedModels || downloadedModels.length === 0) return queryFiltered @@ -246,16 +281,44 @@ export const ModelHubContent = ({ onSort={handleSortPersist} /> - + {engine === 'llamacpp' ? ( + + + Single-file Q4_K_M GGUF models only. Search accepts up to 100 characters + and 50 results. + + {loading && Searching models…} + {!loading && + groupCatalog(displayModels).map(([publisher, models]) => ( + + {publisher} + + + ))} + {!loading && !displayModels.length && ( + No models found + )} + + ) : ( + + )} {!!multiple && ( void + onRetry?: () => void }) { return ( - {formatPullProgressLabel(row)} + {row.engineType === 'llamacpp' && row.status === 'cancelled' + ? 'Cancelled' + : row.engineType === 'llamacpp' && row.status === 'interrupted' + ? 'Download interrupted — retry' + : formatPullProgressLabel(row)} - + {onRetry && (row.status === 'cancelled' || row.status === 'interrupted') ? ( + + ) : ( + + )} ) } diff --git a/desktop/src/ui/components/ModelManager/ModelManager.tsx b/desktop/src/ui/components/ModelManager/ModelManager.tsx index fd7950ea7..053819387 100644 --- a/desktop/src/ui/components/ModelManager/ModelManager.tsx +++ b/desktop/src/ui/components/ModelManager/ModelManager.tsx @@ -2,6 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import { useCallback, useMemo, useState } from 'react' +import { useLlamaInstalledModels } from '@/ui/hooks/useLlamaInstalledModels' +import { useConnectionStore } from '@/ui/stores/connection.store' +import { useNodesStore } from '@/ui/stores/nodes.store' +import { isPairPeer } from '@/shared/utils/node-record-key' +import type { ModelItem } from '@/ui/types/engine-info' import { Button, Flex, Stack, Text } from '@nvidia/foundations-react-core' import type { BackendInfo } from '@/ui/types/engine-info' import { EngineCapabilities } from '@/ui/constants/engine-capabilities' @@ -12,6 +17,7 @@ import { noEngineCapabilities } from '@/shared/utils/engines' import { ConfirmModal } from '@/ui/components/ConfirmModal' import { ModelHubModal } from '@/ui/components/ModelHub/ModelHubModal' import { useEngineProgressStore } from '@/ui/stores/engine-progress.store' +import { useEngineModelsStore } from '@/ui/stores/engine-models.store' import { useEngineStatusStore } from '@/ui/stores/engine-status.store' import { usePendingActionsStore } from '@/ui/stores/pending-actions.store' import { isEnginePullInProgress } from '@/shared/utils/engine-progress' @@ -40,8 +46,40 @@ export function ModelManager({ backend, nodeId }: { backend: BackendInfo; nodeId useEngineStatusStore( state => state.statusByNode.get(nodeId)?.get(backend.type)?.capabilities ) ?? noEngineCapabilities() + const local = useConnectionStore(state => state.selfId === nodeId) + const peer = useNodesStore(state => { + const node = state.nodes.get(nodeId) + return !!node && isPairPeer(node) + }) + const downloadError = useEngineModelsStore( + state => state.models.get(`${nodeId}:${backend.type}`)?.downloadError + ) + const eligible = local || peer + const installed = useLlamaInstalledModels( + nodeId, + backend.type === 'llamacpp' && local && reported.listModels + ) + const hasDownloadError = + backend.type === 'llamacpp' && !!(local ? installed.error : downloadError) + const sourceModels: ModelItem[] = + backend.type === 'llamacpp' && local + ? installed.models.map(model => ({ + name: model.id, + size: model.size, + downloaded: true, + status: 'idle', + parameterSize: '', + quantization: 'Q4_K_M', + family: '', + digest: model.sha256, + sizeVram: null, + expiresAt: null, + expiry: '-1', + capabilities: [] + })) + : backend.models const models = reported.listModels - ? [...backend.models].sort((a, b) => + ? [...sourceModels].sort((a, b) => formatModelDisplayName(a.name, backend.type).localeCompare( formatModelDisplayName(b.name, backend.type) ) @@ -113,6 +151,7 @@ export function ModelManager({ backend, nodeId }: { backend: BackendInfo; nodeId } results.push({ + engineType: backend.type, opId: p.opId, canCancel: p.canCancel, rawModel: p.model, @@ -145,7 +184,12 @@ export function ModelManager({ backend, nodeId }: { backend: BackendInfo; nodeId window.pairApi.engines.unloadModel(backendType, nodeId, modelName) break case 'delete': - if (!reported.deleteModel) break + if ( + (backendType === 'llamacpp' && !eligible) || + !reported.deleteModel || + (backendType === 'llamacpp' && backend.processStatus !== 'stopped') + ) + break if (confirmBeforeDelete) { setModelPendingDelete({ nodeId, @@ -156,16 +200,26 @@ export function ModelManager({ backend, nodeId }: { backend: BackendInfo; nodeId }) break } - window.pairApi.engines.deleteModel(backendType, nodeId, modelName) + if (backendType === 'llamacpp') { + void window.pairApi.engines + .deleteModel(backendType, nodeId, modelName) + .then( + () => installed.reload(), + () => installed.reload() + ) + } else window.pairApi.engines.deleteModel(backendType, nodeId, modelName) break } }, [ backendType, + backend.processStatus, + eligible, backend.displayName, confirmBeforeDelete, nodeId, reported.deleteModel, + installed, displayName ] ) @@ -205,8 +259,8 @@ export function ModelManager({ backend, nodeId }: { backend: BackendInfo; nodeId ) const isRunning = useMemo(() => backend.processStatus === 'running', [backend.processStatus]) const supportsSearch = useMemo( - () => reported.pullModel && (!hasModelSearchOnlyWhenRunning || isRunning), - [reported.pullModel, hasModelSearchOnlyWhenRunning, isRunning] + () => eligible && reported.pullModel && (!hasModelSearchOnlyWhenRunning || isRunning), + [eligible, reported.pullModel, hasModelSearchOnlyWhenRunning, isRunning] ) const handleDownload = useCallback( @@ -226,6 +280,24 @@ export function ModelManager({ backend, nodeId }: { backend: BackendInfo; nodeId return ( + {backendType === 'llamacpp' && ( + <> + {hasDownloadError && ( + + {"Couldn't read llama.cpp's downloaded models"} + + )} + {local && ( + + )} + + )} {transientModel && ( + window.pairApi.engines.pullModel(backendType, nodeId, p.rawModel) + } onCancel={ p.opId ? () => @@ -266,6 +341,7 @@ export function ModelManager({ backend, nodeId }: { backend: BackendInfo; nodeId {!isBusy && ( <> {reported.listModels && + !hasDownloadError && (isRunning || modelOpsWhenStopped) && models.length === 0 && incomingSyncs.length === 0 && ( @@ -310,7 +386,11 @@ export function ModelManager({ backend, nodeId }: { backend: BackendInfo; nodeId } isRunning={isRunning} capabilities={caps} - reported={reported} + reported={ + backendType === 'llamacpp' && !model.digest + ? { ...reported, deleteModel: false } + : reported + } progress={getProgress(nodeId, backend.type, 'pull', model.name)} pendingAction={usePendingActionsStore .getState() diff --git a/desktop/src/ui/constants/engine-capabilities.ts b/desktop/src/ui/constants/engine-capabilities.ts index cfb44503e..d8729a27a 100644 --- a/desktop/src/ui/constants/engine-capabilities.ts +++ b/desktop/src/ui/constants/engine-capabilities.ts @@ -22,7 +22,7 @@ export const EngineCapabilities: Record = { hasPreferredNode: false, hasCrashAlert: false, hasModelSearchOnlyWhenRunning: false, - modelOpsWhenStopped: false + modelOpsWhenStopped: true }, ollama: { // Ollama has no model keep-alive/expiry UI; unload uses unload_model diff --git a/desktop/src/ui/hooks/useLlamaInstalledModels.ts b/desktop/src/ui/hooks/useLlamaInstalledModels.ts new file mode 100644 index 000000000..41ed20c31 --- /dev/null +++ b/desktop/src/ui/hooks/useLlamaInstalledModels.ts @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +import { useCallback, useEffect, useRef, useState } from 'react' +import type { DownloadedModel } from '@/shared/types/engine-api' +import { useEngineProgressStore } from '@/ui/stores/engine-progress.store' +import { usePendingActionsStore } from '@/ui/stores/pending-actions.store' +import { useConnectionStore } from '@/ui/stores/connection.store' + +export function useLlamaInstalledModels(nodeId: string, enabled: boolean) { + const [models, setModels] = useState([]) + const [error, setError] = useState('') + const generation = useRef(0) + const connected = useConnectionStore(state => state.connected) + const progress = useEngineProgressStore(state => + [...state.progress.values()] + .filter(p => p.nodeId === nodeId && p.engineType === 'llamacpp') + .map(p => `${p.opId}:${p.status}`) + .join('|') + ) + const pending = usePendingActionsStore(state => + [...state.pending.keys()].filter(key => key.startsWith(`${nodeId}:llamacpp:`)).join('|') + ) + const reload = useCallback(async () => { + const current = ++generation.current + if (!enabled || !connected) { + setModels([]) + return + } + try { + const result = await window.pairApi.engines.downloadedModels(nodeId) + if (generation.current !== current) return + setModels(result) + setError('') + } catch (cause) { + if (generation.current !== current) return + setModels([]) + setError(cause instanceof Error ? cause.message : 'Could not read installed models.') + } + }, [nodeId, enabled, connected]) + useEffect(() => { + void reload() + return invalidate + function invalidate(): void { + generation.current++ + } + }, [reload, progress, pending]) + return { models, error, reload } +} diff --git a/desktop/src/ui/stores/pending-actions.store.ts b/desktop/src/ui/stores/pending-actions.store.ts index f64dce593..3462ccf85 100644 --- a/desktop/src/ui/stores/pending-actions.store.ts +++ b/desktop/src/ui/stores/pending-actions.store.ts @@ -125,6 +125,7 @@ interface PendingActionsState { pending: Map /** Record an optimistic pending entry for a fired command. No-op for non-lifecycle/non-model commands. */ begin(payload: EngineCommandPayload): void + completeModelAction(nodeId: string, engineType: EngineType, model: string): void /** The pending lifecycle action for an engine, or undefined when none is in flight. */ getLifecyclePending(nodeId: string, engineType: EngineType): EngineCommandType | undefined /** The pending action for a specific model, or undefined when none is in flight. */ @@ -227,6 +228,11 @@ export const usePendingActionsStore = create((set, get) => } }, + completeModelAction: (nodeId, engineType, model) => { + const pending = new Map(get().pending) + if (pending.delete(modelKey(nodeId, engineType, model))) set({ pending }) + }, + getLifecyclePending: (nodeId, engineType) => get().pending.get(lifecycleKey(nodeId, engineType))?.action, diff --git a/desktop/src/ui/types/model-hub.ts b/desktop/src/ui/types/model-hub.ts index df792f76d..f47e6950a 100644 --- a/desktop/src/ui/types/model-hub.ts +++ b/desktop/src/ui/types/model-hub.ts @@ -12,6 +12,7 @@ export interface ModelEntry { id: string name: string author: string + displayName?: string url: string size?: number /** diff --git a/desktop/src/ui/types/model-manager.ts b/desktop/src/ui/types/model-manager.ts index cdbba5939..e71c3ae50 100644 --- a/desktop/src/ui/types/model-manager.ts +++ b/desktop/src/ui/types/model-manager.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 export type IncomingSyncRow = { + engineType?: import('@/shared/types/engines').EngineType opId?: string canCancel?: boolean rawModel: string diff --git a/desktop/src/ui/utils/engine-capability-gates.ts b/desktop/src/ui/utils/engine-capability-gates.ts index 702864af1..a4586ad2e 100644 --- a/desktop/src/ui/utils/engine-capability-gates.ts +++ b/desktop/src/ui/utils/engine-capability-gates.ts @@ -51,11 +51,12 @@ export function modelActionItems({ busy }: ModelActionInput): ModelActionItem[] { const items: ModelActionItem[] = [] - items.push({ - id: 'load', - children: 'Load', - disabled: busy || isLoaded || !downloaded || !isRunning - }) + if (capabilities.hasEject) + items.push({ + id: 'load', + children: 'Load', + disabled: busy || isLoaded || !downloaded || !isRunning + }) if (capabilities.hasEject) { items.push({ id: 'eject', @@ -68,7 +69,12 @@ export function modelActionItems({ }) } if (reported.deleteModel) { - items.push({ id: 'delete', children: 'Delete', danger: true, disabled: busy }) + items.push({ + id: 'delete', + children: !capabilities.hasEject && isRunning ? 'Delete (stop engine first)' : 'Delete', + danger: true, + disabled: busy || (!capabilities.hasEject && isRunning) + }) } return items } diff --git a/desktop/src/ui/utils/format-model-display-name.ts b/desktop/src/ui/utils/format-model-display-name.ts index e16282ed3..b578c6634 100644 --- a/desktop/src/ui/utils/format-model-display-name.ts +++ b/desktop/src/ui/utils/format-model-display-name.ts @@ -19,6 +19,7 @@ const HUB_REGISTRY_ARROW = ' \u2192 ' const MAX_FORMAT_INPUT_CHARS = 256 export function formatModelDisplayName(name: string, engineType?: string | null): string { + if (engineType === 'llamacpp') return name.startsWith('/dev/fd/') ? 'Unavailable model' : name if (!name) { return name } diff --git a/desktop/src/ui/utils/llamacpp-models.ts b/desktop/src/ui/utils/llamacpp-models.ts new file mode 100644 index 000000000..9c5c9f41a --- /dev/null +++ b/desktop/src/ui/utils/llamacpp-models.ts @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +import type { ModelEntry } from '@/ui/types/model-hub' + +// This edits only the model option; Go remains the launch-text validator. +const modelOption = /(?:^|\s)(?:--model|-m)(?:=|\s+)(?:"[^"]*"|'[^']*'|[^\s]+)/g + +export function groupCatalog(models: ModelEntry[]): Array<[string, ModelEntry[]]> { + const groups = new Map() + for (const model of models) { + const publisher = model.author || model.id.split('/')[0] + const rows = groups.get(publisher) ?? [] + rows.push(model) + groups.set(publisher, rows) + } + return [...groups].sort(([a], [b]) => a.localeCompare(b)) +} + +export function selectedLlamaModel(text: string): string { + const match = text.match(/(?:^|\s)(?:--model|-m)(?:=|\s+)("[^"]*"|'[^']*'|[^\s]+)/) + const value = (match?.[1] ?? '').replace(/^["']|["']$/g, '') + return value.startsWith('/dev/fd/') ? '' : value +} + +export function selectLlamaModel(text: string, file: string): string { + const rest = text.replace(modelOption, '').trim() + return [rest, file ? `--model "${file.replace(/["\\]/g, '\\$&')}"` : ''] + .filter(Boolean) + .join(' ') +} diff --git a/desktop/src/ui/utils/model-hub-search.ts b/desktop/src/ui/utils/model-hub-search.ts index 4740b32be..c56e732f1 100644 --- a/desktop/src/ui/utils/model-hub-search.ts +++ b/desktop/src/ui/utils/model-hub-search.ts @@ -45,10 +45,13 @@ function hubModelToEntry(m: EngineHubModel): ModelEntry { * module owns the upstream source (Ollama library scrape, LM Studio community * catalog); the renderer filters/sorts the returned rows locally. */ -export async function searchEngineHub(engine: EngineType): Promise { - if (!getEngineHub(engine)) return [] - const { models } = await window.pairApi.engines.searchHub(engine) - return models.map(hubModelToEntry) +export async function searchEngineHub(engine: EngineType, query?: string): Promise { + if (engine !== 'llamacpp' && !getEngineHub(engine)) return [] + const { models } = await window.pairApi.engines.searchHub(engine, query) + return models.map(model => ({ + ...hubModelToEntry(model), + displayName: engine === 'llamacpp' ? model.id : undefined + })) } /** diff --git a/desktop/tests/modular/engine-settings-bridge.test.ts b/desktop/tests/modular/engine-settings-bridge.test.ts index f9e32bdc0..fb2695fdf 100644 --- a/desktop/tests/modular/engine-settings-bridge.test.ts +++ b/desktop/tests/modular/engine-settings-bridge.test.ts @@ -6,6 +6,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ state: { getSelfId: vi.fn(() => 'local-node'), + getPeerDownloadedModels: vi.fn(() => [ + { id: 'peer-model', file: 'peer.gguf', size: 24, sha256: 'peer-verified' } + ]), getProxyPort: vi.fn(() => 11434), beginLocalEngineOp: vi.fn(), failLocalEngineOp: vi.fn() @@ -15,7 +18,9 @@ const mocks = vi.hoisted(() => ({ callProcess: vi.fn(), callProxy: vi.fn(), sendProcess: vi.fn(), - reportError: vi.fn() + reportError: vi.fn(), + deleteModel: vi.fn(), + modelActionRemote: vi.fn() } })) @@ -61,6 +66,105 @@ describe('unified settings bridge', () => { ) }) + it('relays llama.cpp catalog queries and preserves service errors', async () => { + mocks.supervisor.callProcess.mockResolvedValueOnce({ + models: [{ id: 'ggml-org/Small:Q4_K_M', file: 'hash.gguf', size: 42 }] + }) + const result = await handleServiceBridgeInvoke('engine:search-hub', { + engineType: 'llamacpp', + query: 'Small' + }) + expect(result.models[0]?.name).toBe('ggml-org/Small:Q4_K_M') + expect(mocks.supervisor.callProcess).toHaveBeenCalledWith( + 'broker', + 'engine:model-catalog', + { query: 'Small' }, + 30_000 + ) + mocks.supervisor.callProcess.mockRejectedValueOnce(new Error('not enough disk space')) + await expect( + handleServiceBridgeInvoke('engine:search-hub', { engineType: 'llamacpp' }) + ).rejects.toThrow('not enough disk space') + }) + + it('uses local actions for self and verified discovery inventory for a peer', async () => { + mocks.supervisor.callProcess.mockResolvedValueOnce({ + models: [ + { id: 'ggml-org/Small:Q4_K_M', file: 'hash.gguf', size: 42, sha256: 'verified' } + ] + }) + expect( + await handleServiceBridgeInvoke('engine:downloaded-models', { nodeId: 'local-node' }) + ).toEqual([ + { id: 'ggml-org/Small:Q4_K_M', file: 'hash.gguf', size: 42, sha256: 'verified' } + ]) + expect( + await handleServiceBridgeInvoke('engine:downloaded-models', { nodeId: 'peer' }) + ).toEqual([{ id: 'peer-model', file: 'peer.gguf', size: 24, sha256: 'peer-verified' }]) + expect(mocks.state.getPeerDownloadedModels).toHaveBeenCalledWith('peer') + expect(mocks.supervisor.callProcess).toHaveBeenCalledTimes(1) + }) + + it('awaits the existing remote llama.cpp delete operation', async () => { + let finish = (): void => {} + mocks.supervisor.modelActionRemote.mockImplementation( + () => + new Promise(resolve => { + finish = resolve + }) + ) + let acknowledged = false + const response = handleServiceBridgeInvoke('engine:command', { + command: 'deleteModel', + engineType: 'llamacpp', + nodeId: 'peer', + model: 'peer-model' + }) + void response.then(() => { + acknowledged = true + }) + await Promise.resolve() + expect(acknowledged).toBe(false) + expect(mocks.supervisor.modelActionRemote).toHaveBeenCalledWith( + 'peer', + 'llamacpp', + 'llamacpp', + 'peer-model', + 'deleteModel' + ) + finish() + await response + }) + + it('acknowledges local llama.cpp delete only after the owning operation settles', async () => { + let finish = (): void => {} + mocks.supervisor.deleteModel.mockImplementation( + () => + new Promise(resolve => { + finish = resolve + }) + ) + let acknowledged = false + const response = handleServiceBridgeInvoke('engine:command', { + command: 'deleteModel', + engineType: 'llamacpp', + nodeId: 'local-node', + model: 'ggml-org/Demo:Q4_K_M' + }) + void response.then(() => { + acknowledged = true + }) + await Promise.resolve() + expect(acknowledged).toBe(false) + finish() + await response + expect(mocks.supervisor.deleteModel).toHaveBeenCalledWith( + 'llamacpp', + 'llamacpp', + 'ggml-org/Demo:Q4_K_M' + ) + }) + it('updates llama.cpp through atomic install without uninstalling the working tree', async () => { await handleServiceBridgeInvoke('engine:command', { command: 'update', diff --git a/desktop/tests/modular/engine-settings-panel.test.ts b/desktop/tests/modular/engine-settings-panel.test.ts index d5aad5a92..f376d7811 100644 --- a/desktop/tests/modular/engine-settings-panel.test.ts +++ b/desktop/tests/modular/engine-settings-panel.test.ts @@ -18,6 +18,7 @@ import type { } from '@/shared/types/engine-settings' import type { EngineType } from '@/shared/types/engines' import type { NodeItem } from '@/shared/types/nodes' +import type { DownloadedModel } from '@/shared/types/engine-api' const baseline: EngineSettingsSnapshot = { nodeId: 'local', @@ -51,6 +52,7 @@ const peer: NodeItem = { os: 'Linux', topology: { gpus: [], cpu: { model: '', cores: 0, threads: 0 }, ram: 0, storage: [] } } +const downloadedModels = vi.fn<(nodeId: string) => Promise>() const getSettings = vi.fn<(target: EngineSettingsTarget) => Promise>() const previewSettings = vi.fn<(request: EngineSettingsRequest) => Promise>() const applySettings = vi.fn<(request: EngineSettingsRequest) => Promise>() @@ -93,6 +95,113 @@ async function click(label: string): Promise { await act(async () => button(label).click()) } +it('selects a llama.cpp cache-relative file through preview/apply', async () => { + await render('llamacpp') + const select = container.querySelector('[aria-label="Startup model"]') + expect(select?.textContent).toContain('ggml-org/Demo:Q4_K_M') + expect(select?.textContent).toContain('No model') + await act(async () => { + if (select) { + select.value = 'cache.gguf' + select.dispatchEvent(new Event('change', { bubbles: true })) + } + }) + await click('Apply') + expect(previewSettings.mock.calls[0]?.[0].settings.launchText).toBe('--model "cache.gguf"') + expect(applySettings.mock.calls[0]?.[0].settings.launchText).toBe('--model "cache.gguf"') + expect(container.innerHTML).not.toContain('/dev/fd/') +}) + +it('selects No model and removes the saved llama.cpp model through preview/apply', async () => { + getSettings.mockImplementation(async target => ({ + ...baseline, + ...target, + settings: { + ...baseline.settings, + launchText: '--threads 4 --model "cache.gguf" --ctx-size 4096' + } + })) + await render('llamacpp') + const select = container.querySelector('[aria-label="Startup model"]') + if (!select) throw new Error('Missing startup model picker') + expect(select.value).toBe('cache.gguf') + await act(async () => { + select.value = '' + select.dispatchEvent(new Event('change', { bubbles: true })) + }) + expect(select.selectedOptions[0]?.textContent).toBe('No model (router mode)') + expect(field('Engine arguments').value).toBe('--threads 4 --ctx-size 4096') + await click('Apply') + expect(previewSettings.mock.calls[0]?.[0].settings.launchText).toBe( + '--threads 4 --ctx-size 4096' + ) + expect(applySettings.mock.calls[0]?.[0].settings.launchText).toBe('--threads 4 --ctx-size 4096') +}) + +it('shows a deleted llama.cpp startup model and its preview rejection without applying', async () => { + downloadedModels.mockResolvedValue([]) + getSettings.mockImplementation(async target => ({ + ...baseline, + ...target, + settings: { ...baseline.settings, launchText: '--model deleted.gguf --threads 4' } + })) + previewSettings.mockImplementation(async request => ({ + settings: request.settings, + revision: 1, + errors: { launchText: 'Selected model is no longer installed. Download it again.' }, + restart: false, + rebind: false + })) + await render('llamacpp') + const select = container.querySelector('[aria-label="Startup model"]') + expect(select?.value).toBe('deleted.gguf') + expect(select?.selectedOptions[0]?.textContent).toBe('deleted.gguf') + await change('Engine arguments', '--model deleted.gguf --threads 8') + await click('Apply') + expect(previewSettings.mock.calls[0]?.[0].settings.launchText).toBe( + '--model deleted.gguf --threads 8' + ) + expect(container.querySelector('[role="alert"]')?.textContent).toContain( + 'Selected model is no longer installed. Download it again.' + ) + expect(applySettings).not.toHaveBeenCalled() +}) + +it('ignores delayed llama.cpp inventory after switching devices and returning', async () => { + let finish = (_models: DownloadedModel[]): void => {} + downloadedModels.mockImplementationOnce( + () => + new Promise(resolve => { + finish = resolve + }) + ) + await render('llamacpp') + await render('llamacpp', 'peer') + await render('llamacpp') + expect(container.textContent).toContain('ggml-org/Demo:Q4_K_M') + await act(async () => + finish([{ id: 'ggml-org/Stale:Q4_K_M', file: 'stale.gguf', size: 42, sha256: 'old' }]) + ) + const select = container.querySelector('[aria-label="Startup model"]') + expect(select?.textContent).toContain('ggml-org/Demo:Q4_K_M') + expect(select?.textContent).not.toContain('ggml-org/Stale:Q4_K_M') + expect(select?.value).toBe('') + expect(field('Engine arguments').value).toBe('') + expect(button('Apply').disabled).toBe(true) +}) + +it('shows the installed catalog name when saved arguments select its catalog ID', async () => { + getSettings.mockImplementation(async target => ({ + ...baseline, + ...target, + settings: { ...baseline.settings, launchText: '--model ggml-org/Demo:Q4_K_M' } + })) + await render('llamacpp') + expect(container.querySelector('[aria-label="Startup model"]')?.value).toBe( + 'cache.gguf' + ) +}) + beforeEach(() => { vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) Object.defineProperty(HTMLDialogElement.prototype, 'showModal', { @@ -107,6 +216,11 @@ beforeEach(() => { this.open = false } }) + downloadedModels + .mockReset() + .mockResolvedValue([ + { id: 'ggml-org/Demo:Q4_K_M', file: 'cache.gguf', size: 42, sha256: 'verified' } + ]) getSettings.mockReset().mockImplementation(async target => ({ ...baseline, ...target })) previewSettings.mockReset().mockImplementation(async request => ({ settings: request.settings, @@ -130,6 +244,7 @@ beforeEach(() => { vi.stubGlobal('pairApi', { engines: { getSettings, + downloadedModels, previewSettings, applySettings, onSettingsChanged: (callback: typeof deliver) => { diff --git a/desktop/tests/modular/llamacpp-catalog.test.ts b/desktop/tests/modular/llamacpp-catalog.test.ts new file mode 100644 index 000000000..7ed4daf1d --- /dev/null +++ b/desktop/tests/modular/llamacpp-catalog.test.ts @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// @vitest-environment jsdom +import { act, createElement } from 'react' +import { createRoot } from 'react-dom/client' +import { expect, it, vi } from 'vitest' +import { ModelHubContent } from '@/ui/components/ModelHub/ModelHubContent' + +it('browses llama.cpp by publisher and shows bounded service search errors calmly', async () => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + vi.stubGlobal( + 'ResizeObserver', + class { + observe() { + /* jsdom has no layout observer */ + } + disconnect() { + /* no observer resources */ + } + } + ) + vi.useFakeTimers() + const searchHub = vi.fn(async (_engine: string, query?: string) => { + if (query) throw new Error('query must not exceed 100 characters') + return { + models: ['unsloth/Demo:Q4_K_M', 'ggml-org/Small:Q4_K_M'].map(id => ({ + id, + name: id, + author: id.split('/')[0], + url: '', + downloads: 0, + likes: 0, + updatedAt: '', + tags: [] + })) + } + }) + vi.stubGlobal('pairApi', { engines: { searchHub } }) + const container = document.createElement('div') + document.body.append(container) + const root = createRoot(container) + try { + await act(async () => + root.render( + createElement(ModelHubContent, { + engine: 'llamacpp', + multiple: false, + onSubmit: () => {} + }) + ) + ) + expect(container.textContent).toContain('ggml-org') + expect(container.textContent).toContain('unsloth') + expect(container.textContent).toContain('unsloth/Demo:Q4_K_M') + await act(async () => + root.render( + createElement(ModelHubContent, { + engine: null, + multiple: false, + onSubmit: () => {} + }) + ) + ) + await act(async () => + root.render( + createElement(ModelHubContent, { + engine: 'llamacpp', + multiple: false, + onSubmit: () => {} + }) + ) + ) + expect(searchHub.mock.calls.filter(call => !call[1])).toHaveLength(2) + const input = container.querySelector('input') + await act(async () => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set?.call( + input, + 'x'.repeat(101) + ) + input?.dispatchEvent(new Event('input', { bubbles: true })) + }) + await act(async () => { + await vi.advanceTimersByTimeAsync(450) + }) + expect(container.textContent).toContain('query must not exceed 100 characters') + expect(container.querySelector('[role="alert"]')?.textContent).toContain('100 characters') + } finally { + act(() => root.unmount()) + container.remove() + vi.useRealTimers() + vi.unstubAllGlobals() + } +}) diff --git a/desktop/tests/modular/llamacpp-models.test.ts b/desktop/tests/modular/llamacpp-models.test.ts new file mode 100644 index 000000000..7798e6581 --- /dev/null +++ b/desktop/tests/modular/llamacpp-models.test.ts @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +import { describe, expect, it } from 'vitest' +import { formatModelDisplayName } from '@/ui/utils/format-model-display-name' +import { groupCatalog, selectLlamaModel, selectedLlamaModel } from '@/ui/utils/llamacpp-models' +import { modelActionItems } from '@/ui/utils/engine-capability-gates' +import { EngineCapabilities } from '@/ui/constants/engine-capabilities' +import { noEngineCapabilities } from '@/shared/utils/engines' + +describe('llama.cpp model management', () => { + it('groups catalog IDs by publisher without changing model names', () => { + const rows = ['unsloth/B:Q4_K_M', 'ggml-org/A:Q4_K_M', 'unsloth/C:Q4_K_M'].map(id => ({ + id, + name: id, + author: id.split('/')[0], + url: '' + })) + expect( + groupCatalog(rows).map(([publisher, models]) => [publisher, models.map(m => m.name)]) + ).toEqual([ + ['ggml-org', ['ggml-org/A:Q4_K_M']], + ['unsloth', ['unsloth/B:Q4_K_M', 'unsloth/C:Q4_K_M']] + ]) + }) + it('writes the cache-relative file while preserving inference arguments', () => { + expect(selectLlamaModel('--ctx-size 4096 --model old.gguf --threads 4', 'new.gguf')).toBe( + '--ctx-size 4096 --threads 4 --model "new.gguf"' + ) + expect(selectedLlamaModel('--ctx-size 4096 --model "new.gguf"')).toBe('new.gguf') + expect(selectLlamaModel('-m=old.gguf --threads 4', '')).toBe('--threads 4') + }) + it('shows the exact catalog ID instead of a prettified model name', () => { + expect(formatModelDisplayName('ggml-org/Demo-GGUF:Q4_K_M', 'llamacpp')).toBe( + 'ggml-org/Demo-GGUF:Q4_K_M' + ) + expect(formatModelDisplayName('/dev/fd/3', 'llamacpp')).toBe('Unavailable model') + }) + + it('never retains a descriptor path as a selection', () => { + expect(selectedLlamaModel('--model /dev/fd/3')).toBe('') + expect(selectLlamaModel('--model /dev/fd/3', 'cache.gguf')).not.toContain('/dev/fd/') + }) + it('disables deletion with a reason while llama.cpp is running and offers no Load', () => { + const items = modelActionItems({ + capabilities: EngineCapabilities.llamacpp, + reported: { ...noEngineCapabilities(), deleteModel: true }, + isRunning: true, + isLoaded: false, + downloaded: true, + busy: false + }) + expect(items).toEqual([ + { id: 'delete', children: 'Delete (stop engine first)', danger: true, disabled: true } + ]) + }) +}) diff --git a/desktop/tests/modular/llamacpp-peer-inventory.test.ts b/desktop/tests/modular/llamacpp-peer-inventory.test.ts new file mode 100644 index 000000000..b61179161 --- /dev/null +++ b/desktop/tests/modular/llamacpp-peer-inventory.test.ts @@ -0,0 +1,146 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from 'vitest' +vi.mock('electron', () => ({ BrowserWindow: { getAllWindows: () => [] } })) +vi.mock('@/electron/window', () => ({ createOverviewWindow: vi.fn() })) +import { getModularBridgeState } from '@/electron/service-bridge/modular-state' + +describe('llama.cpp verified peer installed inventory', () => { + it('projects owned and unrecorded cache rows while stopped, not served names', () => { + const state = getModularBridgeState() + state.setSelfId('cache-self') + const node = { + hostUuid: 'cache-peer', + peerRecordId: 'cache-proof', + identityVerification: 'verified', + name: 'Peer', + ipAddress: '192.0.2.90', + port: 14318, + trusted: true, + modelsByEngine: { llamacpp: ['served-only'] }, + downloadedByEngine: { + llamacpp: [ + { + id: 'publisher/model:Q4_K_M', + file: 'owned.gguf', + size: 42, + sha256: 'verified', + owned: true + }, + { id: 'manual.gguf', file: 'manual.gguf', size: 24, sha256: '', owned: false } + ] + } + } + state.handleNotification({ + source: 'broker', + method: 'discovery:nodes-changed', + params: { nodes: [node] } + }) + state.applyRemoteEngineFacts('cache-peer', { + engines: [ + { + engine: 'llamacpp', + installed: true, + running: false, + capabilities: { list_models: true, delete_model: true } + } + ] + }) + const models = () => + state + .getEngineInitialState() + .models.find(row => row.nodeId === 'cache-peer' && row.engineType === 'llamacpp') + ?.models + expect(state.getPeerDownloadedModels('cache-peer')[0]).toMatchObject({ + file: 'owned.gguf', + size: 42 + }) + expect(models()).toMatchObject([ + { name: 'publisher/model:Q4_K_M', size: 42, digest: 'verified', downloaded: true }, + { name: 'manual.gguf', size: 24, digest: '', downloaded: true } + ]) + state.handleNotification({ + source: 'broker', + method: 'discovery:nodes-changed', + params: { nodes: [{ ...node, downloadedByEngine: { llamacpp: [] } }] } + }) + expect(models()).toEqual([]) + }) + + it('projects cache errors only onto llama.cpp and clears them on recovery', () => { + const state = getModularBridgeState() + const node = { + hostUuid: 'cache-error-peer', + peerRecordId: 'proof', + identityVerification: 'verified', + trusted: true, + name: 'Peer', + ipAddress: '192.0.2.91', + port: 14318, + modelsByEngine: { ollama: ['ollama-model'], lmstudio: ['lmstudio-model'] }, + downloadErrorsByEngine: { llamacpp: 'cache_unavailable' } + } + state.handleNotification({ + source: 'broker', + method: 'discovery:nodes-changed', + params: { nodes: [node] } + }) + const rows = () => + state.getEngineInitialState().models.filter(row => row.nodeId === node.hostUuid) + expect(rows().find(row => row.engineType === 'llamacpp')).toMatchObject({ + downloadError: "Couldn't read llama.cpp's downloaded models" + }) + expect(rows().find(row => row.engineType === 'ollama')).toMatchObject({ + models: [{ name: 'ollama-model' }] + }) + expect(rows().find(row => row.engineType === 'lm-studio')).toMatchObject({ + models: [{ name: 'lmstudio-model' }] + }) + for (const row of rows().filter(row => row.engineType !== 'llamacpp')) + expect(row).not.toHaveProperty('downloadError') + state.handleNotification({ + source: 'broker', + method: 'discovery:nodes-changed', + params: { nodes: [{ ...node, downloadErrorsByEngine: {} }] } + }) + expect(rows().find(row => row.engineType === 'llamacpp')).toMatchObject({ + downloadError: undefined + }) + }) + + it('never projects cache inventory from an unverified preview', () => { + const state = getModularBridgeState() + state.handleNotification({ + source: 'broker', + method: 'discovery:nodes-changed', + params: { + nodes: [ + { + hostUuid: 'preview:cache', + peerRecordId: '', + identityVerification: 'unverified', + name: 'Preview', + downloadedByEngine: { + llamacpp: [ + { + id: 'forged', + file: 'forged.gguf', + size: 24, + sha256: 'forged', + owned: true + } + ] + } + } + ] + } + }) + expect( + state + .getEngineInitialState() + .models.some(row => row.models.some(model => model.name === 'forged')) + ).toBe(false) + expect(() => state.getPeerDownloadedModels('preview:cache')).toThrow('verified peer') + }) +}) diff --git a/desktop/tests/modular/model-download-controls.test.ts b/desktop/tests/modular/model-download-controls.test.ts index 90b371a03..4689cc455 100644 --- a/desktop/tests/modular/model-download-controls.test.ts +++ b/desktop/tests/modular/model-download-controls.test.ts @@ -44,6 +44,12 @@ it('renders progress, Canceling, confirmed cancellation and unsupported transfer expect(container.textContent).toContain('Partial data is kept so the download can resume.') expect(container.querySelector('button')).toBeNull() expect(container.querySelector('[role="status"]')).toBeNull() + render({ engineType: 'llamacpp', status: 'cancelled', canCancel: false }) + expect(container.textContent).toContain('Cancelled') + expect(container.textContent).not.toContain('Partial data') + render({ engineType: 'llamacpp', status: 'interrupted', canCancel: false }) + expect(container.textContent).toContain('Download interrupted') + expect(container.textContent).not.toContain('resume') render({ canCancel: false }) expect(container.querySelector('button')).toBeNull() expect(container.textContent).toContain('42%') diff --git a/desktop/tests/modular/model-manager-capabilities.test.ts b/desktop/tests/modular/model-manager-capabilities.test.ts index 75f723a8d..0ceba6724 100644 --- a/desktop/tests/modular/model-manager-capabilities.test.ts +++ b/desktop/tests/modular/model-manager-capabilities.test.ts @@ -5,11 +5,12 @@ import { act, createElement } from 'react' import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import type { EngineCapabilityFlags, EngineType } from '@/shared/types/engines' +import type { EngineCapabilityFlags, EngineType, EngineProcessStatus } from '@/shared/types/engines' import type { BackendInfo, ModelItem } from '@/ui/types/engine-info' import { ModelManager } from '@/ui/components/ModelManager/ModelManager' import { useEngineProgressStore } from '@/ui/stores/engine-progress.store' import type { EngineProgress } from '@/shared/types/engines' +import { useEngineModelsStore } from '@/ui/stores/engine-models.store' import { useEngineStatusStore } from '@/ui/stores/engine-status.store' import { useConnectionStore } from '@/ui/stores/connection.store' import { useNodesStore } from '@/ui/stores/nodes.store' @@ -43,6 +44,9 @@ let container: HTMLDivElement let root: Root const deleteModel = vi.fn() const pullModel = vi.fn() +const downloadedModels = vi.fn(async () => [ + { id: MODEL.name, file: 'cache.gguf', size: 42, sha256: 'verified' } +]) function report(nodeId: string, engineType: EngineType, flags: EngineCapabilityFlags): void { const statusByNode = new Map(useEngineStatusStore.getState().statusByNode) @@ -65,15 +69,22 @@ function report(nodeId: string, engineType: EngineType, flags: EngineCapabilityF act(() => useEngineStatusStore.setState({ statusByNode })) } -function render(nodeId = 'local', engineType: EngineType = 'lm-studio', downloaded = true): void { +function render( + nodeId = 'local', + engineType: EngineType = 'lm-studio', + downloaded = true, + processStatus: EngineProcessStatus = 'running', + digest = '', + models: ModelItem[] = [{ ...MODEL, downloaded, digest }] +): void { const backend: BackendInfo = { type: engineType, displayName: engineType === 'lm-studio' ? 'LM Studio' : 'oMLX', source: 'installed', - processStatus: 'running', + processStatus, port: 1234, proxyPort: 54301, - models: [{ ...MODEL, downloaded }] + models } act(() => root.render(createElement(ModelManager, { backend, nodeId }))) } @@ -110,7 +121,7 @@ describe('ModelManager reported capability gates', () => { this.dispatchEvent(new Event('close')) } }) - vi.stubGlobal('pairApi', { engines: { deleteModel, pullModel } }) + vi.stubGlobal('pairApi', { engines: { deleteModel, pullModel, downloadedModels } }) useEngineStatusStore.setState({ statusByNode: new Map() }) useConnectionStore.setState({ connected: true, selfId: 'local' }) container = document.createElement('div') @@ -125,11 +136,113 @@ describe('ModelManager reported capability gates', () => { Reflect.deleteProperty(HTMLDialogElement.prototype, 'close') useEngineStatusStore.setState({ statusByNode: new Map() }) useEngineProgressStore.setState({ progress: new Map() }) + useEngineModelsStore.setState({ models: new Map() }) useConnectionStore.setState({ connected: false, selfId: null }) useNodesStore.setState({ nodes: new Map() }) vi.unstubAllGlobals() }) + it('shows a peer cache error calmly on llama.cpp only', () => { + report('peer', 'llamacpp', supported) + act(() => + useEngineModelsStore.setState({ + models: new Map([ + [ + 'peer:llamacpp', + { + nodeId: 'peer', + engineType: 'llamacpp', + models: [], + downloadError: "Couldn't read llama.cpp's downloaded models" + } + ] + ]) + }) + ) + render('peer', 'llamacpp', true, 'stopped', '', []) + expect(container.textContent).toContain("Couldn't read llama.cpp's downloaded models") + expect(container.textContent).not.toContain('No model/weight files downloaded') + report('peer', 'lm-studio', supported) + render('peer', 'lm-studio') + expect(container.textContent).not.toContain("Couldn't read llama.cpp's downloaded models") + expect(container.textContent).toContain('demo') + report('peer', 'llamacpp', supported) + render('peer', 'llamacpp', true, 'stopped', '', []) + act(() => useEngineModelsStore.setState({ models: new Map() })) + expect(container.textContent).not.toContain("Couldn't read llama.cpp's downloaded models") + expect(container.textContent).toContain('No model/weight files downloaded') + }) + + it('refreshes the stopped llama.cpp installed list only after deletion finishes', async () => { + let finish = (): void => {} + let removed = false + downloadedModels.mockImplementation(async () => + removed ? [] : [{ id: MODEL.name, file: 'cache.gguf', size: 42, sha256: 'verified' }] + ) + deleteModel.mockImplementation( + () => + new Promise(resolve => { + finish = () => { + removed = true + resolve() + } + }) + ) + report('local', 'llamacpp', { ...supported, deleteRestartsEngine: false }) + await act(async () => render('local', 'llamacpp', true, 'stopped')) + expect(container.textContent).toContain(MODEL.name) + click('Delete') + await act(async () => { + await Promise.resolve() + }) + expect(container.textContent).toContain(MODEL.name) + await act(async () => finish()) + expect(container.textContent).not.toContain(MODEL.name) + expect(container.textContent).toContain('No model/weight files downloaded') + }) + + it('offers owned peer llama.cpp deletion and explains the running gate', async () => { + useNodesStore.setState({ + nodes: new Map([ + [ + 'peer', + { + id: 'peer', + hostUuid: 'peer', + peerRecordId: 'proof', + identityVerification: 'verified', + name: 'Peer', + ipAddress: '192.0.2.10', + port: 14318, + allIpAddresses: [], + status: 'active', + os: 'Linux', + topology: { + gpus: [], + cpu: { model: '', cores: 0, threads: 0 }, + ram: 0, + storage: [] + } + } + ] + ]) + }) + deleteModel.mockResolvedValue(undefined) + report('peer', 'llamacpp', { ...supported, deleteRestartsEngine: false }) + await act(async () => render('peer', 'llamacpp', true, 'stopped', 'verified')) + expect(container.textContent).toContain(MODEL.name) + expect(container.textContent).not.toContain('Only served') + await act(async () => click('Delete')) + expect(deleteModel).toHaveBeenCalledWith('llamacpp', 'peer', MODEL.name) + render('peer', 'llamacpp', true, 'running', 'verified') + const gate = Array.from(container.querySelectorAll('button')).find( + button => button.textContent === 'Delete (stop engine first)' + ) + expect(gate?.disabled).toBe(true) + render('peer', 'llamacpp', true, 'stopped') + expect(labels()).not.toContain('Delete') + }) + it('includes Load options in downloaded local LM Studio model rows only', () => { report('local', 'lm-studio', supported) render() diff --git a/docs/engine-lifecycle.mdx b/docs/engine-lifecycle.mdx index 912ec0f30..75da89006 100644 --- a/docs/engine-lifecycle.mdx +++ b/docs/engine-lifecycle.mdx @@ -10,9 +10,10 @@ run models. PAIR supports Ollama, LM Studio and oMLX. It can install and run those engines for you, or work with a copy you already have. A macOS arm64 **llama.cpp engine** also appears in Engine settings: it can be installed, started, stopped and configured, and is served by its own routing proxy on -**1237** (engine listener **127.0.0.1:8082**). Select a regular GGUF inside PAIR's -llama.cpp model cache with allowlisted `--model`/`-m`; the Go service now -provides catalog/download/cancel/delete RPCs, with desktop/TUI hub screens deferred. Its official b12171 macOS build is **not notarized**, like PAIR itself. +**1237** (engine listener **127.0.0.1:8082**). Use **Models → Add model** to browse its catalog and download a model, +then **Settings → Startup model** to choose the catalog ID and preview/apply the +cache-relative selection. Desktop and TUI expose the existing Go catalog, +download, cancel and delete operations. Its official b12171 macOS build is **not notarized**, like PAIR itself. This page explains what you can expect when you install, start, stop, update, or remove an engine. @@ -72,7 +73,8 @@ Refer also to [Getting started](getting-started.mdx). Install llama.cpp from its node row (not first-run automatic selection). The managed listener defaults to `127.0.0.1:8082`; Settings exposes the server port -and proxy ports and launch arguments/environment. PAIR protects all listener +and proxy ports and allowlisted launch arguments (environment assignments are +not supported for this engine). PAIR protects all listener controls, including Unix-socket host values and port-changing presets. Readiness requires the `/props` router identity (or the inherited model descriptor in single-model mode), not merely HTTP success. Non-loopback @@ -80,7 +82,29 @@ binding still requires the existing explicit `engine_allow_lan_bind` permission. The Go model hub supplies a bounded public Q4_K_M catalog and PAIR-owned verified downloads, asynchronous cancel and manifest-owned delete. See the [model hub contract](../services/nvpair-engine-manager/LLAMACPP.md#go-model-hub-and-owned-cache). -The dedicated proxy routes inference; model hub screens and load profiles remain deferred. +The dedicated proxy routes inference. The desktop's **Models** section groups +the catalog by publisher and displays the service's bounded search errors. +Downloads reuse the ordinary progress/cancel rows. Cancellation acceptance is +not transfer release: wait for **Cancelled**. Interrupted downloads offer +retry, not a promise of resume. Only single-file Q4_K_M GGUF catalog downloads +are supported. Delete requires a stopped engine and a PAIR-owned entry. + +**Settings → Startup model** writes `--model ` through the +existing preview/apply editor and confirms required restarts. Names remain the +catalog IDs; **No model (router mode)** removes the model option. A separately +configured known installed catalog ID is also accepted by the service. Launch +descriptors are never model names. + +Verified peers publish installed llama.cpp cache inventory through the existing +pinned-mTLS discovery path, separate from served-model routing. Desktop and TUI +show owned and unrecorded files, with owned deletion through the existing remote +operation. Stop the peer's engine first; identity checks and refusal reasons are +the same as local deletion. Previews receive no inventory. Startup dropdown +selection remains local; a peer's known installed ID can still be set in its +launch arguments. +Per-model load profiles remain out of scope. See the +[terminal keys](terminal-interface.mdx#llamacpp-model-hub) and +[known limitations](known-issues.mdx#llamacpp-model-hub-limitations). Installation verifies the pinned SHA-256, bounded contained extraction, expected server/libraries/Metal layout, absence of quarantine and existing signature diff --git a/docs/getting-started.mdx b/docs/getting-started.mdx index f8573ebb0..ebec10ffa 100644 --- a/docs/getting-started.mdx +++ b/docs/getting-started.mdx @@ -166,6 +166,35 @@ Refer to [Managing engines](engine-lifecycle.mdx) for details about: - Managing engines in a cluster. - Quitting or relaunching PAIR. +### llama.cpp on macOS arm64 + +Install **llama.cpp** from the local node's engine row after first-run setup. +Its macOS build is not notarized, like PAIR itself. PAIR downloads the engine +and models at runtime; neither is bundled in the application. + +1. Expand **Models → Add model**. Browse the Go catalog grouped by publisher, + or search public models (100 query characters, up to 50 results). +2. Choose a single-file **Q4_K_M GGUF** model and follow its download progress. + **Canceling** means a stop was requested, not confirmed. **Cancelled** confirms + release; **Interrupted** offers retry. llama.cpp does not promise resume. + Checksum, disk-space and file-identity failures appear as service messages. +3. In **Settings → Startup model**, select the downloaded catalog ID and + **Apply**. PAIR previews the change and confirms a restart if required. + **No model (router mode)** removes the startup model; it does not download one. +4. Start the engine. Use the catalog ID shown in the installed list as the + request's model name. The default proxy endpoint is + `http://127.0.0.1:1237/v1`. +5. Stop llama.cpp before deleting an owned downloaded model. Unmanaged files + are not deletable through PAIR. + +Verified peers show installed cache files and support owned deletion through +the existing authenticated remote controls. Stop the peer's llama.cpp engine +before deleting; unrecorded files cannot be deleted by PAIR. Catalog +download/cancel remains available. The startup picker is local: to change a +peer's startup model, enter a known installed catalog ID with `--model` in its +Settings arguments and preview/apply normally. +See [the current limitations](known-issues.mdx#llamacpp-model-hub-limitations). + ## 5. Find Your Endpoint Your applications connect to PAIR on the local machine, not directly to an @@ -175,7 +204,7 @@ address an endpoint. Select **Endpoints** in the top toolbar to open the **API endpoints** window. It lists one copyable URL per engine, labelled by API style: Ollama's native API root (`http://127.0.0.1:`) or the OpenAI-compatible base URL for LM Studio -and oMLX (`http://127.0.0.1:/v1`). The engine header's copy button uses the +oMLX and llama.cpp (`http://127.0.0.1:/v1`). The engine header's copy button uses the same URL and label. Copy the one for the engine you prepared. If it says **No engines are running**, return to step 4 and start an engine. @@ -222,8 +251,9 @@ The engine therefore has to understand the style you send. | Ollama | Works | Works | | LM Studio | Works | Not available | | oMLX | Works | Not available | +| llama.cpp | Works | Not available | -**If you are unsure, send the OpenAI-style request.** It works with all three engines, +**If you are unsure, send the OpenAI-style request.** It works with all four engines, and most tools and SDKs use it. Use the Ollama style only when something you already have is written against Ollama's `/api/...` API. diff --git a/docs/known-issues.mdx b/docs/known-issues.mdx index c60437ea1..5a7ef17bf 100644 --- a/docs/known-issues.mdx +++ b/docs/known-issues.mdx @@ -26,6 +26,34 @@ entirely (avoiding repeated prompts). In the desktop application, navigate to guidance on allowing inbound connections when macOS prompts appear so other nodes can reach this machine. +## llama.cpp Model Hub Limitations + +**Unreleased source, not certification of a published package.** The managed +b12171 engine supports macOS arm64 only and is not notarized, like PAIR. +PAIR's catalog currently admits only public, ungated, single-file Q4_K_M GGUF +text-generation models. Search allows 100 characters and up to 50 results; +service timeouts or validation errors are displayed rather than treated as an +empty successful search. + +Stop llama.cpp before deleting a downloaded model. PAIR refuses deletion while +it is running or a lifecycle/cache mutation is active, and deletes only an +identity-matched owned entry. Unmanaged files are listed but not deletable. +Cancellation is asynchronous; wait for its terminal state. llama.cpp removes +only its own identity-matched transfer temporary file and does not promise +partial-download resume. A checksum, disk-space or identity failure is an error, +not successful cancellation. + +Verified peers expose installed llama.cpp cache files in desktop and TUI, +including while stopped. Unrecorded files are visible but not deletable; +owned deletion uses the existing authenticated remote operation and refuses +while llama.cpp is running. Unverified previews receive no inventory. +Downloaded-model startup pickers remain local, as with the other engines. +The desktop Settings arguments can select a known installed peer catalog ID. + +PAIR downloads the engine and models at runtime; they are not bundled and have +their own upstream terms. Packaged install/update/rollback, real downloads, +inference, stop/restart and multi-node smoke testing remain release gates. + ## Unified-Memory Systems Report Misleading GPU Memory **Implemented in source, pending release testing.** GPU memory now has an explicit diff --git a/docs/release-readiness.mdx b/docs/release-readiness.mdx index 382dc6f4e..990d68a31 100644 --- a/docs/release-readiness.mdx +++ b/docs/release-readiness.mdx @@ -16,9 +16,28 @@ The current source adds `llamacpp-proxy` (default port 1237, engine 8082, service `lc`): **13 supervised workers / 15 executables**. The broker, desktop and TUI inventories, build/install lists and contract documentation include it. Native b12171 Messages passthrough and exact-ID model routing are supported; -the Go model hub provides catalog/download/cancel/delete; desktop/TUI hub -screens remain deferred. The new hub uses fixture/local-server tests, not live -model downloads or packaged certification. Historical packaged-app evidence +the Go model hub provides catalog/download/cancel/delete. Desktop and TUI now +expose publisher-grouped/sorted catalog browsing, bounded search errors, +installed local models, tracked downloads/cancel, stopped-engine owned deletion +and previewed startup model/router-mode selection. TUI retains the existing +unreleased **1.4.0** minor bump relative to v0.9.0's **1.3.0**. +The new hub uses fixture/local-server and client tests, not live model downloads +or packaged certification. + +Verified-peer installed-list/delete parity now reuses the existing pinned-mTLS +engine inventory, scanner/manual enrichment and broker discovery relay. +Desktop and TUI show cache IDs/file names, size and ownership; unrecorded files +cannot be deleted. Remote owned deletion retains the identity/running guards +and visible refusal reasons. Previews receive none. Startup dropdowns remain +local (as for the other engines); peer Settings arguments still accept a known +installed catalog ID. Live-peer and packaged smoke testing remain release gates. + +The license generator documents llama.cpp and models as runtime downloads, +not bundled components; no unbundled engine/model was added to the dependency +attribution manifest. Release smoke testing must cover macOS arm64 install, +update/rollback, real model download/cancel/shutdown, checksum/disk-space/identity +failures, selection/inference, stop/restart and owned deletion. The upstream +binary remains unnotarized; the checksum does not certify publisher trust. Historical packaged-app evidence below predates this worker and does not certify its packaged or multi-node inference behavior. Validate those before release. @@ -82,8 +101,8 @@ The validation notes record these as non-blocking follow-ups, not fixes: stall lock-holding callers. Engine-manager tests can leave fake-engine helpers running, and engine-manager does not build for FreeBSD. -llama.cpp is deferred to a later release because it needs a thirteenth proxy -worker, unsigned-binary review and its own download-cancellation path. +llama.cpp was not included in that historical packaged-app validation. Its +unreleased routing/model surfaces and remaining release gates are tracked above. ## Documentation and Installer Maintenance diff --git a/docs/terminal-interface.mdx b/docs/terminal-interface.mdx index 570a44db7..215c30edb 100644 --- a/docs/terminal-interface.mdx +++ b/docs/terminal-interface.mdx @@ -124,6 +124,39 @@ with `screen -S pair`, detach with `Ctrl-a d`, and reattach with `screen -r pair`. Neither ships with PAIR. Install whichever your distribution provides. +## llama.cpp model hub + +Select llama.cpp in **Engines** (macOS arm64 only; not notarized, like PAIR). +All model controls follow the selected node's reported capabilities. + +| Key | Action | +| --- | --- | +| `m` | Open/close the installed local cache, including while stopped | +| `b` | Switch catalog/installed; catalog rows sort by publisher and model ID | +| `/` | Search the public catalog; Enter submits, Esc cancels the editor | +| `p` | Download the selected catalog ID; otherwise prompt for a model ID | +| `c` | Request cancellation of the newest cancellable transfer | +| `d` | Confirm deletion of a PAIR-owned installed model while stopped | +| `a` | Preview and confirm the selected installed model as the startup model | +| `0` | Preview and confirm **No model (router mode)** | + +Catalog search allows at most 100 characters and 50 results, and only +single-file Q4_K_M GGUF downloads. Progress, cancel acceptance, confirmed stop +and errors use the existing download tracker. **canceling** is not stopped; +an interrupted transfer can be retried. llama.cpp does not promise partial-file +resume. Checksum, disk-space and file-identity errors remain visible. + +Startup selection preserves other tuning arguments, writes a cache-relative +`--model` option, and uses node-owned revisioned settings preview/apply. +Confirmation warns if the engine will restart. Display names are catalog IDs. + +On verified peers, `m` lists installed llama.cpp cache files from authenticated +discovery inventory, including while stopped. `d` confirms owned deletion +through the existing remote operation; stop llama.cpp first. Unrecorded files +are visible but cannot be deleted. Previews receive no inventory. Catalog +download/cancel retain the existing remote methods. The startup picker remains +local; no local inventory is presented as a peer's cache. + ## The Screen The top line shows `NVPAIR TUI` and the connection state on the right: diff --git a/services/nvpair-engine-manager/LLAMACPP.md b/services/nvpair-engine-manager/LLAMACPP.md index d66e5b941..9240892d7 100644 --- a/services/nvpair-engine-manager/LLAMACPP.md +++ b/services/nvpair-engine-manager/LLAMACPP.md @@ -10,7 +10,11 @@ unsupported. PAIR installs, starts, stops and configures the engine and serves it through the dedicated `llamacpp-proxy` worker, default port **1237**. Routing uses exact model IDs from the engine's `/v1/models` inventory, with separate `llamacpp` attribution. The Go model hub supports catalog, downloads, cancel -and delete over JSON-RPC; desktop/TUI hub screens and load profiles are deferred. +and delete over JSON-RPC. Desktop and TUI expose the catalog, local installed +cache, tracked downloads/cancel/delete and startup model/router-mode selection. +Peer installed-cache listing has no service method yet, so the clients label +peer models as served inventory and keep installed-model pickers local. +Per-model load profiles remain deferred. The macOS build is **not notarized**, like this fork's PAIR application. The release is checksum-pinned, not publisher-trusted by that checksum. Its existing diff --git a/services/nvpair-engine-manager/README.md b/services/nvpair-engine-manager/README.md index 2505b0707..055117f6b 100644 --- a/services/nvpair-engine-manager/README.md +++ b/services/nvpair-engine-manager/README.md @@ -5,6 +5,29 @@ SPDX-License-Identifier: Apache-2.0 # nvpair-engine-manager +### Downloaded llama.cpp inventory + +The existing engine-manager `/v1/models` inventory adds `downloadedByEngine`: +`{"llamacpp":[{"id":"catalog ID or file name","file":"cache-relative.gguf","size":24,"sha256":"…","owned":true}]}`. +This cache list is available while stopped and remains separate from served +`models`/`modelsByEngine`, which continue to drive routing. The cache scan is +limited to 4096 entries and uses the existing bounded inventory response. +Scanner enrichment and verified manual peers carry it over pinned cluster mTLS; +the broker preserves it in discovery snapshots. Unverified previews receive none. +A cache-validation/read failure does not fail the served inventory: HTTP remains +200 and `engine:models` still returns all served/loaded engine models. +`downloadErrorsByEngine: {"llamacpp":"cache_unavailable"}` reports that the +llama.cpp cache is unknown (its downloaded list is omitted). This fixed public +code never contains paths or underlying error text. Manual-node updates and +scanner/broker discovery snapshots preserve it separately from engine liveness; +a successful cache read clears it. Genuine whole-inventory failures still fail +the HTTP request with 503 and retain their existing consumer behavior. +Deletion reuses `engine:remote-delete-model` and authenticated ec +`POST /v1/models/delete`, ending in engine-manager's identity-checked +`delete_model`. Unrecorded files are not deletable; a running/busy llama.cpp +engine refuses deletion with its stop-engine reason. + + A config-driven control plane for local inference engines (Ollama today; Intel/others via a dropped-in manifest). It manages everything about an engine **except serving inference**: detect, user-mode install, @@ -25,7 +48,11 @@ to 4 MiB. Existing `engine:action` supports llama.cpp `pull_model`, `list_downloaded` and `delete_model`, including existing operation-ID progress, asynchronous cancel and verified-peer remote management. See [the owned-cache contract](LLAMACPP.md#go-model-hub-and-owned-cache) for checks, inventory and -cleanup. Desktop/TUI catalog screens are deferred to PR 4. +cleanup. Desktop/TUI catalog and local/verified-peer installed-cache controls are +documented in the [engine guide](../../docs/engine-lifecycle.mdx#llamacpp-preview). +Peer cache inventory travels in `downloadedByEngine` on the existing model +inventory and discovery path; deletion uses the existing authenticated remote +operation, not a separate cache-control endpoint. The bundled manifests under `manifests/` are the working reference for manifest authoring. [Pinned archive installs](MANIFEST.md#pinned-macos-archive-installs) diff --git a/services/nvpair-engine-manager/downloaded_inventory_test.go b/services/nvpair-engine-manager/downloaded_inventory_test.go new file mode 100644 index 000000000..b06e39c96 --- /dev/null +++ b/services/nvpair-engine-manager/downloaded_inventory_test.go @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "testing" +) + +func TestDownloadedInventorySkipsUnsupportedLlamaPlatform(t *testing.T) { + reg := NewRegistry() + reg.engines["llamacpp"] = &Manifest{Engine: "llamacpp"} + ex := NewExecutor(reg, NewReporter(nil), nil, t.TempDir()) + result, err := ex.modelsResult(context.Background()) + if err != nil || len(result.DownloadedByEngine) != 0 { + t.Fatalf("unsupported cache broke model inventory: %+v %v", result, err) + } +} diff --git a/services/nvpair-engine-manager/llamacpp_cache_unix.go b/services/nvpair-engine-manager/llamacpp_cache_unix.go index 7040acc60..f9cc529a5 100644 --- a/services/nvpair-engine-manager/llamacpp_cache_unix.go +++ b/services/nvpair-engine-manager/llamacpp_cache_unix.go @@ -66,7 +66,7 @@ func openLlamaCacheAt(root string) (*os.File, error) { next, err := unix.Openat(fd, part, flags, 0) _ = unix.Close(fd) if err != nil { - return nil, fmt.Errorf("cache directory changed or unavailable") + return nil, fmt.Errorf("cache directory changed or unavailable: %w", err) } fd = next } diff --git a/services/nvpair-engine-manager/llamacpp_download.go b/services/nvpair-engine-manager/llamacpp_download.go index 22454aa60..eb06ba81e 100644 --- a/services/nvpair-engine-manager/llamacpp_download.go +++ b/services/nvpair-engine-manager/llamacpp_download.go @@ -9,6 +9,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net" @@ -329,7 +330,7 @@ func (e *Executor) listLlamaModels(ctx context.Context) (json.RawMessage, error) return nil, err } dir, err := openLlamaCache(st.installDir+"-models", false) - if os.IsNotExist(err) { + if errors.Is(err, os.ErrNotExist) { return json.RawMessage(`{"models":[]}`), nil } if err != nil { diff --git a/services/nvpair-engine-manager/llamacpp_peer_test.go b/services/nvpair-engine-manager/llamacpp_peer_test.go new file mode 100644 index 000000000..b0c97995d --- /dev/null +++ b/services/nvpair-engine-manager/llamacpp_peer_test.go @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build !windows + +package main + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "nvpair-shared/clustertrust" +) + +func TestLlamaPeerInventoryIncludesStoppedCache(t *testing.T) { + ex, cache := llamaDownloadExecutor(t, nil, nil) + if err := pullLlamaTest(ex, "peer-inventory"); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cache, "manual.gguf"), llamaTestBytes(), 0600); err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + modelsHandler(ex)(rec, httptest.NewRequest(http.MethodGet, modelsPath, nil)) + var body struct { + Downloaded map[string][]struct { + ID string `json:"id"` + File string `json:"file"` + Size int64 `json:"size"` + Owned bool `json:"owned"` + } `json:"downloadedByEngine"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + rows := body.Downloaded["llamacpp"] + if len(rows) != 2 || rows[0].ID != llamaTestID || rows[0].File != llamaOwnedFilename(llamaTestID) || rows[0].Size != int64(len(llamaTestBytes())) || !rows[0].Owned || rows[1].ID != "manual.gguf" || rows[1].Owned { + t.Fatalf("cache inventory: %s", rec.Body.String()) + } + if len(ex.Models(context.Background())) != 0 { + t.Fatal("stopped cache leaked into served routing inventory") + } +} + +func TestLlamaPeerInventoryBeforeCacheParentExists(t *testing.T) { + ex, _ := llamaDownloadExecutor(t, nil, nil) + state, err := ex.state("llamacpp") + if err != nil { + t.Fatal(err) + } + state.installDir = filepath.Join(t.TempDir(), "not-created", "engine-bin", "llamacpp") + rec := httptest.NewRecorder() + modelsHandler(ex)(rec, httptest.NewRequest(http.MethodGet, modelsPath, nil)) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"downloadedByEngine":{"llamacpp":[]}`) { + t.Fatalf("fresh cache unavailable: %d %s", rec.Code, rec.Body.String()) + } +} + +func TestLlamaCacheFailurePreservesOtherEngineInventory(t *testing.T) { + ex, cache := llamaDownloadExecutor(t, nil, nil) + if err := os.MkdirAll(cache, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cache, llamaManifestName), []byte("invalid"), 0600); err != nil { + t.Fatal(err) + } + mf := testEngineManifest(fakeEngineForTest(t)) + mf.Actions["list_models"] = Action{HTTP: &ActionHTTP{Method: "GET", Path: "/api/tags"}, Result: &ActionResult{Array: "models", Field: "name"}} + ex.reg.engines["fake"] = mf + reconciledTestExecutor(ex) + t.Cleanup(func() { _ = ex.Stop("fake") }) + if err := ex.Start(context.Background(), "fake"); err != nil { + t.Fatal(err) + } + result, err := ex.modelsResult(context.Background()) + if len(result.ByEngine["fake"]) != 1 || result.ByEngine["fake"][0] != "llama3.2:1b" { + t.Fatalf("cache failure erased served inventory: %+v %v", result, err) + } + selfCert, selfKey := mintLeaf(t, "inventory-owner") + peerCert, peerKey := mintLeaf(t, "inventory-caller") + owner := clustertrust.Open(clusterDirFor(t, selfCert, selfKey, map[string][]byte{"inventory-caller": peerCert})) + caller := clustertrust.Open(clusterDirFor(t, peerCert, peerKey, map[string][]byte{"inventory-owner": selfCert})) + _, mux := emSurface(ex, owner) + srv := httptest.NewUnstartedServer(mux) + srv.TLS = owner.ServerTLSConfig() + srv.StartTLS() + defer srv.Close() + cfg, _ := caller.ClientTLSConfig("inventory-owner") + transport := &http.Transport{TLSClientConfig: cfg} + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport} + resp, err := client.Get(srv.URL + modelsPath) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var body struct { + Models []string `json:"models"` + Errors map[string]string `json:"downloadErrorsByEngine"` + } + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil || resp.StatusCode != http.StatusOK || len(body.Models) != 1 || body.Errors["llamacpp"] != "cache_unavailable" || len(body.Errors) != 1 { + t.Fatalf("cache error failed pinned HTTP inventory: status=%d body=%+v err=%v", resp.StatusCode, body, err) + } +} + +func TestLlamaRemoteDeleteUsesIdentityAndRunningGuard(t *testing.T) { + for _, mode := range []string{"delete", "running", "replacement", "unpinned"} { + t.Run(mode, func(t *testing.T) { + ex, cache := llamaDownloadExecutor(t, nil, nil) + if err := pullLlamaTest(ex, "peer-delete"); err != nil { + t.Fatal(err) + } + file := filepath.Join(cache, llamaOwnedFilename(llamaTestID)) + if mode == "running" { + st, _ := ex.state("llamacpp") + st.mu.Lock() + st.running = true + st.mu.Unlock() + } + if mode == "replacement" { + if err := os.Rename(file, file+".original"); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, llamaTestBytes(), 0600); err != nil { + t.Fatal(err) + } + } + selfCert, selfKey := mintLeaf(t, "delete-owner") + peerCert, peerKey := mintLeaf(t, "delete-caller") + pins := map[string][]byte{"delete-caller": peerCert} + if mode == "unpinned" { + pins = nil + } + owner := clustertrust.Open(clusterDirFor(t, selfCert, selfKey, pins)) + caller := clustertrust.Open(clusterDirFor(t, peerCert, peerKey, map[string][]byte{"delete-owner": selfCert})) + server := &controlServer{exec: ex, mesh: owner} + srv := httptest.NewUnstartedServer(server.mux()) + srv.TLS = owner.ServerTLSConfig() + srv.StartTLS() + defer srv.Close() + cfg, ok := caller.ClientTLSConfig("delete-owner") + if !ok { + t.Fatal("missing peer pin") + } + transport := &http.Transport{TLSClientConfig: cfg} + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport} + remote := &remoteClient{http: client, readyHTTP: client, base: srv.URL} + result, err := remote.postJSON(context.Background(), controlDeletePath, "llamacpp", modelActionRequest{Engine: "llamacpp", Model: llamaTestID}) + _, statErr := os.Stat(file) + if mode == "delete" { + if err != nil || !os.IsNotExist(statErr) { + t.Fatalf("delete: %s %v %v", result, err, statErr) + } + } else if err == nil || statErr != nil { + t.Fatalf("unsafe delete: %s %v %v", result, err, statErr) + } + if mode == "running" && !strings.Contains(err.Error(), "stop the engine") { + t.Fatalf("missing refusal reason: %v", err) + } + if mode == "unpinned" && !strings.Contains(err.Error(), "403") { + t.Fatalf("missing peer refusal: %v", err) + } + }) + } +} diff --git a/services/nvpair-engine-manager/models.go b/services/nvpair-engine-manager/models.go index c3bda5c14..024369c75 100644 --- a/services/nvpair-engine-manager/models.go +++ b/services/nvpair-engine-manager/models.go @@ -11,6 +11,8 @@ import ( "slices" "sync" "time" + + "nvpair-shared/noderec" ) // modelsTimeout bounds the whole engine:models sweep so one hung engine can't @@ -40,9 +42,13 @@ const modelsTimeout = 5 * time.Second // ByEngine/Models entry until the next successful sweep. Its own omitempty drops // the whole map when no engine reports loaded state. type ModelsResult struct { - Models []string `json:"models"` - ByEngine map[string][]string `json:"modelsByEngine,omitempty"` - LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` + // DownloadErrorsByEngine contains bounded public codes, never filesystem errors. + // Cache failures do not affect served inventory or engine liveness. + DownloadErrorsByEngine map[string]string `json:"downloadErrorsByEngine,omitempty"` + DownloadedByEngine map[string][]noderec.DownloadedModel `json:"downloadedByEngine,omitempty"` + Models []string `json:"models"` + ByEngine map[string][]string `json:"modelsByEngine,omitempty"` + LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` } // Models returns the union of model names served by every installed, running @@ -201,6 +207,26 @@ func (e *Executor) modelsResult(ctx context.Context) (ModelsResult, error) { res.LoadedByEngine[engineNames[i]] = ld } } + // Cache inventory is available even while the engine is stopped. Keep it + // separate from served models: proxies use the latter to choose owners. + if _, err := e.state("llamacpp"); err == nil { + raw, err := e.listLlamaModels(ctx) + var cache struct { + Models []llamaOwnedModel `json:"models"` + } + if err == nil { + err = json.Unmarshal(raw, &cache) + } + if err != nil { + res.DownloadErrorsByEngine = map[string]string{"llamacpp": "cache_unavailable"} + } else { + rows := make([]noderec.DownloadedModel, 0, len(cache.Models)) + for _, model := range cache.Models { + rows = append(rows, noderec.DownloadedModel{ID: model.ID, File: model.File, Size: model.Size, SHA256: model.SHA256, Owned: model.SHA256 != ""}) + } + res.DownloadedByEngine = map[string][]noderec.DownloadedModel{"llamacpp": rows} + } + } if err := ctx.Err(); err != nil { return res, err } diff --git a/services/nvpair-manual-nodes/README.md b/services/nvpair-manual-nodes/README.md index f8b31e3e9..836b2e46b 100644 --- a/services/nvpair-manual-nodes/README.md +++ b/services/nvpair-manual-nodes/README.md @@ -5,6 +5,29 @@ SPDX-License-Identifier: Apache-2.0 # nvpair-manual-nodes +### Downloaded llama.cpp inventory + +The existing engine-manager `/v1/models` inventory adds `downloadedByEngine`: +`{"llamacpp":[{"id":"catalog ID or file name","file":"cache-relative.gguf","size":24,"sha256":"…","owned":true}]}`. +This cache list is available while stopped and remains separate from served +`models`/`modelsByEngine`, which continue to drive routing. The cache scan is +limited to 4096 entries and uses the existing bounded inventory response. +Scanner enrichment and verified manual peers carry it over pinned cluster mTLS; +the broker preserves it in discovery snapshots. Unverified previews receive none. +A cache-validation/read failure does not fail the served inventory: HTTP remains +200 and `engine:models` still returns all served/loaded engine models. +`downloadErrorsByEngine: {"llamacpp":"cache_unavailable"}` reports that the +llama.cpp cache is unknown (its downloaded list is omitted). This fixed public +code never contains paths or underlying error text. Manual-node updates and +scanner/broker discovery snapshots preserve it separately from engine liveness; +a successful cache read clears it. Genuine whole-inventory failures still fail +the HTTP request with 503 and retain their existing consumer behavior. +Deletion reuses `engine:remote-delete-model` and authenticated ec +`POST /v1/models/delete`, ending in engine-manager's identity-checked +`delete_model`. Unrecorded files are not deletable; a running/busy llama.cpp +engine refuses deletion with its stop-engine reason. + + A Go service for probing explicitly configured PAIR machines or bare engines on networks where mDNS is unavailable. The broker owns persistence; this worker owns live probe state. See [spec.md](spec.md) for the normative trust, intent and replay contract. Node-info JSON decoding reads at most 1 MiB, and model-list decoding at most diff --git a/services/nvpair-manual-nodes/cache_inventory_test.go b/services/nvpair-manual-nodes/cache_inventory_test.go new file mode 100644 index 000000000..873fd1467 --- /dev/null +++ b/services/nvpair-manual-nodes/cache_inventory_test.go @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "os" + "testing" + + "nvpair-shared/engineinventorytest" +) + +func TestPinnedManualPeerCacheErrorPreservesServedInventory(t *testing.T) { + m, _, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "cache.local", "cache-peer") + port, manifest, stop := engineinventorytest.Start(t, peer.clusterDir) + peer.enginePort = port + configureHealthyNode(rt, "cache.local", []string{"ollama-model", "lmstudio-model"}, sampleInfo()) + entry := peer.track(m, "cache-peer") + m.probeNode(entry) + before := m.nodes[nodeID(entry)].status + if len(before.Models) != 2 || !before.OllamaUp || !before.LMStudioUp { + t.Fatalf("fixture: %+v", before) + } + if err := os.WriteFile(manifest, []byte("invalid"), 0600); err != nil { + t.Fatal(err) + } + m.probeNode(entry) + got := m.nodes[nodeID(entry)].status + if got.IdentityVerification != "verified" || got.PeerRecordID != before.PeerRecordID || len(got.Models) != 2 || !got.OllamaUp || !got.LMStudioUp || got.LlamaCPPUp { + t.Fatalf("cache error damaged peer: %+v", got) + } + raw, _ := json.Marshal(got) + var wire struct { + Errors map[string]string `json:"downloadErrorsByEngine"` + } + _ = json.Unmarshal(raw, &wire) + if wire.Errors["llamacpp"] != "cache_unavailable" || len(wire.Errors) != 1 { + t.Fatalf("cache error lost: %s", raw) + } + if err := os.Remove(manifest); err != nil { + t.Fatal(err) + } + m.probeNode(entry) + raw, _ = json.Marshal(m.nodes[nodeID(entry)].status) + wire.Errors = nil + _ = json.Unmarshal(raw, &wire) + if len(wire.Errors) != 0 { + t.Fatalf("recovered cache retained error: %s", raw) + } + stop() + m.probeNode(entry) + got = m.nodes[nodeID(entry)].status + if got.IdentityVerification != "verified" || got.OllamaUp || got.LMStudioUp || len(got.Models) != 0 { + t.Fatalf("whole-inventory failure changed semantics: %+v", got) + } +} diff --git a/services/nvpair-manual-nodes/inventory_fixture_test.go b/services/nvpair-manual-nodes/inventory_fixture_test.go index 5394725e0..0dc40bf4e 100644 --- a/services/nvpair-manual-nodes/inventory_fixture_test.go +++ b/services/nvpair-manual-nodes/inventory_fixture_test.go @@ -49,11 +49,13 @@ func bindTestManager(t *testing.T, m *Manager) { // pin. Node-info advertises this same authenticated endpoint for both services. // Payloads are supplied through rt so tests can change availability between reads. type inventoryPeer struct { - entry ManualEntry - server *httptest.Server - principal string - polls chan string - newConns atomic.Int32 + entry ManualEntry + server *httptest.Server + principal string + clusterDir string + enginePort int + polls chan string + newConns atomic.Int32 } func newInventoryPeer(t *testing.T, m *Manager, rt *fakeRoundTripper, alias, principal string) *inventoryPeer { @@ -66,7 +68,7 @@ func newInventoryPeer(t *testing.T, m *Manager, rt *fakeRoundTripper, alias, pri writePinFromCert(t, peerDir, "uuid-self", filepath.Join(selfDir, "node.crt")) m.mesh = clustertrust.Open(selfDir) peerMesh := clustertrust.Open(peerDir) - p := &inventoryPeer{principal: principal, polls: make(chan string, 64)} + p := &inventoryPeer{principal: principal, clusterDir: peerDir, polls: make(chan string, 64)} srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if _, ok := peerMesh.VerifyClientPin(r); !ok { http.Error(w, "forbidden", http.StatusForbidden) @@ -96,9 +98,13 @@ func newInventoryPeer(t *testing.T, m *Manager, rt *fakeRoundTripper, alias, pri } info.HostUUID = principal info.ClusterUUID = &p.principal + emPort := p.entry.TLSPort + if p.enginePort != 0 { + emPort = p.enginePort + } info.PeerMetadata = &noderec.PeerMetadataParams{HostUUID: principal, Services: map[noderec.ServiceKey]int{ noderec.ServiceNodeInfo: p.entry.TLSPort, - noderec.ServiceEngineManager: p.entry.TLSPort, + noderec.ServiceEngineManager: emPort, noderec.ServiceOllama: 11434, noderec.ServiceLMStudio: 1234, noderec.ServiceOMLX: 1236, diff --git a/services/nvpair-manual-nodes/manager.go b/services/nvpair-manual-nodes/manager.go index 7bb12fd28..7f68309d1 100644 --- a/services/nvpair-manual-nodes/manager.go +++ b/services/nvpair-manual-nodes/manager.go @@ -18,6 +18,7 @@ import ( "net/http" "os" "path/filepath" + "reflect" "strconv" "strings" "sync" @@ -219,10 +220,12 @@ type ManualNodeStatus struct { // keys on. Empty when node-info didn't report one. HostUUID string `json:"hostUuid,omitempty"` // ClusterUUID is the remote node's cluster principal. - ClusterUUID string `json:"clusterUuid,omitempty"` - Models []string `json:"models,omitempty"` - ModelsByEngine map[string][]string `json:"modelsByEngine,omitempty"` - LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` + ClusterUUID string `json:"clusterUuid,omitempty"` + Models []string `json:"models,omitempty"` + ModelsByEngine map[string][]string `json:"modelsByEngine,omitempty"` + LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` + DownloadedByEngine map[string][]noderec.DownloadedModel `json:"downloadedByEngine,omitempty"` + DownloadErrorsByEngine map[string]string `json:"downloadErrorsByEngine,omitempty"` // EngineNative is the remote engine's own report of itself: the // counters, the rates and the active models the node's engine-manager // publishes at GET /v1/engine-stats. It is read over the same transport @@ -845,42 +848,44 @@ func (m *Manager) probeNode(entry ManualEntry) { verification = "verified" } newStatus := ManualNodeStatus{ - Kind: entry.Kind, - FromCluster: fromCluster, - IdentityVerification: verification, - PeerRecordID: info.PeerRecordID, - PeerMetadata: info.PeerMetadata, - ID: id, - Name: entry.Name, - Address: addr, - OllamaUp: ollamaUp, - OllamaPort: 11434, - OllamaModels: ollamaModels, - LMStudioUp: lmStudioUp, - LMStudioPort: lmStudioPort, - LMStudioModels: lmStudioModels, - OMLXUp: omlxUp, - LlamaCPPUp: llamacppUp, - OMLXPort: omlxPort, - LlamaCPPPort: llamacppPort, - OMLXModels: omlxModels, - LlamaCPPModels: llamacppModels, - NodeInfoUp: nodeInfoUp, - NodeInfoPort: nodeInfoPort, - TLSEnabled: entry.TLSPort > 0, - MTLSRequired: entry.TLSPort > 0 && entry.MTLS, - GPUs: info.GPUs, - CPU: info.CPU, - Memory: info.Memory, - TelemetryValid: info.TelemetryValid, - MSSince: msSince, - SampleSeq: info.SampleSeq, - HostUUID: info.HostUUID, - ClusterUUID: clusterUUID, - Models: finalModels, - ModelsByEngine: finalModelsByEngine, - LoadedByEngine: finalLoadedByEngine, - EngineNative: engineNative, + Kind: entry.Kind, + FromCluster: fromCluster, + IdentityVerification: verification, + PeerRecordID: info.PeerRecordID, + PeerMetadata: info.PeerMetadata, + ID: id, + Name: entry.Name, + Address: addr, + OllamaUp: ollamaUp, + OllamaPort: 11434, + OllamaModels: ollamaModels, + LMStudioUp: lmStudioUp, + LMStudioPort: lmStudioPort, + LMStudioModels: lmStudioModels, + OMLXUp: omlxUp, + LlamaCPPUp: llamacppUp, + OMLXPort: omlxPort, + LlamaCPPPort: llamacppPort, + OMLXModels: omlxModels, + LlamaCPPModels: llamacppModels, + NodeInfoUp: nodeInfoUp, + NodeInfoPort: nodeInfoPort, + TLSEnabled: entry.TLSPort > 0, + MTLSRequired: entry.TLSPort > 0 && entry.MTLS, + GPUs: info.GPUs, + CPU: info.CPU, + Memory: info.Memory, + TelemetryValid: info.TelemetryValid, + MSSince: msSince, + SampleSeq: info.SampleSeq, + HostUUID: info.HostUUID, + ClusterUUID: clusterUUID, + Models: finalModels, + ModelsByEngine: finalModelsByEngine, + LoadedByEngine: finalLoadedByEngine, + DownloadedByEngine: emRes.DownloadedByEngine, + DownloadErrorsByEngine: emRes.DownloadErrorsByEngine, + EngineNative: engineNative, } if nodeInfoUp { @@ -956,6 +961,8 @@ func (m *Manager) probeNode(entry ManualEntry) { !sliceEqual(prev.Models, newStatus.Models) || !mapsEqual(prev.ModelsByEngine, newStatus.ModelsByEngine) || !mapsEqual(prev.LoadedByEngine, newStatus.LoadedByEngine) || + !reflect.DeepEqual(prev.DownloadedByEngine, newStatus.DownloadedByEngine) || + !reflect.DeepEqual(prev.DownloadErrorsByEngine, newStatus.DownloadErrorsByEngine) || !gpusEqual(prev.GPUs, newStatus.GPUs) || !cpuEqual(prev.CPU, newStatus.CPU) || !memoryEqual(prev.Memory, newStatus.Memory) || @@ -1028,9 +1035,11 @@ const ( ) type engineManagerModels struct { - Models []string `json:"models"` - ModelsByEngine map[string][]string `json:"modelsByEngine"` - LoadedByEngine map[string][]string `json:"loadedByEngine"` + Models []string `json:"models"` + ModelsByEngine map[string][]string `json:"modelsByEngine"` + LoadedByEngine map[string][]string `json:"loadedByEngine"` + DownloadedByEngine map[string][]noderec.DownloadedModel `json:"downloadedByEngine,omitempty"` + DownloadErrorsByEngine map[string]string `json:"downloadErrorsByEngine,omitempty"` } // engineNativeDue reports whether this node's engine view is due for a fresh diff --git a/services/nvpair-node-scanner/README.md b/services/nvpair-node-scanner/README.md index e181f6acd..edbb42ac5 100644 --- a/services/nvpair-node-scanner/README.md +++ b/services/nvpair-node-scanner/README.md @@ -5,6 +5,29 @@ SPDX-License-Identifier: Apache-2.0 # nvpair-node-scanner +### Downloaded llama.cpp inventory + +The existing engine-manager `/v1/models` inventory adds `downloadedByEngine`: +`{"llamacpp":[{"id":"catalog ID or file name","file":"cache-relative.gguf","size":24,"sha256":"…","owned":true}]}`. +This cache list is available while stopped and remains separate from served +`models`/`modelsByEngine`, which continue to drive routing. The cache scan is +limited to 4096 entries and uses the existing bounded inventory response. +Scanner enrichment and verified manual peers carry it over pinned cluster mTLS; +the broker preserves it in discovery snapshots. Unverified previews receive none. +A cache-validation/read failure does not fail the served inventory: HTTP remains +200 and `engine:models` still returns all served/loaded engine models. +`downloadErrorsByEngine: {"llamacpp":"cache_unavailable"}` reports that the +llama.cpp cache is unknown (its downloaded list is omitted). This fixed public +code never contains paths or underlying error text. Manual-node updates and +scanner/broker discovery snapshots preserve it separately from engine liveness; +a successful cache read clears it. Genuine whole-inventory failures still fail +the HTTP request with 503 and retain their existing consumer behavior. +Deletion reuses `engine:remote-delete-model` and authenticated ec +`POST /v1/models/delete`, ending in engine-manager's identity-checked +`delete_model`. Unrecorded files are not deletable; a running/busy llama.cpp +engine refuses deletion with its stop-engine reason. + + **The consolidated discovery daemon.** Promoted from a plain node-info browser: it is now the single place this host advertises itself on and discovers the LAN through. Every other service registers its port with this daemon instead of advertising its own mDNS service, and consumers query/subscribe to it instead of browsing themselves. Concretely, the daemon: diff --git a/services/nvpair-node-scanner/cache_inventory_test.go b/services/nvpair-node-scanner/cache_inventory_test.go new file mode 100644 index 000000000..a32792b35 --- /dev/null +++ b/services/nvpair-node-scanner/cache_inventory_test.go @@ -0,0 +1,88 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "os" + + "nvpair-shared/engineinventorytest" + "nvpair-shared/noderec" + "testing" +) + +func TestPinnedScannerCacheErrorPreservesServedInventory(t *testing.T) { + var enginePort int + var manifest string + var stop func() + d, srv, _ := secureInventoryFixture(t, "uuid-peer", func(dir string) { enginePort, manifest, stop = engineinventorytest.Start(t, dir) }) + original := srv.Config.Handler + srv.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rec := httptest.NewRecorder() + original.ServeHTTP(rec, r) + if r.URL.Path != "/v1/node-info" || rec.Code != http.StatusOK { + for key, values := range rec.Header() { + w.Header()[key] = values + } + w.WriteHeader(rec.Code) + _, _ = w.Write(rec.Body.Bytes()) + return + } + var info NodeInfoResponse + if err := json.Unmarshal(rec.Body.Bytes(), &info); err != nil { + t.Error(err) + w.WriteHeader(500) + return + } + info.PeerMetadata.Services[noderec.ServiceEngineManager] = enginePort + _ = json.NewEncoder(w).Encode(info) + }) + host, port, _ := net.SplitHostPort(srv.Listener.Addr().String()) + raw := RawNode{ID: "cache-peer", Addresses: []string{host}, TXT: []string{"v=1", "uuid=uuid-peer", "cluster-uuid=uuid-peer", "ip=" + host, "ni=" + port}} + d.onBrowse(DiscoveryEvent{Type: "discovered", Node: raw}) + before, ok := d.dir.get("uuid-peer") + if !ok || !before.ServicePeer() || len(before.Models) != 2 { + t.Fatalf("fixture: %+v", before) + } + if err := os.WriteFile(manifest, []byte("invalid"), 0600); err != nil { + t.Fatal(err) + } + d.enrichModelsAt(&before, host) + if !before.ServicePeer() || len(before.Models) != 2 || len(before.ModelsByEngine["ollama"]) != 1 || len(before.ModelsByEngine["lmstudio"]) != 1 { + t.Fatalf("cache error damaged peer: %+v", before) + } + bytes, _ := json.Marshal(before) + var wire struct { + Errors map[string]string `json:"downloadErrorsByEngine"` + } + _ = json.Unmarshal(bytes, &wire) + if wire.Errors["llamacpp"] != "cache_unavailable" || len(wire.Errors) != 1 { + t.Fatalf("cache error lost: %s", bytes) + } + if !d.refreshNodeModelsCandidates(before.HostUUID, before.IP, []string{host}, enginePort, before.ClusterUUID) { + t.Fatal("cache error alone did not update scanner snapshot") + } + published, _ := d.dir.get(before.HostUUID) + if published.DownloadErrorsByEngine["llamacpp"] != "cache_unavailable" || len(published.Models) != 2 { + t.Fatalf("periodic scanner refresh lost inventory or error: %+v", published) + } + if err := os.Remove(manifest); err != nil { + t.Fatal(err) + } + d.enrichModelsAt(&before, host) + bytes, _ = json.Marshal(before) + wire.Errors = nil + _ = json.Unmarshal(bytes, &wire) + if len(wire.Errors) != 0 { + t.Fatalf("recovered cache retained error: %s", bytes) + } + stop() + d.enrichModelsAt(&before, host) + if len(before.Models) != 2 || !before.ServicePeer() { + t.Fatalf("whole-inventory failure changed scanner preservation: %+v", before) + } +} diff --git a/services/nvpair-node-scanner/connreuse_test.go b/services/nvpair-node-scanner/connreuse_test.go index fab9c6cd9..76c586495 100644 --- a/services/nvpair-node-scanner/connreuse_test.go +++ b/services/nvpair-node-scanner/connreuse_test.go @@ -74,7 +74,7 @@ func TestModelsClient_ReusesPeerConnections(t *testing.T) { // pinned mTLS path instead of the plaintext self-fetch. const rounds = 5 for i := 0; i < rounds; i++ { - models, _, _, ok := d.fetchModels("localhost", port, "uuid-peer") + models, _, _, _, _, ok := d.fetchModels("localhost", port, "uuid-peer") if !ok || len(models) != 1 || models[0] != "m" { t.Fatalf("round %d: ok=%v models=%v", i, ok, models) } diff --git a/services/nvpair-node-scanner/daemon.go b/services/nvpair-node-scanner/daemon.go index c36d84a20..4b0c8a1a5 100644 --- a/services/nvpair-node-scanner/daemon.go +++ b/services/nvpair-node-scanner/daemon.go @@ -125,7 +125,9 @@ type daemon struct { // lastLoadedByEngine caches the per-engine loaded (in-memory) set from the // same /v1/models fetch, so a transient miss reuses it rather than blanking // a remote node's loaded state. Keyed by hostUuid. - lastLoadedByEngine map[string]map[string][]string + lastLoadedByEngine map[string]map[string][]string + lastDownloadedByEngine map[string]map[string][]noderec.DownloadedModel + lastDownloadErrorsByEngine map[string]map[string]string // nodeInfoDown holds the nodes whose last node-info enrichment failed, so // the outage is reported when it starts and when it ends rather than once // per sweep. Guarded by infoMu with the caches it explains: the reason a @@ -819,6 +821,14 @@ func (d *daemon) dropSelf(oldHostUUID, newHostUUID string) { delete(d.lastInfoAt, oldHostUUID) delete(d.lastModels, oldHostUUID) delete(d.lastModelsByEngine, oldHostUUID) + if d.lastDownloadedByEngine != nil { + d.lastDownloadedByEngine[newHostUUID] = d.lastDownloadedByEngine[oldHostUUID] + } + delete(d.lastDownloadedByEngine, oldHostUUID) + if d.lastDownloadErrorsByEngine != nil { + d.lastDownloadErrorsByEngine[newHostUUID] = d.lastDownloadErrorsByEngine[oldHostUUID] + } + delete(d.lastDownloadErrorsByEngine, oldHostUUID) delete(d.lastLoadedByEngine, oldHostUUID) d.infoMu.Unlock() if existed { @@ -1431,17 +1441,19 @@ func (d *daemon) enrichModelsCandidates(node *noderec.DirectoryNode, hosts []str return } ask := func(host string) (modelInventory, bool) { - models, byEngine, loadedByEngine, fetched := d.fetchModels(host, em.Port, node.ClusterUUID) + models, byEngine, loadedByEngine, downloadedByEngine, downloadErrorsByEngine, fetched := d.fetchModels(host, em.Port, node.ClusterUUID) if !fetched { return modelInventory{}, false } - return modelInventory{models: models, byEngine: byEngine, loadedByEngine: loadedByEngine}, true + return modelInventory{models: models, byEngine: byEngine, loadedByEngine: loadedByEngine, downloadedByEngine: downloadedByEngine, downloadErrorsByEngine: downloadErrorsByEngine}, true } key := hostKey{hostUUID: node.HostUUID, service: noderec.ServiceEngineManager} if !d.localNode(node.HostUUID) { _, inventory, ok := askRemembered(&d.enrichHosts, key, hosts, ask) if ok && d.currentPeer(*node) { node.Models, node.ModelsByEngine, node.LoadedByEngine = inventory.models, inventory.byEngine, inventory.loadedByEngine + node.DownloadedByEngine = inventory.downloadedByEngine + node.DownloadErrorsByEngine = inventory.downloadErrorsByEngine } return } @@ -1458,22 +1470,36 @@ func (d *daemon) enrichModelsCandidates(node *noderec.DirectoryNode, hosts []str } d.lastModels[node.HostUUID] = inventory.models d.lastModelsByEngine[node.HostUUID] = inventory.byEngine + if d.lastDownloadedByEngine == nil { + d.lastDownloadedByEngine = make(map[string]map[string][]noderec.DownloadedModel) + } + d.lastDownloadedByEngine[node.HostUUID] = inventory.downloadedByEngine + if d.lastDownloadErrorsByEngine == nil { + d.lastDownloadErrorsByEngine = make(map[string]map[string]string) + } + d.lastDownloadErrorsByEngine[node.HostUUID] = inventory.downloadErrorsByEngine d.lastLoadedByEngine[node.HostUUID] = inventory.loadedByEngine d.infoMu.Unlock() node.Models = inventory.models node.ModelsByEngine = inventory.byEngine node.LoadedByEngine = inventory.loadedByEngine + node.DownloadedByEngine = inventory.downloadedByEngine + node.DownloadErrorsByEngine = inventory.downloadErrorsByEngine return } d.infoMu.Lock() cached, had := d.lastModels[node.HostUUID] cachedByEngine := d.lastModelsByEngine[node.HostUUID] cachedLoaded := d.lastLoadedByEngine[node.HostUUID] + cachedDownloaded := d.lastDownloadedByEngine[node.HostUUID] + cachedDownloadErrors := d.lastDownloadErrorsByEngine[node.HostUUID] d.infoMu.Unlock() if had { node.Models = cached node.ModelsByEngine = cachedByEngine node.LoadedByEngine = cachedLoaded + node.DownloadedByEngine = cachedDownloaded + node.DownloadErrorsByEngine = cachedDownloadErrors } } @@ -1491,41 +1517,45 @@ func (d *daemon) enrichModelsCandidates(node *noderec.DirectoryNode, hosts []str // dialed over loopback (both callers force that for self), which stays plain and // therefore keeps working when this node belongs to no cluster — a standalone // machine still has to show its own models. -func (d *daemon) fetchModels(ip string, port int, clusterUUID string) ([]string, map[string][]string, map[string][]string, bool) { +func (d *daemon) fetchModels(ip string, port int, clusterUUID string) ([]string, map[string][]string, map[string][]string, map[string][]noderec.DownloadedModel, map[string]string, bool) { client, scheme, ok := d.modelsClient(ip, clusterUUID) if !ok { slog.Debug("model enrichment skipped: peer is not a pinned cluster member", "ip", ip, "clusterUuid", clusterUUID) - return nil, nil, nil, false + return nil, nil, nil, nil, nil, false } url := scheme + "://" + net.JoinHostPort(ip, strconv.Itoa(port)) + "/v1/models" if scheme == "https" { var body struct { - Models []string `json:"models"` - ModelsByEngine map[string][]string `json:"modelsByEngine"` - LoadedByEngine map[string][]string `json:"loadedByEngine"` + Models []string `json:"models"` + ModelsByEngine map[string][]string `json:"modelsByEngine"` + LoadedByEngine map[string][]string `json:"loadedByEngine"` + DownloadedByEngine map[string][]noderec.DownloadedModel `json:"downloadedByEngine"` + DownloadErrorsByEngine map[string]string `json:"downloadErrorsByEngine"` } _, _, ok := clustertrust.ReadPeerInventory(context.Background(), d.mesh, client, clusterUUID, url, "", modelsResponseLimit, &body) - return body.Models, body.ModelsByEngine, body.LoadedByEngine, ok + return body.Models, body.ModelsByEngine, body.LoadedByEngine, body.DownloadedByEngine, body.DownloadErrorsByEngine, ok } resp, err := client.Get(url) if err != nil { slog.Debug("model enrichment failed", "ip", ip, "url", url, "err", err) - return nil, nil, nil, false + return nil, nil, nil, nil, nil, false } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, nil, nil, false + return nil, nil, nil, nil, nil, false } var body struct { - Models []string `json:"models"` - ModelsByEngine map[string][]string `json:"modelsByEngine"` - LoadedByEngine map[string][]string `json:"loadedByEngine"` + Models []string `json:"models"` + ModelsByEngine map[string][]string `json:"modelsByEngine"` + LoadedByEngine map[string][]string `json:"loadedByEngine"` + DownloadedByEngine map[string][]noderec.DownloadedModel `json:"downloadedByEngine"` + DownloadErrorsByEngine map[string]string `json:"downloadErrorsByEngine"` } if err := json.NewDecoder(io.LimitReader(resp.Body, modelsResponseLimit)).Decode(&body); err != nil { - return nil, nil, nil, false + return nil, nil, nil, nil, nil, false } - return body.Models, body.ModelsByEngine, body.LoadedByEngine, true + return body.Models, body.ModelsByEngine, body.LoadedByEngine, body.DownloadedByEngine, body.DownloadErrorsByEngine, true } // modelsClient picks the transport for a model fetch: the plain client over @@ -2052,11 +2082,11 @@ func (d *daemon) refreshNodeModelsCandidates(hostUUID, guardIP string, hosts []s hosts = []string{loopbackHost} } ask := func(host string) (modelInventory, bool) { - models, byEngine, loadedByEngine, fetched := d.fetchModels(host, emPort, clusterUUID) + models, byEngine, loadedByEngine, downloadedByEngine, downloadErrorsByEngine, fetched := d.fetchModels(host, emPort, clusterUUID) if !fetched { return modelInventory{}, false } - return modelInventory{models: models, byEngine: byEngine, loadedByEngine: loadedByEngine}, true + return modelInventory{models: models, byEngine: byEngine, loadedByEngine: loadedByEngine, downloadedByEngine: downloadedByEngine, downloadErrorsByEngine: downloadErrorsByEngine}, true } key := hostKey{hostUUID: hostUUID, service: noderec.ServiceEngineManager} _, inventory, fetched := askRemembered(&d.enrichHosts, key, hosts, ask) @@ -2092,10 +2122,18 @@ func (d *daemon) refreshNodeModelsCandidates(hostUUID, guardIP string, hosts []s } d.lastModels[hostUUID] = models d.lastModelsByEngine[hostUUID] = byEngine + if d.lastDownloadedByEngine == nil { + d.lastDownloadedByEngine = make(map[string]map[string][]noderec.DownloadedModel) + } + d.lastDownloadedByEngine[hostUUID] = inventory.downloadedByEngine + if d.lastDownloadErrorsByEngine == nil { + d.lastDownloadErrorsByEngine = make(map[string]map[string]string) + } + d.lastDownloadErrorsByEngine[hostUUID] = inventory.downloadErrorsByEngine d.lastLoadedByEngine[hostUUID] = loadedByEngine d.infoMu.Unlock() - node, changed, valid := d.dir.applyModels(hostUUID, guardIP, emPort, models, byEngine, loadedByEngine) + node, changed, valid := d.dir.applyModels(hostUUID, guardIP, emPort, models, byEngine, loadedByEngine, inventory.downloadedByEngine, inventory.downloadErrorsByEngine) if !valid { return false } @@ -2237,6 +2275,8 @@ func (d *daemon) forget(hostUUID string) { delete(d.lastInfoAt, hostUUID) delete(d.lastModels, hostUUID) delete(d.lastModelsByEngine, hostUUID) + delete(d.lastDownloadedByEngine, hostUUID) + delete(d.lastDownloadErrorsByEngine, hostUUID) delete(d.lastLoadedByEngine, hostUUID) delete(d.nodeInfoDown, hostUUID) d.infoMu.Unlock() diff --git a/services/nvpair-node-scanner/directory.go b/services/nvpair-node-scanner/directory.go index a9fb206ea..ecb21f802 100644 --- a/services/nvpair-node-scanner/directory.go +++ b/services/nvpair-node-scanner/directory.go @@ -191,7 +191,7 @@ func (d *directory) snapshot(filter noderec.ServiceKey) []noderec.DirectoryNode // Returns the (possibly updated) node, whether the inventory changed, and // whether the guarded apply was valid. ok == false means the result is stale // (node gone or re-addressed) and the caller must not cache or emit it. -func (d *directory) applyModels(hostUUID, ip string, emPort int, models []string, byEngine, loadedByEngine map[string][]string) (node noderec.DirectoryNode, changed, ok bool) { +func (d *directory) applyModels(hostUUID, ip string, emPort int, models []string, byEngine, loadedByEngine map[string][]string, downloadedByEngine map[string][]noderec.DownloadedModel, downloadErrorsByEngine map[string]string) (node noderec.DirectoryNode, changed, ok bool) { d.mu.Lock() defer d.mu.Unlock() n, present := d.nodes[hostUUID] @@ -202,12 +202,14 @@ func (d *directory) applyModels(hostUUID, ip string, emPort int, models []string if !has || n.IP != ip || em.Port != emPort { return n, false, false } - if sameStringSet(n.Models, models) && sameByEngine(n.ModelsByEngine, byEngine) && sameByEngine(n.LoadedByEngine, loadedByEngine) { + if sameStringSet(n.Models, models) && sameByEngine(n.ModelsByEngine, byEngine) && sameByEngine(n.LoadedByEngine, loadedByEngine) && reflect.DeepEqual(n.DownloadedByEngine, downloadedByEngine) && reflect.DeepEqual(n.DownloadErrorsByEngine, downloadErrorsByEngine) { return n, false, true } n.Models = models n.ModelsByEngine = byEngine n.LoadedByEngine = loadedByEngine + n.DownloadedByEngine = downloadedByEngine + n.DownloadErrorsByEngine = downloadErrorsByEngine d.nodes[hostUUID] = n return n, true, true } diff --git a/services/nvpair-node-scanner/enrich_hosts.go b/services/nvpair-node-scanner/enrich_hosts.go index 7997a192f..5c795f46c 100644 --- a/services/nvpair-node-scanner/enrich_hosts.go +++ b/services/nvpair-node-scanner/enrich_hosts.go @@ -13,9 +13,11 @@ import ( // modelInventory is the three-part answer a node's engine manager gives, carried // as one value so the concurrent ask can return it. type modelInventory struct { - models []string - byEngine map[string][]string - loadedByEngine map[string][]string + models []string + byEngine map[string][]string + loadedByEngine map[string][]string + downloadedByEngine map[string][]noderec.DownloadedModel + downloadErrorsByEngine map[string]string } // hostKey identifies one node's endpoint for one service. Node-info and the engine diff --git a/services/nvpair-node-scanner/peer_inventory.go b/services/nvpair-node-scanner/peer_inventory.go index f02345492..6a8ab1d2e 100644 --- a/services/nvpair-node-scanner/peer_inventory.go +++ b/services/nvpair-node-scanner/peer_inventory.go @@ -105,6 +105,8 @@ func (d *daemon) publishVerified(node noderec.DirectoryNode, candidateKey string if prior, exists := d.dir.get(node.HostUUID); exists && prior.PeerRecordID == node.PeerRecordID { if node.Models == nil { node.Models, node.ModelsByEngine, node.LoadedByEngine = prior.Models, prior.ModelsByEngine, prior.LoadedByEngine + node.DownloadedByEngine = prior.DownloadedByEngine + node.DownloadErrorsByEngine = prior.DownloadErrorsByEngine } if node.EngineNative == nil { node.EngineNative = prior.EngineNative @@ -137,5 +139,7 @@ func withdrawPeer(node *noderec.DirectoryNode) { node.IdentityVerification = "unverified" node.GPUs, node.CPU, node.Memory = nil, nil, nil node.Models, node.ModelsByEngine, node.LoadedByEngine = nil, nil, nil + node.DownloadedByEngine = nil + node.DownloadErrorsByEngine = nil node.EngineNative = nil } diff --git a/services/nvpair-node-scanner/refresh_test.go b/services/nvpair-node-scanner/refresh_test.go index 0ae19c00e..26e9e9c43 100644 --- a/services/nvpair-node-scanner/refresh_test.go +++ b/services/nvpair-node-scanner/refresh_test.go @@ -440,7 +440,7 @@ func TestApplyModelsGuard(t *testing.T) { d := newDirectory() // Removed mid-sweep: node absent -> not resurrected. - if _, changed, ok := d.applyModels("ghost", "127.0.0.1", 14322, []string{"a"}, nil, nil); ok || changed { + if _, changed, ok := d.applyModels("ghost", "127.0.0.1", 14322, []string{"a"}, nil, nil, nil, nil); ok || changed { t.Errorf("applyModels on an absent node = (changed %v, ok %v), want (false, false)", changed, ok) } if _, present := d.get("ghost"); present { @@ -455,11 +455,11 @@ func TestApplyModelsGuard(t *testing.T) { Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceEngineManager: {Port: 14322}}, Models: []string{"old"}, }) - if _, changed, ok := d.applyModels("peer-E", "127.0.0.1", 14322, []string{"new"}, nil, nil); ok || changed { + if _, changed, ok := d.applyModels("peer-E", "127.0.0.1", 14322, []string{"new"}, nil, nil, nil, nil); ok || changed { t.Errorf("applyModels with a stale IP = (changed %v, ok %v), want (false, false)", changed, ok) } // em port changed -> also discarded. - if _, changed, ok := d.applyModels("peer-E", "10.0.0.9", 99999, []string{"new"}, nil, nil); ok || changed { + if _, changed, ok := d.applyModels("peer-E", "10.0.0.9", 99999, []string{"new"}, nil, nil, nil, nil); ok || changed { t.Errorf("applyModels with a stale em port = (changed %v, ok %v), want (false, false)", changed, ok) } got, _ := d.get("peer-E") diff --git a/services/nvpair-node-scanner/verified_inventory_test.go b/services/nvpair-node-scanner/verified_inventory_test.go index b8d702e81..6165dfb74 100644 --- a/services/nvpair-node-scanner/verified_inventory_test.go +++ b/services/nvpair-node-scanner/verified_inventory_test.go @@ -29,7 +29,7 @@ func TestClaimedPinnedPrincipalCannotPublishServiceInventory(t *testing.T) { d := newMembershipTestDaemon(dir) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/models" { - _ = json.NewEncoder(w).Encode(map[string][]string{"models": {"forged-model"}}) + _ = json.NewEncoder(w).Encode(map[string]any{"models": []string{"forged-model"}, "downloadedByEngine": map[string][]noderec.DownloadedModel{"llamacpp": {{ID: "forged", File: "forged.gguf", Size: 24, Owned: true}}}}) return } _, portString, _ := net.SplitHostPort(r.Host) @@ -51,7 +51,7 @@ func TestClaimedPinnedPrincipalCannotPublishServiceInventory(t *testing.T) { t.Fatal(err) } d.onBrowse(DiscoveryEvent{Type: "discovered", Node: RawNode{ID: "forged-peer", Addresses: []string{host}, TXT: []string{"v=1", "uuid=" + peer, "cluster-uuid=" + peer, "ip=" + host, "ni=" + port, "em=" + port, "ol=11434"}}}) - if n, ok := d.dir.get(peer); ok && (n.Trusted || len(n.Models) > 0 || len(n.ModelsByEngine) > 0) { + if n, ok := d.dir.get(peer); ok && (n.Trusted || len(n.Models) > 0 || len(n.ModelsByEngine) > 0 || len(n.DownloadedByEngine) > 0) { t.Errorf("unauthenticated claim reached trusted/routing inventory: %+v", n) } p, _ := strconv.Atoi(port) @@ -62,7 +62,7 @@ func TestClaimedPinnedPrincipalCannotPublishServiceInventory(t *testing.T) { // secureInventoryFixture uses real mutually pinned cluster leaves and listeners; // no synthetic TLS state or mock transport can accidentally confer provenance. -func secureInventoryFixture(t *testing.T, principal string) (*daemon, *httptest.Server, *atomic.Bool) { +func secureInventoryFixture(t *testing.T, principal string, configure ...func(string)) (*daemon, *httptest.Server, *atomic.Bool) { t.Helper() selfDir, peerDir := t.TempDir(), t.TempDir() clustertrusttest.WriteKeypair(t, selfDir, "uuid-self") @@ -71,6 +71,9 @@ func secureInventoryFixture(t *testing.T, principal string) (*daemon, *httptest. clustertrusttest.WriteAdmission(t, peerDir, "cluster-test", 1) writePinFromCert(t, selfDir, principal, filepath.Join(peerDir, "node.crt")) writePinFromCert(t, peerDir, "uuid-self", filepath.Join(selfDir, "node.crt")) + for _, setup := range configure { + setup(peerDir) + } mesh := clustertrust.Open(peerDir) d := newSelfTestDaemon("uuid-self", "192.0.2.1") d.mesh = clustertrust.Open(selfDir) @@ -83,7 +86,7 @@ func secureInventoryFixture(t *testing.T, principal string) (*daemon, *httptest. return } if r.URL.Path == "/v1/models" { - _ = json.NewEncoder(w).Encode(map[string][]string{"models": {"authenticated-model"}}) + _ = json.NewEncoder(w).Encode(map[string]any{"models": []string{"authenticated-model"}, "downloadedByEngine": map[string][]noderec.DownloadedModel{"llamacpp": {{ID: "peer-model", File: "peer.gguf", Size: 24, SHA256: "verified", Owned: true}}}}) return } if r.URL.Path == "/v1/engine-stats" { @@ -131,7 +134,7 @@ func TestPublicPreviewHasIndependentKeyAndCannotRemoveVerifiedPeer(t *testing.T) d.onBrowse(DiscoveryEvent{Type: "discovered", Node: hint}) key := candidateRecordKey(hint) preview, exists := d.dir.get(key) - if !exists || preview.RecordKey() != key || preview.IdentityVerification != "unverified" || preview.Trusted { + if !exists || preview.RecordKey() != key || preview.IdentityVerification != "unverified" || preview.Trusted || len(preview.DownloadedByEngine) != 0 { t.Fatalf("preview not independently keyed: %+v", preview) } if after, _ := d.dir.get("uuid-peer"); after.PeerRecordID != before.PeerRecordID || !after.ServicePeer() { @@ -156,6 +159,9 @@ func TestPinnedInventoryPromotesOnlyAuthenticatedMetadata(t *testing.T) { if !ok || !n.ServicePeer() || n.PeerRecordID == "" || n.Name != "authenticated-name" || n.Services[noderec.ServiceOllama].Port != 11434 || len(n.Models) != 1 || n.Models[0] != "authenticated-model" { t.Fatalf("authenticated projection = %+v", n) } + if rows := n.DownloadedByEngine["llamacpp"]; len(rows) != 1 || rows[0].ID != "peer-model" || !rows[0].Owned { + t.Fatalf("missing authenticated cache: %+v", n.DownloadedByEngine) + } var out bytes.Buffer d.codec = NewCodec(&out) if !d.refreshNodeTelemetryCandidates(context.Background(), n.HostUUID, []string{host}, port) { @@ -177,7 +183,7 @@ func TestPinnedInventoryPromotesOnlyAuthenticatedMetadata(t *testing.T) { clustertrusttest.WriteAdmission(t, d.clusterDir, "cluster-test", 2) d.reloadTrust() n, _ = d.dir.get("uuid-peer") - if n.Trusted || n.ServicePeer() || len(n.Models) > 0 || len(n.GPUs) > 0 { + if n.Trusted || n.ServicePeer() || len(n.Models) > 0 || len(n.GPUs) > 0 || len(n.DownloadedByEngine) > 0 { t.Fatalf("admission change retained inventory: %+v", n) } } diff --git a/services/nvpair-tui/README.md b/services/nvpair-tui/README.md index db07413f8..a0d7d59ba 100644 --- a/services/nvpair-tui/README.md +++ b/services/nvpair-tui/README.md @@ -28,9 +28,10 @@ only the transfer's terminal update confirms stop. A transport error allows a retry without claiming success. Broker epoch changes, engine-manager restart or PID replacement, peer stream loss, and loss from verified inventory retire the old operation IDs as interrupted: “Download interrupted — retry to resume”. -Late old updates cannot retire a replacement. Confirmed cancellation says partial data is kept -so the download can resume. PAIR removes no cache files. -Ollama and LM Studio have adapters; oMLX's supported manifest has no pull action. +Late old updates cannot retire a replacement. For Ollama and LM Studio, confirmed cancellation says partial data is kept +so the download can resume; PAIR removes no shared engine-cache files. +llama.cpp instead cleans up only its own identity-matched transfer temporary +file and does not promise resume. Ollama, LM Studio and llama.cpp have adapters; oMLX's supported manifest has no pull action. Pulls wait up to 32 min and cancel acceptance 15 s, outside the broker's 31 min and 10 s bounds respectively. See the [engine-manager lifecycle](../nvpair-engine-manager/README.md#model-download-cancellation). @@ -99,6 +100,8 @@ listing but never pull/delete, and its server port is not editable. `n` cycles This node, trusted discovered peers, and cluster members with a host UUID. Peer status uses `engine:remote-get-installed {node:}`, refreshed on selection and after remote actions. Transport/trust errors remain visible. +Changing targets clears pending model-list state; late replies from the previous +target cannot replace the current inventory or leave it loading. Local `i` sends `engine:install {engine}` without `start`; press `s` after installation to start it. Peer `i` installs and starts (`start:true`). Only the existing `engine:remote-start`, `engine:remote-stop`, @@ -129,8 +132,38 @@ PAIR-owned archive installation, keeping the previous tree on failure and preserving the sibling model cache. The engine list reports llama.cpp lifecycle state with a note: its macOS arm64 build is not notarized, like PAIR itself. The Proxies view includes the dedicated llama.cpp worker on default port 1237, -and Overview includes it in the 13-worker registry. Model hub/download/delete -controls remain deferred. +and Overview includes it in the 13-worker registry. + +### llama.cpp model hub + +On a supported local engine, `m` lists the installed PAIR cache even while +stopped. `b` switches between installed models and the Go-owned catalog, sorted +by publisher/model ID; `/` searches (up to 100 characters, 50 results). `p` +downloads the selected catalog model, or prompts for an ID outside the catalog. +Existing progress and `c` cancellation apply; service errors remain visible. +`d` deletes a PAIR-owned model only while llama.cpp is stopped; unmanaged files +are marked and cannot be deleted. + +In the installed list, `a` chooses a startup model; `0` chooses **No model +(router mode)**. Both read the current settings, preview the cache-relative +`--model` change and confirm before Apply, warning when a restart is required. +The model name is its catalog ID, never the inherited launch descriptor. +PAIR downloads llama.cpp and models at runtime, not as bundled software. +Only macOS arm64 and single-file Q4_K_M GGUF catalog downloads are supported. + +On verified peers, `m` lists installed llama.cpp cache files from discovery's +`downloadedByEngine`, including catalog IDs/file names, sizes and ownership. +The additive `downloadErrorsByEngine.llamacpp: "cache_unavailable"` diagnostic +shows “Couldn't read llama.cpp's downloaded models” in llama.cpp's installed +models pane only; other engines' served models and liveness are unchanged. +A successful cache read clears the diagnostic. +Unverified previews receive no cache inventory. `d` deletes PAIR-owned files +through the existing authenticated remote method; running llama.cpp must be +stopped first and its refusal reason remains visible. Catalog downloads and +cancellation retain the existing remote methods. The startup picker remains local; +no local inventory is substituted for a peer. +The existing unreleased TUI 1.4.0 minor bump already covers these additive +features relative to v0.9.0's TUI 1.3.0. All destructive confirmations (including `u`, cluster remove/leave and manual remove) use the shared default-cancel prompt: only `y` dispatches; `n`, Esc and diff --git a/services/nvpair-tui/ui/downloads.go b/services/nvpair-tui/ui/downloads.go index d34c2884d..4d12172e9 100644 --- a/services/nvpair-tui/ui/downloads.go +++ b/services/nvpair-tui/ui/downloads.go @@ -72,8 +72,11 @@ func (v *enginesView) applyDownloadProgress(p downloadProgress) tea.Cmd { v.status = fmt.Sprintf("pull %s %s: %s", p.Engine, p.Model, status) if p.Stage == "interrupted" { v.status = "Download interrupted — retry to resume" + if p.Engine == "llamacpp" { + v.status = "Download interrupted — retry the download" + } } - if p.Stage == "cancelled" { + if p.Stage == "cancelled" && p.Engine != "llamacpp" { v.status += ". Partial data is kept so the download can resume." } if p.Stage == "error" && p.Message != "" { @@ -94,8 +97,12 @@ func (v *enginesView) applyDownloadProgress(p downloadProgress) tea.Cmd { func (v *enginesView) interruptDownloads(node *string) { for i := len(v.downloads) - 1; i >= 0; i-- { if node == nil || v.downloads[i].node == *node { + engine := v.downloads[i].engine v.retireDownload(i) v.status = "Download interrupted — retry to resume" + if engine == "llamacpp" { + v.status = "Download interrupted — retry the download" + } } } } diff --git a/services/nvpair-tui/ui/engines.go b/services/nvpair-tui/ui/engines.go index 3c004e8ef..f5c40ba14 100644 --- a/services/nvpair-tui/ui/engines.go +++ b/services/nvpair-tui/ui/engines.go @@ -162,6 +162,10 @@ func (v *enginesView) SetSize(w, h int) { func (v *enginesView) Update(msg tea.Msg) tea.Cmd { switch msg := msg.(type) { case epochChangedMsg: + v.models.selectionGeneration++ + if v.selectedEngine() == "llamacpp" { + v.confirm = nil + } v.interruptDownloads(nil) v.downloadWorker = nil return nil @@ -211,6 +215,19 @@ func (v *enginesView) Update(msg tea.Msg) tea.Cmd { } return nil + case llamaModelsMsg: + if msg.node != v.target || msg.generation != v.models.generation || v.selectedEngine() != "llamacpp" { + return nil + } + v.models.loading = false + v.models.err, v.models.catalog, v.models.llama = msg.err, msg.catalog, msg.models + v.refreshModels() + return nil + case llamaSettingsMsg: + return v.previewLlamaStartup(msg) + case llamaPreviewMsg: + return v.confirmLlamaStartup(msg) + case modelsLoadedMsg: if msg.node == v.target && msg.generation == v.models.generation { v.models.err = msg.err @@ -298,6 +315,12 @@ func (v *enginesView) Update(msg tea.Msg) tea.Cmd { } switch msg.Msg.Method { case "engine:models-changed": + if v.models.open && v.selectedEngine() == "llamacpp" { + if v.models.catalog { + return nil + } + return v.loadModelsCmd() + } var p struct { Models modelSnapshot `json:"models"` } @@ -338,7 +361,7 @@ func (v *enginesView) Update(msg tea.Msg) tea.Cmd { } func (v *enginesView) CapturingInput() bool { - return v.pulling || v.editingPort || v.confirm.open() + return v.pulling || v.editingPort || v.models.searching || v.confirm.open() } func (v *enginesView) handleKey(msg tea.KeyMsg) tea.Cmd { @@ -348,6 +371,22 @@ func (v *enginesView) handleKey(msg tea.KeyMsg) tea.Cmd { if v.confirm.open() { return v.confirm.Update(msg) } + if v.models.searching { + switch msg.String() { + case "enter": + v.models.searching = false + v.input.Blur() + v.models.query = v.input.Value() + return v.loadLlamaModelsCmd() + case "esc": + v.models.searching = false + v.input.Blur() + return nil + } + var cmd tea.Cmd + v.input, cmd = v.input.Update(msg) + return cmd + } if v.editingPort { switch msg.String() { case "enter": @@ -374,6 +413,37 @@ func (v *enginesView) handleKey(msg tea.KeyMsg) tea.Cmd { v.input, cmd = v.input.Update(msg) return cmd } + if v.selectedEngine() == "llamacpp" { + switch { + case key.Matches(msg, engCatalogKey): + if !v.byName["llamacpp"].Capabilities.PullModel { + return nil + } + v.models.open, v.models.catalog = true, !v.models.catalog + v.models.err, v.models.query = nil, "" + v.models.generation++ + v.models.loading = false + v.models.llama = nil + if !v.models.catalog && v.target != "" { + v.refreshModels() + return nil + } + return v.loadLlamaModelsCmd() + case key.Matches(msg, engCatalogSearchKey): + if !v.models.catalog || !v.byName["llamacpp"].Capabilities.PullModel { + return nil + } + v.models.searching = true + v.input.Placeholder = "Public catalog query (up to 100 characters)" + v.input.SetValue(v.models.query) + v.input.Focus() + return textinput.Blink + case key.Matches(msg, engSelectModelKey): + return v.selectLlamaStartup(false) + case key.Matches(msg, engRouterKey): + return v.selectLlamaStartup(true) + } + } if key.Matches(msg, engTargetKey) { return v.cycleTarget() } @@ -413,6 +483,15 @@ func (v *enginesView) handleKey(msg tea.KeyMsg) tea.Cmd { if engine == "" || !v.byName[engine].Capabilities.PullModel { return nil } + if engine == "llamacpp" && v.models.open && v.models.catalog { + entry, ok := v.selectedLlamaEntry() + if !ok { + return nil + } + v.pullEngine, v.pullNode = engine, v.target + v.input.SetValue(entry.ID) + return v.submitPull() + } v.pullEngine = engine v.pullNode = v.target v.pulling = true @@ -624,7 +703,7 @@ func (v *enginesView) View() string { out = v.modelsView() } if v.selectedEngine() == "llamacpp" { - out += "\n" + footerStyle.Render("llama.cpp is not notarized, like PAIR itself.") + out += "\n" + footerStyle.Render("llama.cpp: macOS arm64 only; not notarized, like PAIR. Single-file Q4_K_M GGUF catalog.") } if v.confirm.open() { out += "\n" + v.confirm.View() @@ -632,6 +711,9 @@ func (v *enginesView) View() string { if downloads := v.downloadsView(); downloads != "" { out += "\n" + downloads } + if v.models.searching { + out += "\nsearch catalog: " + v.input.View() + } if v.pulling { out += "\npull model: " + v.input.View() } @@ -677,6 +759,20 @@ func (v *enginesView) Help() []key.Binding { bindings = append(bindings, engUpdateKey) } } + if v.selectedEngine() == "llamacpp" { + if e.Capabilities.PullModel { + bindings = append(bindings, engCatalogKey) + if v.models.catalog { + bindings = append(bindings, engCatalogSearchKey) + } + } + if v.target == "" && e.Capabilities.ListModels { + bindings = append(bindings, engRouterKey) + if v.models.open && !v.models.catalog { + bindings = append(bindings, engSelectModelKey) + } + } + } if e.Capabilities.PullModel { bindings = append(bindings, engPullKey) } @@ -713,6 +809,12 @@ func (v *enginesView) cycleTarget() tea.Cmd { } } v.target = next + v.models.loading = false + v.models.searching = false + v.models.query = "" + v.models.catalog = false + v.models.llama = nil + v.models.selectionGeneration++ v.models.generation++ v.models.inventory = modelSnapshot{} v.models.err = nil diff --git a/services/nvpair-tui/ui/llamacpp_models.go b/services/nvpair-tui/ui/llamacpp_models.go new file mode 100644 index 000000000..f43f92490 --- /dev/null +++ b/services/nvpair-tui/ui/llamacpp_models.go @@ -0,0 +1,214 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package ui + +import ( + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "time" + + settings "nvpair-shared/enginesettings" + "nvpair-tui/rpc" + + "github.com/charmbracelet/bubbles/key" + "github.com/charmbracelet/bubbles/table" + tea "github.com/charmbracelet/bubbletea" +) + +var ( + engCatalogKey = key.NewBinding(key.WithKeys("b"), key.WithHelp("b", "catalog / installed")) + engCatalogSearchKey = key.NewBinding(key.WithKeys("/"), key.WithHelp("/", "search catalog")) + engSelectModelKey = key.NewBinding(key.WithKeys("a"), key.WithHelp("a", "startup model")) + engRouterKey = key.NewBinding(key.WithKeys("0"), key.WithHelp("0", "router mode")) + llamaModelOption = regexp.MustCompile(`(?:^|\s)(?:--model|-m)(?:=|\s+)(?:"[^"]*"|'[^']*'|[^\s]+)`) +) + +type llamaModel struct { + ID string `json:"id"` + File string `json:"file"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + Owned bool `json:"owned"` +} + +type llamaModelsMsg struct { + node string + generation uint64 + catalog bool + models []llamaModel + err error +} + +type llamaSettingsMsg struct { + node, file, label string + generation uint64 + snapshot settings.Snapshot + err error +} + +type llamaPreviewMsg struct { + node, label string + generation uint64 + request settings.Request + preview settings.Preview + err error +} + +func (v *enginesView) loadLlamaModelsCmd() tea.Cmd { + v.models.generation++ + v.models.loading = true + v.models.llama = nil + v.models.err = nil + v.refreshModels() + node, generation, catalog := v.target, v.models.generation, v.models.catalog + method := "engine:action" + params := map[string]any{"engine": "llamacpp", "action": "list_downloaded"} + timeout := 15 * time.Second + if catalog { + method, params, timeout = "engine:model-catalog", map[string]any{"query": v.models.query}, 30*time.Second + } + return callWithTimeout(v.client, method, params, timeout, func(msg *rpc.Message, err error) tea.Msg { + var result struct { + Models []llamaModel `json:"models"` + } + if err == nil { + err = decodeParams(msg.Result, &result) + } + return llamaModelsMsg{node: node, generation: generation, catalog: catalog, models: result.Models, err: err} + }) +} + +func (v *enginesView) refreshLlamaRows() { + selected := v.selectedModel() + models := append([]llamaModel(nil), v.models.llama...) + sort.Slice(models, func(i, j int) bool { return models[i].ID < models[j].ID }) + rows := make([]table.Row, 0, len(models)) + for _, model := range models { + state := "installed" + if v.models.catalog { + state = strings.SplitN(model.ID, "/", 2)[0] + } + if !v.models.catalog && model.SHA256 == "" { + state = "unmanaged" + } + rows = append(rows, table.Row{model.ID, state}) + } + v.models.table.SetRows(rows) + cursor := 0 + for i, row := range rows { + if row[0] == selected { + cursor = i + break + } + } + v.models.table.SetCursor(cursor) +} + +func (v *enginesView) selectedLlamaEntry() (llamaModel, bool) { + row := v.models.table.SelectedRow() + if len(row) == 0 { + return llamaModel{}, false + } + for _, model := range v.models.llama { + if model.ID == row[0] { + return model, true + } + } + return llamaModel{}, false +} + +func (v *enginesView) selectLlamaStartup(router bool) tea.Cmd { + if v.selectedEngine() != "llamacpp" || v.target != "" || !v.byName["llamacpp"].Capabilities.ListModels { + return nil + } + file, label := "", "No model (router mode)" + if !router { + if !v.models.open { + return nil + } + model, ok := v.selectedLlamaEntry() + if !ok || v.models.catalog { + return nil + } + file, label = model.File, model.ID + } + v.models.selectionGeneration++ + generation, node := v.models.selectionGeneration, v.target + v.status = "Reading startup settings…" + return call(v.client, "engine:get-settings", map[string]string{"engine": "llamacpp"}, func(msg *rpc.Message, err error) tea.Msg { + var snapshot settings.Snapshot + if err == nil { + err = decodeParams(msg.Result, &snapshot) + } + return llamaSettingsMsg{node: node, generation: generation, file: file, label: label, snapshot: snapshot, err: err} + }) +} + +func (v *enginesView) previewLlamaStartup(msg llamaSettingsMsg) tea.Cmd { + if msg.node != v.target || msg.generation != v.models.selectionGeneration || v.selectedEngine() != "llamacpp" { + return nil + } + if msg.err != nil { + v.status = msg.err.Error() + return nil + } + if !msg.snapshot.Editable || msg.snapshot.Format != "pair-arguments-v1" { + v.status = "Startup settings are not editable on this device." + return nil + } + config := msg.snapshot.Settings + config.LaunchText = strings.TrimSpace(llamaModelOption.ReplaceAllString(config.LaunchText, "")) + if msg.file != "" { + config.LaunchText = strings.TrimSpace(config.LaunchText + " --model " + strconv.Quote(msg.file)) + } + request := settings.Request{Engine: "llamacpp", ExpectedRevision: msg.snapshot.Revision, Settings: config} + return call(v.client, "engine:preview-settings", request, func(reply *rpc.Message, err error) tea.Msg { + var preview settings.Preview + if err == nil { + err = decodeParams(reply.Result, &preview) + } + return llamaPreviewMsg{node: msg.node, label: msg.label, generation: msg.generation, request: request, preview: preview, err: err} + }) +} + +func (v *enginesView) confirmLlamaStartup(msg llamaPreviewMsg) tea.Cmd { + if msg.node != v.target || msg.generation != v.models.selectionGeneration || v.selectedEngine() != "llamacpp" { + return nil + } + err := msg.err + if err == nil { + err = msg.preview.ValidationError() + } + if err == nil && msg.preview.Revision != msg.request.ExpectedRevision { + err = fmt.Errorf("Settings changed elsewhere; select the startup model again.") + } + if err != nil { + v.status = err.Error() + return nil + } + request := msg.request + request.Settings, request.RequestID = msg.preview.Settings, downloadID() + detail := msg.label + if msg.preview.Restart { + detail += "\nThis restarts llama.cpp and interrupts inference." + } + cmd := callWithTimeout(v.client, "engine:apply-settings", request, 14*time.Minute, func(reply *rpc.Message, err error) tea.Msg { + if err == nil { + var receipt struct { + Phase string `json:"phase"` + } + err = decodeParams(reply.Result, &receipt) + if err == nil && receipt.Phase != "succeeded" { + err = fmt.Errorf("Settings %s; reload settings before retrying.", receipt.Phase) + } + } + return engineOpMsg{node: msg.node, what: "startup model", engine: "llamacpp", err: err} + }) + v.status = "" + v.confirm = newConfirm("Choose startup model?", detail, cmd) + return nil +} diff --git a/services/nvpair-tui/ui/llamacpp_models_test.go b/services/nvpair-tui/ui/llamacpp_models_test.go new file mode 100644 index 000000000..e245c6b42 --- /dev/null +++ b/services/nvpair-tui/ui/llamacpp_models_test.go @@ -0,0 +1,187 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package ui + +import ( + "encoding/json" + "fmt" + tea "github.com/charmbracelet/bubbletea" + settings "nvpair-shared/enginesettings" + "strings" + "testing" +) + +func TestLlamaCatalogBrowse(t *testing.T) { + v, c := engineWire(t) + modelEngine(v, "llamacpp") + cmd := v.Update(engineKey("b")) + if cmd == nil { + t.Fatal("catalog browse must request the Go catalog") + } + v.Update(engineRequest(t, cmd, c, "engine:model-catalog", `{"query":""}`, `{"models":[{"id":"unsloth/Demo:Q4_K_M","file":"a.gguf"},{"id":"ggml-org/Small:Q4_K_M","file":"b.gguf"}]}`)) + if !strings.Contains(v.View(), "ggml-org/Small:Q4_K_M") || !strings.Contains(v.View(), "Catalog") { + t.Fatalf("missing catalog: %s", v.View()) + } +} + +func TestLlamaDeleteRequiresStoppedEngine(t *testing.T) { + v := newEnginesView(nil) + v.SetSize(100, 20) + modelEngine(v, "llamacpp") + e := v.byName["llamacpp"] + e.Running = true + v.merge(e) + v.models.open = true + v.models.llama = []llamaModel{{ID: "ggml-org/Demo:Q4_K_M", File: "cache.gguf", SHA256: "verified"}} + v.refreshModels() + v.Update(engineKey("d")) + if v.confirm.open() { + t.Fatal("running llama.cpp must not offer deletion") + } + if !strings.Contains(v.status, "Stop llama.cpp") { + t.Fatalf("missing reason: %s", v.status) + } +} + +func TestLlamaInstalledSelectionAndDelete(t *testing.T) { + v, c := engineWire(t) + modelEngine(v, "llamacpp") + v.Update(engineRequest(t, v.Update(engineKey("m")), c, "engine:action", `{"engine":"llamacpp","action":"list_downloaded"}`, `{"models":[{"id":"ggml-org/Demo:Q4_K_M","file":"cache.gguf","sha256":"verified","size":42}]}`)) + if !strings.Contains(v.View(), "ggml-org/Demo:Q4_K_M") { + t.Fatal(v.View()) + } + msg := engineRequest(t, v.Update(engineKey("a")), c, "engine:get-settings", `{"engine":"llamacpp"}`, `{"editable":true,"format":"pair-arguments-v1","revision":2,"settings":{"serverPort":8082,"proxyPort":1237,"launchText":"--threads 4"}}`) + preview := v.Update(msg) + v.Update(engineRequest(t, preview, c, "engine:preview-settings", `{"engine":"llamacpp","expectedRevision":2,"settings":{"serverPort":8082,"proxyPort":1237,"launchText":"--threads 4 --model \"cache.gguf\""}}`, `{"revision":2,"settings":{"serverPort":8082,"proxyPort":1237,"launchText":"--threads 4 --model \"cache.gguf\""}}`)) + if !v.confirm.open() || strings.Contains(v.View(), "/dev/fd/") { + t.Fatal(v.View()) + } + done := make(chan tea.Msg, 1) + cmd := v.Update(engineKey("y")) + go func() { done <- cmd() }() + q, err := c.Read() + if err != nil { + t.Fatal(err) + } + var request settings.Request + if err := json.Unmarshal(q.Params, &request); err != nil { + t.Fatal(err) + } + if q.Method != "engine:apply-settings" || request.RequestID == "" || request.ExpectedRevision != 2 || request.Settings.LaunchText != `--threads 4 --model "cache.gguf"` { + t.Fatalf("wrong settings request: %+v", request) + } + if err := c.Respond(q.ID, json.RawMessage(`{"phase":"succeeded","revision":3}`)); err != nil { + t.Fatal(err) + } + v.Update(<-done) + if !strings.Contains(v.status, "startup model llamacpp ok") { + t.Fatal(v.status) + } + v.Update(engineKey("d")) + if !v.confirm.open() { + t.Fatal("owned stopped model should offer delete") + } + v.Update(engineKey("n")) + v.models.llama[0].SHA256 = "" + v.Update(engineKey("d")) + if v.confirm.open() { + t.Fatal("unmanaged model cannot be deleted") + } +} + +func TestLlamaCatalogPullUsesExistingTracker(t *testing.T) { + v, c := engineWire(t) + modelEngine(v, "llamacpp") + v.Update(engineRequest(t, v.Update(engineKey("b")), c, "engine:model-catalog", `{"query":""}`, `{"models":[{"id":"ggml-org/Demo:Q4_K_M","file":"cache.gguf"}]}`)) + cmd := v.Update(engineKey("p")) + if len(v.downloads) != 1 { + t.Fatal("missing operation tracking") + } + id := v.downloads[0].id + params := fmt.Sprintf(`{"engine":"llamacpp","action":"pull_model","params":{"name":"ggml-org/Demo:Q4_K_M","model":"ggml-org/Demo:Q4_K_M"},"opId":%q}`, id) + engineRequest(t, cmd, c, "engine:action", params, `{"status":"ok"}`) + v.applyDownloadProgress(downloadProgress{OpID: id, Engine: "llamacpp", Model: "ggml-org/Demo:Q4_K_M", Stage: "downloading", Percent: 42, CanCancel: true}) + if !strings.Contains(v.View(), "42%") { + t.Fatal(v.View()) + } + v.downloads[0].status = "canceling" + v.applyDownloadProgress(downloadProgress{OpID: id, Engine: "llamacpp", Model: "ggml-org/Demo:Q4_K_M", Stage: "downloading", Percent: 50, CanCancel: true}) + if !strings.Contains(v.View(), "canceling") { + t.Fatal(v.View()) + } + v.applyDownloadProgress(downloadProgress{OpID: id, Engine: "llamacpp", Model: "ggml-org/Demo:Q4_K_M", Stage: "cancelled"}) + if len(v.downloads) != 0 || strings.Contains(v.status, "Partial data") { + t.Fatal(v.status) + } +} + +func TestLlamaInterruptedDownloadDoesNotPromiseResume(t *testing.T) { + v := newEnginesView(nil) + v.downloads = []engineDownload{{id: "old", engine: "llamacpp", model: "demo", status: "canceling"}} + v.Update(epochChangedMsg{}) + if len(v.downloads) != 0 || v.status != "Download interrupted — retry the download" { + t.Fatal(v.status) + } +} + +func TestLlamaCatalogSearchErrorsAndStaleResults(t *testing.T) { + v, c := engineWire(t) + modelEngine(v, "llamacpp") + v.Update(engineRequest(t, v.Update(engineKey("b")), c, "engine:model-catalog", `{"query":""}`, `{"models":[]}`)) + v.Update(engineKey("/")) + v.input.SetValue(strings.Repeat("x", 101)) + cmd := v.Update(tea.KeyMsg{Type: tea.KeyEnter}) + msg := engineErrorReply(t, cmd, c, "engine:model-catalog", fmt.Sprintf(`{"query":%q}`, strings.Repeat("x", 101)), "query must not exceed 100 characters") + v.Update(msg) + if !strings.Contains(v.View(), "query must not exceed 100 characters") { + t.Fatal(v.View()) + } + v.Update(llamaModelsMsg{generation: v.models.generation - 1, catalog: true, models: []llamaModel{{ID: "stale"}}}) + if strings.Contains(v.View(), "stale") { + t.Fatal("stale catalog replaced latest result") + } +} + +func TestLlamaCatalogSwitchClearsStaleActionRows(t *testing.T) { + v, c := engineWire(t) + modelEngine(v, "llamacpp") + v.Update(engineRequest(t, v.Update(engineKey("b")), c, "engine:model-catalog", `{"query":""}`, `{"models":[{"id":"ggml-org/Demo:Q4_K_M","file":"catalog.gguf","sha256":"metadata"}]}`)) + v.Update(engineKey("b")) // cache request has not completed + if v.selectedModel() != "" { + t.Fatal("catalog result remained selectable as installed inventory") + } + if cmd := v.Update(engineKey("a")); cmd != nil { + t.Fatal("stale catalog row became a startup choice") + } + v.Update(engineKey("d")) + if v.confirm.open() { + t.Fatal("stale catalog row became deletable") + } +} + +func TestLlamaCatalogIgnoresServedInventoryPush(t *testing.T) { + v, c := engineWire(t) + modelEngine(v, "llamacpp") + v.Update(engineRequest(t, v.Update(engineKey("b")), c, "engine:model-catalog", `{"query":""}`, `{"models":[{"id":"ggml-org/Demo:Q4_K_M","file":"catalog.gguf"}]}`)) + generation := v.models.generation + modelPush(v, `{"modelsByEngine":{"llamacpp":["served"]}}`) + if v.models.generation != generation { + t.Fatal("served inventory push invalidated the independent catalog") + } +} + +func TestLlamaRouterSelectionStartsWithSettingsRead(t *testing.T) { + v, c := engineWire(t) + modelEngine(v, "llamacpp") + cmd := v.Update(engineKey("0")) + if cmd == nil { + t.Fatal("router selection must use existing settings protocol") + } + msg := engineRequest(t, cmd, c, "engine:get-settings", `{"engine":"llamacpp"}`, `{"engine":"llamacpp","editable":true,"format":"pair-arguments-v1","revision":2,"settings":{"serverPort":8082,"proxyPort":1237,"launchText":"--model old.gguf --threads 4"}}`) + preview := v.Update(msg) + v.Update(engineRequest(t, preview, c, "engine:preview-settings", `{"engine":"llamacpp","expectedRevision":2,"settings":{"serverPort":8082,"proxyPort":1237,"launchText":"--threads 4"}}`, `{"revision":2,"restart":true,"settings":{"serverPort":8082,"proxyPort":1237,"launchText":"--threads 4"}}`)) + if !v.confirm.open() || !strings.Contains(v.View(), "router mode") { + t.Fatalf("missing selection confirmation: %s", v.View()) + } +} diff --git a/services/nvpair-tui/ui/llamacpp_peer_test.go b/services/nvpair-tui/ui/llamacpp_peer_test.go new file mode 100644 index 000000000..29174a0d9 --- /dev/null +++ b/services/nvpair-tui/ui/llamacpp_peer_test.go @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package ui + +import ( + "encoding/json" + "nvpair-tui/rpc" + "strings" + "testing" +) + +const llamaPeerInventory = `[{"hostUuid":"peer-uuid","name":"GPU-box","trusted":true,"identityVerification":"verified","peerRecordId":"proof","modelsByEngine":{"llamacpp":["served-only"]},"downloadedByEngine":{"llamacpp":[{"id":"ggml-org/Demo:Q4_K_M","file":"cache.gguf","size":42,"sha256":"verified","owned":true},{"id":"manual.gguf","file":"manual.gguf","size":24,"owned":false}]}}]` + +func TestLlamaPeerCacheErrorIsEngineScoped(t *testing.T) { + v, c := engineWire(t) + switchEnginePeer(t, v, c) + modelEngine(v, "llamacpp") + v.table.SetCursor(len(v.order) - 1) + payload := strings.Replace(llamaPeerInventory, `"downloadedByEngine":`, `"downloadErrorsByEngine":{"llamacpp":"cache_unavailable"},"downloadedByEngine":`, 1) + payload = strings.Replace(payload, `"modelsByEngine":{`, `"modelsByEngine":{"ollama":["ollama-model"],"lmstudio":["lmstudio-model"],`, 1) + v.Update(NotificationMsg{Msg: &rpc.Message{Method: "discovery:nodes-changed", Params: json.RawMessage(payload)}}) + v.Update(engineKey("m")) + if !strings.Contains(v.modelsView(), "Couldn't read llama.cpp's downloaded models") { + t.Fatalf("missing cache error: %s", v.modelsView()) + } + for _, name := range []string{"ollama", "lmstudio"} { + modelEngine(v, name) + for i, engine := range v.order { + if engine == name { + v.table.SetCursor(i) + } + } + v.refreshModels() + if view := v.modelsView(); strings.Contains(view, "Couldn't read llama.cpp") || !strings.Contains(view, name+"-model") { + t.Fatalf("cache error damaged %s view: %s", name, view) + } + } + modelEngine(v, "llamacpp") + for i, engine := range v.order { + if engine == "llamacpp" { + v.table.SetCursor(i) + } + } + v.Update(NotificationMsg{Msg: &rpc.Message{Method: "discovery:nodes-changed", Params: json.RawMessage(llamaPeerInventory)}}) + if strings.Contains(v.modelsView(), "Couldn't read llama.cpp") { + t.Fatal("recovered cache retained error") + } +} + +func TestLlamaPeerInstalledListAndDelete(t *testing.T) { + v, c := engineWire(t) + switchEnginePeer(t, v, c) + modelEngine(v, "llamacpp") + v.table.SetCursor(len(v.order) - 1) + v.Update(NotificationMsg{Msg: &rpc.Message{Method: "discovery:nodes-changed", Params: json.RawMessage(llamaPeerInventory)}}) + if cmd := v.Update(engineKey("m")); cmd != nil { + t.Fatal("peer list must use discovery, not local cache RPC") + } + if view := v.View(); !strings.Contains(view, "ggml-org/Demo:Q4_K_M") || !strings.Contains(view, "manual.gguf") || strings.Contains(view, "served-only") || strings.Contains(view, "unavailable; served") { + t.Fatalf("wrong peer list: %s", view) + } + v.Update(engineKey("d")) + if !v.confirm.open() { + t.Fatal("owned stopped peer cache row cannot be deleted") + } + reply := engineRequest(t, v.Update(engineKey("y")), c, "engine:remote-delete-model", `{"node":"peer-uuid","engine":"llamacpp","model":"ggml-org/Demo:Q4_K_M"}`, `{"ok":true}`) + refresh := v.Update(reply) + v.Update(engineRequest(t, refresh, c, "discovery:get-nodes", `null`, `{"nodes":[{"hostUuid":"peer-uuid","trusted":true,"identityVerification":"verified","peerRecordId":"proof","downloadedByEngine":{"llamacpp":[]}}]}`)) + if !strings.Contains(v.View(), "No models") { + t.Fatalf("delete retained rows: %s", v.View()) + } +} + +func TestLlamaPeerDeleteRunningUnownedAndPreviewGuards(t *testing.T) { + v, c := engineWire(t) + switchEnginePeer(t, v, c) + modelEngine(v, "llamacpp") + v.table.SetCursor(len(v.order) - 1) + v.Update(NotificationMsg{Msg: &rpc.Message{Method: "discovery:nodes-changed", Params: json.RawMessage(llamaPeerInventory)}}) + v.Update(engineKey("m")) + e := v.byName["llamacpp"] + e.Running = true + v.merge(e) + v.Update(engineKey("d")) + if v.confirm.open() || !strings.Contains(v.status, "Stop llama.cpp") { + t.Fatalf("missing running refusal: %s", v.status) + } + e.Running = false + v.merge(e) + v.models.table.SetCursor(1) + v.Update(engineKey("d")) + if v.confirm.open() || !strings.Contains(v.status, "PAIR-owned") { + t.Fatalf("unrecorded file deletable: %s", v.status) + } + unverified := strings.ReplaceAll(llamaPeerInventory, `"identityVerification":"verified"`, `"identityVerification":"unverified"`) + v.Update(NotificationMsg{Msg: &rpc.Message{Method: "discovery:nodes-changed", Params: json.RawMessage(unverified)}}) + if strings.Contains(v.modelsView(), "ggml-org/Demo") || strings.Contains(v.modelsView(), "manual.gguf") { + t.Fatal("preview acquired cache inventory") + } +} diff --git a/services/nvpair-tui/ui/llamacpp_target_test.go b/services/nvpair-tui/ui/llamacpp_target_test.go new file mode 100644 index 000000000..bcd9c399b --- /dev/null +++ b/services/nvpair-tui/ui/llamacpp_target_test.go @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package ui + +import ( + "strings" + "testing" +) + +func TestLlamaPendingModelsDoNotLeakAcrossTargets(t *testing.T) { + for _, key := range []string{"m", "b"} { + t.Run(key, func(t *testing.T) { + v, c := engineWire(t) + modelEngine(v, "llamacpp") + pending := v.Update(engineKey(key)) + if !strings.Contains(v.View(), "Loading models…") { + t.Fatal("request did not enter loading state") + } + next := v.Update(engineKey("n")) + v.Update(engineRequest(t, next, c, "engine:remote-get-installed", `{"node":"peer-uuid"}`, `{"engines":[{"engine":"llamacpp","capabilities":{"list_models":true,"pull_model":true}}]}`)) + v.Update(discoverySnapshotMsg{nodes: []availableNode{{HostUUID: "peer-uuid", Trusted: true, IdentityVerification: "verified", PeerRecordID: "proof", DownloadedByEngine: map[string][]llamaModel{"llamacpp": {{ID: "peer-model", File: "peer.gguf", Size: 24, Owned: true, SHA256: "verified"}}}}}}) + v.Update(engineKey("m")) + method, params := "engine:action", `{"engine":"llamacpp","action":"list_downloaded"}` + if key == "b" { + method, params = "engine:model-catalog", `{"query":""}` + } + v.Update(engineRequest(t, pending, c, method, params, `{"models":[{"id":"stale-local-model","file":"stale.gguf"}]}`)) + if strings.Contains(v.View(), "Loading models…") || !strings.Contains(v.View(), "peer-model") || strings.Contains(v.View(), "stale-local-model") { + t.Fatalf("pending local request contaminated peer pane: %s", v.View()) + } + }) + } +} + +func TestLlamaPendingCatalogDoesNotLeakIntoReopenedPeerInventory(t *testing.T) { + v, c := engineWire(t) + next := v.Update(engineKey("n")) + v.Update(engineRequest(t, next, c, "engine:remote-get-installed", `{"node":"peer-uuid"}`, `{"engines":[{"engine":"llamacpp","capabilities":{"list_models":true,"pull_model":true}}]}`)) + v.Update(discoverySnapshotMsg{nodes: []availableNode{{HostUUID: "peer-uuid", Trusted: true, IdentityVerification: "verified", PeerRecordID: "proof", DownloadedByEngine: map[string][]llamaModel{"llamacpp": {{ID: "peer-model", File: "peer.gguf", Size: 24, Owned: true, SHA256: "verified"}}}}}}) + pending := v.Update(engineKey("b")) + v.Update(engineKey("m")) // close the pending catalog + v.Update(engineKey("m")) // reopen cached peer inventory + v.Update(engineRequest(t, pending, c, "engine:model-catalog", `{"query":""}`, `{"models":[{"id":"stale-catalog-model","file":"stale.gguf"}]}`)) + if strings.Contains(v.View(), "Loading models…") || !strings.Contains(v.View(), "peer-model") || strings.Contains(v.View(), "stale-catalog-model") || strings.Contains(v.View(), "Catalog —") { + t.Fatalf("pending catalog contaminated reopened peer pane: %s", v.View()) + } +} diff --git a/services/nvpair-tui/ui/models.go b/services/nvpair-tui/ui/models.go index a274fe66a..ce3bf624a 100644 --- a/services/nvpair-tui/ui/models.go +++ b/services/nvpair-tui/ui/models.go @@ -17,16 +17,24 @@ import ( // modelSnapshot preserves map-key presence: a missing engine is unavailable, // whereas a present empty list is a known empty inventory. type modelSnapshot struct { - ModelsByEngine map[string][]string `json:"modelsByEngine"` - LoadedByEngine map[string][]string `json:"loadedByEngine"` + ModelsByEngine map[string][]string `json:"modelsByEngine"` + LoadedByEngine map[string][]string `json:"loadedByEngine"` + DownloadedByEngine map[string][]llamaModel `json:"downloadedByEngine"` + DownloadErrorsByEngine map[string]string `json:"downloadErrorsByEngine,omitempty"` } type modelsPane struct { - open bool - table table.Model - inventory modelSnapshot - generation uint64 - err error + loading bool + catalog bool + searching bool + query string + llama []llamaModel + selectionGeneration uint64 + open bool + table table.Model + inventory modelSnapshot + generation uint64 + err error } type modelsLoadedMsg struct { @@ -49,7 +57,7 @@ var ( ) func (v *enginesView) sizeModels(w, h int) { - v.models.table.SetColumns([]table.Column{{Title: "MODEL", Width: clampWidth(w-12, 10)}, {Title: "STATE", Width: 8}}) + v.models.table.SetColumns([]table.Column{{Title: "MODEL", Width: clampWidth(w-16, 10)}, {Title: "STATE", Width: 12}}) v.models.table.SetWidth(w) v.models.table.SetHeight(clampWidth(h-3, 1)) } @@ -58,6 +66,9 @@ func (v *enginesView) sizeModels(w, h int) { // discovery; never invent a remote list-models call or query the local engine // for facts about a peer. func (v *enginesView) loadModelsCmd() tea.Cmd { + if v.selectedEngine() == "llamacpp" && (v.target == "" || v.models.catalog) { + return v.loadLlamaModelsCmd() + } v.models.generation++ node, generation := v.target, v.models.generation method := "engine:models" @@ -88,13 +99,21 @@ func (v *enginesView) loadModelsCmd() tea.Cmd { func peerInventory(node string, nodes []availableNode) modelSnapshot { for _, n := range nodes { if n.HostUUID == node { - return modelSnapshot{ModelsByEngine: n.ModelsByEngine, LoadedByEngine: n.LoadedByEngine} + inventory := modelSnapshot{ModelsByEngine: n.ModelsByEngine, LoadedByEngine: n.LoadedByEngine} + if n.IdentityVerification == "verified" && n.Trusted && n.PeerRecordID != "" { + inventory.DownloadedByEngine = n.DownloadedByEngine + inventory.DownloadErrorsByEngine = n.DownloadErrorsByEngine + } + return inventory } } return modelSnapshot{} } func (v *enginesView) peerModelsChanged() { + if v.selectedEngine() == "llamacpp" && v.models.catalog { + return + } if v.target != "" { v.models.generation++ // invalidate any getter issued before this full push v.models.err = nil @@ -104,6 +123,18 @@ func (v *enginesView) peerModelsChanged() { } func (v *enginesView) refreshModels() { + if v.selectedEngine() == "llamacpp" { + if v.target != "" && !v.models.catalog { + v.models.llama = append([]llamaModel(nil), v.models.inventory.DownloadedByEngine["llamacpp"]...) + for i := range v.models.llama { + if !v.models.llama[i].Owned { + v.models.llama[i].SHA256 = "" + } + } + } + v.refreshLlamaRows() + return + } selected := v.selectedModel() rows := []table.Row{} for _, name := range v.models.inventory.ModelsByEngine[v.selectedEngine()] { @@ -139,11 +170,17 @@ func (v *enginesView) selectedModel() string { } func (v *enginesView) toggleModels() tea.Cmd { + // A closed pane abandons its request; reopening peer inventory may not + // issue another RPC, so neither loading nor a late catalog may survive. + v.models.generation++ + v.models.loading = false + v.models.llama = nil v.models.open = !v.models.open if !v.models.open { return nil } v.models.err = nil + v.models.catalog = false if v.target != "" { v.models.inventory = peerInventory(v.target, v.discovered) v.refreshModels() @@ -155,6 +192,24 @@ func (v *enginesView) toggleModels() tea.Cmd { func (v *enginesView) modelsView() string { out := "Models — " + v.selectedEngine() + "\n" + if v.selectedEngine() == "llamacpp" && v.byName["llamacpp"].Capabilities.ListModels { + if v.models.catalog { + out = "Catalog — llama.cpp (single-file Q4_K_M; search: 100 characters / 50 results)\n" + } + if v.models.loading { + return out + footerStyle.Render("Loading models…") + } + if v.models.err != nil { + return out + statusErrStyle.Render(v.models.err.Error()) + } + if !v.models.catalog && v.models.inventory.DownloadErrorsByEngine["llamacpp"] == "cache_unavailable" { + return out + footerStyle.Render("Couldn't read llama.cpp's downloaded models") + } + if len(v.models.llama) == 0 { + return out + footerStyle.Render("No models") + } + return out + v.models.table.View() + } if !v.byName[v.selectedEngine()].Capabilities.ListModels { return out + footerStyle.Render("Model list unavailable (engine stopped or unsupported)") } @@ -177,6 +232,17 @@ func (v *enginesView) confirmDeleteModel() tea.Cmd { if !v.models.open || model == "" || !e.Capabilities.DeleteModel { return nil } + if engine == "llamacpp" { + if e.Running { + v.status = "Stop llama.cpp before deleting a model." + return nil + } + entry, ok := v.selectedLlamaEntry() + if v.models.catalog || !ok || entry.SHA256 == "" { + v.status = "Only PAIR-owned installed models can be deleted." + return nil + } + } label := "This node" if node != "" { label = nodeLabel(node, v.identity, v.discovered, v.members) diff --git a/services/nvpair-tui/ui/nodes.go b/services/nvpair-tui/ui/nodes.go index a42f9cd0d..a3aeadba0 100644 --- a/services/nvpair-tui/ui/nodes.go +++ b/services/nvpair-tui/ui/nodes.go @@ -30,10 +30,12 @@ type availableNode struct { // to some cluster (advertises a cluster-uuid), whether or not we're paired // with it. Either one makes it non-invitable — an already-clustered peer // rejects a fresh pairing (it must leave/be removed first). - Trusted bool `json:"trusted"` - Clustered bool `json:"clustered"` - ModelsByEngine map[string][]string `json:"modelsByEngine"` - LoadedByEngine map[string][]string `json:"loadedByEngine"` + Trusted bool `json:"trusted"` + Clustered bool `json:"clustered"` + ModelsByEngine map[string][]string `json:"modelsByEngine"` + LoadedByEngine map[string][]string `json:"loadedByEngine"` + DownloadedByEngine map[string][]llamaModel `json:"downloadedByEngine"` + DownloadErrorsByEngine map[string]string `json:"downloadErrorsByEngine,omitempty"` } // key is the node's stable identity (hostUuid) for cluster actions; the name diff --git a/services/nvpair-ui-broker/README.md b/services/nvpair-ui-broker/README.md index fdc72c854..e87e24755 100644 --- a/services/nvpair-ui-broker/README.md +++ b/services/nvpair-ui-broker/README.md @@ -5,6 +5,29 @@ SPDX-License-Identifier: Apache-2.0 # nvpair-ui-broker +### Downloaded llama.cpp inventory + +The existing engine-manager `/v1/models` inventory adds `downloadedByEngine`: +`{"llamacpp":[{"id":"catalog ID or file name","file":"cache-relative.gguf","size":24,"sha256":"…","owned":true}]}`. +This cache list is available while stopped and remains separate from served +`models`/`modelsByEngine`, which continue to drive routing. The cache scan is +limited to 4096 entries and uses the existing bounded inventory response. +Scanner enrichment and verified manual peers carry it over pinned cluster mTLS; +the broker preserves it in discovery snapshots. Unverified previews receive none. +A cache-validation/read failure does not fail the served inventory: HTTP remains +200 and `engine:models` still returns all served/loaded engine models. +`downloadErrorsByEngine: {"llamacpp":"cache_unavailable"}` reports that the +llama.cpp cache is unknown (its downloaded list is omitted). This fixed public +code never contains paths or underlying error text. Manual-node updates and +scanner/broker discovery snapshots preserve it separately from engine liveness; +a successful cache read clears it. Genuine whole-inventory failures still fail +the HTTP request with 503 and retain their existing consumer behavior. +Deletion reuses `engine:remote-delete-model` and authenticated ec +`POST /v1/models/delete`, ending in engine-manager's identity-checked +`delete_model`. Unrecorded files are not deletable; a running/busy llama.cpp +engine refuses deletion with its stop-engine reason. + + A Go service that exposes the NVIDIA Personal AI Router API to **UI processes** and other clients (CLIs, dashboards, mobile companions, scripts, tests, etc.). The shipped graphical UI is bundled alongside the backend services and launches diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go index 7635369af..d55f25d9e 100644 --- a/services/nvpair-ui-broker/broker.go +++ b/services/nvpair-ui-broker/broker.go @@ -125,7 +125,9 @@ type AvailableNode struct { // peer's engine-manager /v1/models loadedByEngine field. Lets a per-engine // consumer show which of a remote node's models are loaded. Omitted // when no engine reports loaded state. - LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` + LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` + DownloadedByEngine map[string][]noderec.DownloadedModel `json:"downloadedByEngine,omitempty"` + DownloadErrorsByEngine map[string]string `json:"downloadErrorsByEngine,omitempty"` // EngineNative is the engine's own report of itself: the counters, the // rates and the models it lists as active, as the engine published them. // Enriched by the daemon from the node's engine-manager diff --git a/services/nvpair-ui-broker/discovery.go b/services/nvpair-ui-broker/discovery.go index 299b574a8..0956bc277 100644 --- a/services/nvpair-ui-broker/discovery.go +++ b/services/nvpair-ui-broker/discovery.go @@ -10,6 +10,7 @@ import ( "io" "log/slog" "os/exec" + "reflect" "slices" "sort" "sync" @@ -76,7 +77,9 @@ type EnrichedNode struct { // (normally a subset of ModelsByEngine), enriched by the daemon from the // peer's engine-manager /v1/models loadedByEngine field. Carried through to // AvailableNode so a remote node's cards can reflect loaded state. - LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` + LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` + DownloadedByEngine map[string][]noderec.DownloadedModel `json:"downloadedByEngine,omitempty"` + DownloadErrorsByEngine map[string]string `json:"downloadErrorsByEngine,omitempty"` // EngineNative is the engine's own report of itself: the counters, the // rates and the models it lists as active, as the engine published them. // Enriched by the daemon from the node's engine-manager @@ -112,24 +115,26 @@ func directoryToEnriched(n noderec.DirectoryNode) EnrichedNode { port = ni.Port } return EnrichedNode{ - ID: n.Name, - HostUUID: n.HostUUID, - Host: n.Name, - Port: port, - Addresses: addrs, - TXT: txt, - GPUs: n.GPUs, - CPU: n.CPU, - Memory: n.Memory, - Trusted: n.Trusted, - PeerRecordID: n.PeerRecordID, - IdentityVerification: n.IdentityVerification, - ClusterUUID: n.ClusterUUID, - Clustered: n.Clustered(), - Models: n.Models, - ModelsByEngine: n.ModelsByEngine, - LoadedByEngine: n.LoadedByEngine, - EngineNative: n.EngineNative, + ID: n.Name, + HostUUID: n.HostUUID, + Host: n.Name, + Port: port, + Addresses: addrs, + TXT: txt, + GPUs: n.GPUs, + CPU: n.CPU, + Memory: n.Memory, + Trusted: n.Trusted, + PeerRecordID: n.PeerRecordID, + IdentityVerification: n.IdentityVerification, + ClusterUUID: n.ClusterUUID, + Clustered: n.Clustered(), + Models: n.Models, + ModelsByEngine: n.ModelsByEngine, + LoadedByEngine: n.LoadedByEngine, + DownloadedByEngine: n.DownloadedByEngine, + DownloadErrorsByEngine: n.DownloadErrorsByEngine, + EngineNative: n.EngineNative, } } @@ -352,6 +357,9 @@ func enrichedNodeEqual(a, b EnrichedNode) bool { return false } } + if !reflect.DeepEqual(a.DownloadedByEngine, b.DownloadedByEngine) || !reflect.DeepEqual(a.DownloadErrorsByEngine, b.DownloadErrorsByEngine) { + return false + } if len(a.LoadedByEngine) != len(b.LoadedByEngine) { return false } @@ -413,21 +421,23 @@ func (sn storedNode) toAvailable() AvailableNode { hostUUID = n.storeKey() } return AvailableNode{ - PeerRecordID: n.PeerRecordID, - IdentityVerification: n.IdentityVerification, - ID: n.ID, - Name: n.ID, - HostUUID: hostUUID, - IPAddress: primary, - IPAddresses: candidates, - Port: n.Port, - LastSeen: sn.lastSeen.Unix(), - Trusted: n.Trusted, - Clustered: n.Clustered, - Models: n.Models, - ModelsByEngine: n.ModelsByEngine, - LoadedByEngine: n.LoadedByEngine, - EngineNative: n.EngineNative, + PeerRecordID: n.PeerRecordID, + IdentityVerification: n.IdentityVerification, + ID: n.ID, + Name: n.ID, + HostUUID: hostUUID, + IPAddress: primary, + IPAddresses: candidates, + Port: n.Port, + LastSeen: sn.lastSeen.Unix(), + Trusted: n.Trusted, + Clustered: n.Clustered, + Models: n.Models, + ModelsByEngine: n.ModelsByEngine, + LoadedByEngine: n.LoadedByEngine, + DownloadedByEngine: n.DownloadedByEngine, + DownloadErrorsByEngine: n.DownloadErrorsByEngine, + EngineNative: n.EngineNative, } } @@ -722,6 +732,8 @@ func (s *scannerProcess) handleNotify(method string, params json.RawMessage) { ev.Node.IdentityVerification = "unverified" } ev.Node.Models, ev.Node.ModelsByEngine, ev.Node.LoadedByEngine = nil, nil, nil + ev.Node.DownloadedByEngine = nil + ev.Node.DownloadErrorsByEngine = nil ev.Node.EngineNative = nil } if s.store != nil && method != noderec.NotifyNodeRemoved && ev.Node.ServicePeer() { diff --git a/services/nvpair-ui-broker/downloaded_inventory_test.go b/services/nvpair-ui-broker/downloaded_inventory_test.go new file mode 100644 index 000000000..9b43ddb66 --- /dev/null +++ b/services/nvpair-ui-broker/downloaded_inventory_test.go @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + + "nvpair-shared/clustertrusttest" + "nvpair-shared/noderec" +) + +func TestManualCacheErrorSurvivesDiscoveryAndRelayUntilRecovery(t *testing.T) { + b := inventoryTestBroker(t) + status := manualNodeStatus{ + ID: "manual-peer", Address: "192.0.2.2", HostUUID: "peer", ClusterUUID: "peer", + PeerRecordID: b.mesh.PeerRecordID("peer"), IdentityVerification: "verified", + PeerMetadata: &noderec.PeerMetadataParams{HostUUID: "peer", Name: "peer", Services: map[noderec.ServiceKey]int{noderec.ServiceOllama: 11434}}, + Models: []string{"ollama-model", "lmstudio-model"}, + ModelsByEngine: map[string][]string{"ollama": {"ollama-model"}, "lmstudio": {"lmstudio-model"}}, + DownloadErrorsByEngine: map[string]string{"llamacpp": "cache_unavailable"}, + } + publications := 0 + b.store.onChange = func() { publications++ } + b.upsertManualNode(status) + nodes := b.store.Snapshot() + if len(nodes) != 1 || !nodes[0].Trusted || len(nodes[0].Models) != 2 || nodes[0].DownloadErrorsByEngine["llamacpp"] != "cache_unavailable" { + t.Fatalf("manual cache error damaged discovery: %+v", nodes) + } + relayed := b.relayDir.Snapshot(noderec.ServiceOllama) + if len(relayed) != 1 || len(relayed[0].Models) != 2 || relayed[0].DownloadErrorsByEngine["llamacpp"] != "cache_unavailable" { + t.Fatalf("manual cache error damaged relay: %+v", relayed) + } + before := publications + status.DownloadErrorsByEngine = nil + b.upsertManualNode(status) + nodes = b.store.Snapshot() + relayed = b.relayDir.Snapshot(noderec.ServiceOllama) + if publications != before+1 || len(nodes) != 1 || len(relayed) != 1 || len(nodes[0].Models) != 2 || len(nodes[0].DownloadErrorsByEngine) != 0 || len(relayed[0].DownloadErrorsByEngine) != 0 { + t.Fatalf("cache recovery was not published: nodes=%+v relay=%+v", nodes, relayed) + } + status.DownloadErrorsByEngine = map[string]string{"llamacpp": "cache_unavailable"} + clustertrusttest.RemovePeerPin(t, b.clusterDir, "peer") + if retired := b.normalizeManualStatus(status); len(retired.DownloadErrorsByEngine) != 0 || len(retired.Models) != 0 { + t.Fatalf("retired proof retained inventory diagnostics: %+v", retired) + } +} + +func TestDownloadedInventoryDiscoveryProjection(t *testing.T) { + node := noderec.DirectoryNode{HostUUID: "peer", Name: "peer", DownloadedByEngine: map[string][]noderec.DownloadedModel{"llamacpp": {{ID: "catalog-id", File: "cache.gguf", Size: 24, Owned: true}}}} + enriched := directoryToEnriched(node) + available := (storedNode{scanner: &enriched}).toAvailable() + if rows := available.DownloadedByEngine["llamacpp"]; len(rows) != 1 || rows[0].File != "cache.gguf" || !rows[0].Owned { + t.Fatalf("lost cache metadata: %+v", available) + } + withError := node + withError.DownloadErrorsByEngine = map[string]string{"llamacpp": "cache_unavailable"} + projectedError := directoryToEnriched(withError) + if (storedNode{scanner: &projectedError}).toAvailable().DownloadErrorsByEngine["llamacpp"] != "cache_unavailable" || enrichedNodeEqual(enriched, projectedError) { + t.Fatal("cache error was lost or did not change discovery snapshot") + } + changed := enriched + changed.DownloadedByEngine = map[string][]noderec.DownloadedModel{"llamacpp": {}} + if enrichedNodeEqual(enriched, changed) { + t.Fatal("cache deletion did not change discovery snapshot") + } +} diff --git a/services/nvpair-ui-broker/manualnodes.go b/services/nvpair-ui-broker/manualnodes.go index d7df35d85..c6915661e 100644 --- a/services/nvpair-ui-broker/manualnodes.go +++ b/services/nvpair-ui-broker/manualnodes.go @@ -60,10 +60,12 @@ type manualNodeStatus struct { // machine is also discovered). Empty until the node-info probe succeeds. HostUUID string `json:"hostUuid,omitempty"` // ClusterUUID is the remote's cluster principal. - ClusterUUID string `json:"clusterUuid,omitempty"` - Models []string `json:"models,omitempty"` - ModelsByEngine map[string][]string `json:"modelsByEngine,omitempty"` - LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` + ClusterUUID string `json:"clusterUuid,omitempty"` + Models []string `json:"models,omitempty"` + ModelsByEngine map[string][]string `json:"modelsByEngine,omitempty"` + LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` + DownloadedByEngine map[string][]noderec.DownloadedModel `json:"downloadedByEngine,omitempty"` + DownloadErrorsByEngine map[string]string `json:"downloadErrorsByEngine,omitempty"` // EngineNative is the remote engine's own report of itself, as the // manual-node prober read it over the peer's pinned mTLS path. It is // the same field a discovered node's record carries, so a manual node @@ -501,22 +503,24 @@ func manualToEnriched(s manualNodeStatus) EnrichedNode { modelsByEngine = manualModelsByEngine(s) } en := EnrichedNode{ - ID: s.ID, - HostUUID: hostUUID, - Host: s.Address, - Port: s.NodeInfoPort, - GPUs: s.GPUs, - CPU: s.CPU, - Memory: s.Memory, - Models: models, - ModelsByEngine: modelsByEngine, - LoadedByEngine: s.LoadedByEngine, - EngineNative: s.EngineNative, - Clustered: s.ClusterUUID != "", - PeerRecordID: recordID, - IdentityVerification: verification, - ClusterUUID: s.ClusterUUID, - ManualEngine: verification == "manual", + ID: s.ID, + HostUUID: hostUUID, + Host: s.Address, + Port: s.NodeInfoPort, + GPUs: s.GPUs, + CPU: s.CPU, + Memory: s.Memory, + Models: models, + ModelsByEngine: modelsByEngine, + LoadedByEngine: s.LoadedByEngine, + DownloadedByEngine: s.DownloadedByEngine, + DownloadErrorsByEngine: s.DownloadErrorsByEngine, + EngineNative: s.EngineNative, + Clustered: s.ClusterUUID != "", + PeerRecordID: recordID, + IdentityVerification: verification, + ClusterUUID: s.ClusterUUID, + ManualEngine: verification == "manual", } if s.Address != "" { en.Addresses = []string{s.Address} @@ -557,7 +561,7 @@ func (b *Broker) manualToDirectoryNode(s manualNodeStatus, key string) noderec.D for service, port := range s.PeerMetadata.Services { services[service] = noderec.ServiceStatus{Port: port} } - return noderec.DirectoryNode{HostUUID: s.HostUUID, Name: s.PeerMetadata.Name, IP: s.Address, IPs: []string{s.Address}, ClusterUUID: s.ClusterUUID, Trusted: true, IdentityVerification: "verified", PeerRecordID: s.PeerRecordID, Services: services, GPUs: s.GPUs, CPU: s.CPU, Memory: s.Memory, Models: s.Models, ModelsByEngine: s.ModelsByEngine, LoadedByEngine: s.LoadedByEngine, EngineNative: s.EngineNative, LastSeen: time.Now().Unix()} + return noderec.DirectoryNode{HostUUID: s.HostUUID, Name: s.PeerMetadata.Name, IP: s.Address, IPs: []string{s.Address}, ClusterUUID: s.ClusterUUID, Trusted: true, IdentityVerification: "verified", PeerRecordID: s.PeerRecordID, Services: services, GPUs: s.GPUs, CPU: s.CPU, Memory: s.Memory, Models: s.Models, ModelsByEngine: s.ModelsByEngine, LoadedByEngine: s.LoadedByEngine, DownloadedByEngine: s.DownloadedByEngine, DownloadErrorsByEngine: s.DownloadErrorsByEngine, EngineNative: s.EngineNative, LastSeen: time.Now().Unix()} } hostUUID := s.HostUUID if hostUUID == "" { @@ -653,23 +657,25 @@ func (b *Broker) manualToDirectoryNode(s manualNodeStatus, key string) noderec.D recordID = noderec.ManualRecordID(s.ID) } return noderec.DirectoryNode{ - PeerRecordID: recordID, - IdentityVerification: verification, - HostUUID: hostUUID, - Name: s.ID, - IP: s.Address, - IPs: ips, - ClusterUUID: clusterUUID, - Trusted: trusted, - Services: services, - GPUs: s.GPUs, - CPU: s.CPU, - Memory: s.Memory, - Models: models, - ModelsByEngine: modelsByEngine, - LoadedByEngine: s.LoadedByEngine, - EngineNative: s.EngineNative, - LastSeen: time.Now().Unix(), + PeerRecordID: recordID, + IdentityVerification: verification, + HostUUID: hostUUID, + Name: s.ID, + IP: s.Address, + IPs: ips, + ClusterUUID: clusterUUID, + Trusted: trusted, + Services: services, + GPUs: s.GPUs, + CPU: s.CPU, + Memory: s.Memory, + Models: models, + ModelsByEngine: modelsByEngine, + LoadedByEngine: s.LoadedByEngine, + DownloadedByEngine: s.DownloadedByEngine, + DownloadErrorsByEngine: s.DownloadErrorsByEngine, + EngineNative: s.EngineNative, + LastSeen: time.Now().Unix(), } } diff --git a/services/nvpair-ui-broker/peer_inventory.go b/services/nvpair-ui-broker/peer_inventory.go index d55a244ad..ca4fff9f1 100644 --- a/services/nvpair-ui-broker/peer_inventory.go +++ b/services/nvpair-ui-broker/peer_inventory.go @@ -49,6 +49,8 @@ func (b *Broker) normalizeManualStatus(s manualNodeStatus) manualNodeStatus { s.LlamaCPPUp = false s.Models, s.OllamaModels, s.LMStudioModels, s.OMLXModels, s.LlamaCPPModels = nil, nil, nil, nil, nil s.ModelsByEngine, s.LoadedByEngine = nil, nil + s.DownloadedByEngine = nil + s.DownloadErrorsByEngine = nil s.GPUs, s.CPU, s.Memory = nil, nil, nil s.EngineNative = nil s.TelemetryValid = false diff --git a/services/nvpair-ui-broker/relay/relay.go b/services/nvpair-ui-broker/relay/relay.go index fa6f8b904..6671bfb64 100644 --- a/services/nvpair-ui-broker/relay/relay.go +++ b/services/nvpair-ui-broker/relay/relay.go @@ -286,6 +286,8 @@ func (d *Directory) serviceView(n noderec.DirectoryNode) noderec.DirectoryNode { } n.Services = services n.Models, n.ModelsByEngine, n.LoadedByEngine = nil, nil, nil + n.DownloadedByEngine = nil + n.DownloadErrorsByEngine = nil n.EngineNative = nil return n } diff --git a/services/shared/engineinventorytest/inventory.go b/services/shared/engineinventorytest/inventory.go new file mode 100644 index 000000000..4c6086b11 --- /dev/null +++ b/services/shared/engineinventorytest/inventory.go @@ -0,0 +1,158 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package engineinventorytest runs the real inventory producer against isolated +// fake engines. Only tests import it; no installed engines or user data are used. +package engineinventorytest + +import ( + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +// Start returns the actual engine-manager inventory port, its private cache +// manifest, and an idempotent stop. The caller supplies the peer's pinned identity. +func Start(t *testing.T, clusterDir string) (int, string, func()) { + t.Helper() + root := t.TempDir() + base := root + if runtime.GOOS == "darwin" { + base = filepath.Join(root, "Library", "Application Support") + } + app := filepath.Join(base, "Nvidia Corporation", "Personal AI Router") + manifests := filepath.Join(app, "engines") + if err := os.MkdirAll(manifests, 0700); err != nil { + t.Fatal(err) + } + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + for _, engine := range []string{"ollama", "lmstudio", "llamacpp", "omlx"} { + var actions = map[string]any{} + port := 0 + detect := filepath.Join(root, "not-installed") + if engine == "ollama" || engine == "lmstudio" { + name := engine + "-model" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/ready" { + w.WriteHeader(http.StatusOK) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"models": []map[string]string{{"name": name}}}) + })) + t.Cleanup(srv.Close) + _, p, _ := net.SplitHostPort(srv.Listener.Addr().String()) + port, _ = strconv.Atoi(p) + detect = executable + actions["list_models"] = map[string]any{"http": map[string]string{"method": "GET", "path": "/inventory"}, "result": map[string]string{"array": "models", "field": "name"}} + } + manifest := map[string]any{"engine": engine, "display_name": engine, "manifest_version": 1, "actions": actions, + "platforms": map[string]any{runtime.GOOS + "/" + runtime.GOARCH: map[string]any{"detect": []string{detect}, "detect_requires": nil, "runtime": map[string]any{"mode": "process", "bin": detect, "port": port, "health": nil, "ready": map[string]any{"http": "http://127.0.0.1:{port}/ready", "status": 200, "json_match": nil}}}}} + data, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(manifests, engine+".json"), data, 0600); err != nil { + t.Fatal(err) + } + } + cache := filepath.Join(app, "engine-bin", "llamacpp-models") + if err := os.MkdirAll(cache, 0700); err != nil { + t.Fatal(err) + } + _, source, _, _ := runtime.Caller(0) + binary := filepath.Join(root, "engine-manager") + if runtime.GOOS == "windows" { + binary += ".exe" + } + build := exec.CommandContext(t.Context(), "go", "build", "-o", binary, ".") + build.Dir = filepath.Join(filepath.Dir(source), "..", "..", "nvpair-engine-manager") + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("build inventory producer: %v\n%s", err, output) + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := listener.Addr().(*net.TCPAddr).Port + _ = listener.Close() + cmd := exec.CommandContext(t.Context(), binary, "--http-port", strconv.Itoa(port), "--cluster-dir", clusterDir, "--loaded-poll-interval", "0", "--log-level", "info") + cmd.Env = os.Environ() + for _, key := range []string{"HOME", "XDG_CONFIG_HOME", "APPDATA", "LOCALAPPDATA", "USERPROFILE"} { + for i := len(cmd.Env) - 1; i >= 0; i-- { + if strings.HasPrefix(cmd.Env[i], key+"=") { + cmd.Env = append(cmd.Env[:i], cmd.Env[i+1:]...) + } + } + cmd.Env = append(cmd.Env, key+"="+root) + } + stdin, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + cmd.Stdout = io.Discard + log, err := os.Create(filepath.Join(root, "producer.log")) + if err != nil { + t.Fatal(err) + } + cmd.Stderr = log + if err := cmd.Start(); err != nil { + _ = log.Close() + t.Fatal(err) + } + var once sync.Once + stop := func() { + once.Do(func() { + _ = stdin.Close() + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + select { + case <-done: + case <-time.After(5 * time.Second): + _ = cmd.Process.Kill() + <-done + } + _ = log.Close() + }) + } + t.Cleanup(stop) + client := &http.Client{Timeout: time.Second} + defer client.CloseIdleConnections() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + for { + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://127.0.0.1:"+strconv.Itoa(port)+"/v1/models", nil) + resp, err := client.Do(req) + if err == nil { + var body struct { + Models []string `json:"models"` + } + decodeErr := json.NewDecoder(resp.Body).Decode(&body) + _ = resp.Body.Close() + if resp.StatusCode == http.StatusOK && decodeErr == nil && len(body.Models) == 2 { + break + } + } + select { + case <-ctx.Done(): + bytes, _ := os.ReadFile(filepath.Join(root, "producer.log")) + t.Fatalf("inventory fixture not ready: %v\n%s", ctx.Err(), bytes) + case <-time.After(20 * time.Millisecond): + } + } + return port, filepath.Join(cache, ".pair-models.json"), stop +} diff --git a/services/shared/noderec/downloaded.go b/services/shared/noderec/downloaded.go new file mode 100644 index 000000000..0061adab3 --- /dev/null +++ b/services/shared/noderec/downloaded.go @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package noderec + +// DownloadedModel describes a cache file, not a model currently served for routing. +// Owned means PAIR recorded and verified its file identity. Deletion still checks +// that identity again on the owning node. +type DownloadedModel struct { + ID string `json:"id"` + File string `json:"file"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + Owned bool `json:"owned"` +} diff --git a/services/shared/noderec/noderec.go b/services/shared/noderec/noderec.go index e2baa1e39..d7d7a76a4 100644 --- a/services/shared/noderec/noderec.go +++ b/services/shared/noderec/noderec.go @@ -606,7 +606,9 @@ type DirectoryNode struct { // model cards can reflect loaded state too. An engine key with an // empty list means "running, nothing loaded"; a missing key means the peer // didn't report loaded state for it. Omitted when the peer reports none. - LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` + LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` + DownloadedByEngine map[string][]DownloadedModel `json:"downloadedByEngine,omitempty"` + DownloadErrorsByEngine map[string]string `json:"downloadErrorsByEngine,omitempty"` // EngineNative is the engine's own report of itself: the counters, the // rates and the models it lists as active, as the engine published them. // Enriched from engine-manager's em /v1/engine-stats, over the same