diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc361b208..7273814a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,6 +163,11 @@ jobs: go test -v -count=1 -timeout=15m -json ./... 2>&1 \ | tee "$RUNNER_TEMP/cross-process.jsonl" \ | jq --unbuffered -j 'select(.Output != null) | .Output' + - name: Require llama.cpp broker routing coverage + if: success() + shell: bash + run: | + jq -e 'select(.Action == "pass" and .Test == "TestBrokerLlamaCPPProxyRoutesOpenAI")' "$RUNNER_TEMP/cross-process.jsonl" - name: Upload cross-process diagnostics if: always() uses: actions/upload-artifact@v4.6.2 diff --git a/AGENTS.md b/AGENTS.md index 63db04fc1..d3d2b5488 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,7 @@ change belongs in a Go service instead. service. Operator tools are separate from this supervised service inventory. - The broker supervises the other Go workers: discovery, proxies, engines, cluster, settings, manual nodes, workloads, errors, and the scheduler. -- The broker attempts to start 12 workers, including `omlx-proxy`. Standalone, +- The broker attempts to start 13 workers, including `omlx-proxy` and `llamacpp-proxy`. Standalone, only the scanner is required: missing default auxiliary binaries and initial auxiliary spawn failures are non-fatal. Invalid explicit worker paths are fatal; scanner's initial spawn failure is also fatal. Optional initial spawn @@ -61,9 +61,9 @@ change belongs in a Go service instead. interval. Two consecutive failures report `degraded`; a successful probe restores `running` and clears the failure count. Three failures restart scanner, errors, settings, node-info, workload-manager, manual-nodes, cluster-manager, - and scheduler. Engine-manager and the three proxies never auto-restart on a + and scheduler. Engine-manager and the four proxies never auto-restart on a hang; desktop shows "Not responding — restart manually". -- `workers:get-status` and full-snapshot `workers:update` expose all 12 workers; +- `workers:get-status` and full-snapshot `workers:update` expose all 13 workers; `workers:restart` requests an explicit retry/restart, including after exhaustion. Desktop renders these under Settings → Service → Workers; TUI Overview uses the same registry, with `R` and confirmation to restart. Unix cross-process @@ -102,7 +102,7 @@ Never edit `desktop/docs/services-api.md` by hand. It is generated by ## Services (`services/`) -Fourteen service executables: broker + 12 workers + TUI. Each component is its +Fifteen service executables: broker + 13 workers + TUI. Each component is its own module, with its tests beside its source and a `README.md` describing its JSON-RPC surface. Shared packages live in `shared/`, and `tests/` holds cross-process tests that drive real binaries. diff --git a/SECURITY.md b/SECURITY.md index 71723df5e..beb7cddc7 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -40,7 +40,7 @@ Remove unrelated personal information, prompts, model data, and secrets. - Electron's renderer reaches main through a typed preload bridge. Electron starts only `nvpair-ui-broker`; the terminal client starts its own broker. -- The broker supervises **12 workers**: scanner, node-info, three engine proxies, +- The broker supervises **13 workers**: scanner, node-info, four engine proxies, engine manager, cluster manager, workload manager, errors, node settings, manual nodes, and job scheduler. Scanner startup is required; others are optional. Workers run under the user's operating-system account. diff --git a/desktop/README.md b/desktop/README.md index cb32209b9..a2c7dd7be 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -27,7 +27,7 @@ docs/ Desktop-specific architecture and contract documentation The renderer reaches services only through the preload bridge; it never talks to a Go worker directly. Electron starts `nvpair-ui-broker` and the broker -supervises up to 12 workers. The service bundle contains 14 executables, including +supervises up to 13 workers. The service bundle contains 15 executables, including `omlx-proxy` and the separate, unsupervised `nvpair-tui`. Desktop prevalidates broker, scanner, Ollama proxy, node-info, and node-settings; standalone broker requirements are narrower. diff --git a/desktop/docs/architecture.md b/desktop/docs/architecture.md index 85df05df8..bde0dc836 100644 --- a/desktop/docs/architecture.md +++ b/desktop/docs/architecture.md @@ -45,6 +45,7 @@ The canonical runtime inventory is | `ollama-proxy` | Broker | Ollama-compatible proxy and cluster routing | | `lmstudio-proxy` | Broker, optional | LM Studio OpenAI-compatible proxy | | `omlx-proxy` | Broker, optional | oMLX OpenAI-compatible proxy | +| `llamacpp-proxy` | Broker, optional | llama.cpp OpenAI-compatible proxy | | `nvpair-node-scanner` | Broker | LAN discovery and announcement | | `nvpair-node-info` | Broker | Node metadata and telemetry endpoint | | `nvpair-workload-manager` | Broker, optional | Workload replication | @@ -122,9 +123,9 @@ from worker crash supervision or hang detection. The Go broker separately probes `worker/health` every 30 s (5 s deadline), reports `degraded` after two failures, and restarts the eight eligible workers -after three. Engine-manager and the three proxies never auto-restart on a hang. +after three. Engine-manager and the four proxies never auto-restart on a hang. The bridge recovers `workers:get-status` on each epoch and replaces its cache on -full `workers:update` snapshots. Settings → Service → Workers renders all twelve +full `workers:update` snapshots. Settings → Service → Workers renders all thirteen rows and relays `workers:restart`; ineligible degraded rows say "Not responding — restart manually". Engine-manager and proxy restarts require confirmation because they may interrupt engines or inference. Health, initial-spawn retry, missing diff --git a/desktop/docs/services-api.md b/desktop/docs/services-api.md index 93be826dc..c972f764d 100644 --- a/desktop/docs/services-api.md +++ b/desktop/docs/services-api.md @@ -16,6 +16,10 @@ - none ✅ ### Requests the backend handles but the bridge never calls (unused capability) +- ⚠️ llamacpp-proxy → node/selected +- ⚠️ llamacpp-proxy → node/set-local-backend +- ⚠️ llamacpp-proxy → telemetry/configure +- ⚠️ llamacpp-proxy → telemetry/status - ⚠️ lmstudio-proxy → node/selected - ⚠️ lmstudio-proxy → node/set-local-backend - ⚠️ lmstudio-proxy → telemetry/configure @@ -70,6 +74,34 @@ ### Backend binaries not listed in `modular-binaries.ts` - none ✅ +## llamacpp-proxy + +| Method | Direction | In bridge? | +|---|---|---| +| `error` | notification (we consume) | ✅ yes | +| `errors:clear` | notification (we consume) | ✅ yes | +| `errors:report` | notification (we consume) | ✅ yes | +| `node/discovered` | notification (we consume) | ✅ yes | +| `node/removed` | notification (we consume) | ✅ yes | +| `node/selection-changed` | notification (we consume) | ➖ ignored | +| `node/updated` | notification (we consume) | ✅ yes | +| `proxy/request` | notification (we consume) | ✅ yes | +| `proxy/request-started` | notification (we consume) | ➖ ignored | +| `ready` | notification (we consume) | ✅ yes | +| `workload:progress` | notification (we consume) | ➖ ignored | +| `node/add-manual` | request (we call) | ✅ yes | +| `node/remove-manual` | request (we call) | ✅ yes | +| `node/select` | request (we call) | ✅ yes | +| `node/selected` | request (we call) | ⚠️ not called | +| `node/set-local-backend` | request (we call) | ⚠️ not called | +| `node/set-priority` | request (we call) | ✅ yes | +| `nodes/list` | request (we call) | ✅ yes | +| `telemetry/configure` | request (we call) | ⚠️ not called | +| `telemetry/status` | request (we call) | ⚠️ not called | + +**Dynamic / unresolved notify sites (verify by hand — `npm run service-contracts` prints the line numbers):** +- `method (var) (proxy.go)` + ## lmstudio-proxy | Method | Direction | In bridge? | @@ -403,6 +435,7 @@ Client surface: requests are sent to the broker; notifications are consumed from **Dynamic / unresolved notify sites (verify by hand — `npm run service-contracts` prints the line numbers):** - `proxy:* (broker.go)` +- `llamacpp-proxy:* (llamacppproxy.go)` - `lmstudio-proxy:* (lmstudioproxy.go)` - `omlx-proxy:* (omlxproxy.go)` - `WorkerRunning (var) (supervisor.go, 2 sites)` diff --git a/desktop/docs/services-backend.md b/desktop/docs/services-backend.md index ffdd2f9b2..c2d87d6f8 100644 --- a/desktop/docs/services-backend.md +++ b/desktop/docs/services-backend.md @@ -24,6 +24,7 @@ broker supervises every worker and relays its control plane. | `ollama-proxy` | Ollama-compatible routing proxy with cluster-mTLS ingress | | `lmstudio-proxy` | LM Studio routing proxy with cluster-mTLS ingress | | `omlx-proxy` | oMLX routing proxy with cluster-mTLS ingress | +| `llamacpp-proxy` | llama.cpp routing proxy with cluster-mTLS ingress | | `nvpair-node-scanner` | Discovery and node announcement | | `nvpair-node-info` | Node metadata and telemetry | | `nvpair-manual-nodes` | User-managed node entries | @@ -46,7 +47,7 @@ flowchart TB Broker["nvpair-ui-broker"] Scanner["nvpair-node-scanner"] NodeInfo["nvpair-node-info"] - Proxies["ollama-proxy / lmstudio-proxy / omlx-proxy"] + Proxies["ollama-proxy / lmstudio-proxy / omlx-proxy / llamacpp-proxy"] Engines["nvpair-engine-manager"] Cluster["nvpair-cluster-manager"] Settings["nvpair-node-settings"] @@ -137,12 +138,12 @@ inbound-silence deadline and frontend-owned recovery. This is not per-worker han Worker health is independently broker-owned: `worker/health` probes every 30 s with a 5 s deadline; two failures degrade, three restart eligible workers only. -Engine-manager and the three proxies never auto-restart on a hang. Optional +Engine-manager and the four proxies never auto-restart on a hang. Optional initial spawn failures share the bounded crash/hang budget, while missing default binaries remain `missing` until explicit retry. The bridge hydrates `workers:get-status` on every epoch, replaces rows on full `workers:update` snapshots, and relays `workers:restart {name}`. Settings → Service → Workers -shows all twelve, attempts and last errors, including "Not responding — restart +shows all thirteen, attempts and last errors, including "Not responding — restart manually" where applicable. Engine-manager and proxy restarts ask for confirmation. No health/retry policy is duplicated in TypeScript; see the [broker contract](../../services/nvpair-ui-broker/README.md#worker-lifecycle-registry). @@ -161,7 +162,7 @@ consent policies, loopback-only Arize Phoenix deployment, and trace storage purg | ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------- | | `app:ready` | Complete broker startup and refresh snapshots | `state:request-refresh` | | `discovery:nodes-changed` | Replace discovery snapshot and diff nodes | `discovery:nodes-changed`, `nodes:upsert`, `nodes:remove` | -| `proxy:ready` / `lmstudio-proxy:ready` / `omlx-proxy:ready` | Record engine proxy port | `engines:state-changed` | +| `proxy:ready` / `lmstudio-proxy:ready` / `omlx-proxy:ready` / `llamacpp-proxy:ready` | Record engine proxy port | `engines:state-changed` | | proxy `node/*` | Update per-engine node presence; the advertised port is the peer's promoted proxy port (not the engine's private loopback port) | node and engine pushes | | `engine:ready` / `engine:state-changed` | Update engine facts and models | `engines:state-changed` | | `engine:install-progress` / `engine:remote-progress` | Update operation progress | engine progress pushes | @@ -213,7 +214,7 @@ the orphan-reclaim path, for a process whose `exec.Cmd` handle was lost. Personal AI Router calls `engine:prepare-shutdown` before broker teardown so local processes stop without clearing their persisted desired state. The broker also -first closes its three proxies concurrently, then self-initiates +first closes its four proxies concurrently, then self-initiates `engine:prepare-shutdown` and joins started workers concurrently. Worker joins use bounded stdin-close, terminate, and kill escalation, clipped together with the engine shutdown request to a shared 10 s budget. Ingress (including the heartbeat diff --git a/desktop/docs/services-parity.md b/desktop/docs/services-parity.md index 1c70e510c..2bfbd0dd2 100644 --- a/desktop/docs/services-parity.md +++ b/desktop/docs/services-parity.md @@ -188,7 +188,7 @@ Personal AI Router supports local: Before shutdown, Personal AI Router calls `engine:prepare-shutdown`. This stops managed engine processes without changing the persisted desired state; the broker restores enabled engines on the next launch. The broker also self-initiates -`engine:prepare-shutdown` after closing all three proxies concurrently. Started +`engine:prepare-shutdown` after closing all four proxies concurrently. Started workers then join concurrently with bounded stdin-close, terminate, and kill escalation. All waits share a 10 s teardown deadline. The first 2 s bound ingress and the heartbeat join (proxy EOF grace 1 s, terminate/kill waits 500 ms each), diff --git a/desktop/native/PrivilegedHelper/main.swift b/desktop/native/PrivilegedHelper/main.swift index c4be032f0..702a5d437 100644 --- a/desktop/native/PrivilegedHelper/main.swift +++ b/desktop/native/PrivilegedHelper/main.swift @@ -39,6 +39,7 @@ enum Firewall { "ollama-proxy", "lmstudio-proxy", "omlx-proxy", + "llamacpp-proxy", "nvpair-node-info", "nvpair-node-scanner", "nvpair-workload-manager", diff --git a/desktop/scripts/build/installer.nsh b/desktop/scripts/build/installer.nsh index 9cb87ed76..53db4075d 100644 --- a/desktop/scripts/build/installer.nsh +++ b/desktop/scripts/build/installer.nsh @@ -19,6 +19,7 @@ nsExec::ExecToLog 'taskkill /F /T /IM "nvpair-tui.exe"' nsExec::ExecToLog 'taskkill /F /T /IM "ollama-proxy.exe"' nsExec::ExecToLog 'taskkill /F /T /IM "lmstudio-proxy.exe"' + nsExec::ExecToLog 'taskkill /F /T /IM "llamacpp-proxy.exe"' nsExec::ExecToLog 'taskkill /F /T /IM "nvpair-node-info.exe"' nsExec::ExecToLog 'taskkill /F /T /IM "nvpair-node-scanner.exe"' nsExec::ExecToLog 'taskkill /F /T /IM "nvpair-manual-nodes.exe"' @@ -122,6 +123,7 @@ DetailPrint "Adding Personal AI Router firewall rules..." nsExec::ExecToLog 'netsh advfirewall firewall add rule name="Personal AI Router Ollama Proxy" dir=in action=allow program="$INSTDIR\resources\cli-bin\ollama-proxy.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="Personal AI Router LM Studio Proxy" dir=in action=allow program="$INSTDIR\resources\cli-bin\lmstudio-proxy.exe" enable=yes profile=any remoteip=localsubnet' + nsExec::ExecToLog 'netsh advfirewall firewall add rule name="Personal AI Router llama.cpp Proxy" dir=in action=allow program="$INSTDIR\resources\cli-bin\llamacpp-proxy.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="Personal AI Router Node Info" dir=in action=allow program="$INSTDIR\resources\cli-bin\nvpair-node-info.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="Personal AI Router Node Scanner" dir=in action=allow program="$INSTDIR\resources\cli-bin\nvpair-node-scanner.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="Personal AI Router Workload Manager" dir=in action=allow program="$INSTDIR\resources\cli-bin\nvpair-workload-manager.exe" enable=yes profile=any remoteip=localsubnet' @@ -130,6 +132,7 @@ nsExec::ExecToLog 'netsh advfirewall firewall add rule name="Personal AI Router Engine Manager" dir=in action=allow program="$INSTDIR\resources\cli-bin\nvpair-engine-manager.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="Personal AI Router mDNS (UDP 5353)" dir=in action=allow protocol=UDP localport=5353 program="$INSTDIR\resources\cli-bin\ollama-proxy.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="Personal AI Router mDNS LM Studio Proxy (UDP 5353)" dir=in action=allow protocol=UDP localport=5353 program="$INSTDIR\resources\cli-bin\lmstudio-proxy.exe" enable=yes profile=any remoteip=localsubnet' + nsExec::ExecToLog 'netsh advfirewall firewall add rule name="Personal AI Router mDNS llama.cpp Proxy (UDP 5353)" dir=in action=allow protocol=UDP localport=5353 program="$INSTDIR\resources\cli-bin\llamacpp-proxy.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="Personal AI Router mDNS Node Info (UDP 5353)" dir=in action=allow protocol=UDP localport=5353 program="$INSTDIR\resources\cli-bin\nvpair-node-info.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="Personal AI Router mDNS Node Scanner (UDP 5353)" dir=in action=allow protocol=UDP localport=5353 program="$INSTDIR\resources\cli-bin\nvpair-node-scanner.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="Personal AI Router mDNS Workload Manager (UDP 5353)" dir=in action=allow protocol=UDP localport=5353 program="$INSTDIR\resources\cli-bin\nvpair-workload-manager.exe" enable=yes profile=any remoteip=localsubnet' @@ -143,6 +146,7 @@ DetailPrint "Removing Personal AI Router firewall rules..." nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router Ollama Proxy"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router LM Studio Proxy"' + nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router llama.cpp Proxy"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router Node Info"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router Node Scanner"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router Workload Manager"' @@ -151,6 +155,7 @@ nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router Engine Manager"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router mDNS (UDP 5353)"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router mDNS LM Studio Proxy (UDP 5353)"' + nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router mDNS llama.cpp Proxy (UDP 5353)"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router mDNS Node Info (UDP 5353)"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router mDNS Node Scanner (UDP 5353)"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="Personal AI Router mDNS Workload Manager (UDP 5353)"' @@ -161,6 +166,7 @@ ; Remove pre-rebrand rules left by older installations. nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="PAIR Ollama Proxy"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="PAIR LM Studio Proxy"' + nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="PAIR llama.cpp Proxy"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="PAIR Node Info"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="PAIR Node Scanner"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="PAIR Workload Manager"' @@ -169,6 +175,7 @@ nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="PAIR Engine Manager"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="PAIR mDNS (UDP 5353)"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="PAIR mDNS LM Studio Proxy (UDP 5353)"' + nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="PAIR mDNS llama.cpp Proxy (UDP 5353)"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="PAIR mDNS Node Info (UDP 5353)"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="PAIR mDNS Node Scanner (UDP 5353)"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="PAIR mDNS Workload Manager (UDP 5353)"' diff --git a/desktop/scripts/build/macos/uninstall.sh b/desktop/scripts/build/macos/uninstall.sh index 9cf1e4027..3577b854d 100644 --- a/desktop/scripts/build/macos/uninstall.sh +++ b/desktop/scripts/build/macos/uninstall.sh @@ -59,6 +59,7 @@ for proc in \ "ollama-proxy" \ "lmstudio-proxy" \ "omlx-proxy" \ + "llamacpp-proxy" \ "nvpair-node-info" \ "nvpair-node-scanner" \ "nvpair-manual-nodes" \ @@ -75,7 +76,7 @@ sleep 1 FW=/usr/libexec/ApplicationFirewall/socketfilterfw if [ -x "$FW" ]; then - for bin in ollama-proxy lmstudio-proxy omlx-proxy nvpair-node-info nvpair-node-scanner \ + for bin in ollama-proxy lmstudio-proxy omlx-proxy llamacpp-proxy nvpair-node-info nvpair-node-scanner \ nvpair-workload-manager nvpair-errors nvpair-cluster-manager nvpair-engine-manager; do "$FW" --remove "$APP_PATH/Contents/Resources/cli-bin/$bin" >/dev/null 2>&1 || true done diff --git a/desktop/src/electron/service-bridge/empty-handlers.ts b/desktop/src/electron/service-bridge/empty-handlers.ts index 0d1a89403..6d8d86859 100644 --- a/desktop/src/electron/service-bridge/empty-handlers.ts +++ b/desktop/src/electron/service-bridge/empty-handlers.ts @@ -1096,6 +1096,8 @@ function parseManualNodeStatus(raw: JsonValue | undefined): ManualNodeStatus | n lmstudio_port: numberValue(obj.lmstudio_port), omlx_up: booleanValue(obj.omlx_up), omlx_port: numberValue(obj.omlx_port), + llamacpp_up: booleanValue(obj.llamacpp_up), + llamacpp_port: numberValue(obj.llamacpp_port), node_info_up: booleanValue(obj.node_info_up), node_info_port: numberValue(obj.node_info_port), tls_enabled: booleanValue(obj.tls_enabled), diff --git a/desktop/src/electron/service-bridge/modular-state.ts b/desktop/src/electron/service-bridge/modular-state.ts index 886ea10ea..9df92a32c 100644 --- a/desktop/src/electron/service-bridge/modular-state.ts +++ b/desktop/src/electron/service-bridge/modular-state.ts @@ -66,21 +66,22 @@ import { serviceLogLevel } from './service-log-level' // Live node sources are the two reverse proxies, relayed through the broker, // and the broker's consolidated discovery snapshot. Electron does not consume // worker discovery protocols directly. -type ProxyNodeSource = 'ollama-proxy' | 'lmstudio-proxy' | 'omlx-proxy' +type ProxyNodeSource = 'ollama-proxy' | 'lmstudio-proxy' | 'omlx-proxy' | 'llamacpp-proxy' type BrokerNodeSource = ProxyNodeSource | 'broker' /** * Engines surfaced by the broker's proxy plane. Other engine-manager engines * are not currently routed across nodes. */ -export type ProxyEngine = Extract -export const PROXY_ENGINES: readonly ProxyEngine[] = ['ollama', 'lm-studio', 'omlx'] +export type ProxyEngine = Extract +export const PROXY_ENGINES: readonly ProxyEngine[] = ['ollama', 'lm-studio', 'omlx', 'llamacpp'] /** Map a proxy node source onto the engine it describes. */ const PROXY_SOURCE_ENGINE: Record = { 'ollama-proxy': 'ollama', 'lmstudio-proxy': 'lm-studio', - 'omlx-proxy': 'omlx' + 'omlx-proxy': 'omlx', + 'llamacpp-proxy': 'llamacpp' } function proxyEngineToSource(engine: ProxyEngine): ProxyNodeSource { @@ -91,6 +92,8 @@ function proxyEngineToSource(engine: ProxyEngine): ProxyNodeSource { return 'lmstudio-proxy' case 'omlx': return 'omlx-proxy' + case 'llamacpp': + return 'llamacpp-proxy' } } @@ -218,7 +221,12 @@ function emptyPresence(): EnginePresence { } function emptyEngines(): Record { - return { ollama: emptyPresence(), 'lm-studio': emptyPresence(), omlx: emptyPresence() } + return { + ollama: emptyPresence(), + 'lm-studio': emptyPresence(), + omlx: emptyPresence(), + llamacpp: emptyPresence() + } } /** Immutably set one engine's presence, preserving the other. */ @@ -230,7 +238,8 @@ function setEngine( return { ollama: engine === 'ollama' ? presence : engines.ollama, 'lm-studio': engine === 'lm-studio' ? presence : engines['lm-studio'], - omlx: engine === 'omlx' ? presence : engines.omlx + omlx: engine === 'omlx' ? presence : engines.omlx, + llamacpp: engine === 'llamacpp' ? presence : engines.llamacpp } } @@ -859,7 +868,9 @@ export function parseNodeMetricsSnapshot( /** True for an engine fronted by a broker-supervised reverse proxy. */ export function isProxyEngine(engine: EngineType): engine is ProxyEngine { - return engine === 'ollama' || engine === 'lm-studio' || engine === 'omlx' + return ( + engine === 'ollama' || engine === 'lm-studio' || engine === 'omlx' || engine === 'llamacpp' + ) } const PENDING_OP_IDLE_TIMEOUT_MS = 90_000 @@ -1551,7 +1562,12 @@ class ModularBridgeState { // Per-engine bound proxy port reported by the broker. 0 = not reported yet; // we never fabricate a default — an unknown port surfaces as null, not a // guess. `ollama` is the `ollama-proxy`, `lm-studio` is the `lmstudio-proxy`. - private proxyPorts: Record = { ollama: 0, 'lm-studio': 0, omlx: 0 } + private proxyPorts: Record = { + ollama: 0, + 'lm-studio': 0, + omlx: 0, + llamacpp: 0 + } private selfId: string | null = null /** * Authoritative local-engine facts from `nvpair-engine-manager`, keyed by @@ -2998,6 +3014,10 @@ class ModularBridgeState { this.handleProxyNotification(notification, 'omlx') return } + if (notification.source === 'llamacpp-proxy') { + this.handleProxyNotification(notification, 'llamacpp') + return + } if (notification.source === 'broker') { this.handleBrokerNotification(notification) } diff --git a/desktop/src/electron/service-bridge/modular-supervisor.ts b/desktop/src/electron/service-bridge/modular-supervisor.ts index fa88c9868..1d8f1db19 100644 --- a/desktop/src/electron/service-bridge/modular-supervisor.ts +++ b/desktop/src/electron/service-bridge/modular-supervisor.ts @@ -330,6 +330,7 @@ function proxyEngineFromManagerId(id: string): ProxyEngine | null { if (id === 'ollama') return 'ollama' if (id === 'lmstudio') return 'lm-studio' if (id === 'omlx') return 'omlx' + if (id === 'llamacpp') return 'llamacpp' return null } @@ -342,6 +343,8 @@ function proxyRelayPrefix(engine: ProxyEngine): string { return 'lmstudio-proxy' case 'omlx': return 'omlx-proxy' + case 'llamacpp': + return 'llamacpp-proxy' } } @@ -1101,6 +1104,7 @@ class ModularSupervisor { passPath('--proxy-path', 'proxy') passPath('--lmstudio-proxy-path', 'lmstudio-proxy') passPath('--omlx-proxy-path', 'omlx-proxy') + passPath('--llamacpp-proxy-path', 'llamacpp-proxy') passPath('--workload-manager-path', 'workload-manager') passPath('--cluster-manager-path', 'cluster-manager') passPath('--settings-path', 'node-settings') @@ -1130,6 +1134,7 @@ class ModularSupervisor { await subscribe('proxy:subscribe', 'subscribe to broker ollama-proxy relay') await subscribe('lmstudio-proxy:subscribe', 'subscribe to broker lmstudio-proxy relay') await subscribe('omlx-proxy:subscribe', 'subscribe to broker omlx-proxy relay') + await subscribe('llamacpp-proxy:subscribe', 'subscribe to broker llamacpp-proxy relay') // Engine events are opt-in and replay no baseline — subscribe then hydrate. await subscribe('engine:subscribe', 'subscribe to broker engine relay') await subscribe('workloads:subscribe', 'subscribe to broker workloads stream') @@ -1704,7 +1709,9 @@ class ModularSupervisor { ? 'lm-studio' : event.source === 'omlx-proxy' ? 'omlx' - : null + : event.source === 'llamacpp-proxy' + ? 'llamacpp' + : null if (proxyEngine && event.method === 'ready') { // A (re)bound proxy starts with an empty manual-node set, so forget // what we think we bridged and re-push the local node if applicable. @@ -1760,6 +1767,13 @@ class ModularSupervisor { params: notification.params } } + if (notification.method.startsWith('llamacpp-proxy:')) { + return { + source: 'llamacpp-proxy', + method: notification.method.slice('llamacpp-proxy:'.length), + params: notification.params + } + } if (notification.method.startsWith('lmstudio-proxy:')) { return { source: 'lmstudio-proxy', diff --git a/desktop/src/shared/constants/modular-binaries.ts b/desktop/src/shared/constants/modular-binaries.ts index d4b38e66c..3fe728bf3 100644 --- a/desktop/src/shared/constants/modular-binaries.ts +++ b/desktop/src/shared/constants/modular-binaries.ts @@ -7,6 +7,7 @@ export type ModularProcessName = | 'proxy' | 'lmstudio-proxy' | 'omlx-proxy' + | 'llamacpp-proxy' | 'broker' | 'node-info' | 'scanner' @@ -89,6 +90,14 @@ export const MODULAR_RUNTIME_BINARIES: ModularRuntimeBinary[] = [ needsFirewallAccess: true, optional: true }, + { + processName: 'llamacpp-proxy', + baseName: 'llamacpp-proxy', + args: [], + launchOwner: 'broker', + needsFirewallAccess: true, + optional: true + }, { processName: 'scanner', baseName: 'nvpair-node-scanner', diff --git a/desktop/src/shared/types/manual-nodes.ts b/desktop/src/shared/types/manual-nodes.ts index a224cdd8b..f6295c020 100644 --- a/desktop/src/shared/types/manual-nodes.ts +++ b/desktop/src/shared/types/manual-nodes.ts @@ -33,6 +33,9 @@ export interface ManualNodeStatus { omlx_up?: boolean omlx_port?: number omlx_models?: string[] + llamacpp_up?: boolean + llamacpp_port?: number + llamacpp_models?: string[] node_info_up?: boolean node_info_port?: number tls_enabled?: boolean diff --git a/desktop/src/ui/components/BackendRow/BackendRow.tsx b/desktop/src/ui/components/BackendRow/BackendRow.tsx index f02ea6d9b..1396676c1 100644 --- a/desktop/src/ui/components/BackendRow/BackendRow.tsx +++ b/desktop/src/ui/components/BackendRow/BackendRow.tsx @@ -228,10 +228,7 @@ export function BackendRow({ onInstall={handleInstall} /> {backend.type === 'llamacpp' && ( - - Not notarized, like PAIR itself. Does not serve requests through PAIR - yet. - + Not notarized, like PAIR itself. )} {expanded && } {content} diff --git a/desktop/src/ui/components/BackendRow/EngineSettingsSection.tsx b/desktop/src/ui/components/BackendRow/EngineSettingsSection.tsx index 2cd89490a..e09f1fea9 100644 --- a/desktop/src/ui/components/BackendRow/EngineSettingsSection.tsx +++ b/desktop/src/ui/components/BackendRow/EngineSettingsSection.tsx @@ -94,7 +94,6 @@ function SettingsEditor({ target, disabled }: { target: EngineSettingsTarget; di snapshot.format !== 'pair-arguments-v1' const locked = readOnly || disabled || applying const isOmlx = engine === 'omlx' - const hasProxy = engine !== 'llamacpp' useEffect(() => watchEngineSettings(), []) useEffect(() => { @@ -205,9 +204,9 @@ function SettingsEditor({ target, disabled }: { target: EngineSettingsTarget; di } const id = ++generation.current const serverPort = Number(draft.serverPort) - const proxyPort = hasProxy ? Number(draft.proxyPort) : 0 + const proxyPort = Number(draft.proxyPort) if ( - !(hasProxy ? [serverPort, proxyPort] : [serverPort]).every( + ![serverPort, proxyPort].every( port => Number.isInteger(port) && port >= 1 && port <= 65535 ) ) { @@ -290,7 +289,7 @@ function SettingsEditor({ target, disabled }: { target: EngineSettingsTarget; di aria-invalid={!!errors.serverPort} /> - {hasProxy && ( + { - )} + } {isOmlx ? ( diff --git a/desktop/src/ui/components/ServiceSettings/WorkersCard.tsx b/desktop/src/ui/components/ServiceSettings/WorkersCard.tsx index 5a9b47730..063f06fb3 100644 --- a/desktop/src/ui/components/ServiceSettings/WorkersCard.tsx +++ b/desktop/src/ui/components/ServiceSettings/WorkersCard.tsx @@ -31,7 +31,8 @@ const RESTART_WARNINGS: Record = { 'Engines PAIR started may keep running or stop; running inference may be interrupted.', proxy: 'Running inference through this proxy will be interrupted.', 'lmstudio-proxy': 'Running inference through this proxy will be interrupted.', - 'omlx-proxy': 'Running inference through this proxy will be interrupted.' + 'omlx-proxy': 'Running inference through this proxy will be interrupted.', + 'llamacpp-proxy': 'Running inference through this proxy will be interrupted.' } function WorkerRow({ diff --git a/desktop/src/ui/constants/welcome.ts b/desktop/src/ui/constants/welcome.ts index 601ce4996..24ff16e2d 100644 --- a/desktop/src/ui/constants/welcome.ts +++ b/desktop/src/ui/constants/welcome.ts @@ -21,7 +21,7 @@ export const WELCOME_ENGINE_DEFAULT_SELECTED: Record = { export function getWelcomeEngineCandidates(os: PlatformDisplayName): EngineType[] { const platform = displayNameToPlatform(os) return EnabledEngineTypes.filter(t => { - if (t === 'llamacpp') return false // lifecycle-only preview; install from its node row + if (t === 'llamacpp') return false // install from its node row; model hub is not available yet const caps = EngineCapabilities[t] if (!caps.hasInstall.length) return false if (caps.hasInstallPath) return false diff --git a/desktop/tests/modular/backend-row-capabilities.test.ts b/desktop/tests/modular/backend-row-capabilities.test.ts index 707104b23..dc5a731d9 100644 --- a/desktop/tests/modular/backend-row-capabilities.test.ts +++ b/desktop/tests/modular/backend-row-capabilities.test.ts @@ -115,7 +115,7 @@ function expand(): void { } describe('BackendRow reported capability gates', () => { - it('discloses llama.cpp trust and deferred routing before installation', () => { + it('discloses llama.cpp trust before installation', () => { act(() => root.render( createElement(BackendRow, { @@ -131,7 +131,6 @@ describe('BackendRow reported capability gates', () => { ) ) expect(container.textContent).toContain('Not notarized, like PAIR itself.') - expect(container.textContent).toContain('Does not serve requests through PAIR yet.') expect(container.querySelector('img[alt="llama.cpp"]')).not.toBeNull() }) diff --git a/desktop/tests/modular/engine-settings-panel.test.ts b/desktop/tests/modular/engine-settings-panel.test.ts index 5fb401979..d5aad5a92 100644 --- a/desktop/tests/modular/engine-settings-panel.test.ts +++ b/desktop/tests/modular/engine-settings-panel.test.ts @@ -161,23 +161,23 @@ afterEach(async () => { }) describe('engine launch settings panel', () => { - it('edits llama.cpp server and launch settings without a proxy port', async () => { + it('edits llama.cpp server, proxy and launch settings', async () => { getSettings.mockImplementation(async target => ({ ...baseline, ...target, - settings: { serverPort: 8082, proxyPort: 0, launchText: '' }, + settings: { serverPort: 8082, proxyPort: 1237, launchText: '' }, effectiveServerPort: 8082, - effectiveProxyPort: 0 + effectiveProxyPort: 1237 })) await render('llamacpp') - expect(container.querySelector('[aria-label="Proxy port"]')).toBeNull() + expect(field('Proxy port').disabled).toBe(false) expect(field('Server port').disabled).toBe(false) - await change('Engine arguments', 'CUSTOM=value --threads 4') + await change('Engine arguments', '--threads 4') await click('Apply') expect(previewSettings).toHaveBeenCalledWith( expect.objectContaining({ engine: 'llamacpp', - settings: { serverPort: 8082, proxyPort: 0, launchText: 'CUSTOM=value --threads 4' } + settings: { serverPort: 8082, proxyPort: 1237, launchText: '--threads 4' } }) ) expect(applySettings).toHaveBeenCalledOnce() diff --git a/desktop/tests/modular/epoch-resync.test.ts b/desktop/tests/modular/epoch-resync.test.ts index 14b638d67..9a970cbcd 100644 --- a/desktop/tests/modular/epoch-resync.test.ts +++ b/desktop/tests/modular/epoch-resync.test.ts @@ -53,6 +53,7 @@ const EXPECTED_RESYNC_CALLS = [ 'proxy:get-status', 'lmstudio-proxy:get-status', 'omlx-proxy:get-status', + 'llamacpp-proxy:get-status', 'telemetry/get-status' ] diff --git a/desktop/tests/modular/llamacpp-lifecycle.test.ts b/desktop/tests/modular/llamacpp-lifecycle.test.ts index c5f7f51c2..22ce7263f 100644 --- a/desktop/tests/modular/llamacpp-lifecycle.test.ts +++ b/desktop/tests/modular/llamacpp-lifecycle.test.ts @@ -8,7 +8,7 @@ import { getModularBridgeState, isProxyEngine } from '@/electron/service-bridge/ import { parseEngineSettings } from '@/electron/service-bridge/engine-settings' const capabilities = { - list_models: false, + list_models: true, pull_model: false, delete_model: false, delete_restarts_engine: false, @@ -17,10 +17,15 @@ const capabilities = { uninstall: true } -describe('llama.cpp lifecycle-only projection', () => { - it('shows local installed and running facts without a proxy or model capabilities', () => { +describe('llama.cpp routing projection', () => { + it('shows local engine facts and the independently reported proxy port', () => { const state = getModularBridgeState() state.setSelfId('llama-self') + state.handleNotification({ + source: 'llamacpp-proxy', + method: 'ready', + params: { port: 1237 } + }) for (const running of [false, true]) { state.applyEngineManagerStatus({ engine: 'llamacpp', @@ -35,11 +40,11 @@ describe('llama.cpp lifecycle-only projection', () => { expect(status).toMatchObject({ processStatus: running ? 'running' : 'stopped', enginePort: 8082, - proxyPort: null, - capabilities: { install: true, listModels: false, pullModel: false } + proxyPort: 1237, + capabilities: { install: true, listModels: true, pullModel: false } }) } - expect(isProxyEngine('llamacpp')).toBe(false) + expect(isProxyEngine('llamacpp')).toBe(true) }) it('shows remote lifecycle facts without attributing a routed endpoint', () => { @@ -84,11 +89,11 @@ describe('llama.cpp lifecycle-only projection', () => { state .getEngineInitialState() .models.filter(s => s.nodeId === 'llama-peer' && s.engineType === 'llamacpp') - ).toEqual([]) + ).toMatchObject([{ engineType: 'llamacpp', nodeId: 'llama-peer', models: [] }]) expect(state.isRemoteEngineRunning('llama-peer', 'llamacpp')).toBe(true) }) - it('accepts the zero-proxy-port settings snapshot without adding load profiles', () => { + it('accepts a routed proxy-port settings snapshot without adding load profiles', () => { expect( parseEngineSettings({ nodeId: 'llama-self', @@ -98,14 +103,14 @@ describe('llama.cpp lifecycle-only projection', () => { sequence: 1, revision: 1, appliedRevision: 1, - settings: { serverPort: 8082, proxyPort: 0, launchText: '' }, + settings: { serverPort: 8082, proxyPort: 1237, launchText: '' }, effectiveServerPort: 8082, - effectiveProxyPort: 0, + effectiveProxyPort: 1237, running: false, adopted: false, editable: true, format: 'pair-arguments-v1' }) - ).toMatchObject({ engine: 'llamacpp', settings: { proxyPort: 0 } }) + ).toMatchObject({ engine: 'llamacpp', settings: { proxyPort: 1237 } }) }) }) diff --git a/desktop/tests/modular/manual-nodes-bridge.test.ts b/desktop/tests/modular/manual-nodes-bridge.test.ts index 10afa3433..ad6b9b601 100644 --- a/desktop/tests/modular/manual-nodes-bridge.test.ts +++ b/desktop/tests/modular/manual-nodes-bridge.test.ts @@ -80,6 +80,8 @@ describe('manual-nodes service bridge handlers', () => { lmstudio_port: 0, omlx_up: false, omlx_port: 0, + llamacpp_up: false, + llamacpp_port: 0, node_info_up: false, node_info_port: 0, tls_enabled: false, diff --git a/desktop/tests/modular/workers-card.test.ts b/desktop/tests/modular/workers-card.test.ts index b85585158..c1a2cc478 100644 --- a/desktop/tests/modular/workers-card.test.ts +++ b/desktop/tests/modular/workers-card.test.ts @@ -93,7 +93,7 @@ it('sends no engine-manager restart on cancel, then exactly one on confirm throu }) }) -it.each(['proxy', 'lmstudio-proxy', 'omlx-proxy'])( +it.each(['proxy', 'lmstudio-proxy', 'omlx-proxy', 'llamacpp-proxy'])( 'confirms interruption before restarting %s', async name => { await act(async () => useWorkersStore.setState({ workers: [{ ...worker, name }] })) diff --git a/docs/architecture.mdx b/docs/architecture.mdx index 446d20c22..97d4f75f9 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -79,7 +79,7 @@ flowchart TB ``` Electron starts only `nvpair-ui-broker` as a Go service. The service inventory -contains 14 executables: broker + 12 workers (including `omlx-proxy`) + TUI. +contains 15 executables: broker + 13 workers (including `omlx-proxy` and `llamacpp-proxy`) + TUI. The broker attempts to start available workers, probes their control health, reports lifecycle state, and retries crashes, optional initial spawn failures, and eligible hangs within a shared budget. Workers log to stderr; stdout is @@ -111,7 +111,7 @@ The checks have separate meanings at each layer: | The broker | `ping` plus a 15-second notification heartbeat. Desktop detects 45 seconds of inbound silence and owns bounded stop/start recovery and rehydration. This does not monitor worker hangs. | | Peer nodes | Reachability, not health: missed announcements, then recent inference activity, recent telemetry, or a probe of the node-info or engine-manager port before eviction. Peer inference proxies report activity; their HTTP listeners are not probed for peer reachability. | -Each of the twelve workers answers `worker/health` with `{"ok":true}`. The broker +Each of the thirteen workers answers `worker/health` with `{"ok":true}`. The broker probes every **30 seconds**, with a **5-second deadline**. The diagnostic flag `--worker-health-interval` defaults to `30s` and requires at least `1s`; the probe deadline is capped at the interval. **Two consecutive failed probes** @@ -122,7 +122,7 @@ stopped through bounded escalation and restarted: - `scanner`, `errors`, `settings`, `node-info` - `workload-manager`, `manual-nodes`, `cluster-manager`, `scheduler` -**Engine-manager and all three proxies are never auto-restarted on a hang.** +**Engine-manager and all four proxies are never auto-restarted on a hang.** Engine-manager restart can orphan engines; proxy HTTP inference runs separately from stdio control, and restart would interrupt live streams. They still accept explicit restarts. Their desktop rows show "Not responding — restart manually". @@ -138,13 +138,13 @@ exhaustion leaves the worker down until an explicit retry. `nvpair-errors` cannot report its own death into itself, so its crashes go to stderr instead. The broker registry is independent of the errors service. `workers:get-status` -returns all twelve workers with `running`, `degraded`, `restarting`, `exhausted`, +returns all thirteen workers with `running`, `degraded`, `restarting`, `exhausted`, or `missing` state. Unconditional `workers:update` notifications carry full snapshots, not deltas. `workers:restart {name}` initiates restart/retry, including from exhaustion; a missing default is rechecked and only starts if found. Desktop **Settings → Service → Workers** renders state, PID, attempts and last error; engine-manager and proxy restarts require confirmation. TUI **Overview** -shows all twelve workers and uses **`R`** with confirmation. Both clients recover +shows all thirteen workers and uses **`R`** with confirmation. Both clients recover worker snapshots on epoch changes; neither owns health/retry policy. Cross-process hang tests are **Unix-only**; Windows hang recovery remains @@ -755,7 +755,7 @@ browsing themselves. The scanner advertises one `_nvpair-node._tcp` multicast DNS (mDNS) record for each host, and that record carries two kinds of content: -- The ports its sibling services registered: node-info, all three proxies, +- The ports its sibling services registered: node-info, all four proxies, errors, workloads, cluster manager, and engine manager - The node's identity: `uuid=`, `cluster-uuid=` after clustering, and where to reach it — `ip=` for the address the node ranks first, and `ips=` for the whole @@ -891,7 +891,7 @@ an unverified address if pairing fails. See [Routed networks](remote-networks.md ## Observability and Distributed Tracing PAIR includes built-in OpenTelemetry (OTel) tracing and Arize Phoenix observability. -Every proxy (`ollama-proxy`, `lmstudio-proxy`, `omlx-proxy`) instruments local inference +Every proxy (`ollama-proxy`, `lmstudio-proxy`, `omlx-proxy`, `llamacpp-proxy`) instruments local inference requests, scheduler routing decisions, and engine dispatch operations. ### Local-Only Tracing Topology @@ -1178,8 +1178,8 @@ desktop build compiles the sibling tree directly. | Build In | Output | Contents | | --- | --- | --- | -| `desktop/` | `desktop/cli-bin/` | 14 service executables including the unsupervised TUI, one OS/arch | -| `services/` | `services/build/bin/` | All 14 service executables, for standalone use | +| `desktop/` | `desktop/cli-bin/` | 15 service executables including the unsupervised TUI, one OS/arch | +| `services/` | `services/build/bin/` | All 15 service executables, for standalone use | `desktop/scripts/build-modular-binaries.ts` compiles the runtime inventory for a selected target and writes a manifest recording the source identity, versions, diff --git a/docs/building.mdx b/docs/building.mdx index 9ce07f3d4..169bef181 100644 --- a/docs/building.mdx +++ b/docs/building.mdx @@ -119,7 +119,7 @@ credentials or approvals supplied by this checkout. Downloading an app with ## Build the Services Alone The service scripts read `services/versions.json`, stamp each version, build 14 -service executables (broker + 12 workers, including `omlx-proxy`, + TUI), and +service executables (broker + 13 workers, including `omlx-proxy` and `llamacpp-proxy`, + TUI), and stage them together in `services/build/bin/`. Linux and macOS: diff --git a/docs/developing.mdx b/docs/developing.mdx index 104511821..73fd7cc5e 100644 --- a/docs/developing.mdx +++ b/docs/developing.mdx @@ -130,7 +130,7 @@ successful probe since spawn. Missing defaults wait for `workers:restart`. `workers:get-status` after epoch changes, render the reported state, and relay `workers:restart` without implementing policy. Settings → Service → Workers and TUI Overview (`R`, with confirmation) expose the controls. Engine-manager -and the three proxies never auto-restart on a hang; see +and the four proxies never auto-restart on a hang; see [Health and Failure Detection](architecture.mdx#health-and-failure-detection) for the eight eligible workers and the Unix-only cross-process test boundary. diff --git a/docs/engine-lifecycle.mdx b/docs/engine-lifecycle.mdx index 928d9f9e3..d14ebe00e 100644 --- a/docs/engine-lifecycle.mdx +++ b/docs/engine-lifecycle.mdx @@ -8,9 +8,11 @@ SPDX-License-Identifier: Apache-2.0 An **engine** is the local inference runtime Personal AI Router (PAIR) uses to run models. PAIR supports Ollama, LM Studio and oMLX. It can install and run those engines for you, or work with a copy you already have. A macOS arm64 -**llama.cpp lifecycle-only preview** also appears in Engine settings: it can be -installed, started, stopped and configured, but **does not serve requests through -PAIR yet**. Its official b12171 macOS build is **not notarized**, like PAIR itself. +**llama.cpp engine** also appears in Engine settings: it can be installed, +started, stopped and configured, and is served by its own routing proxy on +**1237** (engine listener **127.0.0.1:8082**). Select a regular GGUF inside PAIR's +llama.cpp model cache with allowlisted `--model`/`-m`; model hub/download/delete +are not available yet. Its official b12171 macOS build is **not notarized**, like PAIR itself. This page explains what you can expect when you install, start, stop, update, or remove an engine. diff --git a/docs/known-issues.mdx b/docs/known-issues.mdx index ed9a00caf..c60437ea1 100644 --- a/docs/known-issues.mdx +++ b/docs/known-issues.mdx @@ -77,9 +77,9 @@ hardware would help. Refer to ## Worker Health: Manual Recovery and Platform Limits -The broker probes all twelve workers, marks them `degraded` after two consecutive +The broker probes all thirteen workers, marks them `degraded` after two consecutive failed probes, and restarts eight eligible workers after three. Engine-manager -and the three proxies are **never auto-restarted on a hang**: doing so could +and the four proxies are **never auto-restarted on a hang**: doing so could orphan engines or interrupt inference streams. Their degraded desktop rows show "Not responding — restart manually" under **Settings → Service → Workers**. Restarting engine-manager may leave engines running or stop them; proxy restarts diff --git a/docs/overview.mdx b/docs/overview.mdx index 8aaf13593..08ccdeed0 100644 --- a/docs/overview.mdx +++ b/docs/overview.mdx @@ -176,10 +176,10 @@ It stages the generated binaries in `desktop/cli-bin/`. ### `services/` -The Go services tree produces 14 executables: broker + 12 workers + TUI: +The Go services tree produces 15 executables: broker + 13 workers + TUI: - `nvpair-ui-broker`: service entry point, worker supervisor, and JSON-RPC API -- `ollama-proxy`, `lmstudio-proxy`, and `omlx-proxy`: compatible inference proxies +- `ollama-proxy`, `lmstudio-proxy`, `omlx-proxy`, and `llamacpp-proxy`: compatible inference proxies - `nvpair-node-scanner` and `nvpair-node-info`: discovery and host telemetry - `nvpair-manual-nodes`: user-specified nodes - `nvpair-engine-manager`: local engine and model lifecycle @@ -190,7 +190,7 @@ The Go services tree produces 14 executables: broker + 12 workers + TUI: - `nvpair-job-scheduler`: node-wide pending-work and GPU-pressure rankings - `nvpair-tui`: standalone terminal interface for headless systems -The broker supervises up to 12 worker processes. Standalone, only the scanner +The broker supervises up to 13 worker processes. Standalone, only the scanner is required; missing default auxiliary binaries or initial auxiliary spawn failures leave reduced capabilities. Invalid explicit paths and scanner initial spawn failure are fatal. Optional initial failures use bounded retries; missing @@ -199,7 +199,7 @@ prevalidates broker, scanner, Ollama proxy, node-info, and node-settings binarie Worker health and retry state appear under **Settings → Service → Workers** and in TUI **Overview**, where `R` requests restart with confirmation. The broker owns health policy: eight workers can restart on a hang, while engine-manager -and the three proxies only degrade and require manual restart if unresponsive. +and the four proxies only degrade and require manual restart if unresponsive. See [Health and Failure Detection](architecture.mdx#health-and-failure-detection). `nvpair-tui` is a separate client that starts its own broker, not a worker. diff --git a/docs/release-readiness.mdx b/docs/release-readiness.mdx index 203d33686..b459453c8 100644 --- a/docs/release-readiness.mdx +++ b/docs/release-readiness.mdx @@ -10,6 +10,16 @@ notice audit. It describes the current checkout, not a certification of the next release. No release was published and no installed app was replaced by this audit. +## Unreleased llama.cpp routing + +The current source adds `llamacpp-proxy` (default port 1237, engine 8082, +service `lc`): **13 supervised workers / 15 executables**. The broker, desktop +and TUI inventories, build/install lists and contract documentation include it. +Native b12171 Messages passthrough and exact-ID model routing are supported; +model hub/download/delete remain deferred. Historical packaged-app evidence +below predates this worker and does not certify its packaged or multi-node +inference behavior. Validate those before release. + ## Packaged-app validation (2026-11-05) **Owner's source build, not certification of a published artifact.** Notes dated @@ -80,7 +90,7 @@ worker, unsigned-binary review and its own download-cancellation path. - [x] Separate supported build targets from published assets. The fork's latest release is an unsigned/unnotarized macOS arm64 candidate, even though GitHub marks it regular/latest. -- [x] Correct process inventory: 14 service executables, 12 supervised workers, +- [x] Correct current source inventory: 15 service executables, 13 supervised workers, and a separate TUI; distinguish standalone/desktop startup requirements. - [x] Document relay v2 envelopes, epoch recovery, broker-client silence handling, three-engine behavior, current telemetry configuration, and idle polling. @@ -211,13 +221,13 @@ tracing-enabled profiling scenarios are still open. **Implemented, pending release testing.** -- All twelve workers answer `worker/health`; the broker probes every 30 s with a +- All thirteen workers answer `worker/health`; the broker probes every 30 s with a 5 s deadline (`--worker-health-interval`, minimum 1 s; deadline capped at the interval). Two consecutive failures report `degraded`; a successful probe restores `running` and clears the failure count. - Three failures restart only scanner, errors, settings, node-info, workload-manager, manual-nodes, cluster-manager, and scheduler. Engine-manager - and the three proxies never auto-restart on a hang: orphan/stream safety is not + and the four proxies never auto-restart on a hang: orphan/stream safety is not established. Desktop shows "Not responding — restart manually" for those rows. - Optional initial spawn failures, crashes and eligible hang restarts share 1 s → 16 s backoff and five replacement attempts. Healthy reset needs 60 s @@ -225,7 +235,7 @@ tracing-enabled profiling scenarios are still open. `missing` with no automatic retry until `workers:restart` finds them. Scanner initial failure and invalid explicit paths remain fatal. - `workers:get-status`, `workers:restart`, and full `workers:update` snapshots - drive Settings → Service → Workers and TUI Overview (all twelve workers, `R` + drive Settings → Service → Workers and TUI Overview (all thirteen workers, `R` with confirmation). Clients recover worker snapshots after epoch changes; the 15 s broker heartbeat / 45 s client-silence layer remains separate. - Cross-process hang tests are **Unix-only; Windows behavior remains unproven**. @@ -405,7 +415,7 @@ remaining platform and live-system checks below are still release gates. | Limitation | Disposition | Remaining boundary / release testing | | --- | --- | --- | -| Worker hangs | **Implemented, pending release testing.** Auto-restart for eight child-free workers; engine-manager and the three proxies degrade and require manual restart on a hang. | [Worker Health](#worker-health) defines the eligible workers, probe thresholds and shared retry budget. Windows cross-process hang recovery remains unproven; verify packaged recovery and confirmation on release platforms. | +| Worker hangs | **Implemented, pending release testing.** Auto-restart for eight child-free workers; engine-manager and the four proxies degrade and require manual restart on a hang. | [Worker Health](#worker-health) defines the eligible workers, probe thresholds and shared retry budget. Windows cross-process hang recovery remains unproven; verify packaged recovery and confirmation on release platforms. | | Optional initial spawn failure | **Implemented, pending release testing.** Optional spawn failures retry under the same bounded budget as crashes and eligible hangs. | Missing default binaries stay `missing` until explicit retry; scanner initial failure and invalid explicit paths remain fatal. Verify packaged startup/retry and exhaustion; see [Worker Health](#worker-health). | | macOS unclustered reachability | **Accepted and documented for this release; not fixed.** | Needs controlled second-machine reproduction before selecting a remedy. Restart PAIR as the documented workaround; see [known issue](known-issues.mdx#macos-can-stop-answering-lan-connections-while-unclustered). | | GPU memory semantics | **Implemented, pending release testing.** Dedicated/unified/shared/unavailable labels preserve unknown readings; Intel integrated GPUs on Macs are labelled shared. | Linux AMD/Intel remain **Memory unavailable** (names-only); Windows runtime is untested on this host. Adapter capacities, independent counters and history need release-platform checks; see [GPU Memory Semantics](#gpu-memory-semantics). | diff --git a/docs/terminal-interface.mdx b/docs/terminal-interface.mdx index 7906126e0..570a44db7 100644 --- a/docs/terminal-interface.mdx +++ b/docs/terminal-interface.mdx @@ -157,7 +157,7 @@ Tab switching and `q` do not work until you do. | # | Tab | What It Shows | | --- | --- | --- | -| 1 | **Overview** | Broker uptime/version and all twelve workers: state, attempts, last error; `R` restarts with confirmation | +| 1 | **Overview** | Broker uptime/version and all thirteen workers: state, attempts, last error; `R` restarts with confirmation | | 2 | **Errors** | Active service errors by severity, age, node, and message | | 3 | **Nodes** | Nodes discovered on the network, with `Connected` or `In cluster` status | | 4 | **Proxies** | Ollama, LM Studio and oMLX proxies: listening port, discovered upstreams, and which node is selected | diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx index 7a8cf1e1a..85b5737f3 100644 --- a/docs/troubleshooting.mdx +++ b/docs/troubleshooting.mdx @@ -200,7 +200,7 @@ body bytes. The `120s` response-header timeout does not cut off this wait. Pre-commit connection/body read failures still retry on another candidate by default, but silence after headers alone does not. -An optional, separate first-body limit is available in all three proxies: +An optional, separate first-body limit is available in all four proxies: `--first-body-timeout 10m` for standalone use, or `NVPAIR_PROXY_FIRST_BODY_TIMEOUT=10m` in PAIR's inherited environment, followed by a restart. On macOS, an app opened from the Finder can inherit it via diff --git a/scripts/wipe-app-data.ps1 b/scripts/wipe-app-data.ps1 index c6cfc5b9a..33d57e363 100644 --- a/scripts/wipe-app-data.ps1 +++ b/scripts/wipe-app-data.ps1 @@ -136,6 +136,7 @@ $PairProcs = @( 'nvpair-ui-broker', 'ollama-proxy', 'lmstudio-proxy', + 'llamacpp-proxy', 'nvpair-node-info', 'nvpair-node-scanner', 'nvpair-manual-nodes', diff --git a/scripts/wipe-app-data.sh b/scripts/wipe-app-data.sh index 335415def..b72de1ecd 100755 --- a/scripts/wipe-app-data.sh +++ b/scripts/wipe-app-data.sh @@ -148,6 +148,7 @@ PAIR_PROCS=( nvpair-ui-broker ollama-proxy lmstudio-proxy + llamacpp-proxy nvpair-node-info nvpair-node-scanner nvpair-manual-nodes diff --git a/services/.gitignore b/services/.gitignore index 4dd2ff5e1..2084efbc3 100644 --- a/services/.gitignore +++ b/services/.gitignore @@ -12,6 +12,7 @@ dist/ # happen to share a name with a component directory. ollama-proxy/ollama-proxy lmstudio-proxy/lmstudio-proxy +llamacpp-proxy/llamacpp-proxy omlx-proxy/omlx-proxy nvpair-node-info/nvpair-node-info nvpair-node-scanner/nvpair-node-scanner diff --git a/services/build.bat b/services/build.bat index d1e03d27d..7856f97cd 100644 --- a/services/build.bat +++ b/services/build.bat @@ -41,6 +41,7 @@ for /f "delims=" %%V in ('jq -r ".product" "%VERSIONS_FILE%"') for /f "delims=" %%V in ('jq -r --arg k "ollama-proxy" ".components[$k]" "%VERSIONS_FILE%"') do set "V_PROXY=%%V" for /f "delims=" %%V in ('jq -r --arg k "lmstudio-proxy" ".components[$k]" "%VERSIONS_FILE%"') do set "V_LMPROXY=%%V" for /f "delims=" %%V in ('jq -r --arg k "omlx-proxy" ".components[$k]" "%VERSIONS_FILE%"') do set "V_OMPROXY=%%V" +for /f "delims=" %%V in ('jq -r --arg k "llamacpp-proxy" ".components[$k]" "%VERSIONS_FILE%"') do set "V_LCPROXY=%%V" for /f "delims=" %%V in ('jq -r --arg k "nvpair-node-info" ".components[$k]" "%VERSIONS_FILE%"') do set "V_NINFO=%%V" for /f "delims=" %%V in ('jq -r --arg k "nvpair-node-scanner" ".components[$k]" "%VERSIONS_FILE%"') do set "V_NSCAN=%%V" for /f "delims=" %%V in ('jq -r --arg k "nvpair-manual-nodes" ".components[$k]" "%VERSIONS_FILE%"') do set "V_MNODES=%%V" @@ -63,6 +64,7 @@ echo product = %V_PRODUCT% echo ollama-proxy = %V_PROXY% echo lmstudio-proxy = %V_LMPROXY% echo omlx-proxy = %V_OMPROXY% +echo llamacpp-proxy = %V_LCPROXY% echo nvpair-node-info = %V_NINFO% echo nvpair-node-scanner = %V_NSCAN% echo nvpair-manual-nodes = %V_MNODES% @@ -81,76 +83,81 @@ echo Building all components echo ======================================== echo. -echo [1/14] Building ollama-proxy (v%V_PROXY%)... +echo [1/15] Building ollama-proxy (v%V_PROXY%)... cd /d "%ROOT%ollama-proxy" go build -ldflags "-X main.Version=%V_PROXY%" -o ollama-proxy.exe . || goto :fail echo OK -echo [2/14] Building lmstudio-proxy (v%V_LMPROXY%)... +echo [2/15] Building lmstudio-proxy (v%V_LMPROXY%)... cd /d "%ROOT%lmstudio-proxy" go build -ldflags "-X main.Version=%V_LMPROXY%" -o lmstudio-proxy.exe . || goto :fail echo OK -echo [3/14] Building omlx-proxy (v%V_OMPROXY%)... +echo [3/15] Building omlx-proxy (v%V_OMPROXY%)... cd /d "%ROOT%omlx-proxy" go build -ldflags "-X main.Version=%V_OMPROXY%" -o omlx-proxy.exe . || goto :fail echo OK -echo [4/14] Building nvpair-node-info (v%V_NINFO%)... +echo [4/15] Building nvpair-node-info (v%V_NINFO%)... cd /d "%ROOT%nvpair-node-info" go build -ldflags "-X main.Version=%V_NINFO%" -o nvpair-node-info.exe . || goto :fail echo OK -echo [5/14] Building nvpair-node-scanner (v%V_NSCAN%)... +echo [5/15] Building nvpair-node-scanner (v%V_NSCAN%)... cd /d "%ROOT%nvpair-node-scanner" go build -ldflags "-X main.Version=%V_NSCAN%" -o nvpair-node-scanner.exe . || goto :fail echo OK -echo [6/14] Building nvpair-manual-nodes (v%V_MNODES%)... +echo [6/15] Building nvpair-manual-nodes (v%V_MNODES%)... cd /d "%ROOT%nvpair-manual-nodes" go build -ldflags "-X main.Version=%V_MNODES%" -o nvpair-manual-nodes.exe . || goto :fail echo OK -echo [7/14] Building nvpair-workload-manager (v%V_WLMGR%)... +echo [7/15] Building nvpair-workload-manager (v%V_WLMGR%)... cd /d "%ROOT%nvpair-workload-manager" go build -ldflags "-X main.Version=%V_WLMGR%" -o nvpair-workload-manager.exe . || goto :fail echo OK -echo [8/14] Building nvpair-errors (v%V_ERRORS%)... +echo [8/15] Building nvpair-errors (v%V_ERRORS%)... cd /d "%ROOT%nvpair-errors" go build -ldflags "-X main.Version=%V_ERRORS%" -o nvpair-errors.exe . || goto :fail echo OK -echo [9/14] Building nvpair-engine-manager (v%V_ENGMGR%)... +echo [9/15] Building nvpair-engine-manager (v%V_ENGMGR%)... cd /d "%ROOT%nvpair-engine-manager" go build -ldflags "-X main.Version=%V_ENGMGR%" -o nvpair-engine-manager.exe . || goto :fail echo OK -echo [10/14] Building nvpair-node-settings (v%V_NSETTINGS%)... +echo [10/15] Building nvpair-node-settings (v%V_NSETTINGS%)... cd /d "%ROOT%nvpair-node-settings" go build -ldflags "-X main.Version=%V_NSETTINGS%" -o nvpair-node-settings.exe . || goto :fail echo OK -echo [11/14] Building nvpair-ui-broker (v%V_BROKER%)... +echo [11/15] Building nvpair-ui-broker (v%V_BROKER%)... cd /d "%ROOT%nvpair-ui-broker" go build -ldflags "-X main.Version=%V_BROKER%" -o nvpair-ui-broker.exe . || goto :fail echo OK -echo [12/14] Building nvpair-cluster-manager (v%V_CLUMGR%)... +echo [12/15] Building nvpair-cluster-manager (v%V_CLUMGR%)... cd /d "%ROOT%nvpair-cluster-manager" go build -ldflags "-X main.Version=%V_CLUMGR%" -o nvpair-cluster-manager.exe . || goto :fail echo OK -echo [13/14] Building nvpair-job-scheduler (v%V_SCHED%)... +echo [13/15] Building nvpair-job-scheduler (v%V_SCHED%)... cd /d "%ROOT%nvpair-job-scheduler" go build -ldflags "-X main.Version=%V_SCHED%" -o nvpair-job-scheduler.exe . || goto :fail echo OK -echo [14/14] Building nvpair-tui (v%V_TUI%)... +echo [14/15] Building nvpair-tui (v%V_TUI%)... cd /d "%ROOT%nvpair-tui" go build -ldflags "-X main.Version=%V_TUI%" -o nvpair-tui.exe . || goto :fail echo OK +echo [15/15] Building llamacpp-proxy (v%V_LCPROXY%)... +cd /d "%ROOT%llamacpp-proxy" +go build -ldflags "-X main.Version=%V_LCPROXY%" -o llamacpp-proxy.exe . || goto :fail +echo OK + echo. echo ======================================== echo Copying binaries to %BIN_OUT% @@ -165,6 +172,7 @@ mkdir "%BIN_OUT%" copy /y "%ROOT%ollama-proxy\ollama-proxy.exe" "%BIN_OUT%\ollama-proxy.exe" >nul || goto :fail copy /y "%ROOT%lmstudio-proxy\lmstudio-proxy.exe" "%BIN_OUT%\lmstudio-proxy.exe" >nul || goto :fail copy /y "%ROOT%omlx-proxy\omlx-proxy.exe" "%BIN_OUT%\omlx-proxy.exe" >nul || goto :fail +copy /y "%ROOT%llamacpp-proxy\llamacpp-proxy.exe" "%BIN_OUT%\llamacpp-proxy.exe" >nul || goto :fail copy /y "%ROOT%nvpair-node-info\nvpair-node-info.exe" "%BIN_OUT%\nvpair-node-info.exe" >nul || goto :fail copy /y "%ROOT%nvpair-node-scanner\nvpair-node-scanner.exe" "%BIN_OUT%\nvpair-node-scanner.exe" >nul || goto :fail copy /y "%ROOT%nvpair-manual-nodes\nvpair-manual-nodes.exe" "%BIN_OUT%\nvpair-manual-nodes.exe" >nul || goto :fail @@ -185,6 +193,7 @@ echo. echo Proxy: %BIN_OUT%\ollama-proxy.exe echo LM Studio Proxy: %BIN_OUT%\lmstudio-proxy.exe echo oMLX Proxy: %BIN_OUT%\omlx-proxy.exe +echo llama.cpp Proxy: %BIN_OUT%\llamacpp-proxy.exe echo Node Info: %BIN_OUT%\nvpair-node-info.exe echo Node Scanner: %BIN_OUT%\nvpair-node-scanner.exe echo Manual Nodes: %BIN_OUT%\nvpair-manual-nodes.exe diff --git a/services/build.sh b/services/build.sh index bdf26c85d..5e3700318 100755 --- a/services/build.sh +++ b/services/build.sh @@ -69,6 +69,7 @@ V_CLUMGR=$( jq -r --arg k 'nvpair-cluster-manager' '.components[$k]' "$VERSIONS_ V_SCHED=$( jq -r --arg k 'nvpair-job-scheduler' '.components[$k]' "$VERSIONS_FILE") V_TUI=$( jq -r --arg k 'nvpair-tui' '.components[$k]' "$VERSIONS_FILE") V_OMPROXY=$(jq -r --arg k 'omlx-proxy' '.components[$k]' "$VERSIONS_FILE") +V_LCPROXY=$(jq -r --arg k 'llamacpp-proxy' '.components[$k]' "$VERSIONS_FILE") if [[ -z "$V_PRODUCT" || "$V_PRODUCT" == "null" ]]; then echo "ERROR: failed to parse versions.json" >&2 @@ -79,6 +80,7 @@ printf ' product = %s\n' "$V_PRODUCT" printf ' ollama-proxy = %s\n' "$V_PROXY" printf ' lmstudio-proxy = %s\n' "$V_LMPROXY" printf ' omlx-proxy = %s\n' "$V_OMPROXY" +printf ' llamacpp-proxy = %s\n' "$V_LCPROXY" printf ' nvpair-node-info = %s\n' "$V_NINFO" printf ' nvpair-node-scanner = %s\n' "$V_NSCAN" printf ' nvpair-manual-nodes = %s\n' "$V_MNODES" @@ -99,7 +101,7 @@ echo build_subbinary() { local idx="$1" name="$2" version="$3" - echo "[$idx/14] Building $name (v$version)..." + echo "[$idx/15] Building $name (v$version)..." (cd "$ROOT/$name" && go build -ldflags "-X main.Version=$version" -o "$name" .) echo " OK" } @@ -117,6 +119,7 @@ build_subbinary 11 nvpair-ui-broker "$V_BROKER" build_subbinary 12 nvpair-cluster-manager "$V_CLUMGR" build_subbinary 13 nvpair-job-scheduler "$V_SCHED" build_subbinary 14 nvpair-tui "$V_TUI" +build_subbinary 15 llamacpp-proxy "$V_LCPROXY" BIN_OUT="$ROOT/build/bin" @@ -136,6 +139,7 @@ mkdir -p "$BIN_OUT" cp "$ROOT/ollama-proxy/ollama-proxy" "$BIN_OUT/ollama-proxy" cp "$ROOT/lmstudio-proxy/lmstudio-proxy" "$BIN_OUT/lmstudio-proxy" cp "$ROOT/omlx-proxy/omlx-proxy" "$BIN_OUT/omlx-proxy" +cp "$ROOT/llamacpp-proxy/llamacpp-proxy" "$BIN_OUT/llamacpp-proxy" cp "$ROOT/nvpair-node-info/nvpair-node-info" "$BIN_OUT/nvpair-node-info" cp "$ROOT/nvpair-node-scanner/nvpair-node-scanner" "$BIN_OUT/nvpair-node-scanner" cp "$ROOT/nvpair-manual-nodes/nvpair-manual-nodes" "$BIN_OUT/nvpair-manual-nodes" @@ -156,6 +160,7 @@ echo printf ' Proxy: %s\n' "$BIN_OUT/ollama-proxy" printf ' LM Studio Proxy: %s\n' "$BIN_OUT/lmstudio-proxy" printf ' oMLX Proxy: %s\n' "$BIN_OUT/omlx-proxy" +printf ' llama.cpp Proxy: %s\n' "$BIN_OUT/llamacpp-proxy" printf ' Node Info: %s\n' "$BIN_OUT/nvpair-node-info" printf ' Node Scanner: %s\n' "$BIN_OUT/nvpair-node-scanner" printf ' Manual Nodes: %s\n' "$BIN_OUT/nvpair-manual-nodes" diff --git a/services/installer/nvpair-setup.nsi b/services/installer/nvpair-setup.nsi index cafba527b..c1f6949ca 100644 --- a/services/installer/nvpair-setup.nsi +++ b/services/installer/nvpair-setup.nsi @@ -110,6 +110,7 @@ FunctionEnd nsExec::ExecToLog 'taskkill /F /IM "ollama-proxy.exe"' nsExec::ExecToLog 'taskkill /F /IM "lmstudio-proxy.exe"' nsExec::ExecToLog 'taskkill /F /IM "omlx-proxy.exe"' + nsExec::ExecToLog 'taskkill /F /IM "llamacpp-proxy.exe"' nsExec::ExecToLog 'taskkill /F /IM "nvpair-node-info.exe"' nsExec::ExecToLog 'taskkill /F /IM "nvpair-node-scanner.exe"' nsExec::ExecToLog 'taskkill /F /IM "nvpair-manual-nodes.exe"' @@ -166,6 +167,7 @@ Section "Install" ; (loopback plaintext + cluster mTLS ingress); it listens for clients and ; browses mDNS, so it gets firewall rules below. File "..\build\bin\omlx-proxy.exe" + File "..\build\bin\llamacpp-proxy.exe" File "..\build\bin\nvpair-node-info.exe" File "..\build\bin\nvpair-node-scanner.exe" File "..\build\bin\nvpair-manual-nodes.exe" @@ -237,6 +239,7 @@ Section "Install" nsExec::ExecToLog 'netsh advfirewall firewall add rule name="NVPAIR Ollama Proxy" dir=in action=allow program="$INSTDIR\bin\ollama-proxy.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="NVPAIR LM Studio Proxy" dir=in action=allow program="$INSTDIR\bin\lmstudio-proxy.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="NVPAIR oMLX Proxy" dir=in action=allow program="$INSTDIR\bin\omlx-proxy.exe" enable=yes profile=any remoteip=localsubnet' + nsExec::ExecToLog 'netsh advfirewall firewall add rule name="NVPAIR llama.cpp Proxy" dir=in action=allow program="$INSTDIR\bin\llamacpp-proxy.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="NVPAIR Node Info" dir=in action=allow program="$INSTDIR\bin\nvpair-node-info.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="NVPAIR Node Scanner" dir=in action=allow program="$INSTDIR\bin\nvpair-node-scanner.exe" enable=yes profile=any remoteip=localsubnet' @@ -245,6 +248,7 @@ Section "Install" nsExec::ExecToLog 'netsh advfirewall firewall add rule name="NVPAIR mDNS (UDP 5353)" dir=in action=allow protocol=UDP localport=5353 program="$INSTDIR\bin\ollama-proxy.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="NVPAIR mDNS LM Studio Proxy (UDP 5353)" dir=in action=allow protocol=UDP localport=5353 program="$INSTDIR\bin\lmstudio-proxy.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="NVPAIR mDNS oMLX Proxy (UDP 5353)" dir=in action=allow protocol=UDP localport=5353 program="$INSTDIR\bin\omlx-proxy.exe" enable=yes profile=any remoteip=localsubnet' + nsExec::ExecToLog 'netsh advfirewall firewall add rule name="NVPAIR mDNS llama.cpp Proxy (UDP 5353)" dir=in action=allow protocol=UDP localport=5353 program="$INSTDIR\bin\llamacpp-proxy.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="NVPAIR mDNS Node Info (UDP 5353)" dir=in action=allow protocol=UDP localport=5353 program="$INSTDIR\bin\nvpair-node-info.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="NVPAIR mDNS Node Scanner (UDP 5353)" dir=in action=allow protocol=UDP localport=5353 program="$INSTDIR\bin\nvpair-node-scanner.exe" enable=yes profile=any remoteip=localsubnet' nsExec::ExecToLog 'netsh advfirewall firewall add rule name="NVPAIR mDNS Workload Manager (UDP 5353)" dir=in action=allow protocol=UDP localport=5353 program="$INSTDIR\bin\nvpair-workload-manager.exe" enable=yes profile=any remoteip=localsubnet' @@ -283,10 +287,12 @@ Section "Uninstall" nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="NVPAIR Ollama Proxy"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="NVPAIR LM Studio Proxy"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="NVPAIR oMLX Proxy"' + nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="NVPAIR llama.cpp Proxy"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="NVPAIR Node Info"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="NVPAIR mDNS (UDP 5353)"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="NVPAIR mDNS LM Studio Proxy (UDP 5353)"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="NVPAIR mDNS oMLX Proxy (UDP 5353)"' + nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="NVPAIR mDNS llama.cpp Proxy (UDP 5353)"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="NVPAIR Node Scanner"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="NVPAIR mDNS Node Info (UDP 5353)"' nsExec::ExecToLog 'netsh advfirewall firewall delete rule name="NVPAIR mDNS Node Scanner (UDP 5353)"' @@ -304,6 +310,7 @@ Section "Uninstall" Delete "$INSTDIR\bin\ollama-proxy.exe" Delete "$INSTDIR\bin\lmstudio-proxy.exe" Delete "$INSTDIR\bin\omlx-proxy.exe" + Delete "$INSTDIR\bin\llamacpp-proxy.exe" Delete "$INSTDIR\bin\nvpair-node-info.exe" Delete "$INSTDIR\bin\nvpair-node-scanner.exe" Delete "$INSTDIR\bin\nvpair-manual-nodes.exe" diff --git a/services/llamacpp-proxy/README.md b/services/llamacpp-proxy/README.md new file mode 100644 index 000000000..91ce702ed --- /dev/null +++ b/services/llamacpp-proxy/README.md @@ -0,0 +1,59 @@ + + +# llama.cpp proxy + +A separate supervised OpenAI-compatible proxy for llama.cpp, copied from +[`lmstudio-proxy`](../lmstudio-proxy/README.md). Its control protocol, routing, +strict routing, streaming, cancellation, bounded request bodies, scheduler +priority, workload identity, telemetry, stall retry/cooldown and opt-in +`--first-body-timeout` behavior match that worker. LM Studio is unchanged. + +Default proxy listener: **1237**. The managed engine listens on **127.0.0.1:8082**. +1237 is separate from Ollama (11434), LM Studio (1234), oMLX (1236), their +engine listeners, and node services (14318–14323). `--port` overrides the default; +`set-port` persists to `llamacpp-proxy-port.json`; `--ignore-persisted-port` +selects the broker's accepted settings instead. Stored engine port 8082 is not +restored as a proxy listener. Shared listen conflict checks apply. + +`GET /v1/models` and `GET /models` aggregate the OpenAI `data` envelope from +candidate `/models` endpoints. IDs are exact, without implied tags. Model-bearing +inference only considers advertised llama.cpp owners, never another engine's +inventory. Supported inference routes are `POST /v1/chat/completions`, +`/v1/completions`, `/v1/embeddings`, and **`/v1/messages`**. The latter is native +in ggml-org/llama.cpp **b12171** (`tools/server/server.cpp` registers +`routes.post_anthropic_messages`); PAIR does not translate protocols. Streaming +responses pass through without retry after body commitment. + +The worker subscribes to `discovery:subscribe {services:["lc"]}`. Only verified +cluster peers are routing candidates; previews are not. Manual bare engines +follow the existing explicit-manual rules. Pinned mTLS peer ingress is terminal: +it forwards only to the broker-supplied healthy loopback engine, never onward. + +## Control protocol + +Newline-delimited JSON-RPC 2.0 on stdio, or `--ipc`. The broker owns lifecycle +and relays client calls and subscribed events under **`llamacpp-proxy:`**. + +- `worker/health` → `{ok:true}` (control-loop responsiveness only). +- `nodes/list`, `node/select`, `node/add-manual`, `node/remove-manual`. +- `node/set-priority`, `node/set-local-backend`, discovery snapshots. +- `set-port`, `port/cancel`, `log/set-level`, telemetry configuration. +- `ready`, `error`, `node/discovered`, `node/updated`, `node/removed`, + `node/selection-changed`, `proxy/request-started`, `proxy/request`, + `workload:started`, `workload:completed`, `workload:errored`, progress/activity. + +Workloads use `engine:"llamacpp"`, with per-process run identity. The broker +stamps their origin and forwards them to workload-manager. See the LM Studio +worker reference for unchanged payload shapes and flags. `--version` is stamped +from `services/versions.json`; build with `go build .` in this module. + +There is no model hub/download/delete adapter yet. Start llama.cpp with an +allowlisted `--model`/`-m` GGUF inside `{install_dir}-models`; see +[`LLAMACPP.md`](../nvpair-engine-manager/LLAMACPP.md). Starting without a model +retains router mode. + +The model-list rewrite and exact-ID semantics are adapted from +NVIDIA/Personal-AI-Router@6d2d67f's engine profile. diff --git a/services/llamacpp-proxy/abort_completion_test.go b/services/llamacpp-proxy/abort_completion_test.go new file mode 100644 index 000000000..f3c563615 --- /dev/null +++ b/services/llamacpp-proxy/abort_completion_test.go @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "testing" + + "nvpair-shared/prefixhash" +) + +func TestRealServerTruncatedStreamEmitsFailedCompletion(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "999") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("partial")) + })) + defer upstream.Close() + + rec := &prRec{} + discovery := NewDiscovery() + discovery.AddManual(nodeForModel(t, "node-a", upstream.URL, "model")) + p := NewProxy(NewCodec(rec), discovery, 1235) + defer p.closeIdleTransports() + server := httptest.NewServer(http.HandlerFunc(p.handleHTTP)) + defer server.Close() + + body := []byte(`{"model":"model","messages":[{"role":"user","content":"private canary"}]}`) + response, err := server.Client().Post(server.URL+"/v1/chat/completions", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + _, readErr := io.ReadAll(response.Body) + _ = response.Body.Close() + if readErr == nil { + t.Fatal("truncated upstream stream reached the client without an error") + } + if !rec.has(`"method":"proxy/request"`) || !rec.has(`"error":"upstream response stream aborted"`) { + t.Fatal("truncated real-server stream omitted its failed completion event") + } + context := prefixhash.Extract(http.MethodPost, "/v1/chat/completions", body) + if _, ok := p.prt.Lookup("model", context.Full); ok { + t.Fatal("truncated real-server stream registered a cache-affinity hint") + } +} diff --git a/services/llamacpp-proxy/activity_test.go b/services/llamacpp-proxy/activity_test.go new file mode 100644 index 000000000..49a63541c --- /dev/null +++ b/services/llamacpp-proxy/activity_test.go @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// The two inference proxies are held to deliberate parity, so the liveness +// report a streaming response raises is asserted on both. See ollama-proxy's +// activity_test.go for the reasoning behind the signal itself. + +// TestStreamedBytesReportNodeActivity mirrors the ollama-proxy test of the same +// name. +func TestStreamedBytesReportNodeActivity(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"choices":[]}`)) + })) + defer upstream.Close() + + rec := &prRec{} + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "serving-node", upstream.URL, "qwen")) + p := NewProxy(NewCodec(rec), disc, 1235) + + p.handleHTTP(httptest.NewRecorder(), + httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"qwen"}`))) + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) && !rec.has(`"method":"node/activity"`) { + time.Sleep(5 * time.Millisecond) + } + if !rec.has(`"method":"node/activity"`) { + t.Fatal("no node/activity was reported after the upstream streamed a response") + } + if !rec.has(`"hostUuid":"serving-node"`) { + t.Fatal("node/activity did not name the node that served the request") + } +} + +// A node that never wrote a response byte has proved nothing and must not be +// vouched for. +func TestNoActivityReportedWithoutUpstreamBytes(t *testing.T) { + rec := &prRec{} + disc := NewDiscovery() + // A port nothing is listening on: the dial fails, so no upstream byte can + // ever reach the client. It still has to advertise the requested model, or + // candidate pruning rejects the request before anything is dialled and the + // test passes without exercising the dial failure at all. + disc.AddManual(Node{ + ID: "dead-node", + Addresses: []string{"127.0.0.1"}, + Port: closedPortFor(t), + Models: []string{"qwen"}, + }) + p := NewProxy(NewCodec(rec), disc, 1235) + + p.handleHTTP(httptest.NewRecorder(), + httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"qwen"}`))) + + if rec.has(`"method":"node/activity"`) { + t.Fatal("activity was reported for a node that never answered") + } +} + +// closedPortFor returns a port nothing is listening on, by binding and releasing +// it, so the dial is a prompt refusal rather than a timeout. +func closedPortFor(t *testing.T) int { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + port := nodeFor(t, "probe", srv.URL).Port + srv.Close() + return port +} diff --git a/services/llamacpp-proxy/affinity.go b/services/llamacpp-proxy/affinity.go new file mode 100644 index 000000000..39bf11caf --- /dev/null +++ b/services/llamacpp-proxy/affinity.go @@ -0,0 +1,94 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "io" + "strings" + + "nvpair-shared/prefixhash" +) + +// affinityNode matches the longest previously served request prefix. Only +// digests survive the request; generated responses are never retained. +func (p *Proxy) affinityNode(context prefixhash.Context) string { + for _, digest := range context.Prefixes { + if record, ok := p.prt.Lookup(context.Model, digest); ok && record.Engine == workloadEngine { + return record.NodeID + } + } + return "" +} + +const ( + sseDoneMarker = "data: [DONE]" + maxCompletionLineBytes = 64 << 10 +) + +// A 200 status alone is insufficient: the engine body must reach a clean EOF. +// Streaming OpenAI-compatible responses must also carry their terminal SSE +// marker. This also handles ReverseProxy returning after a failed copy in +// direct calls. +type affinityBodyReader struct { + io.ReadCloser + complete bool + requireSSEDone bool + done bool + invalid bool + line []byte +} + +func newAffinityBodyReader(body io.ReadCloser, contentType string) *affinityBodyReader { + return &affinityBodyReader{ + ReadCloser: body, + requireSSEDone: strings.HasPrefix(strings.ToLower(contentType), "text/event-stream"), + } +} + +func (r *affinityBodyReader) Read(b []byte) (int, error) { + n, err := r.ReadCloser.Read(b) + if n > 0 && r.requireSSEDone { + r.observe(b[:n]) + } + if err == io.EOF { + if r.requireSSEDone { + r.inspectLine(r.line) + } + r.complete = !r.requireSSEDone || (r.done && !r.invalid) + } + return n, err +} + +func (r *affinityBodyReader) observe(chunk []byte) { + r.line = append(r.line, chunk...) + for { + end := bytes.IndexByte(r.line, '\n') + if end < 0 { + if len(r.line) > maxCompletionLineBytes { + if r.done { + r.invalid = true + } + r.line = nil + } + return + } + r.inspectLine(r.line[:end]) + r.line = r.line[end+1:] + } +} + +func (r *affinityBodyReader) inspectLine(line []byte) { + line = bytes.TrimSpace(line) + if len(line) == 0 { + return + } + if r.done { + r.invalid = true + return + } + if bytes.Equal(line, []byte(sseDoneMarker)) { + r.done = true + } +} diff --git a/services/llamacpp-proxy/cache_affinity_test.go b/services/llamacpp-proxy/cache_affinity_test.go new file mode 100644 index 000000000..d6b3910aa --- /dev/null +++ b/services/llamacpp-proxy/cache_affinity_test.go @@ -0,0 +1,228 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "nvpair-shared/prefixhash" + "nvpair-shared/schedulerwire" +) + +// A real conversation includes the assistant reply before the next user turn. +// Removing affinity promotion must make the second request follow scheduler B. +func TestCacheAffinityConversationAndFailover(t *testing.T) { + failed := false + engine := func(id string) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if id == "a" && failed { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"%s\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", id) + })) + } + a, b := engine("a"), engine("b") + defer a.Close() + defer b.Close() + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "a", a.URL, "model")) + disc.AddManual(nodeForModel(t, "b", b.URL, "model")) + rec := &prRec{} + p := NewProxy(NewCodec(rec), disc, 1235) + defer p.closeIdleTransports() + messages := []map[string]string{{"role": "system", "content": "private-system-canary"}, {"role": "user", "content": "private-user-canary"}} + for turn := 0; turn < 5; turn++ { + if turn == 2 { + failed = true + } + if turn == 0 { + p.SetPriority([]string{"a", "b"}) + } else { + p.SetPriority([]string{"b", "a"}) + } + body, _ := json.Marshal(map[string]interface{}{"model": "model", "messages": messages, "stream": true}) + w := httptest.NewRecorder() + p.handleHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(string(body)))) + want := "a" + if turn >= 2 { + want = "b" + } + if w.Code != 200 || !strings.Contains(w.Body.String(), `"content":"`+want+`"`) { + t.Fatalf("turn %d routed incorrectly: status %d response %s", turn, w.Code, w.Body.String()) + } + messages = append(messages, map[string]string{"role": "assistant", "content": want}, map[string]string{"role": "user", "content": "next-private-user-canary"}) + } + if rec.has("private-system-canary") || rec.has("private-user-canary") { + t.Fatal("prompt leaked into notifications") + } + if !rec.has(`"cache_affinity":true`) { + t.Fatal("no affinity completion reported") + } +} + +func TestCacheAffinityRespectsRoutingGates(t *testing.T) { + for _, tc := range []struct { + name string + pending, pressure int + pin string + remove, otherModel bool + }{ + {name: "pending", pending: 2}, {name: "pressure", pressure: 3}, {name: "pin", pin: "b"}, {name: "departed", remove: true}, {name: "inventory", otherModel: true}, + } { + t.Run(tc.name, func(t *testing.T) { + engine := func(id string) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, id) })) + } + a, b := engine("a"), engine("b") + defer a.Close() + defer b.Close() + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "a", a.URL, "model")) + disc.AddManual(nodeForModel(t, "b", b.URL, "model")) + p := testProxy(disc, 1235) + defer p.closeIdleTransports() + p.SetPriority([]string{"a", "b"}) + request := func(body string) string { + w := httptest.NewRecorder() + p.handleHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body))) + return w.Body.String() + } + if got := request(`{"model":"model","messages":[{"role":"user","content":"first"}]}`); got != "a" { + t.Fatal(got) + } + if tc.remove { + disc.RemoveManual("a") + } + if tc.otherModel { + disc.AddManual(nodeForModel(t, "a", a.URL, "different")) + } + p.SetSelected(tc.pin) + p.SetPrioritySnapshot(schedulerwire.Priority{Nodes: []string{"b", "a"}, Ranks: []schedulerwire.NodeRank{{ID: "a", Pending: tc.pending, GPUPressure: tc.pressure}, {ID: "b"}}}) + if got := request(`{"model":"model","messages":[{"role":"user","content":"first"},{"role":"assistant","content":"reply"},{"role":"user","content":"next"}]}`); got != "b" { + t.Fatalf("gate routed to %q, want b", got) + } + }) + } +} + +func TestCacheAffinityFailedResponseDoesNotRegister(t *testing.T) { + for _, failure := range []string{"status", "client-write", "truncated"} { + t.Run(failure, func(t *testing.T) { + a := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch failure { + case "status": + w.WriteHeader(400) + case "truncated": + w.Header().Set("Content-Length", "999") + } + io.WriteString(w, "partial") + })) + defer a.Close() + b := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "b") })) + defer b.Close() + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "a", a.URL, "model")) + disc.AddManual(nodeForModel(t, "b", b.URL, "model")) + p := testProxy(disc, 1235) + defer p.closeIdleTransports() + p.SetPriority([]string{"a", "b"}) + first := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"model","messages":[{"role":"user","content":"first"}]}`)) + if failure == "client-write" { + p.handleHTTP(&clientGoneWriter{err: io.ErrClosedPipe}, first) + } else { + p.handleHTTP(httptest.NewRecorder(), first) + } + p.SetPriority([]string{"b", "a"}) + w := httptest.NewRecorder() + p.handleHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"model","messages":[{"role":"user","content":"first"},{"role":"assistant","content":"reply"},{"role":"user","content":"next"}]}`))) + if w.Body.String() != "b" { + t.Fatalf("failed response established affinity: got %q", w.Body.String()) + } + }) + } +} + +func TestCacheAffinityCleanTruncatedSSEDoesNotRegister(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, "data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n") + })) + defer upstream.Close() + + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "a", upstream.URL, "model")) + p := testProxy(disc, 1235) + defer p.closeIdleTransports() + body := []byte(`{"model":"model","messages":[{"role":"user","content":"first"}]}`) + p.handleHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(string(body)))) + context := prefixhash.Extract(http.MethodPost, "/v1/chat/completions", body) + if _, ok := p.prt.Lookup("model", context.Full); ok { + t.Fatal("cleanly truncated SSE stream registered a cache-affinity hint") + } +} + +func TestCacheAffinitySSEPostDoneDataDoesNotRegister(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, "data: [DONE]\n\ndata: {\"choices\":[{\"delta\":{\"content\":\"after-done\"}}]}\n\n") + })) + defer upstream.Close() + + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "a", upstream.URL, "model")) + p := testProxy(disc, 1235) + defer p.closeIdleTransports() + body := []byte(`{"model":"model","messages":[{"role":"user","content":"first"}]}`) + p.handleHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(string(body)))) + context := prefixhash.Extract(http.MethodPost, "/v1/chat/completions", body) + if _, ok := p.prt.Lookup("model", context.Full); ok { + t.Fatal("SSE data after [DONE] registered a cache-affinity hint") + } +} + +func TestCacheAffinitySSEOversizedPostDoneDataDoesNotRegister(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, "data: [DONE]\n\n"+strings.Repeat("x", maxCompletionLineBytes+1)) + })) + defer upstream.Close() + + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "a", upstream.URL, "model")) + p := testProxy(disc, 1235) + defer p.closeIdleTransports() + body := []byte(`{"model":"model","messages":[{"role":"user","content":"first"}]}`) + p.handleHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(string(body)))) + context := prefixhash.Extract(http.MethodPost, "/v1/chat/completions", body) + if _, ok := p.prt.Lookup("model", context.Full); ok { + t.Fatal("oversized SSE data after [DONE] registered a cache-affinity hint") + } +} + +func TestCacheAffinityReservationAndThreshold(t *testing.T) { + p := prProxy(t) + p.SetPriority([]string{"b", "a"}) + for i, want := range []bool{true, true, false} { + candidates, affinity := p.reserveCandidateWithAffinity(reservationCandidates("b", "a"), "a") + if affinity != want { + t.Fatalf("reservation %d affinity %v want %v", i, affinity, want) + } + if want && candidates[0].id != "a" { + t.Fatal("warm node was not promoted") + } + } + p.SetPrioritySnapshot(schedulerwire.Priority{Nodes: []string{"b", "a"}, Ranks: []schedulerwire.NodeRank{{ID: "a", Pending: 1, GPUPressure: 2}}}) + candidates, affinity := p.reserveCandidateWithAffinity(reservationCandidates("b", "a"), "a") + if !affinity || candidates[0].id != "a" { + t.Fatal("pending 1 / pressure 2 should preserve affinity") + } +} diff --git a/services/llamacpp-proxy/choose_reachable_test.go b/services/llamacpp-proxy/choose_reachable_test.go new file mode 100644 index 000000000..1ca29b035 --- /dev/null +++ b/services/llamacpp-proxy/choose_reachable_test.go @@ -0,0 +1,271 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" + "nvpair-shared/reach" +) + +// waitForTarget polls targetURL until it settles on want. +// +// Routing never waits on a handshake — reach.Prefer answers with the node's own +// ranking and confirms behind the request — so the address a multi-homed peer +// settles on is what the requests after the first see. +func waitForTarget(t *testing.T, p *Proxy, n Node, want string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + u := p.targetURL(n) + if u != nil && u.Host == want { + return + } + if time.Now().After(deadline) { + got := "" + if u != nil { + got = u.Host + } + t.Fatalf("targetURL settled on %s, want %s", got, want) + } + time.Sleep(time.Millisecond) + } +} + +// countingChooser installs a target chooser that records how many connection +// attempts routing makes and whether they succeed, so a test can assert on +// confirmation behaviour without opening sockets. +func countingChooser(p *Proxy, accept bool) *atomic.Int32 { + var dials atomic.Int32 + p.targets = reach.NewChooser(reach.WithDial( + func(_, _ string, _ time.Duration) (net.Conn, error) { + dials.Add(1) + if !accept { + return nil, net.ErrClosed + } + c1, c2 := net.Pipe() + _ = c2.Close() + return c1, nil + })) + return &dials +} + +func TestChooseReachableFailsOverForPinnedPeer(t *testing.T) { + const peerUUID = "principal-peer" + const reachable = "192.0.2.11" + clusterDir := filepath.Join(t.TempDir(), "cluster") + clustertrusttest.Join(t, clusterDir, "cluster-xyz", "principal-self", peerUUID) + + p := testProxy(NewDiscovery(), 1235) + p.mesh = clustertrust.Open(clusterDir) + var dials atomic.Int32 + p.targets = reach.NewChooser(reach.WithDial( + func(_, address string, _ time.Duration) (net.Conn, error) { + dials.Add(1) + host, _, _ := net.SplitHostPort(address) + if host != reachable { + return nil, net.ErrClosed + } + c1, c2 := net.Pipe() + _ = c2.Close() + return c1, nil + })) + + n := Node{ + ID: "peer-a", + Port: 1234, + Addresses: []string{"192.0.2.10", "192.0.2.11"}, + ClusterUUID: peerUUID, + } + // The first request is not made to wait for the confirmation, so it uses the + // node's own top-ranked address; the ones behind it use the one that answers. + if u := p.targetURL(n); u == nil || u.Host != net.JoinHostPort("192.0.2.10", "1234") { + t.Fatalf("first selection = %v, want the published ranking without waiting", u) + } + waitForTarget(t, p, n, net.JoinHostPort(reachable, "1234")) + if dials.Load() != 2 { + t.Fatalf("pinned peer triggered %d TCP probes, want both candidates tried", dials.Load()) + } +} + +func TestChooseReachableProbesPlainMultiHomed(t *testing.T) { + p := testProxy(NewDiscovery(), 1235) + dials := countingChooser(p, true) + n := Node{ + ID: "manual-a", + Port: 1234, + Addresses: []string{"192.0.2.10", "192.0.2.11"}, + } + if u := p.targetURL(n); u == nil { + t.Fatal("targetURL returned nil") + } + deadline := time.Now().Add(2 * time.Second) + for dials.Load() == 0 { + if time.Now().After(deadline) { + t.Fatal("plain multi-homed target did not confirm reachability") + } + time.Sleep(time.Millisecond) + } +} + +// TestTargetURLFailsOverToAReachableAddress is the reported defect at the routing +// layer: a peer whose canonical address is a direct-connect link this host cannot +// reach must still be routed to, at the address that answers. +func TestTargetURLFailsOverToAReachableAddress(t *testing.T) { + const reachable = "10.172.55.129" + p := testProxy(NewDiscovery(), 1235) + p.targets = reach.NewChooser(reach.WithDial( + func(_, address string, _ time.Duration) (net.Conn, error) { + host, _, _ := net.SplitHostPort(address) + if host != reachable { + return nil, net.ErrClosed + } + c1, c2 := net.Pipe() + _ = c2.Close() + return c1, nil + })) + + n := Node{ + ID: "spark", + Port: 1234, + // The node's own ranking leads with a link only its cabled neighbour can + // reach; this host is not that neighbour. + Addresses: []string{"192.168.240.1", reachable}, + TXT: []string{"ip=192.168.240.1", "ips=192.168.240.1," + reachable}, + } + waitForTarget(t, p, n, net.JoinHostPort(reachable, "1234")) +} + +// TestNodeCandidatesKeepsPublishedOrder: the node ranked its addresses from +// evidence no observer has, so routing must try them in that order rather than +// re-sorting by address class. +func TestNodeCandidatesKeepsPublishedOrder(t *testing.T) { + n := Node{ + ID: "spark", + Port: 1234, + Addresses: []string{"192.168.240.1", "10.172.55.129"}, + TXT: []string{"ip=10.172.55.129", "ips=10.172.55.129,192.168.240.1"}, + } + got := nodeCandidates(n) + want := []string{ + net.JoinHostPort("10.172.55.129", "1234"), + net.JoinHostPort("192.168.240.1", "1234"), + } + if len(got) != len(want) || got[0] != want[0] || got[1] != want[1] { + t.Fatalf("nodeCandidates = %v, want %v", got, want) + } +} + +// fakeNetwork is a chooser dialer whose accepting address can be moved, so a test +// can describe an address that stops answering and another that starts. Probes run +// on background goroutines, so both fields are read concurrently with the test. +type fakeNetwork struct { + mu sync.Mutex + accepting string + dials atomic.Int32 +} + +func (f *fakeNetwork) accept(address string) { + f.mu.Lock() + defer f.mu.Unlock() + f.accepting = address +} + +func (f *fakeNetwork) install(p *Proxy) { + p.targets = reach.NewChooser(reach.WithDial( + func(_, address string, _ time.Duration) (net.Conn, error) { + f.dials.Add(1) + host, _, _ := net.SplitHostPort(address) + f.mu.Lock() + accepting := f.accepting + f.mu.Unlock() + if host != accepting { + return nil, net.ErrClosed + } + c1, c2 := net.Pipe() + _ = c2.Close() + return c1, nil + })) +} + +// confirmedDeadPeer builds a proxy whose only routing target is a multi-homed peer +// whose confirmed address no longer answers: the address is a loopback endpoint +// whose server is already closed, so a forwarded request fails at the transport +// the way an unplugged link does, while the chooser still believes in it. +func confirmedDeadPeer(t *testing.T, replacement string) (*Proxy, Node, *fakeNetwork) { + t.Helper() + dead := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + n := nodeForModel(t, "peer-a", dead.URL, "llama") + dead.Close() + confirmed := n.Addresses[0] + n.Addresses = append(n.Addresses, replacement) + + disc := NewDiscovery() + disc.AddManual(n) + p := testProxy(disc, 1235) + fake := &fakeNetwork{accepting: confirmed} + fake.install(p) + + waitForTarget(t, p, n, net.JoinHostPort(confirmed, strconv.Itoa(n.Port))) + return p, n, fake +} + +// assertReprobed moves the accepting address and requires the selections after the +// failure to confirm again and land on the replacement. A cached winner that +// outlived the failure keeps answering with the old address and dials nothing. +func assertReprobed(t *testing.T, p *Proxy, n Node, fake *fakeNetwork, replacement string) { + t.Helper() + probesBefore := fake.dials.Load() + fake.accept(replacement) + + waitForTarget(t, p, n, net.JoinHostPort(replacement, strconv.Itoa(n.Port))) + if fake.dials.Load() == probesBefore { + t.Fatal("selection probed nothing: the failed address is still cached") + } +} + +// TestUpstreamTransportFailureReprobesTheNextSelection: a dial failure against a +// multi-homed peer must retire the confirmed address. Without that, every later +// request keeps being sent to the address that just failed, and the peer's other +// published addresses are never tried — which is the whole reason routing confirms +// reachability in the first place. +func TestUpstreamTransportFailureReprobesTheNextSelection(t *testing.T) { + const replacement = "192.0.2.11" + p, n, fake := confirmedDeadPeer(t, replacement) + + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"llama"}`))) + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502 from the only, unreachable candidate", rec.Code) + } + + assertReprobed(t, p, n, fake, replacement) +} + +// TestModelListTransportFailureReprobesTheNextSelection: the aggregated model list +// reaches every candidate directly, so it learns about a dead address before any +// inference request does, and must retire it on the same evidence. +func TestModelListTransportFailureReprobesTheNextSelection(t *testing.T) { + const replacement = "192.0.2.11" + p, n, fake := confirmedDeadPeer(t, replacement) + + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/models", nil)) + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503 when the only inventory source is unreachable", rec.Code) + } + + assertReprobed(t, p, n, fake, replacement) +} diff --git a/services/llamacpp-proxy/cluster_trust_test.go b/services/llamacpp-proxy/cluster_trust_test.go new file mode 100644 index 000000000..39f1d188f --- /dev/null +++ b/services/llamacpp-proxy/cluster_trust_test.go @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "path/filepath" + "testing" + + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" +) + +// TestResolveCandidatesFollowsLivePinSet reproduces the join ordering that made +// a freshly-joined node route to nobody. The peer is discovered BEFORE this node +// has any cluster identity, and its discovery record never changes again — which +// is the normal steady state, because the relay only re-sends a node when its +// mDNS record actually moves. Routing must still pick the peer up the moment the +// pin lands, and drop it again the moment the pin is removed, because it reads +// the live pin set rather than a trust flag cached at discovery time. +func TestResolveCandidatesFollowsLivePinSet(t *testing.T) { + const peerUUID = "principal-peer" + clusterDir := filepath.Join(t.TempDir(), "cluster") + + disc := NewDiscovery() + disc.SetSubscribed([]Node{{ + ID: "peer-a", Host: "peer-a", Port: 1234, + Addresses: []string{"192.0.2.10"}, + IP: "192.0.2.10", + ClusterUUID: peerUUID, + }}) + p := testProxy(disc, 1235) + p.mesh = clustertrust.Open(clusterDir) + + // Pre-join: no identity, no pins, so the peer is not a routable target. + if got := p.resolveCandidates(""); len(got) != 0 { + t.Fatalf("pre-join candidates = %+v, want none", got) + } + + // The cluster-manager lands the join on disk while the proxy is running. No + // new discovery snapshot arrives — the peer's mDNS record has not changed. + clustertrusttest.Join(t, clusterDir, "cluster-xyz", "principal-self", peerUUID) + + cands := p.resolveCandidates("") + if len(cands) != 1 { + t.Fatalf("post-join candidates = %+v, want the peer", cands) + } + if cands[0].id != "peer-a" || cands[0].peerUUID != peerUUID || cands[0].url.Scheme != "https" { + t.Fatalf("post-join candidate = %+v, want peer-a over https pinned to %s", cands[0], peerUUID) + } + + // Removing the peer from the cluster retires it as a target just as promptly, + // again with no discovery event involved. + clustertrusttest.RemovePeerPin(t, clusterDir, peerUUID) + if got := p.resolveCandidates(""); len(got) != 0 { + t.Fatalf("post-removal candidates = %+v, want none", got) + } +} + +// TestResolveCandidatesRejectsUnpinnedClusteredPeer keeps the isolation property +// honest now that trust is read locally: a peer that advertises a cluster +// principal we hold no pin for is not routable, even though this node is itself +// a healthy cluster member. Two clusters on one LAN must not route to each other. +func TestResolveCandidatesRejectsUnpinnedClusteredPeer(t *testing.T) { + clusterDir := filepath.Join(t.TempDir(), "cluster") + clustertrusttest.Join(t, clusterDir, "cluster-ours", "principal-self", "principal-ourpeer") + + disc := NewDiscovery() + disc.SetSubscribed([]Node{{ + ID: "stranger", Host: "stranger", Port: 1234, + Addresses: []string{"192.0.2.30"}, + IP: "192.0.2.30", + ClusterUUID: "principal-stranger", + }}) + p := testProxy(disc, 1235) + p.mesh = clustertrust.Open(clusterDir) + + if got := p.resolveCandidates(""); len(got) != 0 { + t.Fatalf("candidates = %+v, want none for a peer in another cluster", got) + } +} diff --git a/services/llamacpp-proxy/codec.go b/services/llamacpp-proxy/codec.go new file mode 100644 index 000000000..5d6fd177b --- /dev/null +++ b/services/llamacpp-proxy/codec.go @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +// The newline-delimited JSON-RPC 2.0 codec is single-sourced in +// nvpair-shared/jsonrpc. These local aliases keep this package's call sites and +// tests unchanged after removing the copy-pasted per-service codec. + +import "nvpair-shared/jsonrpc" + +type ( + Message = jsonrpc.Message + RPCError = jsonrpc.RPCError + Codec = jsonrpc.Codec +) + +var NewCodec = jsonrpc.NewCodec diff --git a/services/llamacpp-proxy/cooldown_test.go b/services/llamacpp-proxy/cooldown_test.go new file mode 100644 index 000000000..572455631 --- /dev/null +++ b/services/llamacpp-proxy/cooldown_test.go @@ -0,0 +1,404 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "nvpair-shared/schedulerwire" +) + +type stallWire struct { + mu sync.Mutex + b []byte +} + +func (w *stallWire) Read([]byte) (int, error) { return 0, io.EOF } +func (w *stallWire) Write(b []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + w.b = append(w.b, b...) + return len(b), nil +} + +// Removing deadline demotion must send the second request into the frozen +// upstream again. Reset scheduler reservations so they cannot hide that bug. +func TestStalledCandidateSecondRequest(t *testing.T) { + old := proxyResponseTimeout + // Leave scheduling slack between the measured stall and the next request. + // First-body opt-ins below remain shorter and independent of this window. + proxyResponseTimeout = 3 * time.Second + defer func() { proxyResponseTimeout = old }() + + const originalBody = `{"model":"model","messages":[{"role":"user","content":"replay regression"}],"stream":false,"max_tokens":7}` + for _, tc := range []struct { + name, flag, env string + headers bool + }{ + {name: "no-headers-default"}, + {name: "headers-without-body-flag", headers: true, flag: "1s"}, + {name: "headers-without-body-env", headers: true, env: "1s"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(firstBodyTimeoutEnv, tc.env) + oldFirstBody := proxyFirstBodyTimeout + proxyFirstBodyTimeout = resolveFirstBodyTimeout(tc.flag) + t.Cleanup(func() { proxyFirstBodyTimeout = oldFirstBody }) + var stalledHits, goodHits atomic.Int32 + release := make(chan struct{}) + stalled := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + stalledHits.Add(1) + _, _ = io.Copy(io.Discard, r.Body) + if tc.headers { + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + } + <-release + })) + defer stalled.Close() + defer close(release) + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + goodHits.Add(1) + body, err := io.ReadAll(r.Body) + if err != nil || string(body) != originalBody { + t.Error("healthy peer did not receive the complete original request body") + } + _, _ = io.WriteString(w, "healthy") + })) + defer good.Close() + + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "stalled", stalled.URL, "model")) + disc.AddManual(nodeForModel(t, "good", good.URL, "model")) + wire := &stallWire{} + p := NewProxy(NewCodec(wire), disc, 1235) + defer p.closeIdleTransports() + request := func() time.Duration { + t.Helper() + p.SetPrioritySnapshot(schedulerwire.Priority{ + Nodes: []string{"stalled", "good"}, + Ranks: []schedulerwire.NodeRank{{ID: "stalled"}, {ID: "good"}}, + }) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(originalBody)).WithContext(ctx) + rec := httptest.NewRecorder() + start := time.Now() + p.handleHTTP(rec, req) + if rec.Code != http.StatusOK || rec.Body.String() != "healthy" { + t.Fatalf("response = %d %q, want healthy failover", rec.Code, rec.Body.String()) + } + return time.Since(start) + } + + first := request() + if stalledHits.Load() != 1 { + t.Fatalf("first request frozen hits = %d, want 1", stalledHits.Load()) + } + second := request() + if stalledHits.Load() != 1 { + t.Fatalf("second request paid the deadline again: frozen hits=%d; durations first=%s second=%s", + stalledHits.Load(), first, second) + } + if goodHits.Load() != 2 { + t.Fatalf("healthy hits = %d, want 2", goodHits.Load()) + } + t.Logf("first=%s second=%s (header deadline %s, first-body deadline %s)", first, second, proxyResponseTimeout, proxyFirstBodyTimeout) + assertStallWorkloads(t, wire, 2) + }) + } +} + +// A response-header deadline must not become a post-header generation limit. +// Long-context prefill can remain silent well beyond the header timeout. +func TestDefaultFirstBodyWaitIsUnbounded(t *testing.T) { + t.Setenv(firstBodyTimeoutEnv, "") + oldFirstBody := proxyFirstBodyTimeout + proxyFirstBodyTimeout = resolveFirstBodyTimeout("") + defer func() { proxyFirstBodyTimeout = oldFirstBody }() + old := proxyResponseTimeout + proxyResponseTimeout = 100 * time.Millisecond + defer func() { proxyResponseTimeout = old }() + var fallbackHits atomic.Int32 + slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + time.Sleep(3 * proxyResponseTimeout) + _, _ = io.WriteString(w, "prefill-complete") + })) + defer slow.Close() + fallback := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fallbackHits.Add(1) + _, _ = io.WriteString(w, "duplicate") + })) + defer fallback.Close() + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "slow", slow.URL, "model")) + disc.AddManual(nodeForModel(t, "fallback", fallback.URL, "model")) + p := testProxy(disc, 1235) + defer p.closeIdleTransports() + p.SetPriority([]string{"slow", "fallback"}) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"model"}`)).WithContext(ctx)) + if rec.Code != http.StatusOK || rec.Body.String() != "prefill-complete" || fallbackHits.Load() != 0 { + t.Fatalf("default cut off healthy post-header prefill: status=%d body=%q fallback hits=%d", + rec.Code, rec.Body.String(), fallbackHits.Load()) + } + if len(p.coolingSet(time.Now())) != 0 { + t.Fatal("healthy prefill created a default cooldown") + } +} + +func TestCooldownOrderingRecovery(t *testing.T) { + p := prProxy(t, "a", "b", "c", "d") + p.SetPriority([]string{"a", "b", "c", "d"}) + p.SetSelected("a") + now := time.Now() + p.noteStalled("a", now) + p.noteStalled("c", now) + assertOrder(t, candidateIDs(p), []string{"b", "d", "a", "c"}) + p.noteSuccess("a") + assertOrder(t, candidateIDs(p), []string{"a", "b", "d", "c"}) + if len(p.coolingSet(now.Add(proxyResponseTimeout))) != 0 { + t.Fatal("cooldown did not expire without traffic or success") + } + assertOrder(t, candidateIDs(p), []string{"a", "b", "c", "d"}) + other := prProxy(t, "a", "b") + p.noteStalled("a", now) + assertOrder(t, candidateIDs(other), []string{"a", "b"}) +} + +func TestCooldownReservationAndAffinity(t *testing.T) { + for _, tc := range []struct { + name, warm, pin string + ids, ranked, cooled, want []string + }{ + {name: "ranked", ids: []string{"a", "b"}, ranked: []string{"a", "b"}, cooled: []string{"a"}, want: []string{"b", "a"}}, + {name: "affinity", warm: "a", ids: []string{"a", "b"}, ranked: []string{"a", "b"}, cooled: []string{"a"}, want: []string{"b", "a"}}, + {name: "unlisted-peer", warm: "a", ids: []string{"a", "b"}, ranked: []string{"a"}, cooled: []string{"a"}, want: []string{"b", "a"}}, + {name: "manual-selection", pin: "a", ids: []string{"a", "b"}, ranked: []string{"a", "b"}, cooled: []string{"a"}, want: []string{"b", "a"}}, + {name: "all-cooling", ids: []string{"a", "b"}, ranked: []string{"a", "b"}, cooled: []string{"a", "b"}, want: []string{"a", "b"}}, + {name: "sole-cooling", ids: []string{"a"}, ranked: []string{"a"}, cooled: []string{"a"}, want: []string{"a"}}, + } { + t.Run(tc.name, func(t *testing.T) { + p := prProxy(t, tc.ids...) + p.SetPriority(tc.ranked) + p.SetSelected(tc.pin) + for _, id := range tc.cooled { + p.noteStalled(id, time.Now()) + } + got, affinity := p.reserveCandidateWithAffinity(reservationCandidates(tc.ids...), tc.warm) + var ids []string + for _, cand := range got { + ids = append(ids, cand.id) + } + assertOrder(t, ids, tc.want) + if affinity { + t.Fatal("affinity promoted a cooling candidate over a ready peer") + } + }) + } +} + +func TestPrecommitDropRetriesWithoutCooldown(t *testing.T) { + dropped := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // A 200 followed by EOF before the declared body starts. + w.Header().Set("Content-Length", "999") + w.WriteHeader(http.StatusOK) + })) + defer dropped.Close() + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "healthy") + })) + defer good.Close() + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "dropped", dropped.URL, "model")) + disc.AddManual(nodeForModel(t, "good", good.URL, "model")) + wire := &stallWire{} + p := NewProxy(NewCodec(wire), disc, 1235) + defer p.closeIdleTransports() + p.SetPriority([]string{"dropped", "good"}) + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"model"}`))) + if rec.Code != 200 || rec.Body.String() != "healthy" { + t.Fatalf("precommit drop did not fail over: %d %q", rec.Code, rec.Body.String()) + } + if len(p.coolingSet(time.Now())) != 0 { + t.Fatal("fast body failure was incorrectly classified as a stall") + } + assertStallWorkloads(t, wire, 1) +} + +func TestCommittedStreamNeverRetries(t *testing.T) { + var goodHits atomic.Int32 + broken := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "999") + _, _ = io.WriteString(w, "partial") + })) + defer broken.Close() + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + goodHits.Add(1) + _, _ = io.WriteString(w, "healthy") + })) + defer good.Close() + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "broken", broken.URL, "model")) + disc.AddManual(nodeForModel(t, "good", good.URL, "model")) + p := testProxy(disc, 1235) + defer p.closeIdleTransports() + p.SetPriority([]string{"broken", "good"}) + server := httptest.NewServer(http.HandlerFunc(p.handleHTTP)) + defer server.Close() + resp, err := server.Client().Post(server.URL+"/v1/chat/completions", "application/json", + strings.NewReader(`{"model":"model"}`)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err == nil || string(data) != "partial" || goodHits.Load() != 0 { + t.Fatalf("committed stream was retried or not truncated: body=%q error=%v goodHits=%d", data, err, goodHits.Load()) + } +} + +func TestCooldownSuccessAndFastStatus(t *testing.T) { + for _, code := range []int{http.StatusOK, http.StatusInternalServerError} { + t.Run(http.StatusText(code), func(t *testing.T) { + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(code) // empty 200 is a valid complete response + })) + defer engine.Close() + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "only", engine.URL, "model")) + p := testProxy(disc, 1235) + defer p.closeIdleTransports() + p.noteStalled("only", time.Now()) + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"model"}`))) + if rec.Code != code { + t.Fatalf("sole cooling candidate was not tried: status %d, want %d", rec.Code, code) + } + if cooling := p.coolingSet(time.Now())["only"]; cooling != (code != http.StatusOK) { + t.Fatalf("cooling=%v for status %d: only success should clear", cooling, code) + } + }) + } +} + +func TestCallerCancellationDoesNotCoolCandidate(t *testing.T) { + for _, headers := range []bool{false, true} { + t.Run(fmt.Sprint(headers), func(t *testing.T) { + release := make(chan struct{}) + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if headers { + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + } + <-release + })) + defer engine.Close() + defer close(release) + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "only", engine.URL, "model")) + p := testProxy(disc, 1235) + defer p.closeIdleTransports() + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + p.handleHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"model"}`)).WithContext(ctx)) + if len(p.coolingSet(time.Now())) != 0 { + t.Fatal("a caller deadline demoted the candidate") + } + }) + } +} + +func TestFinalBodyStallIsBounded(t *testing.T) { + old := proxyFirstBodyTimeout + proxyFirstBodyTimeout = 100 * time.Millisecond + defer func() { proxyFirstBodyTimeout = old }() + release := make(chan struct{}) + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + <-release + })) + defer engine.Close() + defer close(release) + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "only", engine.URL, "model")) + p := testProxy(disc, 1235) + defer p.closeIdleTransports() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", + strings.NewReader(`{"model":"model"}`)).WithContext(ctx)) + if rec.Code != http.StatusBadGateway || ctx.Err() != nil { + t.Fatalf("final body stall was not bounded before client deadline: status=%d context=%v", rec.Code, ctx.Err()) + } + if !p.coolingSet(time.Now())["only"] { + t.Fatal("final body stall was not remembered") + } +} + +func assertStallWorkloads(t *testing.T, wire *stallWire, count int) { + t.Helper() + wire.mu.Lock() + data := append([]byte(nil), wire.b...) + wire.mu.Unlock() + started := make(map[string]string) + completed := make(map[string]bool) + dec := json.NewDecoder(strings.NewReader(string(data))) + for dec.More() { + var frame struct { + Method string `json:"method"` + Params struct { + WorkloadInfo Workload `json:"workloadInfo"` + } `json:"params"` + } + if err := dec.Decode(&frame); err != nil { + t.Fatal(err) + } + wl := frame.Params.WorkloadInfo + switch frame.Method { + case workloadStartedMethod: + if wl.State != "running" || completed[wl.ID] { + t.Fatalf("invalid running workload event: %+v", wl) + } + started[wl.ID] = wl.ScheduledOn + case workloadCompletedMethod: + if started[wl.ID] != "good" { + t.Fatalf("no running workload event on serving node before completion: id=%s runningOn=%q", wl.ID, started[wl.ID]) + } + if completed[wl.ID] || wl.ScheduledOn != "good" { + t.Fatalf("invalid or duplicate completion: %+v", wl) + } + completed[wl.ID] = true + case workloadErroredMethod: + t.Fatalf("failover emitted a failed workload: %+v", wl) + } + } + if len(started) != count || len(completed) != count { + t.Fatalf("workloads started=%v completed=%v, want %d identities", started, completed, count) + } +} diff --git a/services/llamacpp-proxy/discovery.go b/services/llamacpp-proxy/discovery.go new file mode 100644 index 000000000..9b05581b6 --- /dev/null +++ b/services/llamacpp-proxy/discovery.go @@ -0,0 +1,165 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +// Discovery is the proxy's routing-target set. The proxy runs no mDNS of its +// own: routing targets are pushed down from the broker's discovery relay +// (discovery:nodes snapshots for the lm service) into the subscribed overlay, +// merged with user-added manual nodes. The proxy is itself advertised — as an lm +// service — by the node-scanner daemon's single _nvpair-node record, keyed off +// the engine port the broker's poller registers. +// +// The routable Node projection (IP / withPrimaryIP) and the manual-node +// overlay live here; request-path reachability (TCP-probe + failover) lives in +// proxy.go. + +import ( + "slices" + "sync" + + "nvpair-shared/discovery" + "nvpair-shared/netpick" +) + +// uuidFromTXT extracts a node's stable uuid= from its TXT records. Kept as a +// thin re-export of the shared helper (it was triplicated across the two proxies +// and the scanner before consolidation). +var uuidFromTXT = discovery.UUIDFromTXT + +// Node is the proxy's routable view of a node. It adds a canonical dialable IP +// field over the discovered node shape. +type Node struct { + ID string `json:"id"` + Host string `json:"host"` + Port int `json:"port"` + Addresses []string `json:"addresses"` + TXT []string `json:"txt"` + // Models is the latest model inventory carried by the broker's discovery + // snapshot. Model-bearing inference is eligible only when this list + // advertises the requested model; an empty list stays in discovery but is + // not an inference candidate until a later inventory update. + Models []string `json:"models,omitempty"` + // IP is the single canonical LAN address a consumer should dial/display for + // this node, resolved via the shared netpick ranker: the node's + // own ip= TXT if present, else the best-scored advertised IPv4. It is + // stamped onto outbound node/* notifications only (see withPrimaryIP) so a + // downstream consumer agrees on the same address the proxy routes to. + IP string `json:"ip,omitempty"` + // ClusterUUID is the relay peer's cluster principal (its mTLS cert UUID), + // carried from the discovery DirectoryNode. Non-empty only for a clustered + // peer; it is the key used to pin the peer's server cert when dialing its + // promoted proxy over cluster mTLS. Whether we actually hold that pin is + // resolved against the live mesh at routing time (resolveCandidates), never + // cached here: a cached answer goes stale the moment a peer is paired or + // removed. Internal routing metadata, not part of the proxy's outward node + // contract. + ClusterUUID string `json:"-"` +} + +// withPrimaryIP returns a copy of the node with IP resolved by the shared ranker +// (netpick.Primary over its TXT + Addresses). +func (n Node) withPrimaryIP() Node { + n.IP = netpick.Primary(n.TXT, n.Addresses) + return n +} + +// Discovery holds the proxy's routing targets: the relay-fed subscribed overlay +// and the user-added manual overlay. +type Discovery struct { + mu sync.RWMutex + manualNodes map[string]Node + // subscribedNodes are routing targets pushed down by the broker's discovery + // relay (discovery:nodes snapshots for the lm service), keyed by node ID (the + // directory instance name). + subscribedNodes map[string]Node +} + +func NewDiscovery() *Discovery { + return &Discovery{ + manualNodes: make(map[string]Node), + subscribedNodes: make(map[string]Node), + } +} + +// Nodes returns the merged subscribed + manual node set (manual entries that +// aren't also present via the relay are appended). +func (d *Discovery) Nodes() []Node { + d.mu.RLock() + defer d.mu.RUnlock() + out := make([]Node, 0, len(d.manualNodes)+len(d.subscribedNodes)) + seen := make(map[string]struct{}, len(d.subscribedNodes)) + for id, n := range d.subscribedNodes { + out = append(out, n) + seen[id] = struct{}{} + } + for id, n := range d.manualNodes { + if _, exists := seen[id]; !exists { + out = append(out, n) + } + } + return out +} + +// SetSubscribed replaces the relay-fed routing overlay with the given set and +// reports what changed versus the previous set (keyed by node ID): nodes newly +// present, nodes whose routable details changed, and nodes that dropped out. The +// broker pushes the full filtered snapshot on every change, so the overlay is +// replaced wholesale; the returned diff lets the caller emit node/discovered| +// updated|removed so a consumer (the UI) learns which peers currently run this +// engine. Manual nodes are a separate overlay and are untouched. +func (d *Discovery) SetSubscribed(nodes []Node) (discovered, updated, removed []Node) { + d.mu.Lock() + defer d.mu.Unlock() + next := make(map[string]Node, len(nodes)) + for _, n := range nodes { + next[n.ID] = n + switch prev, ok := d.subscribedNodes[n.ID]; { + case !ok: + discovered = append(discovered, n) + case !nodeEqual(prev, n): + updated = append(updated, n) + } + } + for id, prev := range d.subscribedNodes { + if _, ok := next[id]; !ok { + removed = append(removed, prev) + } + } + d.subscribedNodes = next + return discovered, updated, removed +} + +// nodeEqual reports whether two routable Nodes carry the same routing/display +// identity — the fields a consumer dials or renders. A change in any of them +// warrants a node/updated. +func nodeEqual(a, b Node) bool { + return a.ID == b.ID && a.Host == b.Host && a.Port == b.Port && a.IP == b.IP && + slices.Equal(a.Addresses, b.Addresses) && slices.Equal(a.TXT, b.TXT) && + slices.Equal(a.Models, b.Models) +} + +func (d *Discovery) AddManual(node Node) (added bool) { + d.mu.Lock() + defer d.mu.Unlock() + _, exists := d.manualNodes[node.ID] + d.manualNodes[node.ID] = node + return !exists +} + +func (d *Discovery) RemoveManual(id string) (removed bool) { + d.mu.Lock() + defer d.mu.Unlock() + _, exists := d.manualNodes[id] + if exists { + delete(d.manualNodes, id) + } + return exists +} + +func (d *Discovery) IsManual(id string) bool { + d.mu.RLock() + defer d.mu.RUnlock() + _, exists := d.manualNodes[id] + return exists +} diff --git a/services/llamacpp-proxy/e2e_test.go b/services/llamacpp-proxy/e2e_test.go new file mode 100644 index 000000000..a27f7d6ef --- /dev/null +++ b/services/llamacpp-proxy/e2e_test.go @@ -0,0 +1,217 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "time" +) + +// proxyBin is the real llamacpp-proxy binary, built once in TestMain so the +// e2e test exercises the shipped artifact (not just in-process handlers). +var proxyBin string + +func TestMain(m *testing.M) { + tmp, err := os.MkdirTemp("", "nvpair-lmproxy-e2e-*") + if err != nil { + panic(err) + } + suffix := "" + if runtime.GOOS == "windows" { + suffix = ".exe" + } + proxyBin = filepath.Join(tmp, "llamacpp-proxy"+suffix) + if out, err := exec.Command("go", "build", "-o", proxyBin, ".").CombinedOutput(); err != nil { + panic("build llamacpp-proxy: " + err.Error() + "\n" + string(out)) + } + code := m.Run() + _ = os.RemoveAll(tmp) + os.Exit(code) +} + +type e2eFrame struct { + ID json.RawMessage `json:"id"` + Method string `json:"method"` + Params json.RawMessage `json:"params"` + Result json.RawMessage `json:"result"` + Error json.RawMessage `json:"error"` +} + +func e2eReadFrames(r io.Reader, out chan<- e2eFrame) { + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for sc.Scan() { + var f e2eFrame + if err := json.Unmarshal(sc.Bytes(), &f); err != nil { + continue + } + out <- f + } +} + +func e2eSend(t *testing.T, w io.Writer, id int, method string, params any) { + t.Helper() + msg := map[string]any{"jsonrpc": "2.0", "id": id, "method": method} + if params != nil { + msg["params"] = params + } + data, _ := json.Marshal(msg) + if _, err := w.Write(append(data, '\n')); err != nil { + t.Fatalf("send %s: %v", method, err) + } +} + +func e2eWaitResult(t *testing.T, frames <-chan e2eFrame, id string, timeout time.Duration) { + t.Helper() + deadline := time.After(timeout) + for { + select { + case f := <-frames: + if string(f.ID) != id { + continue + } + if len(f.Error) > 0 && string(f.Error) != "null" { + t.Fatalf("rpc id %s returned error: %s", id, f.Error) + } + return + case <-deadline: + t.Fatalf("timed out waiting for response id %s", id) + } + } +} + +func e2eWaitReadyPort(t *testing.T, frames <-chan e2eFrame, timeout time.Duration) int { + t.Helper() + deadline := time.After(timeout) + for { + select { + case f := <-frames: + if f.Method != "ready" { + continue + } + var p struct { + Port int `json:"port"` + } + if err := json.Unmarshal(f.Params, &p); err != nil { + t.Fatalf("parse ready params: %v", err) + } + return p.Port + case <-deadline: + t.Fatalf("timed out waiting for ready notification") + return 0 + } + } +} + +func e2eFreePort(t *testing.T) int { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port +} + +func e2eSplitHostPort(t *testing.T, serverURL string) (string, int) { + t.Helper() + host, portStr, err := net.SplitHostPort(strings.TrimPrefix(serverURL, "http://")) + if err != nil { + t.Fatalf("split %q: %v", serverURL, err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + t.Fatalf("port %q: %v", portStr, err) + } + return host, port +} + +// TestE2EFailoverOverRealBinary spawns the real llamacpp-proxy binary and +// drives it the way the broker/UI does: register a busy (503) and a healthy +// (200) upstream as manual nodes over JSON-RPC stdio, then send a genuine +// OpenAI inference POST to the proxy's real HTTP port. It asserts the request +// fails over from the busy node to the healthy one, the original body is +// replayed, and CORS headers are present — the whole shipped path (binary + +// stdio control plane + HTTP forwarding + failover) end-to-end, no mocks. +func TestE2EFailoverOverRealBinary(t *testing.T) { + var gotBody string + busy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer busy.Close() + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + gotBody = string(b) + w.WriteHeader(http.StatusOK) + io.WriteString(w, `{"ok":true}`) + })) + defer good.Close() + + port := e2eFreePort(t) + cmd := exec.Command(proxyBin, "--port", strconv.Itoa(port), "--ignore-persisted-port") + stdin, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { + _ = stdin.Close() + _ = cmd.Process.Kill() + _ = cmd.Wait() + }() + + frames := make(chan e2eFrame, 256) + go e2eReadFrames(stdout, frames) + + if got := e2eWaitReadyPort(t, frames, 10*time.Second); got != port { + t.Fatalf("ready port = %d, want %d", got, port) + } + + busyHost, busyPort := e2eSplitHostPort(t, busy.URL) + goodHost, goodPort := e2eSplitHostPort(t, good.URL) + e2eSend(t, stdin, 1, "node/add-manual", map[string]any{"id": "busy", "host": busyHost, "port": busyPort, "addresses": []string{busyHost}, "models": []string{"m"}}) + e2eWaitResult(t, frames, "1", 5*time.Second) + e2eSend(t, stdin, 2, "node/add-manual", map[string]any{"id": "good", "host": goodHost, "port": goodPort, "addresses": []string{goodHost}, "models": []string{"m"}}) + e2eWaitResult(t, frames, "2", 5*time.Second) + // Select the busy node so the failover path is deterministic. + e2eSend(t, stdin, 3, "node/select", map[string]any{"id": "busy"}) + e2eWaitResult(t, frames, "3", 5*time.Second) + + resp, err := http.Post(fmt.Sprintf("http://127.0.0.1:%d/v1/chat/completions", port), "application/json", strings.NewReader(`{"model":"m"}`)) + if err != nil { + t.Fatalf("inference POST: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200 (should fail over from the 503 node)", resp.StatusCode) + } + if got := resp.Header.Get("Access-Control-Allow-Origin"); got != "*" { + t.Errorf("Access-Control-Allow-Origin = %q, want *", got) + } + if gotBody != `{"model":"m"}` { + t.Errorf("healthy upstream got body %q, want the original request body", gotBody) + } + + e2eSend(t, stdin, 9, "shutdown", nil) + e2eWaitResult(t, frames, "9", 5*time.Second) +} diff --git a/services/llamacpp-proxy/failover_test.go b/services/llamacpp-proxy/failover_test.go new file mode 100644 index 000000000..f86dd9f3a --- /dev/null +++ b/services/llamacpp-proxy/failover_test.go @@ -0,0 +1,600 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + "time" +) + +// rwNop is a no-op io.ReadWriter so a Codec can be constructed in tests +// without a real transport: reads hit EOF immediately and writes are +// discarded. handleHTTP only ever writes (notifications), so this is enough. +type rwNop struct{} + +func (rwNop) Read([]byte) (int, error) { return 0, io.EOF } +func (rwNop) Write(p []byte) (int, error) { return len(p), nil } + +func testProxy(disc *Discovery, port int) *Proxy { + return NewProxy(NewCodec(rwNop{}), disc, port) +} + +// nodeFor turns an httptest server URL into a discovery Node pointing at it. +func nodeFor(t *testing.T, id, serverURL string) Node { + t.Helper() + u, err := url.Parse(serverURL) + if err != nil { + t.Fatalf("parse %q: %v", serverURL, err) + } + host, portStr, err := net.SplitHostPort(u.Host) + if err != nil { + t.Fatalf("split %q: %v", u.Host, err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + t.Fatalf("port %q: %v", portStr, err) + } + return Node{ID: id, Addresses: []string{host}, Port: port} +} + +func nodeForModel(t *testing.T, id, serverURL, model string) Node { + t.Helper() + node := nodeFor(t, id, serverURL) + node.Models = []string{model} + return node +} + +// TestHandlePlain_OptionsPreflight: a CORS preflight is answered locally with +// 204 + permissive headers and never forwarded. +func TestHandlePlain_OptionsPreflight(t *testing.T) { + p := testProxy(NewDiscovery(), 11434) + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodOptions, "/v1/chat/completions", nil) + req.RemoteAddr = "127.0.0.1:40000" + req.Header.Set("Access-Control-Request-Headers", "X-Custom-Token") + p.handlePlain(rec, req) + + if rec.Code != http.StatusNoContent { + t.Fatalf("status = %d, want 204", rec.Code) + } + if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" { + t.Errorf("Access-Control-Allow-Origin = %q, want *", got) + } + if rec.Header().Get("Access-Control-Allow-Methods") == "" { + t.Errorf("missing Access-Control-Allow-Methods") + } + if got := rec.Header().Get("Access-Control-Expose-Headers"); got != "*" { + t.Errorf("Access-Control-Expose-Headers = %q, want *", got) + } + // The browser's requested headers are echoed so an arbitrary header clears preflight. + if got := rec.Header().Get("Access-Control-Allow-Headers"); got != "X-Custom-Token" { + t.Errorf("Access-Control-Allow-Headers = %q, want echoed X-Custom-Token", got) + } +} + +// TestHandlePlain_EngineCredentialedPreflightPreserved: when an engine opts an +// exact origin into credentialed CORS, its preflight policy reaches the browser +// instead of being replaced by the proxy's uncredentialed wildcard fallback. +func TestHandlePlain_EngineCredentialedPreflightPreserved(t *testing.T) { + preflightSeen := make(chan struct{}, 1) + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodOptions { + t.Errorf("engine method = %s, want OPTIONS", r.Method) + } + preflightSeen <- struct{}{} + w.Header().Set("Access-Control-Allow-Origin", "https://app.example") + w.Header().Set("Access-Control-Allow-Credentials", "true") + w.Header().Set("Access-Control-Allow-Methods", "POST") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type") + w.WriteHeader(http.StatusNoContent) + })) + defer engine.Close() + + disc := NewDiscovery() + disc.AddManual(nodeFor(t, "engine", engine.URL)) + p := testProxy(disc, 11434) + req := httptest.NewRequest(http.MethodOptions, "/v1/chat/completions", nil) + req.RemoteAddr = "127.0.0.1:40000" + req.Header.Set("Origin", "https://app.example") + req.Header.Set("Access-Control-Request-Method", http.MethodPost) + req.Header.Set("Access-Control-Request-Headers", "Content-Type") + rec := httptest.NewRecorder() + + p.handlePlain(rec, req) + + select { + case <-preflightSeen: + default: + t.Fatal("engine did not receive the credentialed preflight") + } + if rec.Code != http.StatusNoContent { + t.Errorf("status = %d, want %d", rec.Code, http.StatusNoContent) + } + if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example" { + t.Errorf("Access-Control-Allow-Origin = %q, want the engine's exact origin", got) + } + if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" { + t.Errorf("Access-Control-Allow-Credentials = %q, want the engine's true", got) + } +} + +// TestHandleHTTP_EngineCORSPolicyPreserved: an engine that declares its own +// origin policy keeps it. Replacing it with the proxy's wildcard would widen +// what the user configured, and would break a credentialed response outright. +func TestHandleHTTP_EngineCORSPolicyPreserved(t *testing.T) { + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "https://app.example") + w.Header().Set("Access-Control-Allow-Credentials", "true") + w.WriteHeader(http.StatusOK) + io.WriteString(w, `{"done":true}`) + })) + defer engine.Close() + + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "engine", engine.URL, "llama")) + p := testProxy(disc, 11434) + + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"llama"}`))) + + if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example" { + t.Errorf("Access-Control-Allow-Origin = %q, want the engine's own origin", got) + } + if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" { + t.Errorf("Access-Control-Allow-Credentials = %q, want the engine's true", got) + } +} + +// TestHandleHTTP_EngineCredentialsWithoutOriginDropped: an engine (or an +// intermediary in front of it) that sends Allow-Credentials but no origin has +// declared no policy to keep, so the proxy supplies its own. The wildcard it +// writes is invalid next to Allow-Credentials: true, and a browser rejects that +// pair, so the inherited header must not survive the forward. +func TestHandleHTTP_EngineCredentialsWithoutOriginDropped(t *testing.T) { + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Access-Control-Allow-Credentials", "true") + w.WriteHeader(http.StatusOK) + io.WriteString(w, `{"done":true}`) + })) + defer engine.Close() + + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "engine", engine.URL, "llama")) + p := testProxy(disc, 11434) + + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"llama"}`))) + + if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" { + t.Errorf("Access-Control-Allow-Origin = %q, want the proxy's wildcard", got) + } + if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" { + t.Errorf("Access-Control-Allow-Credentials = %q, want cleared alongside the wildcard origin", got) + } +} + +// TestHandleHTTP_HappyPathSingleNode: the common case — one healthy node +// answers directly, body forwarded, CORS present on the success response. +func TestHandleHTTP_HappyPathSingleNode(t *testing.T) { + var gotBody string + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + gotBody = string(b) + w.WriteHeader(http.StatusOK) + io.WriteString(w, `{"done":true}`) + })) + defer good.Close() + + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "good", good.URL, "llama")) + p := testProxy(disc, 11434) + + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"llama"}`))) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + if gotBody != `{"model":"llama"}` { + t.Errorf("node got body %q, want the original request body", gotBody) + } + if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" { + t.Errorf("Access-Control-Allow-Origin = %q, want * on success", got) + } +} + +// TestHandleHTTP_NoRetryOn400: a client error (400) is returned as-is and not +// failed over — retrying elsewhere would return the same error and mask it. +func TestHandleHTTP_NoRetryOn400(t *testing.T) { + hits := 0 + bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits++ + w.WriteHeader(http.StatusBadRequest) + io.WriteString(w, `{"error":"bad request"}`) + })) + defer bad.Close() + other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer other.Close() + + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "bad", bad.URL, "llama")) + disc.AddManual(nodeForModel(t, "other", other.URL, "llama")) + p := testProxy(disc, 11434) + p.SetSelected("bad") + + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"llama"}`))) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 (client errors must not fail over)", rec.Code) + } + if hits != 1 { + t.Errorf("bad node hit %d times, want exactly 1 (no retry on 400)", hits) + } +} + +// TestHandleHTTP_RejectionHasCORS: even the no-node rejection carries CORS so a +// browser sees the real 502 instead of an opaque CORS error. +func TestHandleHTTP_RejectionHasCORS(t *testing.T) { + p := testProxy(NewDiscovery(), 11434) + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"x"}`))) + + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502", rec.Code) + } + if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" { + t.Errorf("Access-Control-Allow-Origin = %q, want * on rejection", got) + } +} + +// TestHandleHTTP_FailoverOn503: a busy first node (503) is skipped and the +// request is filled by the next node, with the original body replayed. +func TestHandleHTTP_FailoverOn503(t *testing.T) { + busy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + io.WriteString(w, `{"error":"loading model"}`) + })) + defer busy.Close() + + var gotBody string + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + gotBody = string(b) + w.WriteHeader(http.StatusOK) + io.WriteString(w, `{"done":true}`) + })) + defer good.Close() + + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "busy", busy.URL, "llama")) + disc.AddManual(nodeForModel(t, "good", good.URL, "llama")) + p := testProxy(disc, 11434) + p.SetSelected("busy") // deterministic: busy is tried first + + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"llama"}`))) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 (should have failed over past the 503)", rec.Code) + } + if gotBody != `{"model":"llama"}` { + t.Errorf("failover node got body %q, want the original request body", gotBody) + } + if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" { + t.Errorf("Access-Control-Allow-Origin = %q, want * on proxied success", got) + } +} + +// TestHandleHTTP_AllNodesDownReturnsError: when every candidate fails at the +// transport, the client gets one clean 502 (not a hang), still with CORS. +func TestHandleHTTP_AllNodesDownReturnsError(t *testing.T) { + // Two servers we immediately close so dials fail. + a := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + b := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + na := nodeForModel(t, "a", a.URL, "llama") + nb := nodeForModel(t, "b", b.URL, "llama") + a.Close() + b.Close() + + disc := NewDiscovery() + disc.AddManual(na) + disc.AddManual(nb) + p := testProxy(disc, 11434) + + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"llama"}`))) + + if rec.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502 when all nodes are down", rec.Code) + } + if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" { + t.Errorf("Access-Control-Allow-Origin = %q, want * on exhausted error", got) + } +} + +// TestHandleHTTP_404FailoverInferenceOnly: a 404 (model-not-found) on an +// inference call fails over to the next advertised owner, but a 404 on a +// non-inference path is returned as-is. +func TestHandleHTTP_404FailoverInferenceOnly(t *testing.T) { + missing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + io.WriteString(w, `{"error":"model not found"}`) + })) + defer missing.Close() + has := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + io.WriteString(w, `{"done":true}`) + })) + defer has.Close() + + newProxy := func() *Proxy { + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "missing", missing.URL, "llama")) + disc.AddManual(nodeForModel(t, "has", has.URL, "llama")) + p := testProxy(disc, 11434) + p.SetSelected("missing") + return p + } + + // Inference POST: 404 on first → fail over → 200. + rec := httptest.NewRecorder() + newProxy().handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"llama"}`))) + if rec.Code != http.StatusOK { + t.Fatalf("inference 404: status = %d, want 200 (should fail over)", rec.Code) + } + + // An ordinary non-inference GET still returns the first node's 404. + rec = httptest.NewRecorder() + newProxy().handleHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/unknown", nil)) + if rec.Code != http.StatusNotFound { + t.Fatalf("non-inference 404: status = %d, want 404 (must NOT fail over)", rec.Code) + } +} + +func TestHandleHTTP_AggregatesModelList(t *testing.T) { + entered := make(chan struct{}, 2) + release := make(chan struct{}) + server := func(body string) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/models" { + t.Errorf("upstream request = %s %s, want GET /models", r.Method, r.URL.Path) + } + if r.Header.Get("Authorization") != "" || r.Header.Get("Cookie") != "" { + t.Errorf("client credentials leaked to fan-out target") + } + entered <- struct{}{} + <-release + _, _ = io.WriteString(w, body) + })) + } + a := server(`{"object":"list","data":[{"id":"a","owned_by":"a-only"},{"id":"shared","owned_by":"first"}]}`) + defer a.Close() + b := server(`{"object":"list","data":[{"id":"shared","owned_by":"second"},{"id":"c","owned_by":"c-only"}]}`) + defer b.Close() + malformed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"object":"list","data":null}`) + })) + defer malformed.Close() + down := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + downNode := nodeFor(t, "down", down.URL) + down.Close() + + disc := NewDiscovery() + disc.AddManual(nodeFor(t, "a", a.URL)) + disc.AddManual(nodeFor(t, "b", b.URL)) + disc.AddManual(downNode) + disc.AddManual(nodeFor(t, "malformed", malformed.URL)) + p := testProxy(disc, 1234) + p.SetSelected("a") + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/v1/models", nil) + req.Header.Set("Authorization", "Bearer client-secret") + req.Header.Set("Cookie", "session=client-secret") + done := make(chan struct{}) + go func() { + p.handleHTTP(rec, req) + close(done) + }() + + for range 2 { + select { + case <-entered: + case <-time.After(5 * time.Second): + close(release) + t.Fatal("model-list requests were not issued concurrently") + } + } + close(release) + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("aggregate request did not finish") + } + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + var got struct { + Object string `json:"object"` + Data []struct { + ID string `json:"id"` + OwnedBy string `json:"owned_by"` + } `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.Object != "list" { + t.Errorf("object = %q, want list", got.Object) + } + if len(got.Data) != 3 || got.Data[0].ID != "a" || got.Data[1].ID != "shared" || got.Data[2].ID != "c" { + t.Fatalf("models = %+v, want a, shared, c", got.Data) + } + if got.Data[1].OwnedBy != "first" { + t.Errorf("duplicate metadata = %q, want deterministic first candidate", got.Data[1].OwnedBy) + } + if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" { + t.Errorf("Access-Control-Allow-Origin = %q, want *", got) + } +} + +func TestHandleHTTP_ModelListEmptyAndUnavailable(t *testing.T) { + empty := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"object":"list","data":[]}`) + })) + emptyNode := nodeFor(t, "empty", empty.URL) + empty.Close() + + disc := NewDiscovery() + disc.AddManual(emptyNode) + rec := httptest.NewRecorder() + testProxy(disc, 1234).handleHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/models", nil)) + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("unavailable status = %d, want 503", rec.Code) + } + + empty = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"object":"list","data":[]}`) + })) + defer empty.Close() + disc = NewDiscovery() + disc.AddManual(nodeFor(t, "empty", empty.URL)) + rec = httptest.NewRecorder() + testProxy(disc, 1234).handleHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/models", nil)) + if rec.Code != http.StatusOK || rec.Body.String() != `{"object":"list","data":[]}` { + t.Fatalf("empty response = %d %s, want 200 native empty list", rec.Code, rec.Body.String()) + } +} + +// TestHandleHTTP_StrictModelRouting proves capability is a gate before +// selection and priority. Unknown and known-missing nodes are excluded from +// inference but remain available for non-inference routes. +func TestHandleHTTP_StrictModelRouting(t *testing.T) { + missHits, unknownHits, matchHits := 0, 0, 0 + miss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + missHits++ + w.WriteHeader(http.StatusOK) + })) + defer miss.Close() + unknown := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + unknownHits++ + w.WriteHeader(http.StatusOK) + })) + defer unknown.Close() + match := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + matchHits++ + body, _ := io.ReadAll(r.Body) + if string(body) != `{"model":"llama"}` { + t.Errorf("matching node got body %q", body) + } + w.WriteHeader(http.StatusOK) + })) + defer match.Close() + + disc := NewDiscovery() + missNode := nodeFor(t, "selected-miss", miss.URL) + missNode.Models = []string{"mistral"} + unknownNode := nodeFor(t, "a-unknown", unknown.URL) + matchNode := nodeFor(t, "z-match", match.URL) + matchNode.Models = []string{"llama"} + disc.AddManual(missNode) + disc.AddManual(unknownNode) + disc.AddManual(matchNode) + p := testProxy(disc, 11434) + p.SetSelected("selected-miss") + p.SetPriority([]string{"a-unknown", "selected-miss", "z-match"}) + candidates := p.resolveCandidates("llama") + if len(candidates) != 1 || candidates[0].id != "z-match" { + t.Fatalf("model candidates = %v, want only z-match", candidates) + } + + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"llama"}`))) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + if missHits != 0 || unknownHits != 0 || matchHits != 1 { + t.Fatalf("hits miss=%d unknown=%d match=%d, want 0/0/1", missHits, unknownHits, matchHits) + } + + // Capability filtering applies only to model-bearing inference. + rec = httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/unknown", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("non-inference status = %d, want 200", rec.Code) + } + if missHits != 1 || unknownHits != 0 || matchHits != 1 { + t.Fatalf("non-inference hits miss=%d unknown=%d match=%d, want 1/0/1", missHits, unknownHits, matchHits) + } +} + +func TestHandleHTTP_NoAdvertisedModelRejectsLocally(t *testing.T) { + hits := 0 + upstream := func() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits++ + w.WriteHeader(http.StatusOK) + })) + } + missing := upstream() + defer missing.Close() + unknown := upstream() + defer unknown.Close() + + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "missing", missing.URL, "mistral")) + disc.AddManual(nodeFor(t, "unknown", unknown.URL)) + events := &prRec{} + p := NewProxy(NewCodec(events), disc, 1234) + p.SetSelected("missing") + + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"llama"}`))) + + if rec.Code != http.StatusBadGateway || + !strings.Contains(rec.Body.String(), "no available node advertises the requested model") { + t.Fatalf("response = %d %s, want actionable local 502", rec.Code, rec.Body.String()) + } + if hits != 0 { + t.Fatalf("ineligible upstreams received %d requests, want 0", hits) + } + if !events.has("no node advertises requested model") { + t.Fatalf("missing rejected request event: %s", events.b) + } +} + +// TestResolveCandidates_SelfGuard: a node resolving to the proxy's own +// listen address is dropped so we never self-forward. +func TestResolveCandidates_SelfGuard(t *testing.T) { + disc := NewDiscovery() + disc.AddManual(Node{ID: "self", Addresses: []string{"127.0.0.1"}, Port: 11434}) + disc.AddManual(Node{ID: "real", Addresses: []string{"192.0.2.10"}, Port: 11434}) + p := testProxy(disc, 11434) + + cands := p.resolveCandidates("") + var haveReal bool + for _, c := range cands { + if c.id == "self" { + t.Errorf("self-target node must be excluded, got candidate %+v", c) + } + if c.id == "real" { + haveReal = true + } + } + if !haveReal { + t.Errorf("expected the real node to survive the self-guard, candidates = %+v", cands) + } +} diff --git a/services/llamacpp-proxy/first_body_timeout_test.go b/services/llamacpp-proxy/first_body_timeout_test.go new file mode 100644 index 000000000..9ae5d8f77 --- /dev/null +++ b/services/llamacpp-proxy/first_body_timeout_test.go @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + "time" +) + +func TestResolveFirstBodyTimeout(t *testing.T) { + for _, tc := range []struct { + name, flag, env string + want time.Duration + }{ + {"default disabled", "", "", 0}, + {"environment opt-in", "", "10m", 10 * time.Minute}, + {"flag opt-in", "7m", "", 7 * time.Minute}, + {"flag beats environment", "5m", "10m", 5 * time.Minute}, + {"explicit zero disables environment", "0s", "10m", 0}, + {"environment zero disables", "", "0", 0}, + {"invalid flag disables", "soon", "10m", 0}, + {"invalid environment disables", "", "soon", 0}, + {"negative flag disables", "-1s", "10m", 0}, + {"negative environment disables", "", "-1m", 0}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(firstBodyTimeoutEnv, tc.env) + if got := resolveFirstBodyTimeout(tc.flag); got != tc.want { + t.Fatalf("first-body timeout = %s, want %s", got, tc.want) + } + }) + } +} + +func TestFirstBodyTimeoutDoesNotChangeHeaderDeadline(t *testing.T) { + old := proxyFirstBodyTimeout + t.Cleanup(func() { proxyFirstBodyTimeout = old }) + proxyFirstBodyTimeout = 7 * time.Minute + if got := newProxyTransport(nil).ResponseHeaderTimeout; got != proxyResponseTimeout { + t.Fatalf("header timeout = %s, want unchanged %s", got, proxyResponseTimeout) + } +} diff --git a/services/llamacpp-proxy/go.mod b/services/llamacpp-proxy/go.mod new file mode 100644 index 000000000..5886a99e8 --- /dev/null +++ b/services/llamacpp-proxy/go.mod @@ -0,0 +1,39 @@ +module llamacpp-proxy + +go 1.25.0 + +require ( + go.opentelemetry.io/otel v1.34.0 + go.opentelemetry.io/otel/sdk v1.34.0 + go.opentelemetry.io/otel/trace v1.34.0 + nvpair-shared v0.0.0-00010101000000-000000000000 +) + +replace nvpair-shared => ../shared + +require ( + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/cenkalti/backoff v2.2.1+incompatible // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grandcat/zeroconf v1.0.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 // indirect + github.com/miekg/dns v1.1.55 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect + go.opentelemetry.io/otel/metric v1.34.0 // indirect + go.opentelemetry.io/proto/otlp v1.5.0 // indirect + golang.org/x/mod v0.38.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + golang.org/x/tools v0.48.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect + google.golang.org/grpc v1.69.4 // indirect + google.golang.org/protobuf v1.36.3 // indirect +) diff --git a/services/llamacpp-proxy/go.sum b/services/llamacpp-proxy/go.sum new file mode 100644 index 000000000..0b5cc426c --- /dev/null +++ b/services/llamacpp-proxy/go.sum @@ -0,0 +1,88 @@ +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6kE= +github.com/grandcat/zeroconf v1.0.0/go.mod h1:lTKmG1zh86XyCoUeIHSA4FJMBwCJiQmGfcP2PdzytEs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 h1:VNqngBF40hVlDloBruUehVYC3ArSgIyScOAyMRqBxRg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1/go.mod h1:RBRO7fro65R6tjKzYgLAFo0t1QEXY1Dp+i/bvpRiqiQ= +github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= +github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= +go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= +go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f h1:gap6+3Gk41EItBuyi4XX/bp4oqJ3UwuIMl25yGinuAA= +google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:Ic02D47M+zbarjYYUlK57y316f2MoN0gjAwI3f2S95o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= +google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= +google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/services/llamacpp-proxy/ingress.go b/services/llamacpp-proxy/ingress.go new file mode 100644 index 000000000..2c74a96d8 --- /dev/null +++ b/services/llamacpp-proxy/ingress.go @@ -0,0 +1,243 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "log/slog" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strconv" + "sync" + "time" + + "nvpair-shared/cors" +) + +const engineIdentityProbeHeader = "X-NVPAIR-Engine-Identity-Probe" + +// nvpairRequestIDHeader carries PAIR's job id from the origin proxy to the +// peer that serves the request. It is the origin's own id for the request, +// and it is not a routing input: the ingress stays strictly terminal, so a +// peer's request is never re-routed onward. +const nvpairRequestIDHeader = "X-NVPAIR-Request-Id" + +// nvpairOriginNodeHeader carries the ORIGIN node's id from the origin proxy +// to the peer that serves the request. +// +// It rides alongside the job id because PAIR's job id is a per-proxy counter: +// two origins can both hold a job "1", so the job id alone does not say +// which job a frame is about. The origin does. +const nvpairOriginNodeHeader = "X-NVPAIR-Origin-Node" + +// nvpairRunIDHeader carries the ORIGIN proxy's run nonce from the origin +// proxy to the peer that serves the request. +// +// It rides alongside the job id because a proxy that restarts counts its jobs +// from 1 again: the run nonce says which run the job id belongs to, so a +// recycled job id is never taken for the job it replaced. +const nvpairRunIDHeader = "X-NVPAIR-Run-Id" + +var plaintextRejectWarned sync.Map + +func logPlaintextReject(r *http.Request) { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + host = r.RemoteAddr + } + if _, loaded := plaintextRejectWarned.LoadOrStore(host, struct{}{}); !loaded { + slog.Warn("rejected non-loopback plaintext request; cluster peers must use mTLS", + "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path) + return + } + slog.Debug("rejected non-loopback plaintext request; cluster peers must use mTLS", + "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path) +} + +// localBackend is the explicit loopback engine the cluster mTLS ingress +// forwards to. It is supplied by the broker over node/set-local-backend and is +// deliberately NOT sourced from the discovery overlay: a request that arrived +// over the LAN mTLS ingress can only ever be dumped on this node's own local +// engine, never re-routed to a peer, so the ingress path is strictly terminal +// and cannot recurse or amplify. +type localBackend struct { + Engine string `json:"engine"` + Host string `json:"host"` + Port int `json:"port"` + Healthy bool `json:"healthy"` +} + +// setLocalBackend records (or, with a zero port / unhealthy flag, effectively +// clears) the local engine the ingress serves. +func (p *Proxy) setLocalBackend(b localBackend) { + p.backendMu.Lock() + p.backend = b + p.backendMu.Unlock() +} + +// localBackendTarget returns the loopback URL of the current local engine, and +// false when none is set/healthy (the ingress then answers 503 rather than +// forwarding). The host defaults to 127.0.0.1 and is always loopback. +func (p *Proxy) localBackendTarget() (*url.URL, bool) { + p.backendMu.RLock() + b := p.backend + p.backendMu.RUnlock() + if b.Port <= 0 || !b.Healthy { + return nil, false + } + host := b.Host + if host == "" { + host = "127.0.0.1" + } + return &url.URL{Scheme: "http", Host: net.JoinHostPort(host, strconv.Itoa(b.Port))}, true +} + +// handlePlain is the plaintext personality: it accepts requests only from +// loopback and hands them to the full local router (handleHTTP). A non-loopback +// caller — any LAN peer — is refused; peers must use the mTLS ingress. This is +// what closes the former open-relay exposure (the listener still binds all +// interfaces for the TLS personality, but plaintext is loopback-only). +func (p *Proxy) handlePlain(w http.ResponseWriter, r *http.Request) { + if !isLoopbackRemote(r.RemoteAddr) { + // Answer a non-loopback preflight ahead of the gate. It grants no access + // on its own; the request that follows still receives the real 403. A + // loopback preflight continues into handleHTTP so an available engine's + // exact origin and credentials policy can be preserved. + if cors.WritePreflight(w, r) { + return + } + logPlaintextReject(r) + writeIngressError(w, http.StatusForbidden, "loopback-only", + "plaintext requests are accepted only from loopback; cluster peers must use the mTLS ingress") + return + } + // Engine-manager marks identity/action requests so the federated model-list + // facade can never satisfy llama.cpp's own /v1/models readiness probe. + if r.Header.Get(engineIdentityProbeHeader) == "1" { + writeIngressError(w, http.StatusConflict, "proxy-facade", "the compatibility facade is not an llama.cpp engine") + return + } + p.handleHTTP(w, r) +} + +// handleClusterIngress is the LAN mTLS personality: it authenticates the caller +// against this node's cluster pins and, once the peer is a trusted cluster +// member, forwards the request straight to the local loopback engine — exactly +// like the local plaintext path, with no route filtering. The mTLS pin is the +// sole authorization boundary (a trusted peer is treated like a local client), +// so the two personalities stay behaviorally identical toward the engine. It +// never calls resolveCandidates, so a peer request cannot be re-routed onward. +func (p *Proxy) handleClusterIngress(w http.ResponseWriter, r *http.Request) { + // Re-derive membership and pins per request so a cluster left, or a peer + // paired or removed, after startup is reflected immediately without a proxy + // restart — a removed peer must stop being accepted right away, which is the + // whole point of the gate. + p.mesh.Refresh() + peer, ok := p.mesh.VerifyClientPin(r) + if !ok { + writeIngressError(w, http.StatusForbidden, "cluster-auth", + "client certificate is not a pinned member of this node's cluster") + return + } + target, ok := p.localBackendTarget() + if !ok { + writeIngressError(w, http.StatusServiceUnavailable, "no-local-backend", + "no local inference backend is available on this node") + return + } + // The job's identity, when the request came from a peer: the origin + // proxy's job id, the origin's node id and the origin's run nonce all + // travelled with the request. The frame the serving side publishes is + // attributed to that origin, not to this node. + job := peerJob{ + jobID: r.Header.Get(nvpairRequestIDHeader), + originNodeID: r.Header.Get(nvpairOriginNodeHeader), + runID: r.Header.Get(nvpairRunIDHeader), + start: time.Now(), + } + slog.Debug("cluster ingress forwarding to local backend", + "peer", peer, "method", r.Method, "path", r.URL.Path, "target", target.Host, + "job_id", job.jobID, "origin_node_id", job.originNodeID, "run_id", job.runID) + p.reverseProxyToLocal(w, r, target, job) +} + +// reverseProxyToLocal streams the request to the local engine, preserving +// cancellation (the request context is the proxy's root context, so a client +// disconnect or shutdown tears down the upstream call and stops generation). +// +// A request a peer forwarded carries the origin's job identity, and the +// serving side publishes the live frame under it: the frame is attributed to +// the origin, so every member of the cluster keys the job's live view the +// same way and the job's terminal event retires it. +func (p *Proxy) reverseProxyToLocal(w http.ResponseWriter, r *http.Request, target *url.URL, job peerJob) { + p.newLocalReverseProxy(target, job).ServeHTTP(w, r) +} + +// newLocalReverseProxy builds the ingress's reverse proxy to the local +// engine. +// +// A peer-served request is measured on the way back: the reader counts the +// bytes the engine sent, and the frame leaves attributed to the origin. A +// request this node received directly is served by its own router, which +// publishes the frame itself. +func (p *Proxy) newLocalReverseProxy(target *url.URL, job peerJob) *httputil.ReverseProxy { + // The peer-served job's own limiter: the frame the serving side + // leaves for it leaves at most once per window. + limiter := newProgressLimiter() + return &httputil.ReverseProxy{ + Director: func(req *http.Request) { + req.URL.Scheme = target.Scheme + req.URL.Host = target.Host + req.Host = target.Host + stripTraceContext(req.Header) + }, + Transport: p.plainHTTPTransport(), + BufferPool: p.bufPool, + ModifyResponse: func(resp *http.Response) error { + if job.jobID == "" { + return nil + } + reader := newPeerProgressReader(resp.Body, job.start) + reader.onProgress = func(prog peerProgress) { + p.publishPeerProgress(job, limiter, prog) + } + resp.Body = reader + return nil + }, + ErrorHandler: func(ew http.ResponseWriter, _ *http.Request, err error) { + slog.Warn("cluster ingress upstream error", "target", target.Host, "err", err) + writeIngressError(ew, http.StatusBadGateway, "backend-error", "local inference backend error") + }, + } +} + +// isLoopbackRemote reports whether an http.Request RemoteAddr (host:port) is a +// loopback address (127.0.0.0/8 or ::1). An unparseable/empty RemoteAddr is not +// loopback, so it fails closed. +func isLoopbackRemote(remoteAddr string) bool { + host, _, err := net.SplitHostPort(remoteAddr) + if err != nil { + host = remoteAddr + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +// writeIngressError writes a small structured JSON error. It never echoes the +// request body or any generated output. CORS headers are included because these +// are the proxy's own rejections: without them a browser client cannot read the +// status or reason, and every one of them looks like a generic CORS failure. +func writeIngressError(w http.ResponseWriter, status int, code, msg string) { + cors.Apply(w.Header()) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(status) + body, err := json.Marshal(map[string]string{"error": msg, "code": code}) + if err != nil { + body = []byte(`{"error":"ingress error"}`) + } + _, _ = w.Write(body) +} diff --git a/services/llamacpp-proxy/ingress_test.go b/services/llamacpp-proxy/ingress_test.go new file mode 100644 index 000000000..07ffdb9d3 --- /dev/null +++ b/services/llamacpp-proxy/ingress_test.go @@ -0,0 +1,111 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" +) + +// TestResolveCandidatesUnclusteredDropsRelayPeers is the core isolation +// assertion: an unclustered node (nil mesh) must not route inference to +// relay-discovered peers, only to explicit user-added manual nodes. +func TestResolveCandidatesUnclusteredDropsRelayPeers(t *testing.T) { + disc := NewDiscovery() + disc.SetSubscribed([]Node{{ + ID: "peer-a", Host: "peer-a", Port: 1234, + Addresses: []string{"192.0.2.10"}, + IP: "192.0.2.10", + ClusterUUID: "cluster-uuid-a", + }}) + disc.AddManual(Node{ + ID: "manual-x", Host: "manual-x", Port: 1234, + Addresses: []string{"192.0.2.20"}, IP: "192.0.2.20", + }) + p := testProxy(disc, 1235) // mesh nil => unclustered + + cands := p.resolveCandidates("") + if len(cands) != 1 { + t.Fatalf("unclustered candidate set = %+v, want exactly the manual node", cands) + } + if cands[0].id != "manual-x" || cands[0].peerUUID != "" || cands[0].url.Scheme != "http" { + t.Fatalf("unclustered candidate = %+v, want plaintext manual-x with no peerUUID", cands[0]) + } +} + +func TestHandlePlainRejectsNonLoopback(t *testing.T) { + p := testProxy(NewDiscovery(), 1235) + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil) + req.RemoteAddr = "192.0.2.50:40000" + rec := httptest.NewRecorder() + + p.handlePlain(rec, req) + if rec.Code != http.StatusForbidden { + t.Fatalf("non-loopback plaintext status = %d, want %d", rec.Code, http.StatusForbidden) + } + // The refusal carries CORS so a browser client reads this 403 and its reason + // instead of an opaque "CORS error" that hides why the call failed. + if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" { + t.Errorf("Access-Control-Allow-Origin = %q, want * on the refusal", got) + } +} + +// TestHandlePlainAnswersPreflightBeforeLoopbackGate: the preflight is answered +// even for a caller the gate will refuse. It authorizes nothing — the request +// that follows is still rejected — but without it the browser never sends that +// request and reports the refusal as a generic CORS failure. +func TestHandlePlainAnswersPreflightBeforeLoopbackGate(t *testing.T) { + p := testProxy(NewDiscovery(), 1235) + req := httptest.NewRequest(http.MethodOptions, "/v1/chat/completions", nil) + req.RemoteAddr = "192.0.2.50:40000" + rec := httptest.NewRecorder() + + p.handlePlain(rec, req) + if rec.Code != http.StatusNoContent { + t.Fatalf("preflight status = %d, want %d", rec.Code, http.StatusNoContent) + } + if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" { + t.Errorf("Access-Control-Allow-Origin = %q, want *", got) + } +} + +func TestHandlePlainRejectsEngineIdentityProbe(t *testing.T) { + p := testProxy(NewDiscovery(), 1235) + req := httptest.NewRequest(http.MethodGet, "/v1/models", nil) + req.RemoteAddr = "127.0.0.1:40000" + req.Header.Set("X-NVPAIR-Engine-Identity-Probe", "1") + rec := httptest.NewRecorder() + + p.handlePlain(rec, req) + if rec.Code != http.StatusConflict { + t.Fatalf("identity probe status = %d, want %d", rec.Code, http.StatusConflict) + } +} + +func TestHandleClusterIngressUnclusteredForbids(t *testing.T) { + p := testProxy(NewDiscovery(), 1235) + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil) + rec := httptest.NewRecorder() + + p.handleClusterIngress(rec, req) + if rec.Code != http.StatusForbidden { + t.Fatalf("unclustered ingress status = %d, want %d", rec.Code, http.StatusForbidden) + } +} + +func TestLocalReverseProxyUsesSharedPlainTransport(t *testing.T) { + p := testProxy(NewDiscovery(), 1235) + shared := p.plainHTTPTransport() + target := &url.URL{Scheme: "http", Host: "127.0.0.1:1"} + rp := p.newLocalReverseProxy(target, peerJob{}) + tr, ok := rp.Transport.(*http.Transport) + if !ok { + t.Fatalf("Transport type = %T, want *http.Transport", rp.Transport) + } + if tr != shared { + t.Fatal("ingress reverse proxy did not use the shared plain Transport") + } +} diff --git a/services/llamacpp-proxy/ipc.go b/services/llamacpp-proxy/ipc.go new file mode 100644 index 000000000..63c92ffb6 --- /dev/null +++ b/services/llamacpp-proxy/ipc.go @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +// The IPC transport (Unix socket / Windows named pipe) is single-sourced in +// nvpair-shared/ipc; the platform split lives there. dialIPC aliases it so call +// sites are unchanged. + +import "nvpair-shared/ipc" + +var dialIPC = ipc.Dial diff --git a/services/llamacpp-proxy/main.go b/services/llamacpp-proxy/main.go new file mode 100644 index 000000000..ff66811c6 --- /dev/null +++ b/services/llamacpp-proxy/main.go @@ -0,0 +1,126 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "log/slog" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "nvpair-shared/applog" + "nvpair-shared/clustertrust" + "nvpair-shared/nodeid" + "nvpair-shared/telemetry" +) + +func main() { + port := flag.Int("port", defaultProxyPort, "HTTP listen port") + ignorePersistedPort := flag.Bool("ignore-persisted-port", false, "use --port even when a persisted port exists") + responseHeaderTimeout := flag.String("response-header-timeout", "", "upstream response-header timeout (Go duration, e.g. 10m); default: $NVPAIR_PROXY_RESPONSE_HEADER_TIMEOUT or 120s") + firstBodyTimeout := flag.String("first-body-timeout", "", "opt-in inference first-body timeout (Go duration, e.g. 10m; 0 disables); default: $NVPAIR_PROXY_FIRST_BODY_TIMEOUT or disabled") + ipcPath := flag.String("ipc", "", "IPC endpoint: Unix domain socket path or Windows named pipe (default: stdin/stdout)") + clusterDir := flag.String("cluster-dir", "", "cluster trust directory (node.crt/key + trusted pins); enables the LAN mTLS inference ingress when this node is clustered") + showVersion := flag.Bool("version", false, "print version and exit") + resolveLevel := applog.RegisterFlag(nil, slog.LevelInfo) + flag.Parse() + proxyResponseTimeout = resolveResponseHeaderTimeout(*responseHeaderTimeout) + proxyFirstBodyTimeout = resolveFirstBodyTimeout(*firstBodyTimeout) + + if *showVersion { + fmt.Println(Version) + os.Exit(0) + } + + applog.Init("llamacpp-proxy", resolveLevel()) + + var transport io.ReadWriteCloser + if *ipcPath != "" { + conn, err := dialIPC(*ipcPath) + if err != nil { + log.Fatalf("failed to connect to IPC endpoint %q: %v", *ipcPath, err) + } + transport = conn + log.Printf("using IPC transport: %s", *ipcPath) + } else { + transport = newStdioTransport() + log.Print("using stdio transport") + } + defer transport.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + go func() { + select { + case sig := <-sigCh: + log.Printf("received %s, shutting down", sig) + cancel() + case <-ctx.Done(): + } + }() + + // Restore a previously chosen port (set via set-port) over the + // --port/default, so the proxy comes back up where the user last put it. + persisted, hasPersisted := loadPersistedPort() + effectivePort := chooseStartupPort(*port, *ignorePersistedPort, persisted, hasPersisted) + if hasPersisted && !*ignorePersistedPort && effectivePort == persisted { + log.Printf("restored persisted proxy port %d", persisted) + } + + tel := telemetry.NewManager("nvpair-llamacpp-proxy", telemetry.OTLPExporter) + + codec := NewCodec(transport) + defer codec.Close() // flush accepted frames before closing the transport + disc := NewDiscovery() + proxy := NewProxy(codec, disc, effectivePort, tel) + defer func() { + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 3*time.Second) + defer shutdownCancel() + _ = tel.Shutdown(shutdownCtx) + }() + // Open a live view of this node's cluster mTLS trust fabric. While unclustered + // the proxy serves only the loopback plaintext personality; once this node is + // a member the same listener also serves the pin-gated LAN mTLS ingress, and + // peers become routable candidates. The proxy needs no restart to notice + // either transition — it re-derives membership per request and on a watch. + // + // Membership is gated on an active admission or a pin, never on keypair + // presence: a left/removed node keeps its keypair by design, and would + // otherwise keep logging cluster_ingress with no cluster peers to serve. + proxy.mesh = clustertrust.Open(*clusterDir) + + // Resolve this node's stable per-host id from the same on-disk identity + // the broker and the scanner use. A request forwarded to a cluster peer + // carries it, so the peer knows which node the job originated on and + // attributes the live frame it publishes to that origin rather than to + // itself. The base is the parent of the cluster dir (which is + // /cluster), exactly as the broker resolves it. + base := "" + if *clusterDir != "" { + base = filepath.Dir(*clusterDir) + } + proxy.nodeID = nodeid.Resolve(base) + if proxy.nodeID == "" { + slog.Warn("could not resolve this node's id; a forwarded request will not name its origin") + } + go proxy.mesh.Watch(ctx, func(clustered bool) { + slog.Info("cluster inference ingress switched personality", "cluster_ingress", clustered) + proxy.dropUnpinnedPeerTransports() + }) + + if err := proxy.Run(ctx); err != nil && ctx.Err() == nil { + log.Fatalf("proxy error: %v", err) + } + log.Print("shutdown complete") +} diff --git a/services/llamacpp-proxy/manual_overlay_test.go b/services/llamacpp-proxy/manual_overlay_test.go new file mode 100644 index 000000000..98ac09917 --- /dev/null +++ b/services/llamacpp-proxy/manual_overlay_test.go @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" + "nvpair-shared/noderec" + "path/filepath" + "testing" +) + +// A broker-proven peer uses its manually configured overlay hostname, not a +// LAN address from discovery, and still requires a live exact certificate pin. +func TestManualOverlayHostnameKeepsPinnedTransport(t *testing.T) { + dir := filepath.Join(t.TempDir(), "cluster") + clustertrusttest.Join(t, dir, "cluster", "self", "peer") + n, ok := subscribedToNode(noderec.DirectoryNode{ + HostUUID: "peer", ClusterUUID: "peer", Name: "peer", IP: "Workstation.tail.example", + IPs: []string{"Workstation.tail.example"}, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceLlamaCPP: {Port: 1234}}, + }) + if !ok { + t.Fatal("manual address lost in relay projection") + } + disc := NewDiscovery() + disc.SetSubscribed([]Node{n}) + p := testProxy(disc, 1235) + p.mesh = clustertrust.Open(dir) + got := p.resolveCandidates("") + if len(got) != 1 || got[0].url.Scheme != "https" || got[0].url.Host != "Workstation.tail.example:1234" || got[0].peerUUID != "peer" { + t.Fatalf("overlay candidates = %+v", got) + } + clustertrusttest.RemovePeerPin(t, dir, "peer") + if got := p.resolveCandidates(""); len(got) != 0 { + t.Fatalf("removed peer still routable: %+v", got) + } +} diff --git a/services/llamacpp-proxy/messages_test.go b/services/llamacpp-proxy/messages_test.go new file mode 100644 index 000000000..56c6b3004 --- /dev/null +++ b/services/llamacpp-proxy/messages_test.go @@ -0,0 +1,237 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bufio" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestMessagesRouteRole(t *testing.T) { + for _, tc := range []struct { + method, path string + inference bool + }{ + {http.MethodPost, "/v1/messages", true}, + {http.MethodGet, "/v1/messages", false}, + {http.MethodPut, "/v1/messages", false}, + {http.MethodPost, "/v1/messages/count_tokens", false}, + } { + if got := isInferenceRequest(tc.method, tc.path); got != tc.inference { + t.Errorf("%s %s inference = %v, want %v", tc.method, tc.path, got, tc.inference) + } + } +} + +// Exercise real HTTP copying: the first SSE event must arrive while the engine +// is still waiting to finish, including after an advertised owner's 404. +func TestMessagesModelRoutingAndFailover(t *testing.T) { + for _, streaming := range []bool{false, true} { + t.Run(map[bool]string{false: "json", true: "sse"}[streaming], func(t *testing.T) { + body := `{"model":"model","max_tokens":7,"messages":[{"role":"user","content":"test"}],"stream":` + + map[bool]string{false: "false", true: "true"}[streaming] + `}` + var missingHits, goodHits, wrongHits atomic.Int32 + wrong := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + wrongHits.Add(1) + })) + defer wrong.Close() + check := func(r *http.Request) { + t.Helper() + data, err := io.ReadAll(r.Body) + if err != nil || string(data) != body || r.URL.Path != "/v1/messages" { + t.Error("engine did not receive the unchanged Messages request") + } + if r.Header.Get("x-api-key") != "synthetic-test-key" || + r.Header.Get("anthropic-version") != "2023-06-01" { + t.Error("Anthropic request headers were not passed through") + } + } + missing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + missingHits.Add(1) + check(r) + w.WriteHeader(http.StatusNotFound) + })) + defer missing.Close() + release := make(chan struct{}) + var once sync.Once + finish := func() { once.Do(func() { close(release) }) } + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + goodHits.Add(1) + check(r) + if streaming { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, "event: message_start\ndata: {}\n\n") + w.(http.Flusher).Flush() + <-release + _, _ = io.WriteString(w, "event: message_stop\ndata: {}\n\n") + } else { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"type":"message","content":[]}`) + } + })) + defer good.Close() + defer finish() + + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "wrong", wrong.URL, "different-model")) + disc.AddManual(nodeForModel(t, "missing", missing.URL, "model")) + disc.AddManual(nodeForModel(t, "good", good.URL, "model")) + p := testProxy(disc, 0) + defer p.closeIdleTransports() + p.SetSelected("wrong") + p.SetPriority([]string{"missing", "good"}) + server := httptest.NewServer(http.HandlerFunc(p.handleHTTP)) + defer func() { + finish() + server.Close() + }() + client := &http.Client{Timeout: 3 * time.Second} + defer client.CloseIdleConnections() + req, err := http.NewRequest(http.MethodPost, server.URL+"/v1/messages", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-api-key", "synthetic-test-key") + req.Header.Set("anthropic-version", "2023-06-01") + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + if streaming { + if resp.Header.Get("Content-Type") != "text/event-stream" { + t.Fatal("SSE content type changed") + } + reader := bufio.NewReader(resp.Body) + line, err := reader.ReadString('\n') + if err != nil || line != "event: message_start\n" { + t.Fatal("first SSE event did not arrive before engine completion") + } + finish() + rest, err := io.ReadAll(reader) + if err != nil || string(rest) != "data: {}\n\nevent: message_stop\ndata: {}\n\n" { + t.Fatal("SSE response changed") + } + } else { + data, err := io.ReadAll(resp.Body) + if err != nil || string(data) != `{"type":"message","content":[]}` { + t.Fatal("JSON response changed") + } + } + if missingHits.Load() != 1 || goodHits.Load() != 1 || wrongHits.Load() != 0 { + t.Fatal("request did not fail over only between advertised model owners") + } + }) + } +} + +func TestMessagesFirstBodyStallCooldown(t *testing.T) { + oldFirst, oldHeader := proxyFirstBodyTimeout, proxyResponseTimeout + proxyFirstBodyTimeout, proxyResponseTimeout = 50*time.Millisecond, 3*time.Second + defer func() { proxyFirstBodyTimeout, proxyResponseTimeout = oldFirst, oldHeader }() + var stalledHits atomic.Int32 + stalled := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + stalledHits.Add(1) + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + <-r.Context().Done() + })) + defer stalled.Close() + const event = "event: message_stop\ndata: {}\n\n" + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, event) + })) + defer good.Close() + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "stalled", stalled.URL, "model")) + disc.AddManual(nodeForModel(t, "good", good.URL, "model")) + p := testProxy(disc, 0) + defer p.closeIdleTransports() + for i := 0; i < 2; i++ { + p.SetPriority([]string{"stalled", "good"}) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/messages", + strings.NewReader(`{"model":"model","stream":true}`)).WithContext(ctx)) + cancel() + if rec.Code != http.StatusOK || rec.Body.String() != event { + t.Fatal("SSE first-body stall did not fail over") + } + if !p.coolingSet(time.Now())["stalled"] { + t.Fatal("first-body stall did not cool candidate") + } + } + if stalledHits.Load() != 1 { + t.Fatal("second request retried cooling candidate before a ready peer") + } +} + +func TestMessagesCommittedSSENeverRetries(t *testing.T) { + var goodHits atomic.Int32 + const partial = "event: message_start\ndata: {}\n\n" + broken := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Content-Length", "999") + _, _ = io.WriteString(w, partial) + w.(http.Flusher).Flush() + })) + defer broken.Close() + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { goodHits.Add(1) })) + defer good.Close() + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "broken", broken.URL, "model")) + disc.AddManual(nodeForModel(t, "good", good.URL, "model")) + p := testProxy(disc, 0) + defer p.closeIdleTransports() + p.SetPriority([]string{"broken", "good"}) + server := httptest.NewServer(http.HandlerFunc(p.handleHTTP)) + defer server.Close() + client := &http.Client{Timeout: 3 * time.Second} + defer client.CloseIdleConnections() + resp, err := client.Post(server.URL+"/v1/messages", "application/json", strings.NewReader(`{"model":"model","stream":true}`)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err == nil || string(data) != partial || goodHits.Load() != 0 { + t.Fatal("committed SSE was retried or not truncated") + } +} + +func TestMessagesOlderEngineReturnsError(t *testing.T) { + var hits atomic.Int32 + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `{"error":"unsupported endpoint"}`) + })) + defer engine.Close() + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "old", engine.URL, "model")) + p := testProxy(disc, 0) + defer p.closeIdleTransports() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/messages", + strings.NewReader(`{"model":"model","stream":true}`)).WithContext(ctx)) + if rec.Code != http.StatusNotFound || rec.Body.String() != `{"error":"unsupported endpoint"}` || hits.Load() != 1 || ctx.Err() != nil { + t.Fatal("older engine error was not returned cleanly after bounded candidates") + } +} diff --git a/services/llamacpp-proxy/models_test.go b/services/llamacpp-proxy/models_test.go new file mode 100644 index 000000000..d9efdb00c --- /dev/null +++ b/services/llamacpp-proxy/models_test.go @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestLlamaModelListMappingAndExactIDs(t *testing.T) { + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/models" { + t.Errorf("engine path = %q", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[{"id":"model"},{"id":"model:latest"}]}`)) + })) + defer engine.Close() + for _, path := range []string{"/v1/models", "/models"} { + t.Run(path, func(t *testing.T) { + disc := NewDiscovery() + disc.AddManual(nodeFor(t, "local", engine.URL)) + p := testProxy(disc, defaultProxyPort) + rec := httptest.NewRecorder() + p.handleHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"id":"model"`) || !strings.Contains(rec.Body.String(), `"id":"model:latest"`) { + t.Fatalf("model list: %d %s", rec.Code, rec.Body.String()) + } + }) + } + if nodeAdvertisesModel(Node{Models: []string{"model:latest"}}, "model") { + t.Fatal("model IDs must not acquire an implied tag") + } +} diff --git a/services/llamacpp-proxy/portcancel_test.go b/services/llamacpp-proxy/portcancel_test.go new file mode 100644 index 000000000..3efd4ec90 --- /dev/null +++ b/services/llamacpp-proxy/portcancel_test.go @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "testing" + "time" +) + +// A canceled write queued behind httpMu must not bind or persist after a newer +// operation owns the proxy. The cancellation check belongs after lock admission. +func TestCanceledPortWriteCannotApplyAfterListenerLock(t *testing.T) { + redirectConfigDir(t) + p := NewProxy(NewCodec(&bytes.Buffer{}), NewDiscovery(), 11435) + p.httpMu.Lock() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- p.setPortContext(ctx, 11436) }() + cancel() + p.httpMu.Unlock() + select { + case err := <-done: + if err == nil { + t.Fatal("canceled write succeeded") + } + case <-time.After(time.Second): + t.Fatal("canceled write did not leave listener lock") + } + if p.port != 11435 { + t.Fatalf("canceled write changed port to %d", p.port) + } + if port, ok := loadPersistedPort(); ok { + t.Fatalf("canceled write persisted %d", port) + } +} diff --git a/services/llamacpp-proxy/portstore.go b/services/llamacpp-proxy/portstore.go new file mode 100644 index 000000000..f897fc3a5 --- /dev/null +++ b/services/llamacpp-proxy/portstore.go @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + + "nvpair-shared/appdir" +) + +const proxyPortFile = "llamacpp-proxy-port.json" + +const ( + defaultProxyPort = 1237 + reservedEnginePort = 8082 +) + +type persistedPort struct { + Port int `json:"port"` +} + +func chooseStartupPort(flagPort int, ignorePersisted bool, persisted int, hasPersisted bool) int { + if ignorePersisted || !hasPersisted || persisted == reservedEnginePort { + return flagPort + } + return persisted +} + +func proxyPortPath() (string, error) { + return appdir.Path(proxyPortFile) +} + +// loadPersistedPort returns the previously chosen proxy port, if a valid one +// was saved. Any error (no file, bad JSON, out-of-range) reports "none" so +// startup falls back to the --port flag / default. +func loadPersistedPort() (int, bool) { + path, err := proxyPortPath() + if err != nil { + return 0, false + } + data, err := os.ReadFile(path) + if err != nil { + return 0, false + } + var pp persistedPort + if err := json.Unmarshal(data, &pp); err != nil { + return 0, false + } + if pp.Port < 1 || pp.Port > 65535 { + return 0, false + } + return pp.Port, true +} + +// savePersistedPort atomically writes the chosen port (tmp + rename) so a +// crash mid-write can't leave a truncated file behind. +func savePersistedPort(port int) error { + path, err := proxyPortPath() + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + data, err := json.Marshal(persistedPort{Port: port}) + if err != nil { + return err + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return err + } + if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) + return err + } + return nil +} diff --git a/services/llamacpp-proxy/portstore_test.go b/services/llamacpp-proxy/portstore_test.go new file mode 100644 index 000000000..3fc49018d --- /dev/null +++ b/services/llamacpp-proxy/portstore_test.go @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "fmt" + "net" + "os" + "strings" + "testing" + "time" +) + +// redirectConfigDir points os.UserConfigDir() at a temp dir for the test, so +// proxy-port.json reads/writes don't touch the real per-user config. Sets all +// three env vars os.UserConfigDir() consults across platforms: XDG_CONFIG_HOME +// on Linux, $HOME/Library on macOS, and APPDATA on Windows. Missing APPDATA +// meant the Windows-first-class path read/wrote the real %AppData% file — +// clobbering the user's saved port and making the test fail on repeat runs. +func redirectConfigDir(t *testing.T) { + t.Helper() + dir := t.TempDir() + t.Setenv("XDG_CONFIG_HOME", dir) + t.Setenv("HOME", dir) + t.Setenv("APPDATA", dir) + t.Setenv("LOCALAPPDATA", dir) +} + +func freeTCPPort(t *testing.T) int { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("reserve free port: %v", err) + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port +} + +func TestDefaultProxyPort(t *testing.T) { + if defaultProxyPort != 1237 { + t.Fatalf("default proxy port = %d, want 1237", defaultProxyPort) + } +} + +func TestChooseStartupPort(t *testing.T) { + for _, tc := range []struct { + name string + flagPort, persisted int + ignorePersisted, hasPersisted bool + want int + }{ + {"new default", 1237, 0, false, false, 1237}, + {"legacy default migrates", 1237, 8082, false, true, 1237}, + {"custom survives opt-out", 1237, 12400, false, true, 12400}, + {"managed flag wins", 1237, 12400, true, true, 1237}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := chooseStartupPort(tc.flagPort, tc.ignorePersisted, tc.persisted, tc.hasPersisted); got != tc.want { + t.Fatalf("chooseStartupPort() = %d, want %d", got, tc.want) + } + }) + } +} + +func TestPersistedPortRoundTrip(t *testing.T) { + redirectConfigDir(t) + + if _, ok := loadPersistedPort(); ok { + t.Fatal("expected no persisted port before any save") + } + if err := savePersistedPort(11500); err != nil { + t.Fatalf("savePersistedPort: %v", err) + } + if p, ok := loadPersistedPort(); !ok || p != 11500 { + t.Errorf("round-trip: got %d ok=%v, want 11500", p, ok) + } + + // An out-of-range stored value is treated as "none" so startup falls + // back to the flag/default rather than trying to bind port 0. + path, err := proxyPortPath() + if err != nil { + t.Fatalf("proxyPortPath: %v", err) + } + if err := os.WriteFile(path, []byte(`{"port":0}`), 0o644); err != nil { + t.Fatal(err) + } + if _, ok := loadPersistedPort(); ok { + t.Error("port 0 should be treated as none") + } +} + +// TestSetPortRebinds drives a live rebind: the proxy starts serving on one +// port, set-port moves it to another, and afterward the new port accepts +// connections, the old one doesn't, the choice is persisted, and a fresh +// ready notification carries the new port. +func TestSetPortRebinds(t *testing.T) { + redirectConfigDir(t) + + buf := &bytes.Buffer{} + codec := NewCodec(buf) + disc := NewDiscovery() + + portA := freeTCPPort(t) + proxy := NewProxy(codec, disc, portA) + + lnA, err := net.Listen("tcp", fmt.Sprintf(":%d", portA)) + if err != nil { + t.Fatalf("listen on port A: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + proxy.serveHTTP(ctx, lnA) + defer proxy.shutdown(context.Background()) + + portB := freeTCPPort(t) + if err := proxy.setPort(portB); err != nil { + t.Fatalf("setPort: %v", err) + } + + // New port is now serving. + conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", portB), 2*time.Second) + if err != nil { + t.Fatalf("new port %d not listening after rebind: %v", portB, err) + } + conn.Close() + + // Old port stopped accepting. + if c, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", portA), 500*time.Millisecond); err == nil { + c.Close() + t.Errorf("old port %d should be closed after rebind", portA) + } + + // Persisted for next startup. + if p, ok := loadPersistedPort(); !ok || p != portB { + t.Errorf("persisted port: got %d ok=%v, want %d", p, ok, portB) + } + + // A fresh ready notification announced the new port. + if !strings.Contains(buf.String(), fmt.Sprintf("\"port\":%d", portB)) { + t.Errorf("expected ready notification carrying port %d, got %q", portB, buf.String()) + } +} diff --git a/services/llamacpp-proxy/priority_test.go b/services/llamacpp-proxy/priority_test.go new file mode 100644 index 000000000..6661b30bf --- /dev/null +++ b/services/llamacpp-proxy/priority_test.go @@ -0,0 +1,177 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "io" + "strconv" + "sync" + "testing" +) + +// prRec is a thread-safe io.ReadWriter that records codec writes so a test can +// assert the response emitted for a request. Reads hit EOF immediately. +type prRec struct { + mu sync.Mutex + b []byte +} + +func (r *prRec) Read([]byte) (int, error) { return 0, io.EOF } + +func (r *prRec) Write(p []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.b = append(r.b, p...) + return len(p), nil +} + +func (r *prRec) has(s string) bool { + r.mu.Lock() + defer r.mu.Unlock() + return string(r.b) != "" && contains(string(r.b), s) +} + +func contains(hay, needle string) bool { + for i := 0; i+len(needle) <= len(hay); i++ { + if hay[i:i+len(needle)] == needle { + return true + } + } + return false +} + +// prNode builds a discovery Node with a single non-local address so +// resolveCandidates resolves it deterministically (single candidate → no TCP +// probe) and it never trips the loopback rewrite or self-forward guard. The +// octet is derived from the id's first byte so each id gets a distinct valid IP. +func prNode(id string) Node { + return Node{ID: id, Addresses: []string{"192.0.2." + strconv.Itoa(int(id[0]))}, Port: 1234} +} + +// prProxy returns a proxy whose discovery holds the given node ids. +func prProxy(t *testing.T, ids ...string) *Proxy { + t.Helper() + disc := NewDiscovery() + for _, id := range ids { + disc.AddManual(prNode(id)) + } + return testProxy(disc, 1235) +} + +func candidateIDs(p *Proxy) []string { + return candidateIDsForModel(p, "") +} + +func candidateIDsForModel(p *Proxy, model string) []string { + cands := p.resolveCandidates(model) + out := make([]string, 0, len(cands)) + for _, c := range cands { + out = append(out, c.id) + } + return out +} + +func assertOrder(t *testing.T, got, want []string) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("candidate order = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("candidate order = %v, want %v", got, want) + } + } +} + +// TestResolveCandidates_PriorityOrder: the priority list dictates auto order. +func TestResolveCandidates_PriorityOrder(t *testing.T) { + p := prProxy(t, "a", "b", "c") + p.SetPriority([]string{"c", "a", "b"}) + assertOrder(t, candidateIDs(p), []string{"c", "a", "b"}) +} + +// TestResolveCandidates_UnlistedFallback: nodes absent from the priority list +// come last, in stable ID order. +func TestResolveCandidates_UnlistedFallback(t *testing.T) { + p := prProxy(t, "a", "b", "c") + p.SetPriority([]string{"b"}) + assertOrder(t, candidateIDs(p), []string{"b", "a", "c"}) +} + +// TestResolveCandidates_UnknownIgnored: an id not in discovery is skipped. +func TestResolveCandidates_UnknownIgnored(t *testing.T) { + p := prProxy(t, "a", "b", "c") + p.SetPriority([]string{"zzz", "c"}) + assertOrder(t, candidateIDs(p), []string{"c", "a", "b"}) +} + +// TestResolveCandidates_ManualPinOverridesPriority: an explicit node/select pin +// wins over the priority list; the rest follow priority order. +func TestResolveCandidates_ManualPinOverridesPriority(t *testing.T) { + p := prProxy(t, "a", "b", "c") + p.SetPriority([]string{"a", "b", "c"}) + p.SetSelected("b") + assertOrder(t, candidateIDs(p), []string{"b", "a", "c"}) +} + +func TestResolveCandidates_FiltersBeforeSelectionAndPriority(t *testing.T) { + disc := NewDiscovery() + a, c, d := prNode("a"), prNode("c"), prNode("d") + a.Models, c.Models, d.Models = []string{"llama"}, []string{"llama"}, []string{"mistral"} + for _, n := range []Node{a, prNode("b"), c, d} { + disc.AddManual(n) + } + p := testProxy(disc, 1235) + p.SetSelected("d") + p.SetPriority([]string{"d", "b", "c", "a"}) + assertOrder(t, candidateIDsForModel(p, "llama"), []string{"c", "a"}) + + p.SetSelected("a") + assertOrder(t, candidateIDsForModel(p, "llama"), []string{"a", "c"}) +} + +// TestResolveCandidates_EmptyReverts: an empty priority list reverts to the +// default stable ID order. +func TestResolveCandidates_EmptyReverts(t *testing.T) { + p := prProxy(t, "c", "a", "b") + p.SetPriority([]string{"c", "a"}) + p.SetPriority(nil) // clear + assertOrder(t, candidateIDs(p), []string{"a", "b", "c"}) +} + +// TestSetPriority_CountAndCopy: SetPriority returns the stored length and +// PriorityList hands back an independent copy. +func TestSetPriority_CountAndCopy(t *testing.T) { + p := prProxy(t, "a") + if n := p.SetPriority([]string{"a", "b", "c"}); n != 3 { + t.Fatalf("SetPriority count = %d, want 3", n) + } + got := p.PriorityList() + got[0] = "mutated" + if again := p.PriorityList(); again[0] != "a" { + t.Fatalf("PriorityList returned an aliased slice: %v", again) + } +} + +// TestHandleSetPriority_Response: the node/set-priority request returns {count}. +func TestHandleSetPriority_Response(t *testing.T) { + rec := &prRec{} + p := NewProxy(NewCodec(rec), NewDiscovery(), 1235) + + id := json.RawMessage(`7`) + p.handleMessage(&Message{ + JSONRPC: "2.0", + ID: &id, + Method: "node/set-priority", + Params: json.RawMessage(`{"nodes":["x","y"]}`), + }) + + if !rec.has(`"count":2`) { + t.Fatalf("expected response with count=2, got: %s", rec.b) + } + if got := p.PriorityList(); len(got) != 2 || got[0] != "x" || got[1] != "y" { + t.Fatalf("stored priority = %v, want [x y]", got) + } +} diff --git a/services/llamacpp-proxy/progress.go b/services/llamacpp-proxy/progress.go new file mode 100644 index 000000000..67faf0a21 --- /dev/null +++ b/services/llamacpp-proxy/progress.go @@ -0,0 +1,252 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "io" + "log/slog" + "sync" + "time" + + "nvpair-shared/inferencemetrics" +) + +// progressMethod is the notification method a live progress frame travels +// on. The frame is a snapshot of a running request, not a state transition: +// it never enters the workload catalog and is never deduped against the +// lifecycle event that shares its job id. +const progressMethod = "workload:progress" + +// progressInterval is the smallest gap between two progress frames for one +// request. The limit is at the source, so a cluster with twenty running +// jobs carries at most ten frames per second whatever the engines' chunk +// rates are. +// +// It is a var (not a const) only so a test can shorten it; production never +// reassigns it. +var progressInterval = 2 * time.Second + +// ProgressFrame is the live view of one in-flight inference request. +// +// It carries only what PAIR measured exactly: the elapsed time, the number +// of content chunks received, the number of response bytes received, and the +// time to the first byte. There is no token count and no rate — PAIR does +// not know the engine's token count until the response ends, and a chunk is +// not a token. +// +// The frame is engine-agnostic: an engine that reports live numbers of its +// own publishes them under the same job id, on a frame of the same shape. +// This engine reports no live numbers, so the frame never carries any. +// +// Every field but the job id is optional: a frame that carries only part of +// the view says so instead of claiming zeros. +type ProgressFrame struct { + // JobID is PAIR's own id for the request. It is the same id the + // workload lifecycle events carry, so a frame joins a job card without + // a lookup. + JobID string `json:"jobId"` + // NodeID is the node that is serving the request. For a request this + // proxy forwarded to a peer it is the peer's id, not this node's: the + // frame describes where the work runs. + NodeID string `json:"nodeId,omitempty"` + // OriginatedFrom is the node the request entered the cluster on. It is + // the origin of the job, and the half of the identity a job is keyed + // by: PAIR's job id is a per-proxy counter, so two origins can both + // hold a job "1". + // + // It is set only for a request this proxy serves on behalf of a peer: + // the origin's id travelled with the request. A request this proxy + // received directly leaves it empty, and the origin's own broker stamps + // this node's id onto the frame. + OriginatedFrom string `json:"originatedFrom,omitempty"` + // Engine is the engine that serves the job. It is part of the job's + // identity, not decoration: every proxy counts its own requests from + // 1, so an llama.cpp job "1" and an oMLX job "1" on the same node are + // two different jobs, and only the engine tells them apart. + Engine string `json:"engine"` + // RunID is this proxy's per-process run nonce — the same value the + // workload carries as `runId`. It tells a recycled job id from the job + // it replaced: a proxy that restarts counts its jobs from 1 again. + RunID string `json:"runId,omitempty"` + // ElapsedMs is the time since the request reached the proxy. + ElapsedMs *int64 `json:"elapsedMs,omitempty"` + // Chunks is the number of content chunks received so far. + Chunks *int64 `json:"chunks,omitempty"` + // Bytes is the number of response body bytes received so far. + Bytes *int64 `json:"bytes,omitempty"` + // TTFBMs is the time to the first response byte. It is absent until a + // byte has arrived. + TTFBMs *int64 `json:"ttfbMs,omitempty"` +} + +// progressLimiter lets at most one frame per request leave every +// progressInterval. It is per request, not per proxy: one busy job must not +// silence another job's frames. +type progressLimiter struct { + mu sync.Mutex + last time.Time +} + +// newProgressLimiter returns the limiter of one job. +func newProgressLimiter() *progressLimiter { + return &progressLimiter{} +} + +// due reports whether a frame may be emitted now, and records the emission. +func (l *progressLimiter) due(now time.Time) bool { + l.mu.Lock() + defer l.mu.Unlock() + if !l.last.IsZero() && now.Sub(l.last) < progressInterval { + return false + } + l.last = now + return true +} + +// emitProgress publishes a live progress frame. A failed write is logged and +// dropped: a frame is a snapshot, and the next one carries the newer +// numbers. +func (p *Proxy) emitProgress(frame ProgressFrame) { + if err := p.codec.Notify(progressMethod, frame); err != nil { + slog.Debug("failed to emit progress frame", "job_id", frame.JobID, "err", err) + } +} + +// publishProgress publishes the PAIR-derived live facts of an in-flight +// request, rate-limited to one frame every progressInterval. +// +// It runs on the request's own goroutine — the one copying the response +// body — so a frame leaves with the chunk that triggered it and no separate +// ticker is involved. +func (p *Proxy) publishProgress(jobID, nodeID string, start time.Time, limiter *progressLimiter, prog LiveProgress) { + now := time.Now() + if !limiter.due(now) { + return + } + frame := ProgressFrame{ + JobID: jobID, + NodeID: nodeID, + Engine: workloadEngine, + RunID: p.runID, + ElapsedMs: inferencemetrics.Int64Ptr(now.Sub(start).Milliseconds()), + Chunks: inferencemetrics.Int64Ptr(prog.Chunks), + Bytes: inferencemetrics.Int64Ptr(prog.Bytes), + } + if prog.HasByte { + frame.TTFBMs = inferencemetrics.Int64Ptr(prog.TTFBMs) + } + p.emitProgress(frame) +} + +// peerJob is the identity of a job another node forwarded, and the instant +// the request reached the serving proxy. +// +// The identity travelled with the request: the origin proxy's job id, the +// origin's node id and the origin's run nonce. +type peerJob struct { + // jobID is the origin's id for the request. + jobID string + // originNodeID is the node the request entered the cluster on. + originNodeID string + // runID is the origin proxy's per-process run nonce. + runID string + // start is the instant the request reached this proxy, so the elapsed + // time and the time to the first byte are the request's own. + start time.Time +} + +// peerProgress is what the serving side measured of a peer-served request's +// response: the bytes the engine sent and the instant the first of them +// arrived. +// +// It carries no chunk count: the serving side relays the engine's stream +// without parsing it, so it has no content chunks to count. The origin's own +// frame carries them, and the two partial views are merged field by field. +type peerProgress struct { + // bytes is the number of response body bytes the engine sent. + bytes int64 + // hasByte reports whether a response byte has arrived at all. + hasByte bool + // ttfbMs is the time to the first response byte, in milliseconds. + ttfbMs int64 +} + +// peerProgressReader counts the bytes the engine sent to a peer-served +// request and notes the instant the first of them arrived. +// +// It runs on the goroutine that copies the response back to the peer, and it +// reports on every read, so the frame leaves with the bytes that triggered +// it. +type peerProgressReader struct { + io.ReadCloser + // start is the instant the request reached the proxy, so the elapsed + // time and the time to the first byte are the request's own. + start time.Time + onProgress func(peerProgress) + + mu sync.Mutex + bytes int64 + firstByteAt time.Time +} + +// newPeerProgressReader wraps a response body with the reader that counts +// the bytes the engine sent. +func newPeerProgressReader(body io.ReadCloser, start time.Time) *peerProgressReader { + return &peerProgressReader{ReadCloser: body, start: start} +} + +// Read reads the response body and counts what arrived. +func (r *peerProgressReader) Read(b []byte) (int, error) { + n, err := r.ReadCloser.Read(b) + if n > 0 { + r.mu.Lock() + if r.firstByteAt.IsZero() { + r.firstByteAt = time.Now() + } + r.bytes += int64(n) + prog := peerProgress{ + bytes: r.bytes, + hasByte: true, + ttfbMs: r.firstByteAt.Sub(r.start).Milliseconds(), + } + r.mu.Unlock() + if r.onProgress != nil { + r.onProgress(prog) + } + } + return n, err +} + +// publishPeerProgress publishes the live facts the serving side measured for +// a job another node forwarded. +// +// The frame is attributed to the job's ORIGIN: the job id, the origin's node +// id and the origin's run nonce all travelled with the request, and a live +// view is keyed by (origin, engine, job id). A frame the serving side +// published without the origin would be stamped with the serving node's own +// id by the serving node's broker, and the job's terminal event — which +// names the origin — would never retire that view. +// +// The frame carries only what the serving side measured: the elapsed time, +// the bytes the engine sent and the time to the first byte. It carries no +// chunk count, because the serving side does not parse the engine's stream. +func (p *Proxy) publishPeerProgress(job peerJob, limiter *progressLimiter, prog peerProgress) { + now := time.Now() + if !limiter.due(now) { + return + } + frame := ProgressFrame{ + JobID: job.jobID, + NodeID: p.nodeID, + OriginatedFrom: job.originNodeID, + Engine: workloadEngine, + RunID: job.runID, + ElapsedMs: inferencemetrics.Int64Ptr(now.Sub(job.start).Milliseconds()), + Bytes: inferencemetrics.Int64Ptr(prog.bytes), + } + if prog.hasByte { + frame.TTFBMs = inferencemetrics.Int64Ptr(prog.ttfbMs) + } + p.emitProgress(frame) +} diff --git a/services/llamacpp-proxy/progress_test.go b/services/llamacpp-proxy/progress_test.go new file mode 100644 index 000000000..ab47b5eb4 --- /dev/null +++ b/services/llamacpp-proxy/progress_test.go @@ -0,0 +1,267 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "nvpair-shared/telemetry" +) + +// progressNotifications decodes the live progress frames the proxy wrote. +func progressNotifications(t *testing.T, rec *prRec) []ProgressFrame { + t.Helper() + rec.mu.Lock() + raw := string(rec.b) + rec.mu.Unlock() + var frames []ProgressFrame + for _, line := range strings.Split(raw, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + var note struct { + Method string `json:"method"` + Params json.RawMessage `json:"params"` + } + if err := json.Unmarshal([]byte(line), ¬e); err != nil { + t.Fatalf("the proxy wrote a line that is not JSON: %q", line) + } + if note.Method != progressMethod { + continue + } + var frame ProgressFrame + if err := json.Unmarshal(note.Params, &frame); err != nil { + t.Fatalf("a progress frame is not readable: %v (%s)", err, note.Params) + } + frames = append(frames, frame) + } + return frames +} + +// TestTelemetryReaderCountsLiveProgress drives the reader with a known SSE +// stream and checks that its live counters are exact: the content chunks are +// the chunks that carried text, the bytes are the bytes the engine sent, and +// the first-byte and first-content instants are recorded. +func TestTelemetryReaderCountsLiveProgress(t *testing.T) { + stream := "data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n" + + "data: {\"choices\":[{\"delta\":{\"content\":\" there\"}}]}\n\n" + + "data: {\"choices\":[{\"delta\":{\"role\":\"assistant\"}}]}\n\n" + + "data: [DONE]\n\n" + + session := telemetry.NewManager("nvpair-llamacpp-proxy-test", telemetry.OTLPExporter).Session() + reader := newTelemetryBodyReader(io.NopCloser(strings.NewReader(stream)), nil, time.Now(), time.Now(), session) + buf := make([]byte, 13) + for { + if _, err := reader.Read(buf); err != nil { + if err == io.EOF { + break + } + t.Fatal(err) + } + } + + prog := reader.Progress() + if prog.Chunks != 2 { + t.Fatalf("content chunks = %d, want 2 (the role-only chunk carries no text)", prog.Chunks) + } + if prog.Bytes != int64(len(stream)) { + t.Fatalf("response bytes = %d, want %d (every byte the engine sent)", prog.Bytes, len(stream)) + } + if !prog.HasByte || !prog.HasContent { + t.Fatal("the first byte and the first content chunk were not both recorded") + } + if prog.TTFBMs < 0 { + t.Fatalf("time to first byte = %d ms, want a measured non-negative duration", prog.TTFBMs) + } + if prog.TTFCMs < prog.TTFBMs { + t.Fatalf("time to first content = %d ms, want it at or after the first byte (%d ms)", prog.TTFCMs, prog.TTFBMs) + } +} + +// TestInFlightRequestEmitsOneProgressFrame: an in-flight request publishes a +// live progress frame, and a burst of chunks inside one window produces one +// frame — not one per chunk. +func TestInFlightRequestEmitsOneProgressFrame(t *testing.T) { + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + // The whole burst in one write, so the reader sees it in one read + // and the frame's counts are the engine's counts. + _, _ = io.WriteString(w, "data: {\"choices\":[{\"delta\":{\"content\":\"a\"}}]}\n\n"+ + "data: {\"choices\":[{\"delta\":{\"content\":\"b\"}}]}\n\n"+ + "data: {\"choices\":[{\"delta\":{\"content\":\"c\"}}]}\n\n"+ + "data: [DONE]\n\n") + })) + defer engine.Close() + + rec := &prRec{} + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "serving-node", engine.URL, "llama")) + p := NewProxy(NewCodec(rec), disc, 1234) + + p.handleHTTP(httptest.NewRecorder(), + httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"llama"}`))) + + frames := progressNotifications(t, rec) + if len(frames) != 1 { + t.Fatalf("progress frames = %d, want 1 for a burst of chunks inside one window", len(frames)) + } + frame := frames[0] + if frame.JobID == "" { + t.Fatal("the progress frame carries no job id") + } + if frame.NodeID != "serving-node" { + t.Fatalf("the progress frame names node %q, want the serving node", frame.NodeID) + } + // The engine is part of the job's identity: every proxy counts its own + // requests from 1, so an llamacpp job "1" and another engine's job "1" on + // the same node are two different jobs. + if frame.Engine != workloadEngine { + t.Fatalf("the progress frame names engine %q, want %q", frame.Engine, workloadEngine) + } + // The run nonce says which run the job id belongs to, so a proxy that + // restarted and counts from 1 again never takes the old job for the + // new one. + if frame.RunID != p.runID { + t.Fatalf("the progress frame carries run id %q, want the proxy's run nonce %q", frame.RunID, p.runID) + } + if frame.Chunks == nil || *frame.Chunks != 3 { + t.Fatalf("the progress frame reports chunks %v, want the 3 content chunks the engine sent", frame.Chunks) + } + if frame.Bytes == nil || *frame.Bytes <= 0 { + t.Fatalf("the progress frame reports bytes %v, want the bytes received", frame.Bytes) + } + if frame.ElapsedMs == nil { + t.Fatal("the progress frame reports no elapsed time") + } + // A chunk is not a token: the frame must carry no token count and no + // rate derived from one. + raw, err := json.Marshal(frame) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"oken", "PerSecond", "perSecond"} { + if strings.Contains(string(raw), forbidden) { + t.Fatalf("the progress frame carries an estimated number %q: %s", forbidden, raw) + } + } +} + +// TestForwardedPeerRequestCarriesTheJobIdentity: a request forwarded to a +// cluster peer carries the origin's job id, the origin's node id and the +// origin's run nonce. The serving peer needs all three to publish the job's +// live frame under the origin's identity. A request that is not forwarded to +// a peer carries none of them — the peer is the only node that needs them. +func TestForwardedPeerRequestCarriesTheJobIdentity(t *testing.T) { + p := testProxy(NewDiscovery(), 1235) + p.nodeID = "node-origin" + + peer := candidate{ + id: "peer-a", + url: &url.URL{Scheme: "https", Host: "192.0.2.10:1234"}, + peerUUID: "principal-peer", + } + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil) + p.forwardDirector(peer, "42")(req) + if got := req.Header.Get(nvpairRequestIDHeader); got != "42" { + t.Fatalf("%s = %q, want the job id 42", nvpairRequestIDHeader, got) + } + if got := req.Header.Get(nvpairOriginNodeHeader); got != "node-origin" { + t.Fatalf("%s = %q, want the origin's node id", nvpairOriginNodeHeader, got) + } + if got := req.Header.Get(nvpairRunIDHeader); got != p.runID { + t.Fatalf("%s = %q, want the proxy's run nonce %q", nvpairRunIDHeader, got, p.runID) + } + + local := candidate{ + id: "local-node", + url: &url.URL{Scheme: "http", Host: "127.0.0.1:8000"}, + } + req = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil) + p.forwardDirector(local, "42")(req) + if got := req.Header.Get(nvpairRequestIDHeader); got != "" { + t.Fatalf("%s = %q, want it absent on a request that is not forwarded to a peer", nvpairRequestIDHeader, got) + } + if got := req.Header.Get(nvpairOriginNodeHeader); got != "" { + t.Fatalf("%s = %q, want it absent on a request that is not forwarded to a peer", nvpairOriginNodeHeader, got) + } + if got := req.Header.Get(nvpairRunIDHeader); got != "" { + t.Fatalf("%s = %q, want it absent on a request that is not forwarded to a peer", nvpairRunIDHeader, got) + } +} + +// TestTheServingSideAttributesTheFrameToTheOrigin: a request a peer +// forwarded is measured on the serving side too, and the frame the serving +// side publishes is attributed to the ORIGIN of the job. A frame published +// without the origin would be stamped with the serving node's own id by the +// serving node's broker, and the job's terminal event — which names the +// origin — would never retire that view. +func TestTheServingSideAttributesTheFrameToTheOrigin(t *testing.T) { + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, "data: {\"choices\":[{\"delta\":{\"content\":\"a\"}}]}\n\n"+ + "data: {\"choices\":[{\"delta\":{\"content\":\"b\"}}]}\n\n"+ + "data: [DONE]\n\n") + })) + defer engine.Close() + + rec := &prRec{} + p := NewProxy(NewCodec(rec), NewDiscovery(), 1235) + p.nodeID = "node-peer" + + target, err := url.Parse(engine.URL) + if err != nil { + t.Fatal(err) + } + rp := p.newLocalReverseProxy(target, peerJob{ + jobID: "job-99", + originNodeID: "origin-a", + runID: "run-1", + start: time.Now(), + }) + rp.ServeHTTP(httptest.NewRecorder(), + httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"m"}`))) + + frames := progressNotifications(t, rec) + if len(frames) == 0 { + t.Fatal("the serving side published no live frame for the peer-served job") + } + frame := frames[0] + if frame.JobID != "job-99" { + t.Fatalf("the frame's job id = %q, want the origin's job id 99", frame.JobID) + } + if frame.OriginatedFrom != "origin-a" { + t.Fatalf("the frame's origin = %q, want the origin that forwarded the request", frame.OriginatedFrom) + } + if frame.RunID != "run-1" { + t.Fatalf("the frame's run id = %q, want the origin's run nonce", frame.RunID) + } + if frame.NodeID != "node-peer" { + t.Fatalf("the frame names node %q, want the serving node", frame.NodeID) + } + if frame.Engine != workloadEngine { + t.Fatalf("the frame names engine %q, want %q", frame.Engine, workloadEngine) + } + if frame.Bytes == nil || *frame.Bytes <= 0 { + t.Fatalf("the frame reports bytes %v, want the bytes the engine sent", frame.Bytes) + } + if frame.ElapsedMs == nil { + t.Fatal("the frame reports no elapsed time") + } + // The serving side relays the engine's stream without parsing it, so it + // counts no content chunks: a frame must never claim a chunk count it + // did not measure. + if frame.Chunks != nil { + t.Fatalf("the frame reports chunks %v, want none — the serving side does not parse the engine's stream", *frame.Chunks) + } +} diff --git a/services/llamacpp-proxy/protocol_terminal_test.go b/services/llamacpp-proxy/protocol_terminal_test.go new file mode 100644 index 000000000..cf0aee517 --- /dev/null +++ b/services/llamacpp-proxy/protocol_terminal_test.go @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "errors" + "io" + "nvpair-shared/jsonrpc" + "strings" + "testing" + "time" +) + +type terminalTestStream struct { + io.Reader + io.Writer +} + +func TestProtocolMismatchStopsOpenReadLoop(t *testing.T) { + for _, frame := range []string{ + `{"jsonrpc":"2.0","method":"ready"}`, + `{"jsonrpc":"2.0","method":"ready","epoch":"x"}`, + `{"jsonrpc":"2.0","method":"ready","seq":1}`, + `{"jsonrpc":"2.0","id":1,"method":"log/set-level","epoch":"x","seq":1}`, + `{"jsonrpc":"2.0","id":1,"result":{},"epoch":"x","seq":1}`, + } { + t.Run(frame, func(t *testing.T) { + held, release := io.Pipe() + defer held.Close() + defer release.Close() + var out bytes.Buffer + input := io.MultiReader(strings.NewReader(frame+"\n"+`{"jsonrpc":"2.0","id":2,"method":"log/set-level","params":{"level":"invalid"}}`+"\n"), held) + p := &Proxy{codec: NewCodec(terminalTestStream{input, &out})} + done := make(chan error, 1) + go func() { done <- p.readLoop(context.Background()) }() + select { + case err := <-done: + var mismatch *jsonrpc.ProtocolMismatchError + if !errors.As(err, &mismatch) { + t.Fatalf("expected named mismatch, got %v", err) + } + if out.Len() != 0 { + t.Fatal("trailing command executed") + } + case <-time.After(time.Second): + t.Fatal("read loop remained alive after mismatch with input held open") + } + }) + } +} + +func TestReadLoopRecoversMalformedAndAcceptsStampedControl(t *testing.T) { + var input, output bytes.Buffer + input.WriteString("{not json}\n") + writer := jsonrpc.NewCodec(terminalTestStream{strings.NewReader(""), &input}) + if err := writer.Notify("log/set-level", map[string]string{"level": "invalid"}); err != nil { + t.Fatal(err) + } + input.WriteString(`{"jsonrpc":"2.0","id":2,"method":"log/set-level","params":{"level":"invalid"}}` + "\n") + p := &Proxy{codec: NewCodec(terminalTestStream{&input, &output})} + if err := p.readLoop(context.Background()); err != nil { + t.Fatal(err) + } + response, err := jsonrpc.NewCodec(terminalTestStream{&output, io.Discard}).Read() + if err != nil || !response.IsResponse() || string(*response.ID) != "2" { + t.Fatalf("valid control after malformed JSON was not executed: %v %v", response, err) + } +} diff --git a/services/llamacpp-proxy/proxy.go b/services/llamacpp-proxy/proxy.go new file mode 100644 index 000000000..2edb75643 --- /dev/null +++ b/services/llamacpp-proxy/proxy.go @@ -0,0 +1,2442 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/tls" + "encoding/hex" + "encoding/json" + stderrors "errors" + "fmt" + "io" + "log" + "log/slog" + "net" + "net/http" + "net/http/httputil" + "net/url" + "os" + "sort" + "strconv" + "sync" + "sync/atomic" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + "nvpair-shared/applog" + "nvpair-shared/bufpool" + "nvpair-shared/clustertrust" + "nvpair-shared/cors" + "nvpair-shared/errors" + "nvpair-shared/inferencemetrics" + "nvpair-shared/jsonrpc" + "nvpair-shared/listen" + "nvpair-shared/netmon" + "nvpair-shared/netpick" + "nvpair-shared/nodeactivity" + "nvpair-shared/noderec" + "nvpair-shared/prefixhash" + "nvpair-shared/proxyport" + "nvpair-shared/reach" + "nvpair-shared/routingmetrics" + "nvpair-shared/schedulerwire" + "nvpair-shared/splitlisten" + "nvpair-shared/telemetry" + "nvpair-shared/workerhealth" +) + +// Version is stamped at build time via -ldflags "-X main.Version=...". +// See versions.json at the repo root for the source of truth. +var Version = "dev" + +type ReadyParams struct { + Version string `json:"version"` + Port int `json:"port"` +} + +// ErrorParams is sent as a JSON-RPC "error" notification when the +// proxy encounters a fatal startup-time condition it wants to surface +// to the orchestrator before exiting. Code is a short machine-readable +// tag ("bind-failed" today); Message is a human-friendly string suitable +// for an error bar. +type ErrorParams struct { + Code string `json:"code"` + Message string `json:"message"` + Port int `json:"port,omitempty"` +} + +type NodesResult struct { + Nodes []Node `json:"nodes"` +} + +type SelectParams struct { + ID string `json:"id"` +} + +type SelectedResult struct { + ID string `json:"id"` +} + +// RequestStartedEvent is emitted as a `proxy/request-started` +// notification the moment we've resolved a target node and are about +// to forward the request to it. Pairs with the existing +// `proxy/request` completion event by ID so the orchestrator can +// track in-flight requests per target — increment on start, decrement +// on the matching completion. Rejection-path requests (no active +// node) never get a started event because they were never in flight; +// they go straight to a completion event with an unmatched ID. +// +// NodeID is the chosen node's identifier in the discovery list. It's +// the authoritative way to attribute activity to a node card in the +// UI: Target (host:port) would be ambiguous whenever the proxy +// rewrote a local-interface address to 127.0.0.1 (see nodeURL), so +// multiple nodes could plausibly match the same Target string. +// Cluster model-list fan-out has no single node, so it reports an empty +// NodeID and the explicit Target "cluster". +type RequestStartedEvent struct { + ID string `json:"id"` + NodeID string `json:"node_id,omitempty"` + Method string `json:"method"` + Path string `json:"path"` + Target string `json:"target"` +} + +// RequestEvent is emitted as a `proxy/request` notification when a +// proxied request finishes (or is rejected before forwarding). The +// ID is unique within one proxy process lifetime — paired with the +// matching RequestStartedEvent so consumers can pop it from an +// in-flight map. ID is always populated even on the rejection path +// (where no Started event was emitted) so consumers don't need a +// separate code path for ID-less completions. +// +// NodeID is empty on the rejection path (no target was resolved) and +// on cluster model-list fan-out, and is the chosen node's identifier otherwise. See RequestStartedEvent +// for why attribution by NodeID is needed instead of by Target. +// +// TTFB is the time-to-first-byte: milliseconds from the moment we +// started forwarding to the node until its HTTP response status line +// came back, captured via ReverseProxy.ModifyResponse. Omitted +// (serialized as absent rather than zero) when not applicable: +// rejection path (no forward happened) and upstream-error path +// (ModifyResponse is never called on connection/dial failures). This +// is the "is the node snappy?" signal — distinct from Duration, +// which for streaming llama.cpp responses is dominated by token +// generation time and so doesn't really reflect latency at all. +type RequestEvent struct { + CacheAffinity bool `json:"cache_affinity"` + ID string `json:"id"` + NodeID string `json:"node_id,omitempty"` + Method string `json:"method"` + Path string `json:"path"` + Target string `json:"target"` + Status int `json:"status"` + Duration int64 `json:"duration_ms"` + TTFB int64 `json:"ttfb_ms,omitempty"` + Error string `json:"error,omitempty"` + + // Cumulative routing totals for this proxy process, carried so consumers + // replace their displayed totals instead of accumulating their own copy of + // the routing filter. Reset on restart with the process, exactly as the + // per-consumer counters they replace did. + Requests uint64 `json:"requests"` + CacheAffinityHits uint64 `json:"cache_affinity_hits"` +} + +// Workload lifecycle method names (workload-manager spec 7). The proxy is +// a workload *producer*: it emits one of these per forwarded inference +// request so the broker can stamp the origin (originatedFrom) and forward it to the +// workload-manager, which broadcasts it cluster-wide. We don't emit +// workload:submitted (the proxy never queues — it forwards immediately) or +// workloads:remove (retirement is a broker concern). +const ( + workloadStartedMethod = "workload:started" + workloadCompletedMethod = "workload:completed" + workloadErroredMethod = "workload:errored" + + // workloadEngine is the opaque engine identifier carried in every + // workload this proxy produces. This proxy only ever fronts llama.cpp. + workloadEngine = "llamacpp" +) + +// inferenceEndpoints is the set of request paths that count as cluster +// workloads. Health checks, model listings (/v1/models), and other control +// traffic are deliberately excluded so we don't flood the cluster with +// non-inference noise. llama.cpp serves the OpenAI-compatible API, so these +// are the OpenAI inference routes. +var inferenceEndpoints = map[string]bool{ + "/v1/chat/completions": true, + "/v1/completions": true, + "/v1/embeddings": true, + "/v1/messages": true, +} + +// isInferenceRequest reports whether a request should be tracked as a +// workload — a POST to one of the known inference endpoints. +func isInferenceRequest(method, path string) bool { + return method == http.MethodPost && inferenceEndpoints[path] +} + +// Workload mirrors the workload-manager spec 6 object. The proxy populates +// the fields it can observe: originatedFrom is intentionally left empty for +// the broker to stamp with the authoritative local node id (exactly like +// errors:report), while scheduledOn is set to the node this proxy actually +// routed the request to (the served candidate's node id — the same +// authoritative attribution handle), so a consumer can attribute the workload +// to where it ran rather than to where it came from. requesterId is omitted. +// Pointer fields serialize as JSON null when unset, matching the spec's +// nullable columns. The embedded measurement is the exception: its fields are +// omitted entirely when the engine did not report them. +type Workload struct { + ID string `json:"id"` + Model string `json:"model"` + Engine string `json:"engine"` + RunID string `json:"runId"` + State string `json:"state"` + OriginatedFrom string `json:"originatedFrom"` + ScheduledOn string `json:"scheduledOn,omitempty"` + CreatedAt int64 `json:"createdAt"` + StartedAt *int64 `json:"startedAt"` + CompletedAt *int64 `json:"completedAt"` + Error *string `json:"error"` + RequesterID *string `json:"requesterId"` + + // Measurement carries the engine's own prefill/decode numbers for this + // request. It is embedded anonymously so its fields appear inline in + // the workload JSON, and it stays empty for a workload that never + // reached an engine (a rejected or cancelled request). + inferencemetrics.Measurement +} + +// workloadParams is the params envelope for a workload:* notification +// (spec 7.1): a single workloadInfo carrying the full Workload. +type workloadParams struct { + WorkloadInfo Workload `json:"workloadInfo"` +} + +// bufferBodyAndModel reads the request body once and returns the raw bytes +// (so each failover attempt can replay it — see the loop in handleHTTP) along +// with the JSON "model" field for workload tracking. Inference bodies are +// small (prompt + model), so full buffering is cheap. Returns (nil, "") when +// the body is absent and an empty model when none is parseable. The caller +// restores r.Body from the returned bytes before each forward attempt. +func bufferBodyAndModel(r *http.Request) ([]byte, string) { + if r.Body == nil { + return nil, "" + } + body, err := io.ReadAll(r.Body) + _ = r.Body.Close() + if err != nil { + return body, "" + } + var probe struct { + Model string `json:"model"` + } + if err := json.Unmarshal(body, &probe); err != nil { + return body, "" + } + return body, probe.Model +} + +type statusCapture struct { + http.ResponseWriter + status int + + // idle bounds how long a single write of streamed bytes to the client may + // block before it's abandoned. Zero disables the deadline. See + // idleClientWriteTimeout for the rationale (killed-client / half-open + // socket zombie jobs). + idle time.Duration + rc *http.ResponseController + // wroteErr retains the first error returned when writing the response body + // to the client (e.g. a dead client's send buffer filling and the write + // deadline tripping), so handleHTTP can mark the workload failed rather + // than misreporting the truncated stream as completed. + wroteErr error + + // upstreamAlive is called after each successful body write, but only once the + // upstream has committed — handleHTTP sets it in ModifyResponse, so it stays + // nil while the only thing this writer could carry is the proxy's own error + // body. Past that point every byte written came from the node serving the + // request, which is proof that node is working: the liveness evidence + // discovery cannot obtain for itself while the node is too busy to answer a + // probe. Called on the reverse proxy's copy goroutine, so it must be cheap. + upstreamAlive func() +} + +// Unwrap exposes the underlying ResponseWriter so http.ResponseController can +// reach the connection for SetWriteDeadline (and Flush) through this wrapper. +func (sc *statusCapture) Unwrap() http.ResponseWriter { return sc.ResponseWriter } + +func (sc *statusCapture) WriteHeader(code int) { + sc.status = code + sc.ResponseWriter.WriteHeader(code) +} + +// Write bounds each streamed write to the client with a deadline so a write +// blocked on a dead/half-open client fails promptly instead of hanging the +// reverse-proxy copy indefinitely. The deadline is cleared after every +// successful write, so a legitimately slow generation with long gaps between +// tokens is never penalized — only a write actively stuck on a gone client +// trips it. The first write error is retained (wroteErr) for the caller. +func (sc *statusCapture) Write(b []byte) (int, error) { + if sc.idle > 0 { + if sc.rc == nil { + sc.rc = http.NewResponseController(sc.ResponseWriter) + } + _ = sc.rc.SetWriteDeadline(time.Now().Add(sc.idle)) + } + n, err := sc.ResponseWriter.Write(b) + if err != nil { + if sc.wroteErr == nil { + sc.wroteErr = err + } + } else if sc.rc != nil { + _ = sc.rc.SetWriteDeadline(time.Time{}) + } + // Reported on every chunk rather than once per response so a long generation + // keeps vouching for its node for as long as it streams. The reporter + // coalesces, so the cost of calling this per chunk is a mutex and a clock + // read. + if err == nil && sc.upstreamAlive != nil { + sc.upstreamAlive() + } + return n, err +} + +// FlushError makes the streamed flush deadline-aware. For a streaming +// (chunked) upstream, ReverseProxy flushes after every write via +// http.NewResponseController(w).Flush — and because a small chunk buffers on +// Write without touching the socket, the actual network write for it happens +// here in Flush, not in Write. Without this method that flush reaches the +// underlying connection through Unwrap with no deadline and blocks unbounded on +// a stalled client (the same zombie the Write deadline guards against). So arm +// the same idle deadline around the flush, clear it on success, and retain a +// real flush error so the workload is classified failed. Implementing +// FlushError (which also satisfies the Flusher path via the ResponseController) +// means the flush routes through here instead of unwrapping past us. +func (sc *statusCapture) FlushError() error { + if sc.rc == nil { + sc.rc = http.NewResponseController(sc.ResponseWriter) + } + if sc.idle > 0 { + _ = sc.rc.SetWriteDeadline(time.Now().Add(sc.idle)) + } + err := sc.rc.Flush() + if err != nil { + // A ResponseWriter that genuinely can't flush is not a client failure; + // only retain real I/O errors (e.g. the deadline tripping on a dead + // client) so we don't misreport an unsupported-flush as a failed write. + if !stderrors.Is(err, http.ErrNotSupported) && sc.wroteErr == nil { + sc.wroteErr = err + } + return err + } + if sc.idle > 0 { + _ = sc.rc.SetWriteDeadline(time.Time{}) + } + return nil +} + +type Proxy struct { + portWrites proxyport.Operations + codec *Codec + discovery *Discovery + cancel context.CancelFunc + + // httpMu guards port, the servers, the split listener, and ln across a + // live set-port rebind. The HTTP handlers never read port (they route by + // upstream node), so the only contention is set-port vs set-port + // (serialized) and the initial serveHTTP store vs a later rebind. + httpMu sync.Mutex + port int + plainSrv *http.Server + tlsSrv *http.Server + split *splitlisten.Splitter + ln net.Listener + + // mesh is this node's cluster mTLS trust fabric, loaded from --cluster-dir. + // nil = unclustered: the LAN TLS ingress accepts nothing and the node does + // only loopback-plaintext local routing. Read-only after startup. + mesh *clustertrust.Mesh + + // backendMu guards backend, the explicit loopback engine the cluster mTLS + // ingress forwards to. The broker sets/clears it via node/set-local-backend; + // it is never sourced from discovery, so an ingress request can only ever + // reach this node's own local engine and can never be re-routed to a peer. + backendMu sync.RWMutex + backend localBackend + + selectedMu sync.RWMutex + selectedID string + + // activity coalesces the liveness reports raised when a peer's engine streams + // response bytes back through us (see reportActivity). + activity *nodeactivity.Reporter + + // priorityMu guards the scheduler's authoritative baseline and the + // optimistic reservations made since that snapshot arrived. resolveCandidates + // reads priority to form the failover list; reserveCandidate atomically adds + // local dispatches before forwarding so a concurrent burst cannot repeatedly + // choose from the same stale scheduler state. + prt *prefixhash.PrefixRoutingTable + + priorityMu sync.RWMutex + priority []string + priorityPending map[string]int + priorityGPUPressure map[string]int + priorityReservations map[string]int + + // coolMu guards process-local deadline measurements, never persisted. + coolMu sync.Mutex + coolUntil map[string]time.Time + + // targets remembers, per node, which of its published addresses accepted a + // connection, so a repeated forward costs no confirmation. An entry is + // re-confirmed when the node's candidate list changes and forgotten on an + // upstream error, so the next request fails over to another address. + targets *reach.Chooser + + // transportMu guards the long-lived HTTP transports reused across forwards + // and model-list fetches. Allocating a new http.Transport per request + // defeats connection pooling and leaks idle sockets until GC. + transportMu sync.Mutex + plainTransport *http.Transport + peerTransports map[string]*http.Transport + + // nextRequestID is a monotonic counter for tagging RequestStarted / + // RequestEvent pairs. Atomic add returns the new value, so request + // IDs start at 1 and never collide within a single proxy lifetime. + // IDs deliberately reset across restarts — they're only meaningful + // while the orchestrator's in-flight map is also alive, and a + // fresh proxy session always starts that map empty on the + // orchestrator side via the proxy:stopped → proxy:ready event + // pair. + nextRequestID atomic.Uint64 + + // Routing metrics totals, accumulated in nvpair-shared/routingmetrics rather + // than by each consumer. See routing_metrics.go. + routing routingmetrics.Counters + + // runID is a per-process nonce minted at startup and stamped on every + // workload this proxy emits. It makes a workload's identity + // (originatedFrom, engine, runId, id) globally unique even though + // nextRequestID resets to 1 on restart and the Ollama proxy also counts + // from 1 — without it, two concurrent cross-engine jobs, or a reused id + // after a restart, would collide in the broker's store. + runID string + + bufPool *bufpool.Pool + telemetry *telemetry.Manager + + // nodeID is this node's stable per-host id. A request forwarded to a + // cluster peer carries it, so the peer knows which node the job + // originated on and attributes the live frame it publishes to that + // origin rather than to itself. + nodeID string +} + +func NewProxy(codec *Codec, discovery *Discovery, port int, tel ...*telemetry.Manager) *Proxy { + var tp *telemetry.Manager + if len(tel) > 0 { + tp = tel[0] + } + if tp == nil { + tp = telemetry.NewManager("nvpair-llamacpp-proxy", telemetry.OTLPExporter) + } + return &Proxy{ + prt: prefixhash.NewPrefixRoutingTable(10000, 5*time.Minute), + codec: codec, + discovery: discovery, + port: port, + targets: reach.NewChooser(), + runID: newRunID(), + activity: nodeactivity.NewReporter(activityReportInterval), + bufPool: bufpool.New(), + telemetry: tp, + } +} + +// activityReportInterval is how often a single node's streaming may raise a +// liveness report. A generation writes hundreds of chunks and the scanner treats +// a report as good for a minute, so anything finer is pure noise on the broker +// pipe. +const activityReportInterval = 2 * time.Second + +// reportActivity tells the broker a node's engine just returned response bytes, +// so discovery can keep that node even while it is too busy to answer a liveness +// probe. This is the only liveness signal that strengthens under load, which is +// exactly when the probe-based ones fail. +// +// Reports are not filtered to remote nodes here: this proxy knows targets by URL +// and port, not by whether a uuid is its own. The scanner holds that identity and +// drops its own (see noteActivity). +func (p *Proxy) reportActivity(nodeID string) { + if !p.activity.Due(nodeID) { + return + } + if err := p.codec.Notify(noderec.NotifyNodeActivity, noderec.NodeActivityParams{HostUUID: nodeID}); err != nil { + slog.Debug("failed to report node activity", "node_id", nodeID, "err", err) + } +} + +// newRunID returns a short random per-process nonce (hex). A crypto/rand read +// failure falls back to a timestamp — uniqueness matters more than +// unpredictability here. +func newRunID() string { + var b [8]byte + if _, err := rand.Read(b[:]); err != nil { + return strconv.FormatInt(time.Now().UnixNano(), 16) + } + return hex.EncodeToString(b[:]) +} + +func (p *Proxy) Run(ctx context.Context) error { + ctx, cancel := context.WithCancel(ctx) + p.cancel = cancel + defer cancel() + + // Keep the "is this address local?" set fresh as interfaces come and go, + // so the loopback rewrite in nodeURL stays correct after VPN/dock changes + // or a sleep/wake IP reassignment. + startLocalAddrWatch(ctx) + + // Bind synchronously before announcing "ready": if the port is + // already in use, we want the UI to see a real error reason + // instead of being stuck on "Proxy running" while ListenAndServe + // silently fails in a goroutine. + ln, err := p.listen() + if err != nil { + // Best-effort: notify the orchestrator with a structured + // reason. The process will exit non-zero regardless (main.go + // log.Fatalf's on a non-nil Run error), so a failed Notify + // here is not worth surfacing separately. + _ = p.codec.Notify("error", ErrorParams{ + Code: "bind-failed", + Message: fmt.Sprintf("failed to bind port %d: %v", p.port, err), + Port: p.port, + }) + return fmt.Errorf("failed to bind port %d: %w", p.port, err) + } + + if err := p.codec.Notify("ready", ReadyParams{ + Version: Version, + Port: p.port, + }); err != nil { + // Don't leave a dangling listener holding the port if we + // couldn't even tell the orchestrator about it. + _ = ln.Close() + return fmt.Errorf("failed to send ready notification: %w", err) + } + + // Routing targets come from the broker's discovery relay. Subscribe + // for lm nodes; they arrive as discovery:nodes snapshots (handled in + // handleMessage), each replacing the subscribed overlay. Non-fatal: if the + // parent isn't a relay-aware broker the proxy still routes to manual nodes. + slog.Debug("subscribing to discovery relay for routing targets", "service", string(noderec.ServiceLlamaCPP)) + if err := p.codec.Notify(noderec.MethodSubscribe, noderec.SubscribeParams{Services: []noderec.ServiceKey{noderec.ServiceLlamaCPP}}); err != nil { + slog.Warn("failed to subscribe to discovery relay", "err", err) + } + + p.serveHTTP(ctx, ln) + + err = p.readLoop(ctx) + + // The app is going away (stdin closed or ctx cancelled). Stop any + // inference requests still in flight rather than letting them run to + // completion: cancelling the proxy's root context propagates to every + // in-flight request context — and thus the upstream reverse-proxy + // connection — so the target llama.cpp sees the client disconnect and stops + // generating instead of burning the GPU on a result nobody will read. + cancel() + + // srv.Shutdown then waits for the handlers to unwind (now fast, since + // their upstream calls were just cancelled). As each returns it emits its + // own terminal workload:errored, so peers don't keep showing the workload + // as a "running" ghost. A hard kill (SIGKILL) bypasses all of this. + shutCtx, shutCancel := context.WithTimeout(context.Background(), 3*time.Second) + defer shutCancel() + p.shutdown(shutCtx) + + return err +} + +// Timeouts for upstream connections. Logged at startup so they're always +// present in any captured log for post-mortem analysis. +const ( + proxyDialTimeout = 10 * time.Second + proxyKeepAlive = 30 * time.Second + // defaultProxyResponseTimeout is the upstream response-header timeout + // unless --response-header-timeout (or NVPAIR_PROXY_RESPONSE_HEADER_TIMEOUT) + // overrides it. An engine sends no headers until generation starts, so a + // request queued behind other work, or waiting on a slow model load, + // needs longer than this to survive the proxy. + defaultProxyResponseTimeout = 120 * time.Second + proxyMaxIdleConns = 50 + proxyIdleConnTimeout = 90 * time.Second + // Inbound http.Server limits — keep IdleTimeout aligned with client + // IdleConnTimeout so idle keep-alives are reaped on both sides. + proxyReadHeaderTimeout = 10 * time.Second + proxyServerIdleTimeout = 90 * time.Second + maxModelListBytes = 16 << 20 +) + +// responseHeaderTimeoutEnv carries the upstream response-header timeout when +// the --response-header-timeout flag is empty. The broker spawns the proxy as a +// child process, so a value set in the broker's (or the desktop app's) +// environment is inherited; on macOS, `launchctl setenv` reaches an app opened +// from the Finder. +const responseHeaderTimeoutEnv = "NVPAIR_PROXY_RESPONSE_HEADER_TIMEOUT" + +// proxyResponseTimeout is the effective upstream response-header timeout, +// resolved at startup: the flag, then the environment, then the default. +// Transports are built lazily, so assigning it before serving is sufficient. +var proxyResponseTimeout = defaultProxyResponseTimeout + +// resolveResponseHeaderTimeout applies the flag > env > default precedence. A +// missing, unparseable or non-positive value keeps the default, so a bad +// setting can never disable the timeout. +func resolveResponseHeaderTimeout(flagVal string) time.Duration { + raw := flagVal + if raw == "" { + raw = os.Getenv(responseHeaderTimeoutEnv) + } + if raw == "" { + return defaultProxyResponseTimeout + } + d, err := time.ParseDuration(raw) + if err != nil || d <= 0 { + log.Printf("invalid response-header timeout %q, using default %s", raw, defaultProxyResponseTimeout) + return defaultProxyResponseTimeout + } + return d +} + +// idleClientWriteTimeout bounds how long a single write of streamed response +// bytes to the client may block. A killed client can leave a half-open socket +// whose kernel send buffer fills and never drains; without this deadline the +// reverse-proxy copy blocks indefinitely (TCP retransmit backoff runs into +// minutes, and r.Context() never fires when no FIN/RST arrives), so the request +// handler never returns and its terminal workload event is never emitted — the +// "zombie job" left showing as running until PAIR restarts. statusCapture.Write +// resets the deadline after every successful write, so this only trips a write +// that is actively stuck on a gone client, never a slow-but-live generation. +// +// It is a var (not a const) only so a test can shorten it to exercise the +// deadline against a real socket; production never reassigns it. +var idleClientWriteTimeout = 30 * time.Second + +var modelListClient = &http.Client{ + Transport: &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: proxyDialTimeout, + KeepAlive: proxyKeepAlive, + }).DialContext, + ResponseHeaderTimeout: proxyDialTimeout, + MaxIdleConns: proxyMaxIdleConns, + IdleConnTimeout: proxyIdleConnTimeout, + }, + Timeout: proxyDialTimeout, +} + +// listen binds the proxy's TCP listener synchronously so bind failures +// (EADDRINUSE and friends) can be reported through a structured error +// notification before the process exits. The caller is responsible for +// closing the returned listener if it doesn't hand it to serveHTTP. +func (p *Proxy) listen() (net.Listener, error) { + return listen.TCP(fmt.Sprintf(":%d", p.port)) +} + +// serveHTTP takes the already-bound base listener and drives the two proxy +// personalities over it: a plaintext HTTP server (loopback-only, full local +// router) and a LAN mTLS ingress (pin-gated, forwards to the local engine), +// split by the connection's first byte via nvpair-shared/splitlisten. The two +// http.Servers are recorded so set-port can rebind both onto a fresh split +// without tearing the servers down. +func (p *Proxy) serveHTTP(ctx context.Context, ln net.Listener) { + base := func(_ net.Listener) context.Context { return ctx } + plainSrv := &http.Server{ + Handler: http.HandlerFunc(p.handlePlain), + BaseContext: base, + ReadHeaderTimeout: proxyReadHeaderTimeout, + IdleTimeout: proxyServerIdleTimeout, + } + tlsSrv := &http.Server{ + Handler: http.HandlerFunc(p.handleClusterIngress), + BaseContext: base, + ReadHeaderTimeout: proxyReadHeaderTimeout, + IdleTimeout: proxyServerIdleTimeout, + } + + p.httpMu.Lock() + p.plainSrv = plainSrv + p.tlsSrv = tlsSrv + p.ln = ln + p.startSplitLocked(ln) + p.httpMu.Unlock() + + slog.Info("proxy timeouts configured", + "dial_timeout", proxyDialTimeout, + "keep_alive", proxyKeepAlive, + "response_header_timeout", proxyResponseTimeout, + "first_body_timeout", proxyFirstBodyTimeout, + "max_idle_conns", proxyMaxIdleConns, + "idle_conn_timeout", proxyIdleConnTimeout, + ) + slog.Info("HTTP proxy listening", "port", p.port, "addr", ln.Addr().String(), + "cluster_ingress", p.mesh.Clustered()) +} + +// startSplitLocked wraps base in a first-byte splitter and starts both servers +// on its sub-listeners. Caller holds httpMu. Reuses the persistent plainSrv / +// tlsSrv so set-port can call it repeatedly on fresh listeners. +func (p *Proxy) startSplitLocked(base net.Listener) { + split := splitlisten.New(base) + p.split = split + go func() { + if err := p.plainSrv.Serve(split.Plain()); err != nil && err != http.ErrServerClosed { + slog.Error("plaintext HTTP server exited", "err", err) + } + }() + go p.serveTLS(split.TLS()) +} + +// serveTLS terminates cluster mTLS on the split's TLS sub-listener. The server +// certificate is resolved per handshake from the live mesh, so this one +// sub-listener covers both states: while this node is unclustered there is no +// leaf to present and the handshake is refused (it exposes no LAN inference +// surface), and the moment the node becomes a member the same sub-listener +// serves the pin-gated ingress — no rebind, and no process restart to pick up a +// freshly-minted identity. +func (p *Proxy) serveTLS(l net.Listener) { + if err := p.tlsSrv.Serve(tls.NewListener(l, p.mesh.ServerTLSConfig())); err != nil && err != http.ErrServerClosed { + slog.Error("cluster mTLS ingress exited", "err", err) + } +} + +// shutdown gracefully stops both personalities and closes the split listener. +func (p *Proxy) shutdown(ctx context.Context) { + p.httpMu.Lock() + plainSrv, tlsSrv, split := p.plainSrv, p.tlsSrv, p.split + p.httpMu.Unlock() + if plainSrv != nil { + _ = plainSrv.Shutdown(ctx) + } + if tlsSrv != nil { + _ = tlsSrv.Shutdown(ctx) + } + if split != nil { + _ = split.Close() + } + p.closeIdleTransports() +} + +// setPort live-rebinds the HTTP listener onto newPort and persists the choice +// so it survives a restart. It binds the new listener first (so a bind +// failure leaves the current one serving), starts the same server on it, then +// closes the old listener — in-flight connections on the old port drain +// naturally. A fresh `ready` notification announces the new port so the +// orchestrator/UI learn where the proxy is now listening. +func (p *Proxy) setPort(newPort int) error { + return p.setPortContext(context.Background(), newPort) +} + +func (p *Proxy) setPortContext(ctx context.Context, newPort int) error { + p.httpMu.Lock() + defer p.httpMu.Unlock() + if err := ctx.Err(); err != nil { + return err + } + + if newPort == p.port { + return nil + } + newLn, err := listen.TCP(fmt.Sprintf(":%d", newPort)) + if err != nil { + return fmt.Errorf("failed to bind port %d: %w", newPort, err) + } + if err := ctx.Err(); err != nil { + _ = newLn.Close() + return err + } + oldSplit := p.split + p.ln = newLn + p.port = newPort + + // Re-serve both personalities on a fresh split over the new listener, then + // close the old split (and its base listener) so in-flight connections on + // the old port drain naturally. The plaintext and mTLS personalities always + // move together as one unit. + slog.Info("HTTP proxy listening", "port", newPort, "addr", newLn.Addr().String(), + "cluster_ingress", p.mesh.Clustered()) + p.startSplitLocked(newLn) + if oldSplit != nil { + _ = oldSplit.Close() + } + + if err := savePersistedPort(newPort); err != nil { + slog.Warn("failed to persist proxy port", "port", newPort, "err", err) + } + if err := p.codec.Notify("ready", ReadyParams{Version: Version, Port: newPort}); err != nil { + slog.Warn("failed to emit ready after rebind", "err", err) + } + return nil +} + +// emitWorkload sends a workload:* lifecycle notification to the +// orchestrator. The broker stamps the origin (originatedFrom) and forwards it to the +// workload-manager; a failed write is logged but never blocks the request. +func (p *Proxy) emitWorkload(method string, w Workload) { + if err := p.codec.Notify(method, workloadParams{WorkloadInfo: w}); err != nil { + slog.Warn("failed to emit workload notification", "method", method, "err", err) + } +} + +// candidate is one forwarding target: the node's discovery ID (the +// authoritative attribution handle, stable across nodeURL's 127.0.0.1 +// rewrite) and its resolved URL. peerUUID is set for a remote cluster peer: +// the request is dialed over cluster mTLS to the peer's promoted proxy (https), +// pinned to that peer's exact server cert. Empty peerUUID means a plain-HTTP +// dial — the local backend (self) or an explicit manual node. +type candidate struct { + id string + url *url.URL + peerUUID string + gpuPressure int + queueDepth int +} + +func (p *Proxy) nodeMetrics(nodeID string) (int, int) { + p.priorityMu.RLock() + defer p.priorityMu.RUnlock() + gp := 0 + if p.priorityGPUPressure != nil { + gp = p.priorityGPUPressure[nodeID] + } + qd := 0 + if p.priorityPending != nil { + qd = p.priorityPending[nodeID] + } + return gp, qd +} + +// candidateTransport returns the reverse-proxy / model-list transport for a +// candidate. Plain/self/manual candidates share one long-lived Transport. +// Cluster peers share one long-lived mTLS Transport per peerUUID. Callers must +// not CloseIdleConnections on the returned value except via closeIdleTransports. +func (p *Proxy) candidateTransport(c candidate) *http.Transport { + if c.peerUUID == "" { + return p.plainHTTPTransport() + } + return p.peerHTTPTransport(c.peerUUID) +} + +func newProxyTransport(tlsCfg *tls.Config) *http.Transport { + tr := &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: proxyDialTimeout, + KeepAlive: proxyKeepAlive, + }).DialContext, + ResponseHeaderTimeout: proxyResponseTimeout, + MaxIdleConns: proxyMaxIdleConns, + MaxIdleConnsPerHost: proxyMaxIdleConns, + IdleConnTimeout: proxyIdleConnTimeout, + } + if tlsCfg != nil { + tr.TLSClientConfig = tlsCfg + } + return tr +} + +func (p *Proxy) plainHTTPTransport() *http.Transport { + p.transportMu.Lock() + defer p.transportMu.Unlock() + if p.plainTransport == nil { + p.plainTransport = newProxyTransport(nil) + } + return p.plainTransport +} + +func (p *Proxy) peerHTTPTransport(peerUUID string) *http.Transport { + p.transportMu.Lock() + defer p.transportMu.Unlock() + if tr, ok := p.peerTransports[peerUUID]; ok { + if p.mesh != nil && p.mesh.HasPin(peerUUID) { + return tr + } + tr.CloseIdleConnections() + delete(p.peerTransports, peerUUID) + } + if p.mesh == nil { + return unpinnedPeerTransport() + } + cfg, ok := p.mesh.ClientTLSConfig(peerUUID) + if !ok { + return unpinnedPeerTransport() + } + tr := newProxyTransport(cfg) + if p.peerTransports == nil { + p.peerTransports = make(map[string]*http.Transport) + } + p.peerTransports[peerUUID] = tr + return tr +} + +// errPeerUnpinned is the dial error of a fail-closed peer transport. +var errPeerUnpinned = stderrors.New("peer is not a pinned cluster member") + +// unpinnedPeerTransport fails every dial. A peer candidate is resolved only +// while its pin is held, but the pin can go between resolution and dial; the +// peer is then refused outright rather than dialed with a transport that does +// not pin its certificate. +func unpinnedPeerTransport() *http.Transport { + return &http.Transport{ + DialContext: func(context.Context, string, string) (net.Conn, error) { + return nil, errPeerUnpinned + }, + } +} + +// dropUnpinnedPeerTransports closes idle conns for peer Transports whose pins +// are gone. Safe to call from the mesh Watch callback. +func (p *Proxy) dropUnpinnedPeerTransports() { + p.transportMu.Lock() + defer p.transportMu.Unlock() + for uuid, tr := range p.peerTransports { + if p.mesh != nil && p.mesh.HasPin(uuid) { + continue + } + tr.CloseIdleConnections() + delete(p.peerTransports, uuid) + } +} + +func (p *Proxy) closeIdleTransports() { + p.transportMu.Lock() + defer p.transportMu.Unlock() + if p.plainTransport != nil { + p.plainTransport.CloseIdleConnections() + } + for uuid, tr := range p.peerTransports { + tr.CloseIdleConnections() + delete(p.peerTransports, uuid) + } +} + +// retrySignal is returned from ModifyResponse to abort a retryable upstream +// response before its body streams to the client, so handleHTTP can fail over +// to the next candidate. It's a distinct type rather than errors.New(...) +// because this package aliases nvpair-shared/errors as `errors` (which has no New). +type retrySignal struct{} + +func (retrySignal) Error() string { return "llamacpp-proxy: retry next candidate" } + +type modelListItem struct { + key string + raw json.RawMessage +} + +type modelListResult struct { + items []modelListItem + ok bool + err error +} + +// serveModelList queries every llama.cpp candidate concurrently and returns +// the native /v1/models envelope with duplicate IDs removed. Results are +// merged in candidate order, not completion order, so duplicate metadata is +// deterministic while an unavailable peer cannot hide healthy inventories. +func (p *Proxy) serveModelList(w http.ResponseWriter, r *http.Request, candidates []candidate) (int, error) { + writeJSON := func(status int, body []byte) { + cors.Apply(w.Header()) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(status) + _, _ = w.Write(body) + } + results := make([]modelListResult, len(candidates)) + var wg sync.WaitGroup + for i, cand := range candidates { + target := *cand.url + target.Path = "/models" + target.RawPath = "" + target.RawQuery = r.URL.RawQuery + upstream, err := http.NewRequestWithContext(r.Context(), http.MethodGet, target.String(), nil) + if err != nil { + results[i].err = err + continue + } + upstream.Header.Set("Accept", "application/json") + + // A cluster-peer candidate is queried over mTLS to its promoted proxy; + // self/manual candidates use the shared plain client. + client := modelListClient + if cand.peerUUID != "" { + client = &http.Client{Timeout: modelListClient.Timeout, Transport: p.candidateTransport(cand)} + } + + wg.Add(1) + go func(i int, cand candidate, req *http.Request, client *http.Client) { + defer wg.Done() + resp, err := client.Do(req) + if err != nil { + p.targets.Forget(cand.id) + results[i].err = err + return + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + results[i].err = fmt.Errorf("upstream returned %s", resp.Status) + return + } + body, err := io.ReadAll(io.LimitReader(resp.Body, maxModelListBytes+1)) + if err != nil { + results[i].err = err + return + } + if len(body) > maxModelListBytes { + results[i].err = fmt.Errorf("model list exceeds %d bytes", maxModelListBytes) + return + } + var envelope struct { + Data *[]json.RawMessage `json:"data"` + } + if err := json.Unmarshal(body, &envelope); err != nil { + results[i].err = err + return + } + if envelope.Data == nil { + results[i].err = fmt.Errorf("upstream response has no data array") + return + } + models := *envelope.Data + items := make([]modelListItem, 0, len(models)) + for _, raw := range models { + var identity struct { + ID string `json:"id"` + } + if err := json.Unmarshal(raw, &identity); err != nil { + results[i].err = fmt.Errorf("invalid model record: %w", err) + return + } + if identity.ID == "" { + results[i].err = fmt.Errorf("model record has no id") + return + } + items = append(items, modelListItem{key: identity.ID, raw: raw}) + } + results[i] = modelListResult{items: items, ok: true} + }(i, cand, upstream, client) + } + wg.Wait() + + success := false + models := make([]json.RawMessage, 0) + seen := make(map[string]bool) + for i, result := range results { + if !result.ok { + slog.Debug("model list candidate unavailable", + "node_id", candidates[i].id, "target", candidates[i].url.Host, "err", result.err) + continue + } + success = true + for _, item := range result.items { + if !seen[item.key] { + seen[item.key] = true + models = append(models, item.raw) + } + } + } + if !success { + err := fmt.Errorf("no valid model list from %d candidate(s)", len(candidates)) + writeJSON(http.StatusServiceUnavailable, []byte(`{"error":"model inventory unavailable"}`)) + return http.StatusServiceUnavailable, err + } + body, err := json.Marshal(struct { + Object string `json:"object"` + Data []json.RawMessage `json:"data"` + }{Object: "list", Data: models}) + if err != nil { + writeJSON(http.StatusInternalServerError, []byte(`{"error":"failed to encode model inventory"}`)) + return http.StatusInternalServerError, err + } + writeJSON(http.StatusOK, body) + return http.StatusOK, nil +} + +func (p *Proxy) handleHTTP(w http.ResponseWriter, r *http.Request) { + session := p.telemetry.Session() + start := time.Now() + + // Allocate the request ID up front so both code paths (rejection + // and forward) can stamp the same value into their notification. + // The rejection path never emits a Started event, so its ID won't + // appear in any orchestrator in-flight map — that's fine; the + // completion event still bumps the failed counter regardless of + // whether a matching Started was seen. + reqID := strconv.FormatUint(p.nextRequestID.Add(1), 10) + + // Parse the request's model before choosing a node. Model eligibility only + // applies to inference routes; control endpoints retain their existing + // routing behavior even when their JSON happens to contain a model field. + bodyBytes, model := bufferBodyAndModel(r) + isInf := isInferenceRequest(r.Method, r.URL.Path) + + var ctx = r.Context() + var span trace.Span + var finalStatus int + if isInf { + ctx = telemetry.ExtractHTTPContext(r) + var rootSpan trace.Span + ctx, rootSpan = session.Tracer().Start(ctx, "pair.proxy.inference", trace.WithSpanKind(trace.SpanKindServer)) + span = rootSpan + defer func() { + if span != nil { + if finalStatus != 0 { + span.SetAttributes(attribute.Int("http.response.status_code", finalStatus)) + } + span.End() + } + }() + isStreaming := isStreamingRequest(r.URL.Path, bodyBytes) + span.SetAttributes( + attribute.String("gen_ai.system", "pair"), + metadataAttribute("gen_ai.request.model", model), + metadataAttribute("http.request.method", r.Method), + metadataAttribute("http.route", r.URL.Path), + attribute.String("pair.client.protocol", "openai"), + attribute.String("pair.request_id", reqID), + attribute.Bool("pair.is_streaming", isStreaming), + ) + span.SetAttributes(telemetry.RequestAttributes(r.Header)...) + if session.RecordPayloads() { + promptStr, truncated := telemetry.TruncateUTF8(extractPrompt(bodyBytes), telemetry.MaxPayloadBytes) + if promptStr != "" { + span.SetAttributes( + attribute.String("gen_ai.prompt", promptStr), + attribute.String("input.value", promptStr), + ) + if truncated { + span.SetAttributes(attribute.Bool("gen_ai.prompt.truncated", true)) + } + } + } + r = r.WithContext(ctx) + } + + routingModel := "" + if isInf { + routingModel = model + } + candidates := p.resolveCandidates(routingModel) + contextHash := prefixhash.Extract(r.Method, r.URL.Path, bodyBytes) + warmNodeID := p.affinityNode(contextHash) + cacheAffinity := false + if isInf && model != "" { + candidates, cacheAffinity = p.reserveCandidateWithAffinity(candidates, warmNodeID) + } + if isInf { + _, schedSpan := session.Tracer().Start(ctx, "pair.router.schedule", trace.WithSpanKind(trace.SpanKindInternal)) + schedAttrs := []attribute.KeyValue{ + attribute.Int("pair.route.candidate_count", len(candidates)), + attribute.Bool("pair.route.cache_affinity", cacheAffinity), + } + if len(candidates) > 0 { + schedAttrs = append(schedAttrs, + metadataAttribute("pair.route.selected_node_id", candidates[0].id), + attribute.Int("pair.route.gpu_pressure", candidates[0].gpuPressure), + attribute.Int("pair.route.queue_depth", candidates[0].queueDepth), + ) + } + schedSpan.SetAttributes(schedAttrs...) + schedSpan.End() + } + if r.Method == http.MethodGet && (r.URL.Path == "/v1/models" || r.URL.Path == "/models") { + if len(candidates) > 0 { + p.codec.Notify("proxy/request-started", RequestStartedEvent{ + ID: reqID, Method: r.Method, Path: r.URL.Path, Target: "cluster", + }) + } + status, err := p.serveModelList(w, r, candidates) + errText := "" + if err != nil { + errText = err.Error() + } + p.codec.Notify("proxy/request", RequestEvent{ + ID: reqID, Method: r.Method, Path: r.URL.Path, Target: "cluster", + Status: status, Duration: time.Since(start).Milliseconds(), Error: errText, + }) + return + } + if len(candidates) == 0 { + // With no engine to consult, retain the local permissive preflight used + // for engines that do not publish a CORS policy. + if cors.WritePreflight(w, r) { + finalStatus = http.StatusNoContent + return + } + cors.Apply(w.Header()) + rejectionBody := `{"error":"no active node selected or available"}` + rejectionError := "no active node" + if isInf && model != "" { + rejectionBody = `{"error":"no available node advertises the requested model"}` + rejectionError = "no node advertises requested model" + } + slog.Warn("proxy request rejected", + "id", reqID, "method", r.Method, "path", r.URL.Path, + "remote", r.RemoteAddr, "reason", rejectionError) + http.Error(w, rejectionBody, http.StatusBadGateway) + finalStatus = http.StatusBadGateway + p.codec.Notify("proxy/request", RequestEvent{ + ID: reqID, + Method: r.Method, + Path: r.URL.Path, + Status: http.StatusBadGateway, + Duration: time.Since(start).Milliseconds(), + Error: rejectionError, + }) + return + } + // shouldRetry reports whether an upstream status warrants failing over to + // the next candidate: busy/unavailable/gateway statuses, plus a 404 on an + // inference call (an advertised owner's inventory may have become stale). + // Genuine client errors (400/401/422…) are not retried — they'd fail + // identically on every node. + shouldRetry := func(code int) bool { + switch code { + case http.StatusRequestTimeout, + http.StatusTooManyRequests, + http.StatusBadGateway, + http.StatusServiceUnavailable, + http.StatusGatewayTimeout: + return true + case http.StatusNotFound: + return isInf + } + return code >= 500 + } + + var ( + servedNodeID string + servedTarget string + ttfbMs int64 + proxyErr string + started bool + wl *Workload + + // telReader wraps the body of the candidate we committed to and + // carries the engine's own prefill/decode measurement. It is declared + // here, outside the failover loop and before emitTerminal, so the + // measurement is reachable where the workload is finalized below. + // Only the committed candidate sets it. + telReader *telemetryBodyReader + ) + + // Emit workload:started up front, the moment we begin forwarding, naming + // the first candidate we'll try. A burst of concurrent inference requests + // must surface as job cards immediately; the upstream engine serializes + // concurrent requests on a single GPU slot, so gating "started" on the + // upstream response headers (the commit point) left every queued-but- + // forwarded job invisible until the node dequeued it — only one card at a + // time (a prior regression). If failover later commits a different + // node, the commit block re-points scheduledOn; the terminal + // completed/errored transition is emitted once at the end regardless. + if isInf && model != "" { + createdMs := start.UnixMilli() + wl = &Workload{ + ID: reqID, + Model: model, + Engine: workloadEngine, + RunID: p.runID, + State: "running", + ScheduledOn: candidates[0].id, + CreatedAt: createdMs, + StartedAt: &createdMs, + } + p.emitWorkload(workloadStartedMethod, *wl) + } + + // The terminal workload transition (completed/errored) can be reached from + // two places: the normal path after the stream copy unwinds below, and the + // disconnect watcher that fires while the copy is still blocked. terminalOnce + // guarantees exactly one is emitted; wlMu guards the shared wl fields the + // watcher (a separate goroutine) and ModifyResponse's failover re-point both + // touch; terminated suppresses a late started re-point once we've finalized. + var ( + terminalOnce sync.Once + wlMu sync.Mutex + terminated bool + ) + emitTerminal := func(state, errMsg string) { + if wl == nil { + return + } + terminalOnce.Do(func() { + now := time.Now().UnixMilli() + wlMu.Lock() + terminated = true + wl.CompletedAt = &now + wl.State = state + if errMsg != "" { + wl.Error = &errMsg + } + snapshot := *wl + wlMu.Unlock() + method := workloadCompletedMethod + if state != "completed" { + method = workloadErroredMethod + } + p.emitWorkload(method, snapshot) + }) + } + + // Watch for the client going away while the request is in flight. The + // terminal event is otherwise emitted only after the stream copy returns; + // a client that disconnects mid-stream can leave the copy blocked, so we + // emit the terminal here the moment r.Context() is cancelled instead of + // waiting for the unwind. Cancelling r.Context() (client close, or our own + // shutdown) also propagates to the ReverseProxy's upstream request, so the + // engine stops generating. terminalOnce keeps this from double-emitting + // with the normal path. The half-open case (no FIN, r.Context() never + // fires) is caught instead by statusCapture's write deadline below. + if wl != nil { + reqCtx := r.Context() + finished := make(chan struct{}) + defer close(finished) + go func() { + select { + case <-reqCtx.Done(): + emitTerminal("failed", "client disconnected before completion") + case <-finished: + } + }() + } + + // committedSC is the statusCapture of the candidate we committed to + // streaming; its wroteErr tells us after the fact whether the client write + // failed (dead/half-open client) so we can mark the workload failed. + var committedSC *statusCapture + var upstreamBody *affinityBodyReader + + // progressLim rate-limits this request's live frames to one every + // progressInterval. It is per request: one busy job must not silence + // another job's frames. + var progressLim progressLimiter + + // Failover loop: try candidates in order until one returns a + // usable response or the list is exhausted. We can only retry before the + // first byte reaches the client; once a response starts streaming we're + // committed. proxy/request-started fires at that commit point so it names + // the node that actually serves the request, not one we failed over from; + // workload:started was already emitted above (and is re-pointed there on a + // failover). The self-forward guard lives in resolveCandidates. + for i := range candidates { + cand := candidates[i] + last := i == len(candidates)-1 + if bodyBytes != nil { + r.Body = io.NopCloser(bytes.NewReader(bodyBytes)) + } + retry := false + sc := &statusCapture{ResponseWriter: w, status: http.StatusOK, idle: idleClientWriteTimeout} + + var dispatchSpan trace.Span + dispatchStart := time.Now() + if isInf { + _, dispatchSpan = session.Tracer().Start(ctx, "pair.engine.dispatch", trace.WithSpanKind(trace.SpanKindClient)) + host := cand.url.Hostname() + port := 0 + if pStr := cand.url.Port(); pStr != "" { + port, _ = strconv.Atoi(pStr) + } else if cand.url.Scheme == "https" { + port = 443 + } else { + port = 80 + } + isLocal := isLocalHost(cand.url.Host) || cand.peerUUID == "" + dispatchSpan.SetAttributes( + metadataAttribute("pair.route.target_node_id", cand.id), + metadataAttribute("server.address", host), + attribute.Int("server.port", port), + attribute.Bool("pair.route.is_local", isLocal), + ) + if i > 0 { + dispatchSpan.SetAttributes(attribute.Bool("pair.route.failover", true)) + } + } + + proxy := &httputil.ReverseProxy{ + Director: p.forwardDirector(cand, reqID), + // A remote cluster peer is dialed over mTLS (per-peer pinned config); + // self/manual candidates use the plain transport. See candidateTransport. + Transport: stallTransport{base: p.candidateTransport(cand), proxy: p, id: cand.id, timeout: proxyResponseTimeout}, + BufferPool: p.bufPool, + // ModifyResponse fires when the upstream's status line + headers + // have arrived but before the body streams. That's both the retry + // decision point and, on commit, the time-to-first-byte boundary. + ModifyResponse: func(resp *http.Response) error { + if !last && shouldRetry(resp.StatusCode) { + // Abort before streaming: ReverseProxy closes resp.Body and + // calls ErrorHandler with our sentinel, then we try next. + retry = true + return retrySignal{} + } + // Only successful inference waits for content. All commit effects + // below still happen once, for the node that actually serves. + if isInf && resp.StatusCode >= 200 && resp.StatusCode < 300 { + body, err := awaitFirstBody(r.Context(), resp.Body, proxyFirstBodyTimeout) + if err != nil { + if r.Context().Err() == nil && stderrors.Is(err, errFirstBodyTimeout) { + p.noteStalled(cand.id, time.Now()) + } + return err + } + resp.Body = body + } + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + p.noteSuccess(cand.id) + } + // Prefer an engine-declared preflight policy so an exact origin plus + // Allow-Credentials can pass a credentialed browser fetch. Engines + // that publish no policy retain the proxy's permissive 204 fallback. + cors.CompletePreflightFallback(resp) + upstreamBody = newAffinityBodyReader(resp.Body, resp.Header.Get("Content-Type")) + if isInf && dispatchSpan != nil { + reader := newTelemetryBodyReader(upstreamBody, dispatchSpan, dispatchStart, start, session) + reader.onProgress = func(prog LiveProgress) { + p.publishProgress(reqID, cand.id, start, &progressLim, prog) + } + telReader = reader + resp.Body = reader + } else { + resp.Body = upstreamBody + } + // Committing to this candidate — body stream is about to begin. + ttfbMs = time.Since(start).Milliseconds() + servedNodeID = cand.id + servedTarget = cand.url.Host + proxyErr = "" // clear any error recorded from a failed-over candidate + // Arm the liveness report only now. statusCapture also carries + // the proxy's OWN error bodies — ReverseProxy's ErrorHandler + // writes a failed dial's message through it — and those bytes + // prove nothing about the node. Reaching here means the upstream + // returned a status line, so everything written from this point + // came from the node. Same goroutine as the body copy, so no + // synchronization is needed. + sc.upstreamAlive = func() { p.reportActivity(cand.id) } + // The engine may enforce its own origin policy. Honor it: + // overwriting a declared Access-Control-Allow-Origin would + // silently widen the user's policy, and a wildcard is invalid + // alongside Allow-Credentials, so it would break a credentialed + // response outright. An engine that omits the header has + // expressed nothing to preserve, so the proxy supplies its own. + if resp.Header.Get("Access-Control-Allow-Origin") == "" { + cors.Apply(resp.Header) + } + if !started { + started = true + p.codec.Notify("proxy/request-started", RequestStartedEvent{ + ID: reqID, + NodeID: cand.id, + Method: r.Method, + Path: r.URL.Path, + Target: cand.url.Host, + }) + // workload:started was already emitted up front naming the + // first candidate. If failover landed us on a different + // node, re-point scheduledOn so the card — and the terminal + // completed/errored event, which carries the same wl — name + // the node that actually served. Guarded by wlMu against the + // disconnect watcher, and skipped once terminated so a late + // re-point can't resurrect a workload we've already failed. + if wl != nil { + wlMu.Lock() + if !terminated && wl.ScheduledOn != cand.id { + wl.ScheduledOn = cand.id + snapshot := *wl + wlMu.Unlock() + p.emitWorkload(workloadStartedMethod, snapshot) + } else { + wlMu.Unlock() + } + } + } + return nil + }, + ErrorHandler: func(ew http.ResponseWriter, _ *http.Request, err error) { + if _, ok := err.(retrySignal); ok { + return // retryable status — the loop advances to the next candidate + } + // Transport/dial error (not a status-based retry): forget this + // node's confirmed address so the next request re-confirms and + // can fail over to another of its published addresses + // (multi-homed peer). The in-request failover below moves on to + // the next node. + p.targets.Forget(cand.id) + if !last { + // Transport/dial error with candidates left: fail over. + retry = true + proxyErr = err.Error() + slog.Warn("proxy upstream error, failing over", + "id", reqID, "node_id", cand.id, "target", cand.url.Host, + "path", r.URL.Path, "err", err) + return + } + // Last candidate failed at the transport: terminal, surface it. + servedNodeID = cand.id + servedTarget = cand.url.Host + if cors.WritePreflight(ew, r) { + proxyErr = "" + return + } + proxyErr = err.Error() + slog.Warn("proxy upstream error, candidates exhausted", + "id", reqID, "node_id", cand.id, "target", cand.url.Host, + "method", r.Method, "path", r.URL.Path, + "duration_ms", time.Since(start).Milliseconds(), "err", err) + body, mErr := json.Marshal(map[string]string{ + "error": "upstream error: " + err.Error(), + }) + if mErr != nil { + body = []byte(`{"error":"upstream error"}`) + } + cors.Apply(ew.Header()) + ew.Header().Set("Content-Type", "application/json") + ew.Header().Set("X-Content-Type-Options", "nosniff") + ew.WriteHeader(http.StatusBadGateway) + ew.Write(body) + }, + } + + copyAborted := serveReverseProxy(proxy, sc, r) + if isInf && dispatchSpan != nil { + if telReader != nil { + telReader.finalize() + } + if retry || upstreamBody == nil { + dispatchSpan.SetStatus(codes.Error, "upstream candidate failed or retry triggered") + dispatchSpan.SetAttributes(attribute.Int("http.response.status_code", http.StatusBadGateway)) + } else if sc.status != 0 { + dispatchSpan.SetAttributes(attribute.Int("http.response.status_code", sc.status)) + } + dispatchSpan.End() + } + if copyAborted { + // net/http deliberately aborts a real server handler after a + // committed response copy fails. Catch that sentinel so the proxy + // can still emit its terminal telemetry and workload failure. The + // client connection remains closed; this only restores local cleanup. + proxyErr = "upstream response stream aborted" + finalStatus = sc.status + committedSC = sc + break + } + if !retry { + finalStatus = sc.status + committedSC = sc + break + } + } + + if contextHash.Full != (prefixhash.Digest{}) && finalStatus == http.StatusOK && + proxyErr == "" && r.Context().Err() == nil && committedSC != nil && + committedSC.wroteErr == nil && upstreamBody != nil && upstreamBody.complete { + p.prt.Put(model, contextHash.Full, prefixhash.RouteRecord{NodeID: servedNodeID, Engine: workloadEngine}) + } + cacheAffinity = cacheAffinity && servedNodeID == warmNodeID + + slog.Debug("proxy request complete", + "id", reqID, + "node_id", servedNodeID, + "method", r.Method, + "path", r.URL.Path, + "target", servedTarget, + "status", finalStatus, + "duration_ms", time.Since(start).Milliseconds(), + "ttfb_ms", ttfbMs, + "err", proxyErr, + ) + + reqs, hits := p.recordRoutingMetric(r.Method, r.URL.Path, cacheAffinity) + // The proxy's own measured request duration. It is operational metadata + // about the request, not a measurement of the engine: llama.cpp reports + // no total time, so the workload's totalMs stays absent rather than + // carrying a number this proxy made up. + durationMs := time.Since(start).Milliseconds() + p.codec.Notify("proxy/request", RequestEvent{ + ID: reqID, + NodeID: servedNodeID, + CacheAffinity: cacheAffinity, + Method: r.Method, + Path: r.URL.Path, + Target: servedTarget, + Status: finalStatus, + Duration: durationMs, + TTFB: ttfbMs, + Error: proxyErr, + Requests: reqs, + CacheAffinityHits: hits, + }) + + // Attach the engine's own prefill/decode measurement to the workload. The + // measurement is read from the committed reader once the copy unwinds and + // written under wlMu, alongside the other shared workload fields. The + // disconnect watcher may already have emitted the terminal event (the + // request was cancelled before it finished); that event legitimately + // carries no measurement. + if wl != nil { + var measurement inferencemetrics.Measurement + if telReader != nil { + measurement = telReader.Measurement() + } + // The measurement carries the engine's own numbers and nothing + // else. llama.cpp reports no total request time, so totalMs stays + // absent: the proxy's own wall-clock duration would include + // queueing, transport and client write time, and would not mean + // the same thing as the engine-reported total oMLX reports. + wlMu.Lock() + wl.Measurement = measurement + wlMu.Unlock() + } + + // Terminal workload transition pairs with the workload:started emitted at + // the commit point above. Cancellation, an upstream/transport error, a + // failed client write (dead/half-open client), or any non-2xx status is a + // failure; a clean 2xx is a completion. The Workload carries the same id so + // the broker (and peers) can collapse the start/finish pair. Routed through + // emitTerminal so the disconnect watcher and this path emit exactly once. + if wl != nil { + switch { + case r.Context().Err() != nil: + // The request was cancelled before it finished — either the + // client disconnected or, on shutdown, we cancelled it to stop + // the in-flight inference. A mid-stream cancel never reaches + // ErrorHandler (the 200 headers are already sent), so without this + // branch it would be misreported as completed. (The watcher above + // usually beats us to it; emitTerminal makes that a no-op.) + emitTerminal("failed", "request cancelled before completion") + case committedSC != nil && committedSC.wroteErr != nil: + // The response committed but a write to (or flush toward) the + // client failed — typically the idle deadline tripping on a + // dead/half-open client. The stream is truncated, so this is a + // failure, not the completion the 200 status would otherwise + // suggest. + emitTerminal("failed", "client connection lost: "+committedSC.wroteErr.Error()) + case proxyErr != "" || finalStatus >= http.StatusBadRequest: + msg := proxyErr + if msg == "" { + msg = fmt.Sprintf("upstream returned HTTP %d", finalStatus) + } + emitTerminal("failed", msg) + default: + emitTerminal("completed", "") + } + } +} + +// serveReverseProxy turns net/http's post-commit copy sentinel back into a +// result for handleHTTP. It must not swallow application panics. +func serveReverseProxy(proxy *httputil.ReverseProxy, w http.ResponseWriter, r *http.Request) (aborted bool) { + defer func() { + if recovered := recover(); recovered != nil { + if recovered == http.ErrAbortHandler { + aborted = true + return + } + panic(recovered) + } + }() + proxy.ServeHTTP(w, r) + return false +} + +// forwardDirector rewrites a request for the candidate it is about to be +// forwarded to. +// +// A request forwarded to a cluster peer carries PAIR's job id, so the +// serving peer can publish the job's live frame under that id. It carries +// the origin node's id too: the job id is a per-proxy counter, so two +// origins can both hold a job "1", and only the origin tells their jobs +// apart. It carries the run nonce as well: a proxy that restarts counts its +// jobs from 1 again, and the run nonce says which run the job id belongs to. +// +// All three headers are random per-request identifiers that carry no +// content, and they are not routing inputs: the ingress stays strictly +// terminal and never re-routes a peer's request onward. +func (p *Proxy) forwardDirector(cand candidate, jobID string) func(*http.Request) { + return func(req *http.Request) { + req.URL.Scheme = cand.url.Scheme + req.URL.Host = cand.url.Host + req.Host = cand.url.Host + if cand.peerUUID != "" && jobID != "" { + req.Header.Set(nvpairRequestIDHeader, jobID) + if p.nodeID != "" { + req.Header.Set(nvpairOriginNodeHeader, p.nodeID) + } + if p.runID != "" { + req.Header.Set(nvpairRunIDHeader, p.runID) + } + } + stripTraceContext(req.Header) + } +} + +// resolveCandidates returns the ordered list of nodes to try for the current +// request. A model-bearing request first filters a request-local node copy to +// advertised owners. A user-selected eligible node then leads, followed by +// scheduler priority and stable ID fallback. The failover loop walks the +// resulting owner list until a node returns a usable response. +// +// A node that resolves to this proxy's own listen address is dropped +// (self-forward guard): a local llama.cpp advertisement could otherwise +// resolve back to this proxy's own port and loop. +// +// Returns an empty slice when no forwarding target is available; the caller +// treats that as the rejection path. +func (p *Proxy) resolveCandidates(model string) []candidate { + p.selectedMu.RLock() + id := p.selectedID + p.selectedMu.RUnlock() + + priority := p.PriorityList() + + p.httpMu.Lock() + selfPort := p.port + p.httpMu.Unlock() + + // Re-derive membership and pins before resolving so a cluster joined or left, + // and a peer paired or removed, since the last request is reflected without a + // restart: a removed peer stops being a routable candidate immediately, and a + // freshly-paired one becomes one. + p.mesh.Refresh() + + nodes := p.discovery.Nodes() + known := len(nodes) + if model != "" { + owners := make([]Node, 0, len(nodes)) + for _, node := range nodes { + if nodeAdvertisesModel(node, model) { + owners = append(owners, node) + } + } + nodes = owners + } + // Sort by ID so candidate order is stable across calls — Discovery.Nodes() + // iterates a map, whose order is randomized per call, which otherwise bounced + // back-to-back requests between nodes. The ID sort is also the + // fallback order for nodes the scheduler's priority list doesn't mention. + sort.Slice(nodes, func(i, j int) bool { return nodes[i].ID < nodes[j].ID }) + byID := make(map[string]Node, len(nodes)) + for _, n := range nodes { + byID[n.ID] = n + } + + out := make([]candidate, 0, len(nodes)) + // Dedup by resolved backend host: the same physical node can appear under two + // IDs (e.g. a manually-added entry and its relay-discovered record), and + // routing to the same engine twice is wasteful. + seenHost := make(map[string]bool, len(nodes)) + // placed tracks node IDs already considered so the priority-ordered and + // fallback passes don't reconsider one (scheduler ordering). + placed := make(map[string]bool, len(nodes)) + add := func(n Node) { + placed[n.ID] = true + // targetURL picks a reachable address for a multi-homed node (cached, + // TCP-probed), falling back to the first candidate; nil only when the + // node advertises no usable address. + u := p.targetURL(n) + if u == nil { + return + } + peerUUID := "" + switch { + case isSelfTarget(u, selfPort): + // Our own advertised endpoint (lm now points at this proxy). Serve + // it from the explicit local backend — the loopback engine — rather + // than dialing our own mTLS ingress, which would recurse. Ranking + // still used this node's real (discovered) model list above. + lb, ok := p.localBackendTarget() + if !ok { + slog.Debug("resolveCandidates: no local backend for self", "node_id", n.ID) + return + } + u = lb + case p.mesh.HasPin(n.ClusterUUID): + // A pinned cluster peer: reach it only over mTLS to its promoted + // proxy (the lm port now advertises the proxy, not the engine). + // The pin is read from the live mesh refreshed above, not from the + // relayed n.Trusted: that flag is the scanner's answer from whenever + // it last saw this peer's mDNS record, so a peer discovered before + // this node's pins were written stays false until its record next + // changes. It is also strictly weaker than what we hold here — the + // dial itself is gated on ClientTLSConfig finding the same pin — so + // the relayed value can only ever disagree by being stale. + u.Scheme = "https" + peerUUID = n.ClusterUUID + case p.discovery.IsManual(n.ID): + // An explicit user-added manual node: dialed plain to the address + // the user supplied (a deliberate, separately-labeled bypass). + default: + // A relay peer we don't hold a pin for (untrusted, or this node is + // unclustered). Its engine is loopback-only and its proxy refuses + // plaintext from the LAN, so it is not a routable target. + slog.Debug("resolveCandidates: dropping unpinned relay peer", + "node_id", n.ID, "cluster_uuid", n.ClusterUUID) + return + } + // Defensive: the local backend must never resolve back to this proxy. + if isSelfTarget(u, selfPort) { + slog.Debug("resolveCandidates: skipping self-target node", + "node_id", n.ID, "target", u.Host, "self_port", selfPort) + return + } + if seenHost[u.Host] { + return + } + seenHost[u.Host] = true + gp, qd := p.nodeMetrics(n.ID) + out = append(out, candidate{ + id: n.ID, + url: u, + peerUUID: peerUUID, + gpuPressure: gp, + queueDepth: qd, + }) + } + + // Capability has already been enforced. An eligible explicit selection wins, + // then the scheduler's least-loaded order, then unlisted owners by stable ID. + if id != "" { + if n, ok := byID[id]; ok && !placed[id] { + add(n) + } + } + for _, pid := range priority { + if n, ok := byID[pid]; ok && !placed[pid] { + add(n) + } + } + for _, n := range nodes { + if !placed[n.ID] { + add(n) + } + } + + out = demoteCooling(out, p.coolingSet(time.Now())) + + slog.Debug("resolveCandidates resolved", + "selected", id, "priority", len(priority), "candidates", len(out), + "eligible", len(nodes), "known", known) + return out +} + +// reserveCandidate atomically moves the least estimated loaded scheduler-listed +// candidate to the front of this request's failover list. The scheduler's +// pending count and GPU pressure form the authoritative baseline; reservations +// are local dispatches made since that snapshot arrived and have not necessarily +// completed the proxy→broker→scheduler→proxy feedback loop yet. +// +// Model eligibility was enforced before this function receives the list. An +// explicit node/select pin bypasses reservations, and unlisted/manual owners +// retain their existing fallback position. +func (p *Proxy) reserveCandidate(candidates []candidate) []candidate { + ordered, _ := p.reserveCandidateWithAffinity(candidates, "") + return ordered +} + +func (p *Proxy) reserveCandidateWithAffinity(candidates []candidate, warmNodeID string) ([]candidate, bool) { + if len(candidates) == 0 { + return candidates, false + } + cooling := p.coolingSet(time.Now()) + candidates = demoteCooling(candidates, cooling) + hasReady := false + for _, cand := range candidates { + if !cooling[cand.id] { + hasReady = true + break + } + } + if selectedID := p.SelectedID(); selectedID != "" { + for _, cand := range candidates { + if cand.id == selectedID { + return candidates, false + } + } + } + + candidateIndex := make(map[string]int, len(candidates)) + for i, cand := range candidates { + candidateIndex[cand.id] = i + } + + p.priorityMu.Lock() + defer p.priorityMu.Unlock() + if p.priorityReservations == nil { + p.priorityReservations = make(map[string]int) + } + + // Check affinity under the same lock as optimistic reservations. An eligible + // manual pin was handled above. Missing or overloaded hints leave ordering alone. + if index, ok := candidateIndex[warmNodeID]; ok && warmNodeID != "" && + (!hasReady || !cooling[warmNodeID]) && + p.priorityPending[warmNodeID]+p.priorityReservations[warmNodeID] <= 1 && + p.priorityGPUPressure[warmNodeID] < schedulerwire.MaxGPUPressure { + chosen := candidates[index] + p.priorityReservations[warmNodeID]++ + copy(candidates[1:index+1], candidates[:index]) + candidates[0] = chosen + return candidates, true + } + + bestIndex := -1 + bestOrder := len(p.priority) + var bestLoad uint64 + for order, id := range p.priority { + if hasReady && cooling[id] { + continue + } + index, ok := candidateIndex[id] + if !ok { + continue + } + load := uint64(p.priorityPending[id]) + + uint64(p.priorityGPUPressure[id]) + + uint64(p.priorityReservations[id]) + if bestIndex < 0 || load < bestLoad || (load == bestLoad && order < bestOrder) { + bestIndex = index + bestOrder = order + bestLoad = load + } + } + if bestIndex < 0 { + return candidates, false + } + + chosen := candidates[bestIndex] + p.priorityReservations[chosen.id]++ + if bestIndex > 0 { + copy(candidates[1:bestIndex+1], candidates[:bestIndex]) + candidates[0] = chosen + } + return candidates, false +} + +func nodeAdvertisesModel(n Node, model string) bool { + if model == "" { + return false + } + for _, available := range n.Models { + if available == model { + return true + } + } + return false +} + +// isSelfTarget reports whether u points back at this proxy's own listener. +// nodeURL has already rewritten local-interface addresses to 127.0.0.1, so a +// loopback host on our own port is us. +func isSelfTarget(u *url.URL, selfPort int) bool { + host, portStr, err := net.SplitHostPort(u.Host) + if err != nil { + return false + } + port, err := strconv.Atoi(portStr) + if err != nil || port != selfPort { + return false + } + switch host { + case "127.0.0.1", "::1", "localhost": + return true + } + return false +} + +// nodeURL returns the single best forward URL for a node (the first candidate +// in deterministic, loopback-first order). It does no reachability probing — +// p.targetURL is the request-path entry point. Kept as a free function so the +// URL-construction unit test can exercise it without a Proxy. +func nodeURL(n Node) *url.URL { + candidates := nodeCandidates(n) + if len(candidates) == 0 { + return nil + } + return &url.URL{Scheme: "http", Host: candidates[0]} +} + +// targetURL picks the forward URL for a node, preferring an address we can +// actually reach. With a single candidate it's just that candidate; with +// several (a multi-homed peer) the shared reach.Chooser returns the confirmed +// last-good address, or the node's own top-ranked one while it confirms in the +// background. The confirmation is transport-neutral: a TCP accept proves the +// address is reachable, while the real pinned mTLS request still authenticates +// which peer answered there. +// +// reach.Prefer, not a blocking confirmation: this runs once per discovered node per +// request, including nodes this request will not be routed to, so a handshake here +// would charge every request for every node's connectivity. An address that is +// wrong is caught by the ErrorHandler below, which forgets it and fails over. +func (p *Proxy) targetURL(n Node) *url.URL { + candidates := nodeCandidates(n) + if len(candidates) == 0 { + return nil + } + host := p.targets.Prefer(n.ID, candidates) + return &url.URL{Scheme: "http", Host: host} +} + +// nodeCandidates returns the ordered, de-duplicated host:port targets for a node. +// +// Order comes from the node itself: netpick.Candidates keeps the node's published +// ranking, which it derived from evidence no observer has, and appends anything +// else it advertised. Re-sorting here by address class is what previously put a +// two-host direct-connect link ahead of a peer's real LAN address. +// +// Any local-interface address is rewritten to loopback (the engine binds loopback +// only) and floated to the front because it's unambiguously reachable. +func nodeCandidates(n Node) []string { + port := strconv.Itoa(n.Port) + sorted := netpick.Candidates(n.TXT, n.Addresses) + if len(sorted) == 0 { + // A non-IP entry (a .local hostname) that netpick cannot parse. + hosts := n.Addresses + if len(hosts) == 0 { + if n.Host == "" { + return nil + } + hosts = []string{n.Host} + } + sorted = append([]string(nil), hosts...) + } + + seen := make(map[string]bool, len(sorted)) + var loopback, rest []string + for _, h := range sorted { + // If the address belongs to a local interface, use loopback instead; + // connecting via the machine's own external IP would be refused. + if isLocalAddress(h) { + h = "127.0.0.1" + } + // net.JoinHostPort bracket-wraps IPv6 literals (fe80::1 -> [fe80::1]). + hp := net.JoinHostPort(h, port) + if seen[hp] { + continue + } + seen[hp] = true + if ip := net.ParseIP(h); ip != nil && ip.IsLoopback() { + loopback = append(loopback, hp) + } else { + rest = append(rest, hp) + } + } + return append(loopback, rest...) +} + +var ( + localAddrsMu sync.RWMutex + // localAddrs is the set of IPs currently bound to this host's interfaces. + // It's used to decide whether a discovered node is actually us, so we can + // dial loopback instead of our own external IP (the engine binds loopback + // only). The initial value is a one-shot enumeration; startLocalAddrWatch + // then keeps it in sync with live interface changes, so a late VPN/dock + // interface or a sleep/wake IP reassignment can't strand us dialing a + // stale address. + localAddrs = netmon.Enumerate().LocalIPs +) + +func setLocalAddrs(s map[string]bool) { + localAddrsMu.Lock() + localAddrs = s + localAddrsMu.Unlock() +} + +func isLocalAddress(addr string) bool { + localAddrsMu.RLock() + defer localAddrsMu.RUnlock() + return localAddrs[addr] +} + +// startLocalAddrWatch keeps localAddrs in sync with the host's live interface +// set for the lifetime of ctx. If the network monitor can't start, the set +// stays at its initial enumeration rather than failing the proxy. +func startLocalAddrWatch(ctx context.Context) { + mon, err := netmon.Watch(ctx) + if err != nil { + slog.Warn("proxy: network monitor unavailable; local address set is static", "err", err) + return + } + setLocalAddrs(mon.LocalIPs()) + ch := mon.Subscribe() + go func() { + for range ch { + setLocalAddrs(mon.LocalIPs()) + slog.Debug("proxy: refreshed local address set after network change") + } + }() +} + +func (p *Proxy) SelectedID() string { + p.selectedMu.RLock() + defer p.selectedMu.RUnlock() + return p.selectedID +} + +func (p *Proxy) SetSelected(id string) { + p.selectedMu.Lock() + p.selectedID = id + p.selectedMu.Unlock() +} + +// clearSelectionIfNotPresent resets the user-selected node (and notifies the +// client) when it's neither in the relay-fed set nor a manual node, so a stale +// selection can't pin routing to a departed target. +func (p *Proxy) clearSelectionIfNotPresent(present map[string]bool) { + p.selectedMu.Lock() + sel := p.selectedID + p.selectedMu.Unlock() + if sel == "" || present[sel] || p.discovery.IsManual(sel) { + return + } + p.selectedMu.Lock() + cleared := p.selectedID == sel + if cleared { + p.selectedID = "" + } + p.selectedMu.Unlock() + if cleared { + p.codec.Notify("node/selection-changed", SelectedResult{ID: ""}) + } +} + +// PriorityList returns a copy of the current scheduler-supplied priority order. +func (p *Proxy) PriorityList() []string { + p.priorityMu.RLock() + defer p.priorityMu.RUnlock() + return append([]string(nil), p.priority...) +} + +// SetPriority stores the auto-routing priority order (highest first) and returns +// the number of ids stored. The list is kept verbatim — unknown ids are retained +// (a node may appear in discovery later) and only consulted at request time. An +// empty list clears the scheduler's influence. +func (p *Proxy) SetPriority(nodes []string) int { + return p.SetPrioritySnapshot(schedulerwire.Priority{Nodes: nodes}) +} + +// SetPrioritySnapshot replaces the scheduler baseline and clears optimistic +// reservations made against the previous snapshot. Nodes-only callers remain +// valid: a missing rank supplies zero pending and GPU-pressure baselines. +func (p *Proxy) SetPrioritySnapshot(priority schedulerwire.Priority) int { + cleaned := append([]string(nil), priority.Nodes...) + pending := make(map[string]int, len(priority.Ranks)) + gpuPressure := make(map[string]int, len(priority.Ranks)) + for _, rank := range priority.Ranks { + if rank.ID == "" { + continue + } + if rank.Pending < 0 { + rank.Pending = 0 + } + if rank.GPUPressure < 0 { + rank.GPUPressure = 0 + } else if rank.GPUPressure > schedulerwire.MaxGPUPressure { + rank.GPUPressure = schedulerwire.MaxGPUPressure + } + pending[rank.ID] = rank.Pending + gpuPressure[rank.ID] = rank.GPUPressure + } + + p.priorityMu.Lock() + p.priority = cleaned + p.priorityPending = pending + p.priorityGPUPressure = gpuPressure + p.priorityReservations = make(map[string]int) + p.priorityMu.Unlock() + return len(cleaned) +} + +// replaceSubscribed replaces the proxy's relay-fed routing overlay from a +// discovery:nodes snapshot: it projects every node advertising lm with a dialable +// IP into the overlay (dropping the rest) and clears a user selection pinned to a +// node that's no longer routable. The broker sends the full filtered set on every +// change, so this is a wholesale replace, not a per-node apply — a departed node +// is simply absent from the next snapshot. +func (p *Proxy) replaceSubscribed(params json.RawMessage) { + var res noderec.GetNodesResult + if err := json.Unmarshal(params, &res); err != nil { + slog.Warn("invalid discovery:nodes snapshot", "err", err) + return + } + nodes := make([]Node, 0, len(res.Nodes)) + present := make(map[string]bool, len(res.Nodes)) + for _, dn := range res.Nodes { + n, ok := subscribedToNode(dn) + if !ok { + continue + } + nodes = append(nodes, n) + present[n.ID] = true + } + discovered, updated, removed := p.discovery.SetSubscribed(nodes) + // Surface the relay-fed set to the client as node/* events — the signal a + // consumer (the UI) uses to show which peers run this engine — mirroring how + // manual nodes are announced. Without this the routing overlay updates + // silently and peers appear engine-less. A node dropping out is also the + // proxy's "this upstream is gone" signal, surfaced through the errors + // pipeline (the broker forwards these to nvpair-errors); a re-appearance clears + // it. NodeID/Timestamp are left unset so the broker stamps the authoritative + // values. + for _, n := range discovered { + p.codec.Notify("node/discovered", n.withPrimaryIP()) + if err := p.codec.Notify("errors:clear", errors.ClearParams{ID: upstreamUnreachableID(n.ID)}); err != nil { + slog.Debug("failed to send errors:clear", "node", n.ID, "err", err) + } + } + for _, n := range updated { + p.codec.Notify("node/updated", n.withPrimaryIP()) + } + for _, n := range removed { + p.codec.Notify("node/removed", n.withPrimaryIP()) + if err := p.codec.Notify("errors:report", errors.ServiceError{ + ID: upstreamUnreachableID(n.ID), + Message: fmt.Sprintf("Upstream node %q is no longer reachable (dropped from discovery)", n.Host), + Severity: "warning", + Action: "none", + }); err != nil { + slog.Debug("failed to send errors:report", "node", n.ID, "err", err) + } + } + p.clearSelectionIfNotPresent(present) +} + +// upstreamUnreachableID is the canonical ServiceError id for an upstream the +// proxy no longer sees in discovery. Kept as a single function so the report and +// clear can't drift (nvpair-errors matches by literal id). +func upstreamUnreachableID(nodeID string) string { + return "llamacpp-proxy:upstream-unreachable:" + nodeID +} + +// subscribedToNode projects a relay DirectoryNode onto the proxy's routable Node +// for the lm service, returning false when the node doesn't advertise lm or has +// no dialable address. The engine port comes from the lm service key (the real +// llama.cpp port the broker's engine poller registered, not the proxy's listen +// port). +func subscribedToNode(n noderec.DirectoryNode) (Node, bool) { + svc, ok := n.Services[noderec.ServiceLlamaCPP] + if !ok || n.IP == "" { + return Node{}, false + } + // Key routing by the stable per-host UUID, not the hostname: candidate ids, + // scheduledOn, node selection, and the scheduler's priority list are all this + // value, so routing survives a PC rename and never conflates two same-named + // machines. Host stays the hostname (display / dial name). A relay + // DirectoryNode always carries a hostUuid (the scanner guarantees it at the + // browse boundary), so there is no name fallback here. + return Node{ + ID: n.HostUUID, + Host: n.Name, + Port: svc.Port, + // The node's whole ranked address list, not just its canonical one: a + // multi-homed peer's best address from its own vantage point may be a + // direct-connect link this host cannot reach, and routing needs somewhere + // to fail over to when that happens. + Addresses: n.CandidateIPs(), + TXT: n.AddressTXT(), + IP: n.IP, + ClusterUUID: n.ClusterUUID, + // Filter on this node's llama.cpp models only, not the cross-engine union, + // so a model a dual-engine node serves solely via Ollama isn't accepted as + // an llama.cpp owner here (falls back to the union for a peer that sends + // no attribution — see DirectoryNode.EngineModels). + Models: append([]string(nil), n.EngineModels("llamacpp")...), + }, true +} + +func (p *Proxy) readLoop(ctx context.Context) error { + for { + msg, err := p.codec.Read() + if err != nil { + if err == io.EOF || ctx.Err() != nil { + return nil + } + var de *jsonrpc.DecodeError + if stderrors.As(err, &de) { + continue + } + log.Printf("JSON-RPC read error (terminal): %v", err) + return err + } + p.handleMessage(msg) + } +} + +func (p *Proxy) handleMessage(msg *Message) { + if p.portWrites.Handle(msg, p.codec, p.setPortContext, func(port int) any { return ReadyParams{Version: Version, Port: port} }) { + return + } + if msg.Method == applog.SetLevelMethod { + resolved, err := applog.HandleSetLevelParams(msg.Params) + if msg.IsRequest() { + if err != nil { + p.codec.RespondError(msg.ID, -32602, err.Error()) + return + } + p.codec.Respond(msg.ID, map[string]string{"level": resolved}) + } + if err != nil { + slog.Warn("log/set-level rejected", "err", err) + } else { + slog.Info("log level changed", "level", resolved) + } + return + } + + switch msg.Method { + case noderec.NotifyNodes: + p.replaceSubscribed(msg.Params) + return + } + + if !msg.IsRequest() { + if msg.IsNotification() { + log.Printf("ignoring incoming notification: %s", msg.Method) + } + return + } + + switch msg.Method { + case workerhealth.Method: + if err := p.codec.Respond(msg.ID, workerhealth.OK()); err != nil { + slog.Error("failed to respond to worker/health", "err", err) + } + + case "telemetry/configure": + p.handleTelemetryConfigure(msg) + + case "telemetry/status": + p.handleTelemetryStatus(msg) + + case "nodes/list": + nodes := p.discovery.Nodes() + if err := p.codec.Respond(msg.ID, NodesResult{Nodes: nodes}); err != nil { + log.Printf("failed to respond to nodes/list: %v", err) + } + + case "node/select": + var params SelectParams + if err := json.Unmarshal(msg.Params, ¶ms); err != nil { + p.codec.RespondError(msg.ID, -32602, "invalid params: expected {\"id\": \"...\"}") + return + } + if params.ID != "" { + found := false + for _, n := range p.discovery.Nodes() { + if n.ID == params.ID { + found = true + break + } + } + if !found { + p.codec.RespondError(msg.ID, -32602, fmt.Sprintf("node %q not found", params.ID)) + return + } + } + p.SetSelected(params.ID) + log.Printf("node selection changed to %q", params.ID) + if err := p.codec.Respond(msg.ID, SelectedResult{ID: params.ID}); err != nil { + log.Printf("failed to respond to node/select: %v", err) + } + p.codec.Notify("node/selection-changed", SelectedResult{ID: params.ID}) + + case "node/selected": + if err := p.codec.Respond(msg.ID, SelectedResult{ID: p.SelectedID()}); err != nil { + log.Printf("failed to respond to node/selected: %v", err) + } + + case "node/set-priority": + var params schedulerwire.Priority + if err := json.Unmarshal(msg.Params, ¶ms); err != nil { + p.codec.RespondError(msg.ID, -32602, "invalid params: expected {\"nodes\": [\"id\", ...], \"ranks\": [...]}") + return + } + count := p.SetPrioritySnapshot(params) + log.Printf("priority snapshot set (%d nodes, %d ranks): %v", count, len(params.Ranks), params.Nodes) + if err := p.codec.Respond(msg.ID, map[string]int{"count": count}); err != nil { + log.Printf("failed to respond to node/set-priority: %v", err) + } + + case "node/add-manual": + var node Node + if err := json.Unmarshal(msg.Params, &node); err != nil { + p.codec.RespondError(msg.ID, -32602, "invalid params: expected {\"id\",\"host\",\"port\",\"addresses\"}") + return + } + if node.ID == "" || node.Port == 0 || len(node.Addresses) == 0 { + p.codec.RespondError(msg.ID, -32602, "id, port, and at least one address are required") + return + } + added := p.discovery.AddManual(node) + if err := p.codec.Respond(msg.ID, map[string]bool{"added": added}); err != nil { + log.Printf("failed to respond to node/add-manual: %v", err) + } + if added { + log.Printf("manual node added: %s (%s:%d)", node.ID, node.Addresses[0], node.Port) + p.codec.Notify("node/discovered", node.withPrimaryIP()) + } else { + log.Printf("manual node updated: %s (%s:%d)", node.ID, node.Addresses[0], node.Port) + p.codec.Notify("node/updated", node.withPrimaryIP()) + } + + case "node/remove-manual": + var params SelectParams + if err := json.Unmarshal(msg.Params, ¶ms); err != nil { + p.codec.RespondError(msg.ID, -32602, "invalid params: expected {\"id\": \"...\"}") + return + } + removed := p.discovery.RemoveManual(params.ID) + if err := p.codec.Respond(msg.ID, map[string]bool{"removed": removed}); err != nil { + log.Printf("failed to respond to node/remove-manual: %v", err) + } + if removed { + log.Printf("manual node removed: %s", params.ID) + p.selectedMu.Lock() + if p.selectedID == params.ID { + p.selectedID = "" + p.selectedMu.Unlock() + p.codec.Notify("node/selection-changed", SelectedResult{ID: ""}) + } else { + p.selectedMu.Unlock() + } + p.codec.Notify("node/removed", Node{ID: params.ID}) + } + + case "node/set-local-backend": + var b localBackend + if err := json.Unmarshal(msg.Params, &b); err != nil { + p.codec.RespondError(msg.ID, -32602, "invalid params: expected {\"engine\",\"host\",\"port\",\"healthy\"}") + return + } + p.setLocalBackend(b) + slog.Info("local backend updated", "engine", b.Engine, "host", b.Host, "port", b.Port, "healthy", b.Healthy) + if err := p.codec.Respond(msg.ID, map[string]bool{"ok": true}); err != nil { + log.Printf("failed to respond to node/set-local-backend: %v", err) + } + + case "shutdown": + if err := p.codec.Respond(msg.ID, nil); err != nil { + log.Printf("failed to respond to shutdown: %v", err) + } + log.Println("shutdown requested via JSON-RPC") + p.cancel() + + default: + if err := p.codec.RespondError(msg.ID, -32601, fmt.Sprintf("method not found: %s", msg.Method)); err != nil { + log.Printf("failed to send error response: %v", err) + } + } +} diff --git a/services/llamacpp-proxy/proxy_test.go b/services/llamacpp-proxy/proxy_test.go new file mode 100644 index 000000000..d0a029c62 --- /dev/null +++ b/services/llamacpp-proxy/proxy_test.go @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import "testing" + +// TestNodeURL covers the URL-construction part of nodeURL — specifically +// that IPv6 literals are bracket-wrapped and IPv4/hostnames remain +// byte-identical to the pre-JoinHostPort implementation. The +// local-address shortcut is not exercised here because it depends on +// the host's network interfaces (see init() in proxy.go) and would be +// flaky across environments. +func TestNodeURL(t *testing.T) { + // Pick addresses that are unlikely to appear on any local interface. + tests := []struct { + name string + node Node + wantHost string + wantURL string + }{ + { + name: "ipv4", + node: Node{Addresses: []string{"192.0.2.10"}, Port: 11434}, + wantHost: "192.0.2.10:11434", + wantURL: "http://192.0.2.10:11434", + }, + { + name: "ipv6", + node: Node{Addresses: []string{"2001:db8::1"}, Port: 11434}, + wantHost: "[2001:db8::1]:11434", + wantURL: "http://[2001:db8::1]:11434", + }, + { + name: "hostname", + node: Node{Addresses: []string{"gpu-host.lan"}, Port: 11434}, + wantHost: "gpu-host.lan:11434", + wantURL: "http://gpu-host.lan:11434", + }, + { + // Empty Addresses slice — nodeURL should fall back to Host. + name: "fallback to Host", + node: Node{Host: "gpu-host.lan", Port: 11434}, + wantHost: "gpu-host.lan:11434", + wantURL: "http://gpu-host.lan:11434", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + u := nodeURL(tc.node) + if u == nil { + t.Fatalf("nodeURL returned nil") + } + if u.Host != tc.wantHost { + t.Errorf("Host = %q, want %q", u.Host, tc.wantHost) + } + if got := u.String(); got != tc.wantURL { + t.Errorf("String() = %q, want %q", got, tc.wantURL) + } + }) + } +} diff --git a/services/llamacpp-proxy/reservation_test.go b/services/llamacpp-proxy/reservation_test.go new file mode 100644 index 000000000..97049ebf0 --- /dev/null +++ b/services/llamacpp-proxy/reservation_test.go @@ -0,0 +1,235 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "sync" + "testing" + + "nvpair-shared/schedulerwire" +) + +func reservationCandidates(ids ...string) []candidate { + out := make([]candidate, 0, len(ids)) + for _, id := range ids { + out = append(out, candidate{id: id}) + } + return out +} + +func reservedID(p *Proxy, candidates []candidate) string { + candidates = append([]candidate(nil), candidates...) + return p.reserveCandidate(candidates)[0].id +} + +func TestReserveCandidate_ConcurrentEqualLoadHasAtMostOneSkew(t *testing.T) { + p := prProxy(t) + ids := []string{"a", "b", "c", "d"} + ranks := make([]schedulerwire.NodeRank, 0, len(ids)) + for i, id := range ids { + ranks = append(ranks, schedulerwire.NodeRank{ID: id, Rank: i}) + } + p.SetPrioritySnapshot(schedulerwire.Priority{Nodes: ids, Ranks: ranks}) + candidates := reservationCandidates(ids...) + + const requests = 100 + chosen := make(chan string, requests) + var wg sync.WaitGroup + for range requests { + wg.Add(1) + go func() { + defer wg.Done() + chosen <- reservedID(p, candidates) + }() + } + wg.Wait() + close(chosen) + + counts := make(map[string]int, len(ids)) + for id := range chosen { + counts[id]++ + } + min, max := requests, 0 + for _, id := range ids { + if counts[id] < min { + min = counts[id] + } + if counts[id] > max { + max = counts[id] + } + } + if max-min > 1 { + t.Fatalf("100 equal-load reservations are imbalanced: %v", counts) + } +} + +func TestReserveCandidate_ConvergesUnequalPendingDepths(t *testing.T) { + p := prProxy(t) + p.SetPrioritySnapshot(schedulerwire.Priority{ + Nodes: []string{"a", "b", "c"}, + Ranks: []schedulerwire.NodeRank{ + {ID: "a", Pending: 0, Rank: 0}, + {ID: "b", Pending: 2, Rank: 1}, + {ID: "c", Pending: 4, Rank: 2}, + }, + }) + candidates := reservationCandidates("a", "b", "c") + assigned := map[string]int{} + for range 6 { + assigned[reservedID(p, candidates)]++ + } + + total := map[string]int{ + "a": assigned["a"], + "b": 2 + assigned["b"], + "c": 4 + assigned["c"], + } + if total["a"] != 4 || total["b"] != 4 || total["c"] != 4 { + t.Fatalf("unequal depths did not converge: assigned=%v total=%v", assigned, total) + } +} + +func TestReserveCandidate_CombinesPendingPressureAndReservations(t *testing.T) { + p := prProxy(t) + p.SetPrioritySnapshot(schedulerwire.Priority{ + Nodes: []string{"a", "b", "c"}, + Ranks: []schedulerwire.NodeRank{ + {ID: "a", Pending: 0, GPUPressure: 3}, + {ID: "b", Pending: 1, GPUPressure: 0}, + {ID: "c", Pending: 0, GPUPressure: 2}, + }, + }) + candidates := reservationCandidates("a", "b", "c") + got := []string{ + reservedID(p, candidates), + reservedID(p, candidates), + reservedID(p, candidates), + } + want := []string{"b", "b", "c"} + for i := range want { + if got[i] != want[i] { + t.Fatalf("GPU-aware reservations = %v, want %v", got, want) + } + } +} + +func TestSetPrioritySnapshotClampsGPUPressure(t *testing.T) { + p := prProxy(t) + p.SetPrioritySnapshot(schedulerwire.Priority{ + Nodes: []string{"low", "high"}, + Ranks: []schedulerwire.NodeRank{ + {ID: "low", GPUPressure: -1}, + {ID: "high", GPUPressure: schedulerwire.MaxGPUPressure + 1}, + }, + }) + p.priorityMu.RLock() + low := p.priorityGPUPressure["low"] + high := p.priorityGPUPressure["high"] + p.priorityMu.RUnlock() + if low != 0 || high != schedulerwire.MaxGPUPressure { + t.Fatalf("clamped GPU pressure = low:%d high:%d", low, high) + } +} + +func TestReserveCandidate_LegacyNodesOnlyUsesZeroBaseline(t *testing.T) { + p := prProxy(t) + p.SetPriority([]string{"a", "b", "c"}) + candidates := reservationCandidates("a", "b", "c") + counts := map[string]int{} + for range 5 { + counts[reservedID(p, candidates)]++ + } + want := map[string]int{"a": 2, "b": 2, "c": 1} + for id, n := range want { + if counts[id] != n { + t.Fatalf("legacy nodes-only assignments = %v, want %v", counts, want) + } + } +} + +func TestReserveCandidate_UsesEligibleCandidates(t *testing.T) { + p := prProxy(t) + p.SetPrioritySnapshot(schedulerwire.Priority{ + Nodes: []string{"missing", "owner-a", "owner-b", "unknown"}, + Ranks: []schedulerwire.NodeRank{ + {ID: "missing", Pending: 0}, + {ID: "owner-a", Pending: 4}, + {ID: "owner-b", Pending: 5}, + {ID: "unknown", Pending: 0}, + }, + }) + candidates := reservationCandidates("owner-a", "owner-b") + + for range 8 { + got := reservedID(p, candidates) + if got != "owner-a" && got != "owner-b" { + t.Fatalf("reservation escaped eligible candidates to %q", got) + } + } +} + +func TestReserveCandidate_ManualPinBypassesReservations(t *testing.T) { + p := prProxy(t) + p.SetPrioritySnapshot(schedulerwire.Priority{ + Nodes: []string{"a", "b"}, + Ranks: []schedulerwire.NodeRank{{ID: "a"}, {ID: "b"}}, + }) + p.SetSelected("b") + candidates := reservationCandidates("b", "a") // resolveCandidates puts the pin first + if got := reservedID(p, candidates); got != "b" { + t.Fatalf("manual pin resolved to %q, want b", got) + } + p.priorityMu.RLock() + defer p.priorityMu.RUnlock() + if len(p.priorityReservations) != 0 { + t.Fatalf("manual pin created optimistic reservations: %v", p.priorityReservations) + } +} + +func TestReserveCandidate_IneligibleManualPinDoesNotBypassReservations(t *testing.T) { + p := prProxy(t) + p.SetPrioritySnapshot(schedulerwire.Priority{ + Nodes: []string{"owner-b", "owner-a"}, + Ranks: []schedulerwire.NodeRank{{ID: "owner-b"}, {ID: "owner-a"}}, + }) + p.SetSelected("missing") + if got := reservedID(p, reservationCandidates("owner-a", "owner-b")); got != "owner-b" { + t.Fatalf("reservation with ineligible pin = %q, want owner-b", got) + } +} + +func TestReserveCandidate_PreservesFailoverAndSnapshotReset(t *testing.T) { + p := prProxy(t) + p.SetPrioritySnapshot(schedulerwire.Priority{ + Nodes: []string{"b", "a", "c"}, + Ranks: []schedulerwire.NodeRank{ + {ID: "b", Pending: 0}, + {ID: "a", Pending: 5}, + {ID: "c", Pending: 6}, + }, + }) + got := p.reserveCandidate(reservationCandidates("a", "b", "c")) + want := []string{"b", "a", "c"} + for i, id := range want { + if got[i].id != id { + t.Fatalf("reserved failover order = %v, want %v", candidateIDsFrom(got), want) + } + } + + p.SetPrioritySnapshot(schedulerwire.Priority{ + Nodes: []string{"a", "b", "c"}, + Ranks: []schedulerwire.NodeRank{{ID: "a"}, {ID: "b"}, {ID: "c"}}, + }) + if next := reservedID(p, reservationCandidates("a", "b", "c")); next != "a" { + t.Fatalf("new snapshot did not reset reservations: next = %q, want a", next) + } +} + +func candidateIDsFrom(candidates []candidate) []string { + out := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + out = append(out, candidate.id) + } + return out +} diff --git a/services/llamacpp-proxy/response_timeout_test.go b/services/llamacpp-proxy/response_timeout_test.go new file mode 100644 index 000000000..7d5b85574 --- /dev/null +++ b/services/llamacpp-proxy/response_timeout_test.go @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + "time" +) + +func TestResolveResponseHeaderTimeout(t *testing.T) { + for _, tc := range []struct { + name, flag, env string + want time.Duration + }{ + {"default", "", "", defaultProxyResponseTimeout}, + {"environment", "", "10m", 10 * time.Minute}, + {"flag beats environment", "5m", "10m", 5 * time.Minute}, + {"unparseable keeps the default", "soon", "", defaultProxyResponseTimeout}, + {"zero keeps the default", "0s", "", defaultProxyResponseTimeout}, + {"negative keeps the default", "", "-1m", defaultProxyResponseTimeout}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(responseHeaderTimeoutEnv, tc.env) + if got := resolveResponseHeaderTimeout(tc.flag); got != tc.want { + t.Fatalf("resolveResponseHeaderTimeout(%q) with env %q = %s, want %s", tc.flag, tc.env, got, tc.want) + } + }) + } +} + +// Transports are built after main resolves the timeout, so they carry it. +func TestTransportsUseTheResolvedTimeout(t *testing.T) { + old := proxyResponseTimeout + t.Cleanup(func() { proxyResponseTimeout = old }) + proxyResponseTimeout = 7 * time.Minute + if got := newProxyTransport(nil).ResponseHeaderTimeout; got != 7*time.Minute { + t.Fatalf("ResponseHeaderTimeout = %s, want 7m", got) + } +} diff --git a/services/llamacpp-proxy/routing_metrics.go b/services/llamacpp-proxy/routing_metrics.go new file mode 100644 index 000000000..b0dcdf74f --- /dev/null +++ b/services/llamacpp-proxy/routing_metrics.go @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import "nvpair-shared/routingmetrics" + +// recordRoutingMetric folds one *routed* request into this proxy's shared +// counters, and returns the cumulative totals to stamp on the event. +// +// The filter and the accumulation live in nvpair-shared/routingmetrics, not +// here: ollama-proxy, llamacpp-proxy and omlx-proxy must report identical +// numbers, and that package is where the one definition lives. +// +// Called only at the completion emit site, before the notification is written, +// so the event carries the total that includes itself. The 502 rejection path +// (proxy.go ~1088) never calls this, which is what keeps unrouted traffic out +// of the dashboard — structurally, not by inspecting a field. That site is +// precisely what the old consumer-side filter mis-counted, because a rejection +// emits on an inference path with cache_affinity omitted and was counted as a +// routed request. +func (p *Proxy) recordRoutingMetric(method, path string, cacheAffinity bool) (requests, cacheAffinityHits uint64) { + if routingmetrics.Counts(method, path) { + p.routing.Routed(cacheAffinity) + } + return p.routing.Snapshot() +} diff --git a/services/llamacpp-proxy/server_timeout_test.go b/services/llamacpp-proxy/server_timeout_test.go new file mode 100644 index 000000000..39d9b881f --- /dev/null +++ b/services/llamacpp-proxy/server_timeout_test.go @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "net" + "testing" + + "nvpair-shared/clustertrust" +) + +func TestHTTPServersConfigureIdleTimeouts(t *testing.T) { + p := testProxy(NewDiscovery(), 1235) + p.mesh = clustertrust.Open(t.TempDir()) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + p.serveHTTP(context.Background(), ln) + defer p.shutdown(context.Background()) + + if p.plainSrv == nil || p.tlsSrv == nil { + t.Fatal("servers not recorded") + } + for name, srv := range map[string]struct { + readHeader, idle interface{} + }{ + "plain": {p.plainSrv.ReadHeaderTimeout, p.plainSrv.IdleTimeout}, + "tls": {p.tlsSrv.ReadHeaderTimeout, p.tlsSrv.IdleTimeout}, + } { + if srv.readHeader != proxyReadHeaderTimeout { + t.Errorf("%s ReadHeaderTimeout = %v, want %v", name, srv.readHeader, proxyReadHeaderTimeout) + } + if srv.idle != proxyServerIdleTimeout { + t.Errorf("%s IdleTimeout = %v, want %v", name, srv.idle, proxyServerIdleTimeout) + } + } + if proxyServerIdleTimeout != proxyIdleConnTimeout { + t.Fatalf("server IdleTimeout %v != client IdleConnTimeout %v", proxyServerIdleTimeout, proxyIdleConnTimeout) + } +} diff --git a/services/llamacpp-proxy/stall.go b/services/llamacpp-proxy/stall.go new file mode 100644 index 000000000..c88d20bde --- /dev/null +++ b/services/llamacpp-proxy/stall.go @@ -0,0 +1,169 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "errors" + "io" + "log" + "net" + "net/http" + "net/http/httptrace" + "os" + "sync" + "time" +) + +// The response-header deadline does not bound inference prefill after headers. +// First-body timing is opt-in and independent; zero means wait until content, +// an upstream error, or caller cancellation. +const firstBodyTimeoutEnv = "NVPAIR_PROXY_FIRST_BODY_TIMEOUT" + +var proxyFirstBodyTimeout time.Duration + +func resolveFirstBodyTimeout(flagVal string) time.Duration { + raw := flagVal + if raw == "" { + raw = os.Getenv(firstBodyTimeoutEnv) + } + if raw == "" { + return 0 + } + d, err := time.ParseDuration(raw) + if err != nil || d < 0 { + log.Printf("invalid first-body timeout %q, using default (disabled)", raw) + return 0 + } + return d +} + +var errFirstBodyTimeout = errors.New("upstream timed out before first response body byte") + +// A cooldown is evidence of a full silent deadline, not a liveness verdict. +// State is local to this proxy; success or time alone restores normal ordering. +func (p *Proxy) noteStalled(id string, now time.Time) { + p.coolMu.Lock() + defer p.coolMu.Unlock() + if p.coolUntil == nil { + p.coolUntil = make(map[string]time.Time) + } + p.coolUntil[id] = now.Add(proxyResponseTimeout) +} + +func (p *Proxy) noteSuccess(id string) { + p.coolMu.Lock() + defer p.coolMu.Unlock() + delete(p.coolUntil, id) +} + +func (p *Proxy) coolingSet(now time.Time) map[string]bool { + p.coolMu.Lock() + defer p.coolMu.Unlock() + if len(p.coolUntil) == 0 { + return nil + } + cooling := make(map[string]bool, len(p.coolUntil)) + for id, until := range p.coolUntil { + if now.Before(until) { + cooling[id] = true + } else { + delete(p.coolUntil, id) + } + } + return cooling +} + +// Stable partition: keep every candidate, including a sole cooling candidate. +func demoteCooling(candidates []candidate, cooling map[string]bool) []candidate { + if len(cooling) == 0 || len(candidates) < 2 { + return candidates + } + out := make([]candidate, 0, len(candidates)) + for _, cand := range candidates { + if !cooling[cand.id] { + out = append(out, cand) + } + } + for _, cand := range candidates { + if cooling[cand.id] { + out = append(out, cand) + } + } + return out +} + +// Observe the response-header wait after the request was written, not dial or +// TLS time. A shorter caller deadline, cancellation, or fast transport failure +// is not evidence that this candidate spent a full response deadline silent. +type stallTransport struct { + base http.RoundTripper + proxy *Proxy + id string + timeout time.Duration +} + +func (t stallTransport) RoundTrip(req *http.Request) (*http.Response, error) { + var mu sync.Mutex + var written time.Time + trace := &httptrace.ClientTrace{WroteRequest: func(info httptrace.WroteRequestInfo) { + if info.Err == nil { + mu.Lock() + written = time.Now() + mu.Unlock() + } + }} + resp, err := t.base.RoundTrip(req.WithContext(httptrace.WithClientTrace(req.Context(), trace))) + mu.Lock() + waitStart := written + mu.Unlock() + var timeout net.Error + if err != nil && req.Context().Err() == nil && errors.As(err, &timeout) && + timeout.Timeout() && !waitStart.IsZero() && time.Since(waitStart) >= t.timeout { + t.proxy.noteStalled(t.id, time.Now()) + } + return resp, err +} + +type bodyWithPeek struct { + io.Reader + io.Closer +} + +// Hold inference headers until content (or a valid empty EOF) arrives. On an +// error the caller returns from ModifyResponse, so ReverseProxy closes the +// original body and unblocks the pending read. The buffered channel lets that +// reader exit even if the deadline or caller cancellation won. +func awaitFirstBody(ctx context.Context, body io.ReadCloser, budget time.Duration) (io.ReadCloser, error) { + type result struct { + data []byte + err error + } + ready := make(chan result, 1) + go func() { + // Match ReverseProxy's copy buffer. Reading only one byte would split + // the first engine frame and publish progress before it can be parsed. + first := make([]byte, 32*1024) + n, err := body.Read(first) + ready <- result{data: first[:n], err: err} + }() + var deadline <-chan time.Time + if budget > 0 { + timer := time.NewTimer(budget) + defer timer.Stop() + deadline = timer.C + } + select { + case got := <-ready: + if len(got.data) == 0 && got.err != nil && !errors.Is(got.err, io.EOF) { + return nil, got.err + } + return bodyWithPeek{Reader: io.MultiReader(bytes.NewReader(got.data), body), Closer: body}, nil + case <-ctx.Done(): + return nil, ctx.Err() + case <-deadline: + return nil, errFirstBodyTimeout + } +} diff --git a/services/llamacpp-proxy/subscribed_test.go b/services/llamacpp-proxy/subscribed_test.go new file mode 100644 index 000000000..747645d82 --- /dev/null +++ b/services/llamacpp-proxy/subscribed_test.go @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + + "nvpair-shared/noderec" +) + +// TestSubscribedToNode covers the DirectoryNode -> routable Node projection for +// the lm service, including per-engine model attribution: the proxy ranks on the +// node's llama.cpp models only, never the cross-engine union. +func TestSubscribedToNode(t *testing.T) { + withLM := noderec.DirectoryNode{ + HostUUID: "uuid-a", + Name: "host-a", + IP: "10.0.0.5", + Models: []string{"gguf"}, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceLlamaCPP: {Port: 1234}, + }, + } + got, ok := subscribedToNode(withLM) + if !ok { + t.Fatal("node with lm + IP should project") + } + // No attribution -> fall back to the flat union (single-engine peer). + if got.ID != "uuid-a" || got.Port != 1234 || got.IP != "10.0.0.5" || + len(got.Models) != 1 || got.Models[0] != "gguf" { + t.Fatalf("unexpected projection: %+v", got) + } + + noIP := withLM + noIP.IP = "" + if _, ok := subscribedToNode(noIP); ok { + t.Fatal("node without IP should not project") + } + + // A node advertising only a non-lm service must not be an lm routing target. + olOnly := noderec.DirectoryNode{ + Name: "host-b", + IP: "10.0.0.6", + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 11434}}, + } + if _, ok := subscribedToNode(olOnly); ok { + t.Fatal("node without lm should not project") + } + + // Per-engine attribution: a dual-engine node projects ONLY its llama.cpp + // models, never the union — so an Ollama-only model isn't ranked as an + // llama.cpp owner. + dual := noderec.DirectoryNode{ + HostUUID: "uuid-d", + Name: "host-d", + IP: "10.0.0.7", + Models: []string{"ollama-model", "llamacpp-model"}, + ModelsByEngine: map[string][]string{ + "ollama": {"ollama-model"}, + "llamacpp": {"llamacpp-model"}, + }, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{ + noderec.ServiceLlamaCPP: {Port: 1234}, + }, + } + got, ok = subscribedToNode(dual) + if !ok { + t.Fatal("dual-engine node with lm should project") + } + if len(got.Models) != 1 || got.Models[0] != "llamacpp-model" { + t.Fatalf("dual-engine projection Models = %v, want [llamacpp-model] only", got.Models) + } +} + +// TestSubscribedToNodeKeysByHostUUID: the routable Node keys on the stable +// hostUuid, not the hostname, so routing/scheduledOn/selection survive a PC +// rename and never conflate same-named machines. Host stays the hostname for +// display. +func TestSubscribedToNodeKeysByHostUUID(t *testing.T) { + const uuid = "22222222-2222-2222-2222-222222222222" + n := noderec.DirectoryNode{ + HostUUID: uuid, + Name: "host-a", + IP: "10.0.0.5", + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceLlamaCPP: {Port: 1234}}, + } + got, ok := subscribedToNode(n) + if !ok { + t.Fatal("node with lm + IP should project") + } + if got.ID != uuid { + t.Fatalf("ID = %q, want hostUuid %q", got.ID, uuid) + } + if got.Host != "host-a" { + t.Fatalf("Host = %q, want hostname for display", got.Host) + } +} diff --git a/services/llamacpp-proxy/telemetry.go b/services/llamacpp-proxy/telemetry.go new file mode 100644 index 000000000..852b69893 --- /dev/null +++ b/services/llamacpp-proxy/telemetry.go @@ -0,0 +1,443 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net" + "net/http" + "strings" + "sync" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "nvpair-shared/inferencemetrics" + "nvpair-shared/telemetry" +) + +func (p *Proxy) handleTelemetryConfigure(msg *Message) { + revision, settings, err := telemetry.DecodeConfigureRequest(msg.Params) + if err != nil { + p.codec.RespondError(msg.ID, -32602, err.Error()) + return + } + revision, err = p.telemetry.Apply(context.Background(), revision, settings) + if err != nil { + p.codec.RespondError(msg.ID, -32000, "failed to configure telemetry") + return + } + p.codec.Respond(msg.ID, telemetry.ConfigureResult{Revision: revision}) +} + +func (p *Proxy) handleTelemetryStatus(msg *Message) { + p.codec.Respond(msg.ID, telemetry.NewStatusResult(p.telemetry.Status())) +} + +func metadataAttribute(key, value string) attribute.KeyValue { + value, _ = telemetry.TruncateUTF8(value, 256) + return attribute.String(key, value) +} + +// ReverseProxy otherwise copies the original client headers even without an +// injector. Context from local clients must never cross an engine/peer boundary. +func stripTraceContext(header http.Header) { + header.Del("traceparent") + header.Del("tracestate") + header.Del("baggage") +} + +func isStreamingRequest(path string, bodyBytes []byte) bool { + if len(bodyBytes) > 0 { + var payload struct { + Stream *bool `json:"stream"` + } + if err := json.Unmarshal(bodyBytes, &payload); err == nil && payload.Stream != nil { + return *payload.Stream + } + } + return false +} + +func extractPrompt(body []byte) string { + if len(body) == 0 { + return "" + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(body, &raw); err != nil { + return "" + } + if p, ok := raw["prompt"]; ok { + var str string + if err := json.Unmarshal(p, &str); err == nil { + return str + } + return string(p) + } + if m, ok := raw["messages"]; ok { + var str string + if err := json.Unmarshal(m, &str); err == nil { + return str + } + return string(m) + } + if in, ok := raw["input"]; ok { + var str string + if err := json.Unmarshal(in, &str); err == nil { + return str + } + return string(in) + } + return "" +} + +func isLocalHost(hostPort string) bool { + host, _, err := net.SplitHostPort(hostPort) + if err != nil { + host = hostPort + } + if host == "localhost" { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +type telemetryBodyReader struct { + io.ReadCloser + span trace.Span + start time.Time + session *telemetry.Session + completionTruncated bool + firstReadOnce sync.Once + closeOnce sync.Once + lineBuf []byte + completionBuf strings.Builder + promptTokens int + outputTokens int + responseModel string + + // requestStart is the instant the request reached the proxy. The live + // progress frame measures its timings from here, so the elapsed time and + // the time to the first byte are the client's own, not the engine's. + requestStart time.Time + + // progressMu guards the live counters below. They are written by the + // response copy goroutine and read by whoever asks the reader for a + // snapshot, which need not be that goroutine. + progressMu sync.Mutex + chunks int64 + bytes int64 + firstByteAt time.Time + firstContentAt time.Time + + // onProgress is called whenever the reader observes something new. It + // runs on the response copy goroutine, so it must be cheap and must not + // block. + onProgress func(LiveProgress) + + // measurementMu guards measurement, which is accumulated while the + // response streams in and read once the stream is done. The measurement + // is kept independently of the span: with telemetry disabled no span is + // recording, but the engine's numbers are still reported. + measurementMu sync.Mutex + measurement inferencemetrics.Measurement +} + +func newTelemetryBodyReader(body io.ReadCloser, span trace.Span, start, requestStart time.Time, session *telemetry.Session) *telemetryBodyReader { + return &telemetryBodyReader{ + ReadCloser: body, + span: span, + start: start, + requestStart: requestStart, + session: session, + } +} + +// LiveProgress is what the reader has measured of the response so far. +// +// Every number in it is exact. The bytes are the bytes the engine sent, the +// chunks are the content chunks that arrived, and the timings are measured +// instants — never an estimate. There is deliberately no token count and no +// rate: a chunk is not a token, and PAIR does not know the engine's token +// count until the response ends. +type LiveProgress struct { + // Chunks is the number of content chunks received — chunks that carried + // text, not every frame the engine sent. + Chunks int64 + // Bytes is the number of response body bytes received. + Bytes int64 + // TTFBMs is the time to the first response byte, in milliseconds. + TTFBMs int64 + // TTFCMs is the time to the first content chunk, in milliseconds. + TTFCMs int64 + // HasByte reports whether a response byte has arrived at all. + HasByte bool + // HasContent reports whether a content chunk has arrived at all. + HasContent bool +} + +func (r *telemetryBodyReader) Read(b []byte) (int, error) { + n, err := r.ReadCloser.Read(b) + if n > 0 { + r.recordFirstToken() + r.recordBytes(n) + r.observe(b[:n]) + // One notification per read, after the read's bytes have been + // counted and its lines parsed: the snapshot a frame carries is + // the state at the end of the read, not the state before it. + r.notifyProgress() + } + if err == io.EOF { + if len(r.lineBuf) > 0 { + r.inspectLine(r.lineBuf) + r.lineBuf = nil + } + r.finalize() + } + return n, err +} + +func (r *telemetryBodyReader) recordFirstToken() { + r.firstReadOnce.Do(func() { + if r.span != nil && r.span.IsRecording() { + ttftMs := time.Since(r.start).Milliseconds() + r.span.SetAttributes(attribute.Int64("pair.inference.ttft_ms", ttftMs)) + r.span.AddEvent("first_token") + } + }) +} + +// recordBytes counts the response bytes as they arrive and notes the instant +// the first byte came. +func (r *telemetryBodyReader) recordBytes(n int) { + r.progressMu.Lock() + if r.firstByteAt.IsZero() { + r.firstByteAt = time.Now() + } + r.bytes += int64(n) + r.progressMu.Unlock() +} + +// recordContent counts a content chunk — a chunk that carried text — and +// notes the instant the first one came. +func (r *telemetryBodyReader) recordContent() { + r.progressMu.Lock() + r.chunks++ + if r.firstContentAt.IsZero() { + r.firstContentAt = time.Now() + } + r.progressMu.Unlock() +} + +// progressLocked returns the live snapshot. The caller holds progressMu. +func (r *telemetryBodyReader) progressLocked() LiveProgress { + prog := LiveProgress{ + Chunks: r.chunks, + Bytes: r.bytes, + HasByte: !r.firstByteAt.IsZero(), + } + if prog.HasByte { + prog.TTFBMs = r.firstByteAt.Sub(r.requestStart).Milliseconds() + } + if !r.firstContentAt.IsZero() { + prog.HasContent = true + prog.TTFCMs = r.firstContentAt.Sub(r.requestStart).Milliseconds() + } + return prog +} + +// Progress returns the live counters of the response read so far. +func (r *telemetryBodyReader) Progress() LiveProgress { + r.progressMu.Lock() + defer r.progressMu.Unlock() + return r.progressLocked() +} + +// notifyProgress hands the current snapshot to the progress callback, if one +// is installed. It runs on the response copy goroutine. +func (r *telemetryBodyReader) notifyProgress() { + if r.onProgress == nil { + return + } + r.onProgress(r.Progress()) +} + +const maxTelemetryLineBytes = 4 << 20 // 4 MiB max line buffer for large non-streaming frames + +func (r *telemetryBodyReader) observe(chunk []byte) { + if !r.session.RecordPayloads() { + r.completionBuf.Reset() + r.completionTruncated = false + } + r.lineBuf = append(r.lineBuf, chunk...) + for { + end := bytes.IndexByte(r.lineBuf, '\n') + if end < 0 { + if len(r.lineBuf) > maxTelemetryLineBytes { + r.lineBuf = nil + } + return + } + r.inspectLine(r.lineBuf[:end]) + r.lineBuf = r.lineBuf[end+1:] + } +} + +type streamChunk struct { + Model string `json:"model"` + Response string `json:"response"` + Message *struct { + Role string `json:"role"` + Content string `json:"content"` + } `json:"message"` + Choices []struct { + Text string `json:"text"` + Delta struct { + Role string `json:"role"` + Content string `json:"content"` + } `json:"delta"` + Message struct { + Role string `json:"role"` + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + Done bool `json:"done"` + // The token counts llama.cpp reports on its OpenAI-compatible surface. + // They are pointer-typed so a count the engine reported as zero stays a + // measured zero, while a chunk that carries no usage at all leaves the + // field absent. This proxy only ever fronts llama.cpp, so the usage + // block is the engine's own source of truth for both counts. + Usage *struct { + PromptTokens *int `json:"prompt_tokens"` + CompletionTokens *int `json:"completion_tokens"` + } `json:"usage"` +} + +func (r *telemetryBodyReader) inspectLine(line []byte) { + line = bytes.TrimSpace(line) + if len(line) == 0 { + return + } + if bytes.HasPrefix(line, []byte("data: ")) { + line = bytes.TrimPrefix(line, []byte("data: ")) + line = bytes.TrimSpace(line) + if bytes.Equal(line, []byte("[DONE]")) { + return + } + } + var chunk streamChunk + if err := json.Unmarshal(line, &chunk); err != nil { + return + } + if chunk.Model != "" && r.responseModel == "" { + r.responseModel, _ = telemetry.TruncateUTF8(chunk.Model, 256) + } + if chunk.Usage != nil { + if chunk.Usage.PromptTokens != nil && *chunk.Usage.PromptTokens > 0 { + r.promptTokens = *chunk.Usage.PromptTokens + } + if chunk.Usage.CompletionTokens != nil && *chunk.Usage.CompletionTokens > 0 { + r.outputTokens = *chunk.Usage.CompletionTokens + } + // The engine's own token counts. Only what the engine reported is + // recorded: a value it did not report stays absent instead of + // becoming a zero. llama.cpp reports no phase split, so every + // duration stays absent. + r.measurementMu.Lock() + if chunk.Usage.PromptTokens != nil { + r.measurement.PrefillTokens = inferencemetrics.IntPtr(*chunk.Usage.PromptTokens) + } + if chunk.Usage.CompletionTokens != nil { + r.measurement.DecodeTokens = inferencemetrics.IntPtr(*chunk.Usage.CompletionTokens) + } + r.measurementMu.Unlock() + } + // The chunk's text is read whether or not the telemetry session records + // payloads: the live progress frame counts content chunks whatever the + // telemetry settings are, and a chunk that carries text is what makes a + // request a content chunk. + content := chunkContent(chunk) + if content != "" { + r.recordContent() + } + if r.session.RecordPayloads() && !r.completionTruncated { + toAppend := content + part, truncated := telemetry.TruncateUTF8(toAppend, telemetry.MaxPayloadBytes-r.completionBuf.Len()) + r.completionBuf.WriteString(part) + r.completionTruncated = truncated + } +} + +// chunkContent returns the text a streamed chunk carries, if any. A chunk that +// carries no text — a role-only delta, a usage block, a keep-alive — is not +// a content chunk. +func chunkContent(chunk streamChunk) string { + if chunk.Response != "" { + return chunk.Response + } + if chunk.Message != nil && chunk.Message.Content != "" { + return chunk.Message.Content + } + if len(chunk.Choices) > 0 { + if chunk.Choices[0].Delta.Content != "" { + return chunk.Choices[0].Delta.Content + } + if chunk.Choices[0].Message.Content != "" { + return chunk.Choices[0].Message.Content + } + if chunk.Choices[0].Text != "" { + return chunk.Choices[0].Text + } + } + return "" +} + +// Measurement returns the inference measurement parsed from the response +// stream so far. It is readable whether or not a span is recording, so the +// workload event carries the engine's numbers even with telemetry disabled. +func (r *telemetryBodyReader) Measurement() inferencemetrics.Measurement { + r.measurementMu.Lock() + defer r.measurementMu.Unlock() + return r.measurement +} + +func (r *telemetryBodyReader) finalize() { + r.closeOnce.Do(func() { + defer r.completionBuf.Reset() + if r.span != nil && r.span.IsRecording() { + if r.responseModel != "" { + r.span.SetAttributes(metadataAttribute("gen_ai.response.model", r.responseModel)) + } + if r.promptTokens > 0 { + r.span.SetAttributes(attribute.Int("gen_ai.usage.input_tokens", r.promptTokens)) + } + if r.outputTokens > 0 { + r.span.SetAttributes(attribute.Int("gen_ai.usage.output_tokens", r.outputTokens)) + } + if r.session.RecordPayloads() && r.completionBuf.Len() > 0 { + comp, _ := telemetry.TruncateUTF8(r.completionBuf.String(), telemetry.MaxPayloadBytes) + r.span.SetAttributes( + attribute.String("gen_ai.completion", comp), + attribute.String("output.value", comp), + ) + if r.completionTruncated { + r.span.SetAttributes(attribute.Bool("gen_ai.completion.truncated", true)) + } + } + } + }) +} + +func (r *telemetryBodyReader) Close() error { + if len(r.lineBuf) > 0 { + r.inspectLine(r.lineBuf) + r.lineBuf = nil + } + r.finalize() + return r.ReadCloser.Close() +} diff --git a/services/llamacpp-proxy/telemetry_privacy_test.go b/services/llamacpp-proxy/telemetry_privacy_test.go new file mode 100644 index 000000000..773910c59 --- /dev/null +++ b/services/llamacpp-proxy/telemetry_privacy_test.go @@ -0,0 +1,431 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + "unicode/utf8" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" + "nvpair-shared/telemetry" +) + +// Flush the real manager's batch processor without adding a production test API. +func flushTelemetry(t *testing.T, p *Proxy) { + t.Helper() + _, span := p.telemetry.Session().Tracer().Start(context.Background(), "test-flush") + defer span.End() + if tp, ok := span.TracerProvider().(*sdktrace.TracerProvider); ok { + if err := tp.ForceFlush(context.Background()); err != nil { + t.Fatal(err) + } + } +} + +func unconfiguredTelemetryProxy(t *testing.T) (*Proxy, *tracetest.InMemoryExporter, *prRec) { + t.Helper() + exporter := tracetest.NewInMemoryExporter() + nextExporter := exporter + m := telemetry.NewManager("test-llamacpp-proxy", func(context.Context, string) (sdktrace.SpanExporter, error) { + current := nextExporter + nextExporter = tracetest.NewInMemoryExporter() + return current, nil + }) + t.Cleanup(func() { _ = m.Shutdown(context.Background()) }) + rec := &prRec{} + return NewProxy(NewCodec(rec), NewDiscovery(), 1235, m), exporter, rec +} + +func telemetryRPC(t *testing.T, p *Proxy, rec *prRec, method, params string) Message { + t.Helper() + id := json.RawMessage(`123`) + rec.mu.Lock() + start := len(rec.b) + rec.mu.Unlock() + p.handleMessage(&Message{JSONRPC: "2.0", ID: &id, Method: method, Params: json.RawMessage(params)}) + rec.mu.Lock() + data := bytes.Clone(rec.b[start:]) + rec.mu.Unlock() + for _, line := range bytes.Split(data, []byte{'\n'}) { + var msg Message + if json.Unmarshal(line, &msg) == nil && msg.ID != nil && string(*msg.ID) == "123" { + return msg + } + } + t.Fatalf("no RPC response for %s", method) + return Message{} +} + +func configureTelemetry(t *testing.T, p *Proxy, rec *prRec, params string, want uint64) { + t.Helper() + msg := telemetryRPC(t, p, rec, "telemetry/configure", params) + var result struct { + Revision uint64 `json:"revision"` + } + if msg.Error != nil || json.Unmarshal(msg.Result, &result) != nil || result.Revision != want { + t.Fatalf("configure = %s, error %v; want revision %d", msg.Result, msg.Error, want) + } +} + +func telemetryRequest(p *Proxy, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, "/v1/completions", strings.NewReader(body)) + req.RemoteAddr = "127.0.0.1:12345" + rec := httptest.NewRecorder() + p.handlePlain(rec, req) + return rec +} + +func telemetryEngine(t *testing.T, p *Proxy, response string) { + t.Helper() + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, response+"\n") + })) + t.Cleanup(engine.Close) + p.discovery.AddManual(nodeForModel(t, "engine", engine.URL, "llama")) +} + +func TestUnconfiguredProxyExportsNothing(t *testing.T) { + p, exporter, _ := unconfiguredTelemetryProxy(t) + telemetryEngine(t, p, `{"choices":[{"text":"hello","finish_reason":"stop"}]}`) + if got := telemetryRequest(p, `{"model":"llama","prompt":"hello"}`); got.Code != 200 { + t.Fatalf("status = %d", got.Code) + } + flushTelemetry(t, p) + if got := len(exporter.GetSpans()); got != 0 { + t.Fatalf("unconfigured proxy exported %d spans", got) + } +} + +func TestConfigureEnablesAndStaleRevisionIgnored(t *testing.T) { + p, exporter, rec := unconfiguredTelemetryProxy(t) + telemetryEngine(t, p, `{"choices":[{"text":"hello","finish_reason":"stop"}]}`) + configureTelemetry(t, p, rec, `{"revision":2,"enabled":true,"collectorPort":4317,"recordPayloads":false}`, 2) + for i := 0; i < 2; i++ { + if i == 1 { + configureTelemetry(t, p, rec, `{"revision":1,"enabled":false,"collectorPort":4317,"recordPayloads":false}`, 2) + } + exporter.Reset() + if got := telemetryRequest(p, `{"model":"llama","prompt":"hello"}`); got.Code != 200 { + t.Fatal(got.Code) + } + flushTelemetry(t, p) + found := false + for _, span := range exporter.GetSpans() { + found = found || span.Name == "pair.proxy.inference" + } + if !found { + t.Fatal("configured proxy did not export inference") + } + } + msg := telemetryRPC(t, p, rec, "telemetry/status", `{}`) + var status map[string]json.RawMessage + if msg.Error != nil || json.Unmarshal(msg.Result, &status) != nil { + t.Fatalf("status: %s %v", msg.Result, msg.Error) + } + if len(status) != 5 || string(status["revision"]) != "2" || string(status["enabled"]) != "true" || string(status["lastExportError"]) != "null" || string(status["lastExportErrorAt"]) != "null" { + t.Fatalf("status = %s", msg.Result) + } + var lastOK string + if json.Unmarshal(status["lastExportOk"], &lastOK) != nil { + t.Fatal("missing export success time") + } + if stamp, err := time.Parse(time.RFC3339, lastOK); err != nil || stamp.Location() != time.UTC { + t.Fatalf("invalid UTC timestamp %q", lastOK) + } +} + +func TestTelemetryConfigureRejectsInvalidParams(t *testing.T) { + p, _, rec := unconfiguredTelemetryProxy(t) + for _, params := range []string{ + `null`, `{}`, `[]`, `{"revision":-1,"enabled":true,"collectorPort":4317,"recordPayloads":false}`, + `{"revision":1,"enabled":true,"collectorPort":0,"recordPayloads":false}`, + `{"revision":1,"enabled":true,"collectorPort":65536,"recordPayloads":false}`, + `{"revision":1,"enabled":false,"collectorPort":4317,"recordPayloads":true}`, + `{"revision":1,"enabled":"true","collectorPort":4317,"recordPayloads":false}`, + `{"revision":1,"enabled":null,"collectorPort":4317,"recordPayloads":false}`, + } { + msg := telemetryRPC(t, p, rec, "telemetry/configure", params) + if msg.Error == nil || msg.Error.Code != -32602 { + t.Fatalf("params %s: error = %v", params, msg.Error) + } + } + msg := telemetryRPC(t, p, rec, "telemetry/status", `{}`) + var status map[string]json.RawMessage + if msg.Error != nil || json.Unmarshal(msg.Result, &status) != nil || string(status["revision"]) != "0" || string(status["enabled"]) != "false" || string(status["lastExportOk"]) != "null" { + t.Fatalf("invalid requests changed status: %s", msg.Result) + } +} + +func TestDisableMidStreamDropsCompletion(t *testing.T) { + p, exporter, rec := unconfiguredTelemetryProxy(t) + configureTelemetry(t, p, rec, `{"revision":1,"enabled":true,"collectorPort":4317,"recordPayloads":true}`, 1) + first := make(chan struct{}) + finish := make(chan struct{}) + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, `data: {"choices":[{"text":"before-revoke","finish_reason":null}]}`+"\n\n") + w.(http.Flusher).Flush() + close(first) + <-finish + _, _ = io.WriteString(w, `data: {"choices":[{"text":"after-revoke","finish_reason":"stop"}]}`+"\n\ndata: [DONE]\n\n") + })) + defer engine.Close() + defer func() { + select { + case <-finish: + default: + close(finish) + } + }() + p.discovery.AddManual(nodeForModel(t, "engine", engine.URL, "llama")) + done := make(chan *httptest.ResponseRecorder, 1) + go func() { done <- telemetryRequest(p, `{"model":"llama","prompt":"hello","stream":true}`) }() + <-first + configureTelemetry(t, p, rec, `{"revision":2,"enabled":true,"collectorPort":4317,"recordPayloads":false}`, 2) + close(finish) + if got := <-done; got.Code != 200 || !strings.Contains(got.Body.String(), "after-revoke") { + t.Fatal("stream interrupted by capture revocation") + } + flushTelemetry(t, p) + for _, span := range exporter.GetSpans() { + for _, a := range span.Attributes { + if a.Key == "gen_ai.completion" || a.Key == "output.value" { + t.Fatal("completion survived revocation") + } + } + } +} + +// The manager also gates revoked exports. Inspect the reader's span through an +// independent processor so that gate cannot mask an erroneous finalize capture. +func TestTelemetryBodyReaderRevokesBufferedCompletion(t *testing.T) { + p, _, rec := unconfiguredTelemetryProxy(t) + configureTelemetry(t, p, rec, `{"revision":1,"enabled":true,"collectorPort":4317,"recordPayloads":true}`, 1) + exporter := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter)) + defer tp.Shutdown(context.Background()) + _, span := tp.Tracer("reader").Start(context.Background(), "reader") + r := newTelemetryBodyReader(io.NopCloser(strings.NewReader("")), span, time.Now(), time.Now(), p.telemetry.Session()) + r.observe([]byte(`data: {"choices":[{"delta":{"content":"before"}}]}` + "\n")) + if r.completionBuf.Len() == 0 { + t.Fatal("fixture did not buffer completion") + } + configureTelemetry(t, p, rec, `{"revision":2,"enabled":true,"collectorPort":4317,"recordPayloads":false}`, 2) + r.observe([]byte(`data: {"choices":[{"delta":{"content":"after"}}]}` + "\n")) + if r.completionBuf.Len() != 0 { + t.Fatal("revoked reader retained completion") + } + r.Close() + span.End() + for _, a := range exporter.GetSpans()[0].Attributes { + if a.Key == "gen_ai.completion" || a.Key == "output.value" { + t.Fatal("revoked reader finalized payload") + } + } +} + +func TestSensitiveHeadersNeverExported(t *testing.T) { + for _, capture := range []bool{false, true} { + t.Run(map[bool]string{false: "metadata", true: "payloads"}[capture], func(t *testing.T) { + p, exporter := setupTestProxyWithTelemetry(t, capture) + telemetryEngine(t, p, `{"choices":[{"text":"hello","finish_reason":"stop"}]}`) + req := httptest.NewRequest(http.MethodPost, "/v1/completions?api_key=url-secret", strings.NewReader(`{"model":"llama","prompt":"hello"}`)) + req.RemoteAddr = "127.0.0.1:1" + for k, v := range map[string]string{"Authorization": "Bearer auth-secret", "Cookie": "cookie-secret", "X-Secret": "custom-secret"} { + req.Header.Set(k, v) + } + p.handlePlain(httptest.NewRecorder(), req) + flushTelemetry(t, p) + if len(exporter.GetSpans()) < 3 { + t.Fatal("no inference spans") + } + for _, span := range exporter.GetSpans() { + for _, a := range span.Attributes { + text := strings.ToLower(string(a.Key) + a.Value.Emit()) + for _, secret := range []string{"authorization", "cookie", "x-secret", "api_key", "auth-secret", "cookie-secret", "custom-secret", "url-secret"} { + if strings.Contains(text, secret) { + t.Errorf("exported sensitive header or URL query: %s", secret) + } + } + } + } + }) + } +} + +func TestPayloadTruncation(t *testing.T) { + p, exporter := setupTestProxyWithTelemetry(t, true) + // Exactly 20 KiB, with a two-byte rune crossing the 16 KiB boundary. + long := strings.Repeat("a", (16<<10)-1) + "é" + strings.Repeat("b", (4<<10)-1) + responseModel := strings.Repeat("a", 255) + "é" + strings.Repeat("b", 143) + response, _ := json.Marshal(map[string]interface{}{"choices": []map[string]string{{"text": long, "finish_reason": "stop"}}, "model": responseModel}) + telemetryEngine(t, p, string(response)) + body, _ := json.Marshal(map[string]string{"model": "llama", "prompt": long}) + if got := telemetryRequest(p, string(body)); got.Code != 200 { + t.Fatal(got.Code) + } + flushTelemetry(t, p) + seen := map[string]bool{} + for _, span := range exporter.GetSpans() { + attrs := attrMap(span.Attributes) + for _, key := range []string{"gen_ai.prompt", "input.value", "gen_ai.completion", "output.value"} { + if s, ok := attrs[key].(string); ok { + seen[key] = true + if len(s) > 16<<10 || !utf8.ValidString(s) { + t.Errorf("%s is not bounded UTF-8", key) + } + } + } + if span.Name == "pair.proxy.inference" && attrs["gen_ai.prompt.truncated"] != true { + t.Error("missing prompt truncated flag") + } + if span.Name == "pair.engine.dispatch" { + if attrs["gen_ai.completion.truncated"] != true { + t.Error("missing completion truncated flag") + } + if s, ok := attrs["gen_ai.response.model"].(string); !ok || len(s) > 256 || !utf8.ValidString(s) { + t.Error("response model not bounded") + } + } + } + if len(seen) != 4 { + t.Fatalf("missing captured attributes: %v", seen) + } +} + +func TestNoPeerTraceContext(t *testing.T) { + p, exporter := setupTestProxyWithTelemetry(t, true) + dir := t.TempDir() + clustertrusttest.Join(t, dir, "test-cluster", "self", "peer") + p.mesh = clustertrust.Open(dir) + peer, peerExporter := setupTestProxyWithTelemetry(t, true) + peer.mesh = p.mesh + pinData, err := os.ReadFile(filepath.Join(dir, "trusted", "peer.json")) + if err != nil { + t.Fatal(err) + } + var pin struct { + CertPEM string `json:"certPem"` + } + if err := json.Unmarshal(pinData, &pin); err != nil { + t.Fatal(err) + } + block, _ := pem.Decode([]byte(pin.CertPEM)) + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatal(err) + } + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("traceparent") != "" || r.Header.Get("tracestate") != "" || r.Header.Get("baggage") != "" { + t.Error("trace context reached ingress engine") + } + _, _ = io.WriteString(w, `{"choices":[{"text":"hi","finish_reason":"stop"}]}`+"\n") + })) + defer backend.Close() + local := nodeForModel(t, "local", backend.URL, "llama") + peer.setLocalBackend(localBackend{Host: "127.0.0.1", Port: local.Port, Healthy: true}) + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("traceparent") != "" || r.Header.Get("tracestate") != "" || r.Header.Get("baggage") != "" { + t.Error("trace context sent to peer") + } + // Authenticate through the real pin verifier without building a second + // key infrastructure; this certificate is the temporary trusted leaf. + r.TLS = &tls.ConnectionState{PeerCertificates: []*x509.Certificate{cert}} + peer.handleClusterIngress(w, r) + })) + defer server.Close() + node := nodeForModel(t, "peer-node", server.URL, "llama") + node.ClusterUUID = "peer" + p.discovery.SetSubscribed([]Node{node}) + p.peerTransports = map[string]*http.Transport{"peer": server.Client().Transport.(*http.Transport)} + req := httptest.NewRequest(http.MethodPost, "/v1/completions", strings.NewReader(`{"model":"llama","prompt":"hello"}`)) + req.RemoteAddr = "127.0.0.1:1" + req.Header.Set("traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01") + req.Header.Set("tracestate", "vendor=state") + req.Header.Set("baggage", "secret=state") + rec := httptest.NewRecorder() + p.handlePlain(rec, req) + if rec.Code != 200 { + t.Fatalf("peer dispatch failed: %d %s", rec.Code, rec.Body.String()) + } + // A peer can also supply its own tracing headers. The terminal ingress + // must neither extract them nor pass them on to the loopback engine. + ingressReq := httptest.NewRequest(http.MethodPost, "/v1/completions", strings.NewReader(`{"model":"llama","prompt":"peer request"}`)) + ingressReq.TLS = &tls.ConnectionState{PeerCertificates: []*x509.Certificate{cert}} + ingressReq.Header = req.Header.Clone() + ingressRec := httptest.NewRecorder() + peer.handleClusterIngress(ingressRec, ingressReq) + if ingressRec.Code != 200 { + t.Fatalf("direct ingress failed: %d", ingressRec.Code) + } + flushTelemetry(t, p) + flushTelemetry(t, peer) + for _, span := range peerExporter.GetSpans() { + if span.Name != "test-flush" { + t.Errorf("ingress created %s", span.Name) + } + } + found := false + for _, span := range exporter.GetSpans() { + if span.Name == "pair.proxy.inference" { + found = true + if span.SpanContext.TraceID().String() != "4bf92f3577b34da6a3ce929d0e0e4736" || span.Parent.SpanID().String() != "00f067aa0ba902b7" { + t.Error("local client trace context not extracted") + } + } + } + if !found { + t.Fatal("missing local inference span") + } +} + +func TestTelemetryStatusExportFailure(t *testing.T) { + p, _, rec := unconfiguredTelemetryProxy(t) + p.telemetry = telemetry.NewManager("failure", func(context.Context, string) (sdktrace.SpanExporter, error) { return failingTelemetryExporter{}, nil }) + defer p.telemetry.Shutdown(context.Background()) + configureTelemetry(t, p, rec, `{"revision":1,"enabled":true,"collectorPort":4317,"recordPayloads":false}`, 1) + _, span := p.telemetry.Session().Tracer().Start(context.Background(), "export-failure") + span.End() + tp := span.TracerProvider().(*sdktrace.TracerProvider) + if err := tp.ForceFlush(context.Background()); err == nil { + t.Fatal("expected failed export") + } + msg := telemetryRPC(t, p, rec, "telemetry/status", `{}`) + var status struct { + LastOK *string `json:"lastExportOk"` + Error *string `json:"lastExportError"` + At *string `json:"lastExportErrorAt"` + } + if msg.Error != nil || json.Unmarshal(msg.Result, &status) != nil || status.LastOK != nil || status.Error == nil || *status.Error != "collector unavailable" || status.At == nil { + t.Fatalf("failure status = %s", msg.Result) + } + if stamp, err := time.Parse(time.RFC3339, *status.At); err != nil || stamp.Location() != time.UTC { + t.Error("failure timestamp not RFC3339 UTC") + } +} + +type failingTelemetryExporter struct{} + +func (failingTelemetryExporter) ExportSpans(context.Context, []sdktrace.ReadOnlySpan) error { + return errors.New("collector unavailable") +} +func (failingTelemetryExporter) Shutdown(context.Context) error { return nil } diff --git a/services/llamacpp-proxy/telemetry_test.go b/services/llamacpp-proxy/telemetry_test.go new file mode 100644 index 000000000..4e091389b --- /dev/null +++ b/services/llamacpp-proxy/telemetry_test.go @@ -0,0 +1,346 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "go.opentelemetry.io/otel/attribute" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "nvpair-shared/bufpool" + "nvpair-shared/telemetry" +) + +type dummyReadWriteCloser struct { + io.Reader + io.Writer +} + +func (d dummyReadWriteCloser) Close() error { return nil } + +func setupTestProxyWithTelemetry(t *testing.T, recordPayloads bool) (*Proxy, *tracetest.InMemoryExporter) { + t.Helper() + exporter := tracetest.NewInMemoryExporter() + tel := telemetry.NewManager("nvpair-llamacpp-proxy-test", func(context.Context, string) (sdktrace.SpanExporter, error) { + return exporter, nil + }) + t.Cleanup(func() { _ = tel.Shutdown(context.Background()) }) + if _, err := tel.Apply(context.Background(), 1, telemetry.Settings{ + Enabled: true, CollectorPort: 4317, RecordPayloads: recordPayloads, + }); err != nil { + t.Fatal(err) + } + + var inBuf, outBuf bytes.Buffer + rwc := dummyReadWriteCloser{Reader: &inBuf, Writer: &outBuf} + codec := NewCodec(rwc) + disc := NewDiscovery() + p := NewProxy(codec, disc, 1234, tel) + p.bufPool = bufpool.New() + return p, exporter +} + +func TestLlamaCPPProxyTelemetrySpans(t *testing.T) { + mockEngine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + + // First streaming chunk (delta content) + _, _ = w.Write([]byte("data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"model\":\"qwen\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"Hi there\"},\"finish_reason\":null}]}\n\n")) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + time.Sleep(10 * time.Millisecond) + + // Terminal chunk with token stats + _, _ = w.Write([]byte("data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"model\":\"qwen\",\"choices\":[],\"usage\":{\"prompt_tokens\":15,\"completion_tokens\":8}}\n\n")) + _, _ = w.Write([]byte("data: [DONE]\n\n")) + })) + defer mockEngine.Close() + + p, exporter := setupTestProxyWithTelemetry(t, false) + node := nodeForModel(t, "test-node-1", mockEngine.URL, "qwen") + p.discovery.AddManual(node) + p.SetSelected("test-node-1") + + reqBody := `{"model":"qwen","messages":[{"role":"user","content":"hello"}]}` + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewBufferString(reqBody)) + req.RemoteAddr = "127.0.0.1:54321" + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer secret-token-xyz") + req.Header.Set("X-Custom-Client", "open-webui") + rec := httptest.NewRecorder() + + p.handlePlain(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d: %s", rec.Code, rec.Body.String()) + } + flushTelemetry(t, p) + spans := exporter.GetSpans() + if len(spans) < 3 { + t.Fatalf("expected at least 3 spans (schedule, dispatch, inference), got %d", len(spans)) + } + + spanMap := make(map[string]tracetest.SpanStub) + for _, s := range spans { + spanMap[s.Name] = s + } + + // 1. Root inference span + rootSpan, ok := spanMap["pair.proxy.inference"] + if !ok { + t.Fatalf("missing span 'pair.proxy.inference'") + } + rootAttrs := attrMap(rootSpan.Attributes) + if rootAttrs["gen_ai.system"] != "pair" { + t.Errorf("expected gen_ai.system = pair, got %v", rootAttrs["gen_ai.system"]) + } + if rootAttrs["gen_ai.request.model"] != "qwen" { + t.Errorf("expected gen_ai.request.model = qwen, got %v", rootAttrs["gen_ai.request.model"]) + } + if rootAttrs["pair.client.protocol"] != "openai" { + t.Errorf("expected pair.client.protocol = openai, got %v", rootAttrs["pair.client.protocol"]) + } + if rootAttrs["http.response.status_code"] != int64(200) { + t.Errorf("expected http.response.status_code = 200, got %v", rootAttrs["http.response.status_code"]) + } + if rootAttrs["http.request.header.content-type"] != "application/json" { + t.Errorf("expected allowlisted content-type, got %v", rootAttrs["http.request.header.content-type"]) + } + if _, hasPrompt := rootAttrs["gen_ai.prompt"]; hasPrompt { + t.Errorf("expected gen_ai.prompt to be omitted when RecordPayloads=false") + } + + // 2. Scheduler span + schedSpan, ok := spanMap["pair.router.schedule"] + if !ok { + t.Fatalf("missing span 'pair.router.schedule'") + } + schedAttrs := attrMap(schedSpan.Attributes) + if schedAttrs["pair.route.selected_node_id"] != "test-node-1" { + t.Errorf("expected selected node test-node-1, got %v", schedAttrs["pair.route.selected_node_id"]) + } + + // 3. Dispatch span + dispatchSpan, ok := spanMap["pair.engine.dispatch"] + if !ok { + t.Fatalf("missing span 'pair.engine.dispatch'") + } + dispatchAttrs := attrMap(dispatchSpan.Attributes) + if dispatchAttrs["gen_ai.usage.input_tokens"] != int64(15) { + t.Errorf("expected input_tokens = 15, got %v", dispatchAttrs["gen_ai.usage.input_tokens"]) + } + if dispatchAttrs["gen_ai.usage.output_tokens"] != int64(8) { + t.Errorf("expected output_tokens = 8, got %v", dispatchAttrs["gen_ai.usage.output_tokens"]) + } + if ttft, hasTTFT := dispatchAttrs["pair.inference.ttft_ms"]; !hasTTFT || ttft.(int64) < 0 { + t.Errorf("expected valid pair.inference.ttft_ms, got %v", ttft) + } +} + +func TestLlamaCPPProxyPayloadCaptureEnabled(t *testing.T) { + mockEngine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"id":"chatcmpl-1","object":"chat.completion","model":"qwen","choices":[{"index":0,"message":{"role":"assistant","content":"Hello world response"},"finish_reason":"stop"}]}` + "\n")) + })) + defer mockEngine.Close() + + p, exporter := setupTestProxyWithTelemetry(t, true) // RecordPayloads: true + node := nodeForModel(t, "test-node-1", mockEngine.URL, "qwen") + p.discovery.AddManual(node) + p.SetSelected("test-node-1") + + reqBody := `{"model":"qwen","prompt":"Tell me a story"}` + req := httptest.NewRequest(http.MethodPost, "/v1/completions", bytes.NewBufferString(reqBody)) + req.RemoteAddr = "127.0.0.1:54321" + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + + p.handlePlain(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d", rec.Code) + } + + flushTelemetry(t, p) + spans := exporter.GetSpans() + spanMap := make(map[string]tracetest.SpanStub) + for _, s := range spans { + spanMap[s.Name] = s + } + + rootSpan := spanMap["pair.proxy.inference"] + rootAttrs := attrMap(rootSpan.Attributes) + if rootAttrs["input.value"] != "Tell me a story" { + t.Errorf("expected input.value captured, got %v", rootAttrs["input.value"]) + } + + dispatchSpan := spanMap["pair.engine.dispatch"] + dispatchAttrs := attrMap(dispatchSpan.Attributes) + if dispatchAttrs["output.value"] != "Hello world response" { + t.Errorf("expected output.value captured, got %v", dispatchAttrs["output.value"]) + } +} + +func TestLlamaCPPProxyCandidateFailureTelemetry(t *testing.T) { + // Candidate 0: closes immediately so dial fails + badServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + badServer.Close() + + // Candidate 1: succeeds + goodServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"id":"chatcmpl-1","object":"chat.completion","model":"qwen","choices":[{"index":0,"message":{"role":"assistant","content":"success from candidate 1"},"finish_reason":"stop"}]}` + "\n")) + })) + defer goodServer.Close() + + p, exporter := setupTestProxyWithTelemetry(t, false) + badNode := nodeForModel(t, "bad-node", badServer.URL, "qwen") + goodNode := nodeForModel(t, "good-node", goodServer.URL, "qwen") + p.discovery.AddManual(badNode) + p.discovery.AddManual(goodNode) + p.SetSelected("bad-node") // bad-node will be tried first + + reqBody := `{"model":"qwen","messages":[{"role":"user","content":"test failover"}]}` + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewBufferString(reqBody)) + req.RemoteAddr = "127.0.0.1:54321" + rec := httptest.NewRecorder() + + p.handlePlain(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected failover to succeed with 200, got %d", rec.Code) + } + + flushTelemetry(t, p) + spans := exporter.GetSpans() + var dispatchSpans []tracetest.SpanStub + for _, s := range spans { + if s.Name == "pair.engine.dispatch" { + dispatchSpans = append(dispatchSpans, s) + } + } + + if len(dispatchSpans) < 2 { + t.Fatalf("expected at least 2 dispatch spans due to failover, got %d", len(dispatchSpans)) + } + + // First dispatch span should be the failed candidate with error status + failedSpan := dispatchSpans[0] + failedAttrs := attrMap(failedSpan.Attributes) + if failedAttrs["http.response.status_code"] == int64(200) { + t.Errorf("failed candidate dispatch span should NOT have http.response.status_code = 200") + } + if failedSpan.Status.Code.String() != "Error" { + t.Errorf("failed candidate dispatch span status = %v, want Error", failedSpan.Status.Code) + } + + // Second dispatch span should be the successful candidate + successSpan := dispatchSpans[1] + successAttrs := attrMap(successSpan.Attributes) + if successAttrs["http.response.status_code"] != int64(200) { + t.Errorf("successful candidate dispatch span status_code = %v, want 200", successAttrs["http.response.status_code"]) + } + if successAttrs["pair.route.failover"] != true { + t.Errorf("expected pair.route.failover = true on second attempt") + } +} + +// lmStudioEngineWith serves body as an llama.cpp SSE response. +func lmStudioEngineWith(t *testing.T, body string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestWorkloadCarriesLlamaCPPTokensOnly(t *testing.T) { + srv := lmStudioEngineWith(t, "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\","+ + "\"model\":\"qwen\",\"choices\":[],\"usage\":{\"prompt_tokens\":15,\"completion_tokens\":8}}\n\n") + rec := &prRec{} + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "serving-node", srv.URL, "qwen")) + p := NewProxy(NewCodec(rec), disc, 1235) + + p.handleHTTP(httptest.NewRecorder(), + httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"qwen"}`))) + + for _, want := range []string{ + `"method":"workload:completed"`, + `"prefillTokens":15`, `"decodeTokens":8`, + } { + if !rec.has(want) { + t.Fatalf("workload event missing %s", want) + } + } + // llama.cpp reports no phase split, so every duration stays absent. It + // reports no total time either: the measurement carries engine-reported + // values only, so the proxy's own wall-clock duration must not stand in + // for it. + for _, absent := range []string{`"prefillMs"`, `"decodeMs"`, `"ttftMs"`, `"cachedTokens"`, `"totalMs"`} { + if rec.has(absent) { + t.Fatalf("llamacpp reports no phase split: %s", absent) + } + } +} + +func TestLlamaCPPMeasurementPresentWithoutTelemetry(t *testing.T) { + // unconfiguredTelemetryProxy's manager is never enabled, so the session's + // tracer is a noop and no span is recording. The measurement must still + // reach the workload event. + p, _, rec := unconfiguredTelemetryProxy(t) + srv := lmStudioEngineWith(t, "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\","+ + "\"model\":\"qwen\",\"choices\":[],\"usage\":{\"prompt_tokens\":15,\"completion_tokens\":8}}\n\n") + p.discovery.AddManual(nodeForModel(t, "serving-node", srv.URL, "qwen")) + + p.handleHTTP(httptest.NewRecorder(), + httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"qwen"}`))) + + if !rec.has(`"prefillTokens":15`) { + t.Fatal("measurement missing with telemetry disabled") + } +} + +func TestLlamaCPPMeasuredZeroTokenCountIsKept(t *testing.T) { + // A count llama.cpp reported as zero is a measured zero, not an absent + // value. + srv := lmStudioEngineWith(t, "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\","+ + "\"model\":\"qwen\",\"choices\":[],\"usage\":{\"prompt_tokens\":0,\"completion_tokens\":0}}\n\n") + rec := &prRec{} + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "serving-node", srv.URL, "qwen")) + p := NewProxy(NewCodec(rec), disc, 1235) + + p.handleHTTP(httptest.NewRecorder(), + httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"qwen"}`))) + + if !rec.has(`"prefillTokens":0`) { + t.Fatal("a reported zero prefill count must be preserved") + } + if !rec.has(`"decodeTokens":0`) { + t.Fatal("a reported zero decode count must be preserved") + } +} + +func attrMap(kvs []attribute.KeyValue) map[string]any { + out := make(map[string]any, len(kvs)) + for _, kv := range kvs { + out[string(kv.Key)] = kv.Value.AsInterface() + } + return out +} diff --git a/services/llamacpp-proxy/transport.go b/services/llamacpp-proxy/transport.go new file mode 100644 index 000000000..ecc5a269f --- /dev/null +++ b/services/llamacpp-proxy/transport.go @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "io" + "nvpair-shared/jsonrpc" +) + +func newStdioTransport() io.ReadWriteCloser { + transport, err := jsonrpc.NewStdioTransport() + if err != nil { + panic("cannot initialize interruptible JSON-RPC stdio") + } + return transport +} diff --git a/services/llamacpp-proxy/transport_pool_test.go b/services/llamacpp-proxy/transport_pool_test.go new file mode 100644 index 000000000..2dc50f0d0 --- /dev/null +++ b/services/llamacpp-proxy/transport_pool_test.go @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "path/filepath" + "testing" + + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" +) + +func TestCandidateTransportReusesPlainTransport(t *testing.T) { + p := testProxy(NewDiscovery(), 1235) + a := p.candidateTransport(candidate{}) + b := p.candidateTransport(candidate{id: "manual"}) + if a != b { + t.Fatalf("plain candidates returned distinct Transports") + } + if a == nil { + t.Fatal("plain Transport is nil") + } +} + +func TestCandidateTransportReusesPeerTransport(t *testing.T) { + const peerUUID = "principal-peer" + clusterDir := filepath.Join(t.TempDir(), "cluster") + clustertrusttest.Join(t, clusterDir, "cluster-xyz", "principal-self", peerUUID) + + p := testProxy(NewDiscovery(), 1235) + p.mesh = clustertrust.Open(clusterDir) + + a := p.candidateTransport(candidate{peerUUID: peerUUID}) + b := p.candidateTransport(candidate{peerUUID: peerUUID}) + if a != b { + t.Fatalf("same peerUUID returned distinct Transports") + } + if a.TLSClientConfig == nil { + t.Fatal("peer Transport missing TLSClientConfig") + } + + other := p.candidateTransport(candidate{peerUUID: "principal-other"}) + if other == a { + t.Fatal("unpinned peer reused pinned peer Transport") + } +} + +func TestDropUnpinnedPeerTransportsRemovesEntry(t *testing.T) { + const peerUUID = "principal-peer" + clusterDir := filepath.Join(t.TempDir(), "cluster") + clustertrusttest.Join(t, clusterDir, "cluster-xyz", "principal-self", peerUUID) + + p := testProxy(NewDiscovery(), 1235) + p.mesh = clustertrust.Open(clusterDir) + + tr := p.candidateTransport(candidate{peerUUID: peerUUID}) + p.transportMu.Lock() + if _, ok := p.peerTransports[peerUUID]; !ok { + p.transportMu.Unlock() + t.Fatal("peer Transport was not cached") + } + p.transportMu.Unlock() + + clustertrusttest.RemovePeerPin(t, clusterDir, peerUUID) + p.mesh.Refresh() + p.dropUnpinnedPeerTransports() + + p.transportMu.Lock() + _, still := p.peerTransports[peerUUID] + p.transportMu.Unlock() + if still { + t.Fatal("peer Transport remained after pin removal") + } + _ = tr +} diff --git a/services/llamacpp-proxy/unpinned_peer_test.go b/services/llamacpp-proxy/unpinned_peer_test.go new file mode 100644 index 000000000..26e180241 --- /dev/null +++ b/services/llamacpp-proxy/unpinned_peer_test.go @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + stderrors "errors" + "net/http" + "testing" +) + +// A peer whose pin is gone is refused at dial time, never reached over a +// transport that does not pin its certificate. +func TestPeerTransportFailsClosedWithoutAPin(t *testing.T) { + p := &Proxy{} + req, err := http.NewRequest(http.MethodGet, "https://192.0.2.1:11434/api/tags", nil) + if err != nil { + t.Fatal(err) + } + resp, err := p.peerHTTPTransport("peer-without-pin").RoundTrip(req) + if resp != nil { + _ = resp.Body.Close() + } + if !stderrors.Is(err, errPeerUnpinned) { + t.Fatalf("err = %v, want errPeerUnpinned", err) + } +} diff --git a/services/llamacpp-proxy/worker_health_test.go b/services/llamacpp-proxy/worker_health_test.go new file mode 100644 index 000000000..f760ee582 --- /dev/null +++ b/services/llamacpp-proxy/worker_health_test.go @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "log/slog" + "net" + "strings" + "testing" +) + +// TestWorkerHealth catches a missing health dispatch, incorrect correlation or +// payload, and a response missing from the ordered writer. +func TestWorkerHealth(t *testing.T) { + for _, rawID := range []string{"7", `"health-7"`} { + t.Run(rawID, func(t *testing.T) { + var out bytes.Buffer + p := &Proxy{codec: NewCodec(&out)} + id := json.RawMessage(rawID) + msg := &Message{JSONRPC: "2.0", ID: &id, Method: "worker/health"} + p.handleMessage(msg) + _ = p.codec.Flush(t.Context()) + output := out.String() + + var response Message + if err := json.Unmarshal([]byte(output), &response); err != nil { + t.Fatalf("dispatch did not respond inline: %q (%v)", output, err) + } + if response.JSONRPC != "2.0" || !response.IsResponse() || response.ID == nil || string(*response.ID) != rawID { + t.Fatalf("incorrect response envelope: %s", output) + } + if response.Error != nil || string(response.Result) != `{"ok":true}` { + t.Fatalf("worker/health result = %s, error = %v; want {\"ok\":true}", response.Result, response.Error) + } + }) + } +} + +// TestWorkerHealthWriteError catches a failed response being silently discarded. +// The terminal diagnostic must contain no request or response payloads. +func TestWorkerHealthWriteError(t *testing.T) { + var diagnostics bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&diagnostics, nil))) + defer slog.SetDefault(previous) + + local, remote := net.Pipe() + defer local.Close() + if err := remote.Close(); err != nil { + t.Fatal(err) + } + p := &Proxy{codec: NewCodec(local)} + id := json.RawMessage(`"private-health-id"`) + msg := &Message{ + JSONRPC: "2.0", ID: &id, Method: "worker/health", + Params: json.RawMessage(`{"private":"health-params"}`), + } + p.handleMessage(msg) + _ = p.codec.Close() // join asynchronous terminal diagnostic before reading its sink + + output := diagnostics.String() + if !strings.Contains(output, "JSON-RPC stream terminated") || + !strings.Contains(output, `reason="write failed"`) { + t.Fatalf("health write error was not surfaced: %q", output) + } + for _, payload := range []string{"private-health-id", "health-params", `"ok"`} { + if strings.Contains(output, payload) { + t.Fatalf("health diagnostic contains payload %q: %s", payload, output) + } + } +} diff --git a/services/llamacpp-proxy/zombie_test.go b/services/llamacpp-proxy/zombie_test.go new file mode 100644 index 000000000..e4dbe1cbd --- /dev/null +++ b/services/llamacpp-proxy/zombie_test.go @@ -0,0 +1,516 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +// count reports how many times s appears across the recorded codec frames. +// Method names appear once per emitted notification, so this counts emissions. +func (r *prRec) count(s string) int { + r.mu.Lock() + defer r.mu.Unlock() + hay := string(r.b) + n := 0 + for i := 0; i+len(s) <= len(hay); i++ { + if hay[i:i+len(s)] == s { + n++ + } + } + return n +} + +// clientGoneWriter is an http.ResponseWriter whose body Write fails after the +// status line is sent, standing in for a client that vanished mid-stream: the +// idle write deadline trips (statusCapture.Write) and the underlying +// connection write returns an error. It records the status and supports Flush +// so the reverse proxy streams through it. This is the shape of the zombie-job +// bug — the response has committed (200), so without the wroteErr check the +// terminal would be misreported as completed. +type clientGoneWriter struct { + header http.Header + status int + err error + wrote bool +} + +func (c *clientGoneWriter) Header() http.Header { + if c.header == nil { + c.header = make(http.Header) + } + return c.header +} + +func (c *clientGoneWriter) WriteHeader(code int) { c.status = code } + +func (c *clientGoneWriter) Write(b []byte) (int, error) { + c.wrote = true + return 0, c.err +} + +func (c *clientGoneWriter) Flush() {} + +// TestHandleHTTP_ClientWriteError_MarksFailed is the zombie-job regression: a +// streaming inference response that has committed (200 headers sent) but whose +// body write to the client fails — the signature of a killed / half-open client +// whose write deadline tripped — must terminate the workload as FAILED, not be +// silently reported completed. +func TestHandleHTTP_ClientWriteError_MarksFailed(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + io.WriteString(w, `{"choices":[{"delta":{"content":"partial tokens"}}]}`) + })) + defer upstream.Close() + + rec := &prRec{} + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "node-a", upstream.URL, "llama")) + p := NewProxy(NewCodec(rec), disc, 1235) + + cw := &clientGoneWriter{err: errors.New("write tcp: connection reset by peer")} + + done := make(chan struct{}) + go func() { + defer close(done) + p.handleHTTP(cw, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"llama"}`))) + }() + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("handleHTTP did not return after the client write failed (zombie: handler blocked)") + } + + if !cw.wrote { + t.Fatal("reverse proxy never attempted a body write to the client; test did not exercise the streaming path") + } + if got := rec.count("workload:errored"); got != 1 { + t.Fatalf("workload:errored emitted %d times, want exactly 1", got) + } + if rec.has("workload:completed") { + t.Fatal("workload:completed emitted for a request whose client write failed (should be failed)") + } +} + +// TestHandleHTTP_ClientDisconnect_TerminalOnce covers the disconnect watcher: a +// request whose context is cancelled mid-flight must emit exactly one terminal +// (errored) — the watcher and post-handler path are guarded by terminalOnce — +// and handleHTTP must return promptly rather than hang. +func TestHandleHTTP_ClientDisconnect_TerminalOnce(t *testing.T) { + received := make(chan struct{}, 1) + release := make(chan struct{}) + var releaseOnce sync.Once + doRelease := func() { releaseOnce.Do(func() { close(release) }) } + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if f, ok := w.(http.Flusher); ok { + io.WriteString(w, `{"choices":[{"delta":{"content":"first chunk"}}]}`+"\n") + f.Flush() + } + select { + case received <- struct{}{}: + default: + } + <-release + })) + defer upstream.Close() + defer doRelease() + + rec := &prRec{} + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "node-a", upstream.URL, "llama")) + p := NewProxy(NewCodec(rec), disc, 1235) + + ctx, cancel := context.WithCancel(context.Background()) + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"llama"}`)).WithContext(ctx) + + done := make(chan struct{}) + go func() { + defer close(done) + p.handleHTTP(httptest.NewRecorder(), req) + }() + + select { + case <-received: + case <-time.After(5 * time.Second): + t.Fatal("upstream never started streaming") + } + cancel() + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("handleHTTP did not return after client disconnect (zombie: handler blocked)") + } + + if got := rec.count("workload:errored"); got != 1 { + t.Fatalf("workload:errored emitted %d times, want exactly 1 (terminalOnce guard)", got) + } + if rec.has("workload:completed") { + t.Fatal("workload:completed emitted for a cancelled request") + } +} + +// deadlineRW records SetWriteDeadline calls and can force a Write error, so the +// statusCapture write-deadline mechanics can be tested without a real socket. +type deadlineRW struct { + *httptest.ResponseRecorder + deadlines []time.Time + writeErr error + flushErr error + flushed int +} + +func (d *deadlineRW) SetWriteDeadline(t time.Time) error { + d.deadlines = append(d.deadlines, t) + return nil +} + +func (d *deadlineRW) Write(b []byte) (int, error) { + if d.writeErr != nil { + return 0, d.writeErr + } + return d.ResponseRecorder.Write(b) +} + +// FlushError lets a test drive statusCapture.FlushError against a controllable +// flush outcome (recorded so the deadline arm/clear can be asserted). +func (d *deadlineRW) FlushError() error { + d.flushed++ + return d.flushErr +} + +// TestStatusCapture_WriteDeadline verifies statusCapture arms a write deadline +// around each streamed write and clears it after a successful one, and that the +// first write error is retained for the caller to classify the workload failed. +func TestStatusCapture_WriteDeadline(t *testing.T) { + t.Run("armed then cleared on success", func(t *testing.T) { + d := &deadlineRW{ResponseRecorder: httptest.NewRecorder()} + sc := &statusCapture{ResponseWriter: d, status: http.StatusOK, idle: 50 * time.Millisecond} + if _, err := sc.Write([]byte("tokens")); err != nil { + t.Fatalf("Write returned error: %v", err) + } + if len(d.deadlines) != 2 { + t.Fatalf("SetWriteDeadline called %d times, want 2 (arm + clear)", len(d.deadlines)) + } + if d.deadlines[0].IsZero() { + t.Fatal("first SetWriteDeadline should arm a future deadline, got zero") + } + if !d.deadlines[1].IsZero() { + t.Fatal("second SetWriteDeadline should clear the deadline (zero time)") + } + if sc.wroteErr != nil { + t.Fatalf("wroteErr set after a successful write: %v", sc.wroteErr) + } + }) + + t.Run("write error retained, deadline not cleared", func(t *testing.T) { + boom := errors.New("i/o timeout") + d := &deadlineRW{ResponseRecorder: httptest.NewRecorder(), writeErr: boom} + sc := &statusCapture{ResponseWriter: d, status: http.StatusOK, idle: 50 * time.Millisecond} + if _, err := sc.Write([]byte("tokens")); !errors.Is(err, boom) { + t.Fatalf("Write err = %v, want %v", err, boom) + } + if !errors.Is(sc.wroteErr, boom) { + t.Fatalf("wroteErr = %v, want %v", sc.wroteErr, boom) + } + if len(d.deadlines) != 1 { + t.Fatalf("SetWriteDeadline called %d times, want 1 (arm only; not cleared on error)", len(d.deadlines)) + } + }) + + t.Run("no deadline when idle is zero", func(t *testing.T) { + d := &deadlineRW{ResponseRecorder: httptest.NewRecorder()} + sc := &statusCapture{ResponseWriter: d, status: http.StatusOK} + if _, err := sc.Write([]byte("tokens")); err != nil { + t.Fatalf("Write returned error: %v", err) + } + if len(d.deadlines) != 0 { + t.Fatalf("SetWriteDeadline called %d times with idle=0, want 0", len(d.deadlines)) + } + }) +} + +// TestStatusCapture_FlushDeadline verifies the flush path is deadline-aware: +// statusCapture.FlushError arms the idle deadline around the underlying flush, +// clears it after a successful flush, and retains a real flush error (but not an +// unsupported-flush) so a stalled client's blocked flush is classified failed +// rather than hanging forever. +func TestStatusCapture_FlushDeadline(t *testing.T) { + t.Run("armed then cleared on success", func(t *testing.T) { + d := &deadlineRW{ResponseRecorder: httptest.NewRecorder()} + sc := &statusCapture{ResponseWriter: d, status: http.StatusOK, idle: 50 * time.Millisecond} + if err := sc.FlushError(); err != nil { + t.Fatalf("FlushError returned error: %v", err) + } + if d.flushed != 1 { + t.Fatalf("underlying flushed %d times, want 1", d.flushed) + } + if len(d.deadlines) != 2 { + t.Fatalf("SetWriteDeadline called %d times, want 2 (arm + clear)", len(d.deadlines)) + } + if d.deadlines[0].IsZero() { + t.Fatal("flush should arm a future deadline, got zero") + } + if !d.deadlines[1].IsZero() { + t.Fatal("flush should clear the deadline on success (zero time)") + } + if sc.wroteErr != nil { + t.Fatalf("wroteErr set after a successful flush: %v", sc.wroteErr) + } + }) + + t.Run("flush error retained, deadline not cleared", func(t *testing.T) { + boom := errors.New("i/o timeout") + d := &deadlineRW{ResponseRecorder: httptest.NewRecorder(), flushErr: boom} + sc := &statusCapture{ResponseWriter: d, status: http.StatusOK, idle: 50 * time.Millisecond} + if err := sc.FlushError(); !errors.Is(err, boom) { + t.Fatalf("FlushError = %v, want %v", err, boom) + } + if !errors.Is(sc.wroteErr, boom) { + t.Fatalf("wroteErr = %v, want %v", sc.wroteErr, boom) + } + if len(d.deadlines) != 1 { + t.Fatalf("SetWriteDeadline called %d times, want 1 (arm only; not cleared on error)", len(d.deadlines)) + } + }) + + t.Run("unsupported flush is not a client failure", func(t *testing.T) { + d := &deadlineRW{ResponseRecorder: httptest.NewRecorder(), flushErr: http.ErrNotSupported} + sc := &statusCapture{ResponseWriter: d, status: http.StatusOK, idle: 50 * time.Millisecond} + _ = sc.FlushError() + if sc.wroteErr != nil { + t.Fatalf("ErrNotSupported must not be retained as wroteErr, got %v", sc.wroteErr) + } + }) +} + +// TestHandleHTTP_RealSocketWriteDeadline is the end-to-end, OS-level proof of +// the zombie-job fix. It drives handleHTTP over a REAL TCP socket with a client +// that reads the response headers and then stops reading — the shape of a +// killed / half-open client whose receive window closes without a FIN/RST. The +// upstream streams without end, so the proxy's kernel send buffer to the client +// fills and its next write blocks. Before the fix that write blocks ~forever +// (no terminal event; the zombie job), so r.Context() never fires and the +// handler never returns. With the fix, statusCapture arms a real +// SetWriteDeadline that the Go runtime's netpoller enforces on every platform +// (IOCP on Windows, epoll/kqueue elsewhere) regardless of the peer's TCP state, +// so the stuck write fails and the workload terminates as failed. This is the +// piece the in-process tests stub out — here the deadline is genuinely enforced +// by the OS/runtime. +func TestHandleHTTP_RealSocketWriteDeadline(t *testing.T) { + // Shorten the idle write deadline so a stuck write trips quickly; restore + // the production default for any test that runs after this one. + orig := idleClientWriteTimeout + idleClientWriteTimeout = 300 * time.Millisecond + defer func() { idleClientWriteTimeout = orig }() + + // Upstream streams 64 KiB chunks endlessly. Once the proxy stops reading + // from it (because the proxy is itself blocked writing to the stalled + // client), the upstream's own writes block too — no busy loop — and it + // unwinds when the proxy tears the connection down (write error or context + // cancel). + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + flusher, _ := w.(http.Flusher) + chunk := bytes.Repeat([]byte("x"), 64*1024) + for { + if _, err := w.Write(chunk); err != nil { + return + } + if flusher != nil { + flusher.Flush() + } + select { + case <-r.Context().Done(): + return + default: + } + } + })) + defer upstream.Close() + + rec := &prRec{} + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "node-a", upstream.URL, "llama")) + p := NewProxy(NewCodec(rec), disc, 1235) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + srv := &http.Server{Handler: http.HandlerFunc(p.handleHTTP)} + go func() { _ = srv.Serve(ln) }() + defer srv.Close() + + conn, err := net.Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatalf("dial proxy: %v", err) + } + defer conn.Close() + + body := `{"model":"llama"}` + reqText := "POST /v1/chat/completions HTTP/1.1\r\n" + + "Host: localhost\r\n" + + "Content-Type: application/json\r\n" + + fmt.Sprintf("Content-Length: %d\r\n", len(body)) + + "\r\n" + body + if _, err := conn.Write([]byte(reqText)); err != nil { + t.Fatalf("write request: %v", err) + } + + // Read the status line only — enough to confirm the response committed and + // started streaming — then STOP reading so the proxy's send buffer backs + // up. A read deadline guards against a hang if the proxy never responds. + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + statusLine, err := bufio.NewReader(conn).ReadString('\n') + if err != nil { + t.Fatalf("read status line: %v", err) + } + if !strings.Contains(statusLine, "200") { + t.Fatalf("unexpected status line: %q", statusLine) + } + + // The stuck write must trip the deadline and terminate the workload as + // failed within a few multiples of the deadline — never completed. + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && !rec.has("workload:errored") { + time.Sleep(10 * time.Millisecond) + } + if !rec.has("workload:errored") { + t.Fatal("workload never terminated after the client stopped reading (zombie: write deadline did not trip / no terminal emitted)") + } + if rec.has("workload:completed") { + t.Fatal("workload:completed emitted for a client that stopped reading (should be failed)") + } +} + +// TestHandleHTTP_RealSocketFlushDeadline is the flush-path counterpart to the +// write-deadline test. Paced small chunks and constrained downstream TCP buffers +// create an intended flush-pressure fixture over a real socket; chunk boundaries +// may still coalesce. A stalled client must terminate the workload as failed +// rather than leaving a Flush blocked without a deadline. +func TestHandleHTTP_RealSocketFlushDeadline(t *testing.T) { + orig := idleClientWriteTimeout + idleClientWriteTimeout = 300 * time.Millisecond + defer func() { idleClientWriteTimeout = orig }() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/x-ndjson") + w.WriteHeader(http.StatusOK) + flusher, ok := w.(http.Flusher) + if !ok { + t.Errorf("upstream ResponseWriter is not a Flusher") + return + } + chunk := bytes.Repeat([]byte("x"), 1500) + for { + if _, err := w.Write(chunk); err != nil { + return + } + flusher.Flush() + // Pacing reduces coalescing into larger writes, keeping pressure + // on the flush path without assuming exact chunk boundaries. + time.Sleep(2 * time.Millisecond) + select { + case <-r.Context().Done(): + return + default: + } + } + })) + defer upstream.Close() + + rec := &prRec{} + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "node-a", upstream.URL, "llama")) + p := NewProxy(NewCodec(rec), disc, 1235) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + // Bound both downstream buffers so a non-reading client creates pressure + // within the test budget, independently of the host's default TCP buffers. + srv := &http.Server{ + Handler: http.HandlerFunc(p.handleHTTP), + ConnState: func(conn net.Conn, state http.ConnState) { + if state != http.StateNew { + return + } + tcpConn, ok := conn.(*net.TCPConn) + if !ok { + t.Errorf("accepted connection is not TCP: %T", conn) + _ = conn.Close() + return + } + if err := tcpConn.SetWriteBuffer(4096); err != nil { + t.Errorf("set downstream send buffer: %v", err) + _ = conn.Close() + } + }, + } + go func() { _ = srv.Serve(ln) }() + defer srv.Close() + + conn, err := net.Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatalf("dial proxy: %v", err) + } + defer conn.Close() + tcpConn, ok := conn.(*net.TCPConn) + if !ok { + t.Fatalf("client connection is not TCP: %T", conn) + } + if err := tcpConn.SetReadBuffer(4096); err != nil { + t.Fatalf("set downstream receive buffer: %v", err) + } + + body := `{"model":"llama"}` + reqText := "POST /v1/chat/completions HTTP/1.1\r\n" + + "Host: localhost\r\n" + + "Content-Type: application/json\r\n" + + fmt.Sprintf("Content-Length: %d\r\n", len(body)) + + "\r\n" + body + if _, err := conn.Write([]byte(reqText)); err != nil { + t.Fatalf("write request: %v", err) + } + + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + statusLine, err := bufio.NewReader(conn).ReadString('\n') + if err != nil { + t.Fatalf("read status line: %v", err) + } + if !strings.Contains(statusLine, "200") { + t.Fatalf("unexpected status line: %q", statusLine) + } + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && !rec.has("workload:errored") { + time.Sleep(10 * time.Millisecond) + } + if !rec.has("workload:errored") { + t.Fatal("workload never terminated after the client stopped reading (flush path not deadline-aware)") + } + if rec.has("workload:completed") { + t.Fatal("workload:completed emitted for a stalled client (should be failed)") + } +} diff --git a/services/nvpair-engine-manager/LAUNCH_TEXT.md b/services/nvpair-engine-manager/LAUNCH_TEXT.md index 434c9d80e..6f5ca36fd 100644 --- a/services/nvpair-engine-manager/LAUNCH_TEXT.md +++ b/services/nvpair-engine-manager/LAUNCH_TEXT.md @@ -43,7 +43,9 @@ Ollama's `OLLAMA_HOST` and LM Studio's `--port`/`-p`, `--bind`/`--host`, `LMS_SERVER_PORT` and `LMS_SERVER_HOST`. llama.cpp also protects `--host`, `--port`, `LLAMA_ARG_HOST`, `LLAMA_ARG_PORT`, port-changing presets, reuse-port controls and the PAIR-owned cache. See the [complete b12171 audit](LLAMACPP.md#binding-audit-b12171). -llama.cpp has a server-port field only; its proxy port remains zero until a later PR. +llama.cpp has both server-port and proxy-port fields (defaults 8082 and 1237). +`--model`/`-m` accepts only an existing regular GGUF resolved inside the managed +model cache, in both saved settings and previews. Saved overrides undergo the same checks before execution. Launch settings do not change the existing bind policy: default loopback and the explicit live diff --git a/services/nvpair-engine-manager/LLAMACPP.md b/services/nvpair-engine-manager/LLAMACPP.md index 99e2d3dca..5a8682d6d 100644 --- a/services/nvpair-engine-manager/LLAMACPP.md +++ b/services/nvpair-engine-manager/LLAMACPP.md @@ -3,12 +3,14 @@ SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -# llama.cpp: lifecycle-only preview +# llama.cpp PAIR manages the official **b12171 macOS arm64** build. Other platforms are -unsupported. It can install, start, stop and configure the engine, but **llama.cpp -does not serve requests through PAIR yet**. There is no proxy worker, routing, -model hub, download/delete adapter or load profile in this change. +unsupported. PAIR installs, starts, stops and configures the engine and serves +it through the dedicated `llamacpp-proxy` worker, default port **1237**. Routing +uses exact model IDs from the engine's `/v1/models` inventory, with separate +`llamacpp` attribution. There is no model hub, download/delete adapter or load +profile yet. The macOS build is **not notarized**, like this fork's PAIR application. The release is checksum-pinned, not publisher-trusted by that checksum. Its existing @@ -36,8 +38,11 @@ engine to be stopped; repeated `engine:install` applies a changed manifest pin. Default listener: **127.0.0.1:8082**, distinct from the fork's other engine and service ports. Configured-port reservations and occupied-port checks still apply. -Readiness and health require `/props` to return the top-level `"role":"router"` -identity, present in b12171 `tools/server/server-models.cpp` (`get_router_props`). +Without a model, readiness and health require `/props` to return the top-level +`"role":"router"` identity, present in b12171 `tools/server/server-models.cpp` +(`get_router_props`). With `--model`, owned-process probes instead match the +inherited descriptor's `model_path` from `tools/server/server-context.cpp` +(`get_res_props`); settings and previews retain the user's model selection. A different service returning HTTP 200 is not an engine match. Foreground-process shutdown uses the existing process lifecycle, not command-mode `wait_for_exit`. The model cache is `LLAMA_CACHE={install_dir}-models`, a PAIR-managed sibling @@ -51,9 +56,37 @@ host tools, agents, tunnels, MCP, filesystem paths and model downloads. As with accepted. Unknown options, positional arguments, `--`, repeated options (including aliases or opposing switches), non-ASCII and **all** environment assignments are invalid. Saved settings use the same checks; invalid settings are reported, not -dropped or applied. No model/download or template-file option is permitted. +dropped or applied. No download or template-file option is permitted. -Allowed value options (documented short aliases included): `--ctx-size/-c`, +`--model/-m` selects one existing regular `.gguf` file in the PAIR-owned +`{install_dir}-models` cache. Absolute and cache-relative paths are accepted; +both cache and file are resolved with `EvalSymlinks` before containment checks. +Symlink escapes, URLs, directories, missing files and non-GGUF names are rejected +in previews, saved settings and before launch. Previews retain the canonical +cache path. Immediately before spawning, PAIR resolves the cache directory once, +then opens its canonical path from `/` one directory component at a time using +`openat` with `O_DIRECTORY|O_NOFOLLOW|O_CLOEXEC`. It walks the model's +cache-relative subdirectories from that descriptor the same way, rejecting +absolute paths and parent traversal in the relative path. The final file opens +read-only with `O_NOFOLLOW` and is checked by descriptor for regular-file type +and single-file GGUF metadata. A parent or subdirectory replaced with a symlink +is never followed; no fresh pathname resolution can bless an outside file. +Only the child copy of this descriptor is inherited, through +`exec.Cmd.ExtraFiles`; llama-server +receives `--model /dev/fd/3`. The parent's copy closes after spawn, including on +failure. Replacing the original pathname cannot redirect the child's later opens. + +The b12171 loader opens metadata with `gguf_init_from_file`/`fopen("rb")`, then +reopens the path with `llama_file` for reads or `mmap` on that descriptor. macOS +`/dev/fd/3` duplicates the inherited descriptor on every open. Split GGUFs derive +sibling paths, so split filenames and `split.*` metadata (including renamed +shards) are explicitly rejected at spawn. Metadata inspection is bounded to +64 MiB; malformed or unsupported GGUF metadata fails closed. Multi-file models +and mmproj are not supported in this PR. The user supplies a single-file model +for now; PR 3 adds the hub that fills the cache. Omitting `--model` retains router +mode. + +Other allowed value options (documented short aliases included): `--ctx-size/-c`, `--batch-size/-b`, `--ubatch-size/-ub` (context 0–1048576, batches 1–1048576), `--threads/-t`, `--threads-batch/-tb` (1–4096), `--gpu-layers/--n-gpu-layers/-ngl` (0–1048576, `auto`, `all`), @@ -72,7 +105,7 @@ b12171's parser requires them. Presence-only boolean switches: `--cont-batching/ `--metrics`, `--slots`, `--no-slots`. Every allowlisted name is checked against `testdata/llamacpp-b12171-options.txt`, the option names b12171 defines; regenerate it from `common/arg.cpp` when the pinned release changes. -No arbitrary template programs, paths, URLs or non-finite numeric values are allowed. +No arbitrary template programs, other paths, URLs or non-finite numeric values are allowed. ## Binding audit (b12171) diff --git a/services/nvpair-engine-manager/README.md b/services/nvpair-engine-manager/README.md index 6114d1b2f..50db31fec 100644 --- a/services/nvpair-engine-manager/README.md +++ b/services/nvpair-engine-manager/README.md @@ -11,9 +11,10 @@ engine **except serving inference**: detect, user-mode install, start/stop/restart, health, and config-declared actions. Adding an engine is a JSON manifest, not code. -[llama.cpp](LLAMACPP.md) is a macOS arm64 lifecycle-only preview, pinned to -b12171. Its macOS build is not notarized, like PAIR itself. It does not serve -requests through PAIR yet; proxy/routing and model integration are deferred. +[llama.cpp](LLAMACPP.md) is supported on macOS arm64, pinned to b12171. Its +macOS build is not notarized, like PAIR itself. The dedicated `llamacpp-proxy` +serves it on port 1237. Select a cached GGUF with the allowlisted `--model`/`-m`; +`list_models` reads its OpenAI inventory. Hub/download/delete remain deferred. The bundled manifests under `manifests/` are the working reference for manifest authoring. [Pinned archive installs](MANIFEST.md#pinned-macos-archive-installs) diff --git a/services/nvpair-engine-manager/lifecycle.go b/services/nvpair-engine-manager/lifecycle.go index 8f5418226..8bca452f1 100644 --- a/services/nvpair-engine-manager/lifecycle.go +++ b/services/nvpair-engine-manager/lifecycle.go @@ -140,7 +140,7 @@ func (e *Executor) doStart(ctx context.Context, st *engineState, engine string, st.mu.Unlock() pathInstalled, _ := e.Detect(engine) - rt := st.plat.Runtime + rt := llamaRuntimeProbes(st.plat.Runtime, st.installDir) port := st.port if opts.Port > 0 { port = opts.Port @@ -472,7 +472,7 @@ func (e *Executor) Stop(engine string) error { // doStop performs the stop. Callers must hold st.opMu. func (e *Executor) doStop(st *engineState, engine string) error { - rt := st.plat.Runtime + rt := llamaRuntimeProbes(st.plat.Runtime, st.installDir) mode := rt.modeOrDefault() st.mu.Lock() if !st.running { @@ -784,7 +784,7 @@ func (e *Executor) reconcileFailedCommandStop(st *engineState, engine string, ru slog.Warn("engine stop command returned an error, but the endpoint is confirmed down", "engine", engine, "err", stopErr) return nil } - rt := st.plat.Runtime + rt := llamaRuntimeProbes(st.plat.Runtime, st.installDir) var port int var healthCtx context.Context var gen int64 @@ -942,7 +942,7 @@ func (e *Executor) runHealth(ctx context.Context, st *engineState, engine string if plat.Runtime.Health == nil { return } - h := *plat.Runtime.Health + h := *llamaRuntimeProbes(plat.Runtime, st.installDir).Health interval := 5 * time.Second if h.IntervalS > 0 { interval = time.Duration(h.IntervalS) * time.Second diff --git a/services/nvpair-engine-manager/llamacpp_model.go b/services/nvpair-engine-manager/llamacpp_model.go new file mode 100644 index 000000000..64bd57853 --- /dev/null +++ b/services/nvpair-engine-manager/llamacpp_model.go @@ -0,0 +1,180 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bufio" + "encoding/binary" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "regexp" + "strings" +) + +// llamaModelPath admits one existing regular GGUF inside the managed cache. +// Resolve both the cache and candidate before the containment check, including +// symlinks in intermediate directories. Return the canonical absolute path so +// the child never interprets a relative path against its working directory. +func llamaModelPath(cache, value string) (string, error) { + fail := func() (string, error) { + return "", fmt.Errorf("model must be an existing regular .gguf file inside the PAIR llama.cpp model cache") + } + if cache == "" || value == "" || strings.Contains(value, "://") { + return fail() + } + if parsed, err := url.Parse(value); err == nil && parsed.Scheme != "" && !filepath.IsAbs(value) { + return fail() + } + root, err := filepath.Abs(cache) + if err != nil { + return fail() + } + root, err = filepath.EvalSymlinks(root) + if err != nil { + return fail() + } + path := value + if !filepath.IsAbs(path) { + path = filepath.Join(root, path) + } + path, err = filepath.EvalSymlinks(path) + if err != nil { + return fail() + } + rel, err := filepath.Rel(root, path) + if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) { + return fail() + } + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || !strings.EqualFold(filepath.Ext(path), ".gguf") { + return fail() + } + return path, nil +} + +const llamaModelDescriptorPath = "/dev/fd/3" // first exec.Cmd.ExtraFiles entry + +var llamaSplitName = regexp.MustCompile(`(?i)-[0-9]{5}-of-[0-9]{5}\.gguf$`) + +// b12171 opens this path repeatedly (GGUF metadata, fitting, then llama_file +// reads/mmap). /dev/fd duplicates the inherited descriptor on macOS, so every +// open stays on the verified inode even if the original pathname is replaced. +func prepareLlamaModelArgs(args []string, cache string) ([]string, []*os.File, error) { + for i := 0; i+1 < len(args); i++ { + if args[i] != "--model" { + continue + } + if llamaSplitName.MatchString(args[i+1]) { + return nil, nil, fmt.Errorf("split/multi-file GGUF models are not supported yet; select a single-file GGUF") + } + file, err := openLlamaModel(cache, args[i+1]) + if err != nil { + return nil, nil, err + } + childArgs := append([]string(nil), args...) + childArgs[i+1] = llamaModelDescriptorPath + return childArgs, []*os.File{file}, nil + } + return args, nil, nil +} + +// Reject renamed shards as well as conventional split filenames. Read only +// bounded GGUF v2/v3 metadata, never tensors; arrays cannot nest in GGUF. +// This is not a model validator: llama.cpp still validates tensors and shape. +func rejectSplitGGUF(file *os.File) error { + invalid := fmt.Errorf("model must have valid single-file GGUF v2/v3 metadata (maximum 64 MiB)") + r := bufio.NewReader(io.NewSectionReader(file, 0, 64<<20)) + var header struct { + Magic [4]byte + Version uint32 + Tensors uint64 + Keys uint64 + } + if err := binary.Read(r, binary.LittleEndian, &header); err != nil || string(header.Magic[:]) != "GGUF" || (header.Version != 2 && header.Version != 3) || header.Keys > 1<<20 { + return invalid + } + readString := func() (string, error) { + var size uint64 + if err := binary.Read(r, binary.LittleEndian, &size); err != nil || size > 1024 { + return "", invalid + } + data := make([]byte, size) + _, err := io.ReadFull(r, data) + return string(data), err + } + skipString := func() error { + var size uint64 + if err := binary.Read(r, binary.LittleEndian, &size); err != nil || size > 64<<20 { + return invalid + } + _, err := io.CopyN(io.Discard, r, int64(size)) + return err + } + // GGUF enum: u8/i8/u16/i16/u32/i32/f32/bool/string/array/u64/i64/f64. + sizes := [...]int64{1, 1, 2, 2, 4, 4, 4, 1, 0, 0, 8, 8, 8} + for k := uint64(0); k < header.Keys; k++ { + key, err := readString() + if err != nil { + return invalid + } + if strings.HasPrefix(key, "split.") { + return fmt.Errorf("split/multi-file GGUF models are not supported yet; select a single-file GGUF") + } + var kind uint32 + if err := binary.Read(r, binary.LittleEndian, &kind); err != nil || kind >= uint32(len(sizes)) { + return invalid + } + count := uint64(1) + if kind == 9 { + if err := binary.Read(r, binary.LittleEndian, &kind); err != nil || kind >= uint32(len(sizes)) || kind == 9 { + return invalid + } + if err := binary.Read(r, binary.LittleEndian, &count); err != nil || count > 1<<20 { + return invalid + } + } + if kind == 8 { + for n := uint64(0); n < count; n++ { + if err := skipString(); err != nil { + return invalid + } + } + } else if _, err := io.CopyN(io.Discard, r, int64(count)*sizes[kind]); err != nil { + return invalid + } + } + return nil +} + +// b12171 /props reports role=router without a model, but model_path in +// single-model mode (server-context.cpp:get_res_props). Preserve identity +// checking in both modes, without mutating shared manifest probe pointers. +func llamaRuntimeProbes(rt Runtime, installDir string) Runtime { + if !rt.EditableLaunch.llamaCPP() || rt.LaunchArgs == nil { + return rt + } + args, err := normalizeLlamaOptions(*rt.LaunchArgs, installDir+"-models") + if err != nil { + return rt + } // the launch path reports invalid saved settings + for i := 0; i+1 < len(args); i++ { + if args[i] != "--model" { + continue + } + copyProbe := func(p *Probe) *Probe { + if p == nil { + return nil + } + copy := *p + copy.JSONMatch = &ProbeJSONMatch{Field: "model_path", Value: llamaModelDescriptorPath} + return © + } + rt.Ready, rt.Health = copyProbe(rt.Ready), copyProbe(rt.Health) + break + } + return rt +} diff --git a/services/nvpair-engine-manager/llamacpp_model_test.go b/services/nvpair-engine-manager/llamacpp_model_test.go new file mode 100644 index 000000000..6e8f3b052 --- /dev/null +++ b/services/nvpair-engine-manager/llamacpp_model_test.go @@ -0,0 +1,91 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "reflect" + "testing" +) + +func TestLlamaModelPathValidation(t *testing.T) { + install := filepath.Join(t.TempDir(), "llamacpp") + cache := install + "-models" + if err := os.MkdirAll(filepath.Join(cache, "nested"), 0700); err != nil { + t.Fatal(err) + } + model := filepath.Join(cache, "nested", "model.gguf") + for _, path := range []string{model, filepath.Join(cache, "model.txt"), filepath.Join(cache, "file:model.gguf")} { + if err := os.WriteFile(path, []byte("fixture"), 0600); err != nil { + t.Fatal(err) + } + } + outside := filepath.Join(t.TempDir(), "outside.gguf") + if err := os.WriteFile(outside, []byte("fixture"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(cache, "escape.gguf")); err != nil { + t.Fatal(err) + } + canonical, err := filepath.EvalSymlinks(model) + if err != nil { + t.Fatal(err) + } + for _, input := range []string{model, filepath.Join("nested", "model.gguf")} { + for _, name := range []string{"--model", "-m"} { + got, err := normalizeLlamaOptions([]string{name, input}, cache) + if err != nil || !reflect.DeepEqual(got, []string{"--model", canonical}) { + t.Fatalf("%s: %v %v", input, got, err) + } + } + } + for _, input := range []string{outside, "../outside.gguf", "escape.gguf", "model.txt", "missing.gguf", "nested", "https://example.invalid/model.gguf", "file:model.gguf", "file://" + model} { + if _, err := normalizeLlamaOptions([]string{"--model", input}, cache); err == nil { + t.Errorf("accepted %q", input) + } + } + if _, err := normalizeLlamaOptions([]string{"--model", model, "-m", model}, cache); err == nil { + t.Fatal("duplicate aliases accepted") + } + p, _ := llamaManifest(t).PlatformFor("darwin", "arm64") + args := []string{"-m", filepath.Join("nested", "model.gguf")} + p.Runtime.LaunchArgs = &args + cmd, err := resolveProcessLaunch(p.Runtime, "/fixture/server", map[string]string{"install_dir": install, "host": "127.0.0.1", "port": "8082"}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(cmd.Args[len(cmd.Args)-2:], []string{"--model", canonical}) { + t.Fatalf("argv: %v", cmd.Args) + } +} + +func TestLlamaModelProbeIdentity(t *testing.T) { + p, _ := llamaManifest(t).PlatformFor("darwin", "arm64") + install := filepath.Join(t.TempDir(), "llamacpp") + if err := os.MkdirAll(install+"-models", 0700); err != nil { + t.Fatal(err) + } + model := filepath.Join(install+"-models", "model.gguf") + if err := os.WriteFile(model, []byte("fixture"), 0600); err != nil { + t.Fatal(err) + } + model, err := filepath.EvalSymlinks(model) + if err != nil { + t.Fatal(err) + } + args := []string{"--model", model} + p.Runtime.LaunchArgs = &args + rt := llamaRuntimeProbes(p.Runtime, install) + if rt.Ready.JSONMatch.Field != "model_path" || rt.Ready.JSONMatch.Value != llamaModelDescriptorPath || rt.Health.JSONMatch.Value != llamaModelDescriptorPath { + t.Fatalf("model probe: %+v", rt.Ready) + } + if p.Runtime.Ready.JSONMatch.Field != "role" { + t.Fatal("mutated manifest probe") + } + p.Runtime.LaunchArgs = nil + if llamaRuntimeProbes(p.Runtime, install).Ready.JSONMatch.Field != "role" { + t.Fatal("router mode lost") + } +} diff --git a/services/nvpair-engine-manager/llamacpp_model_unix.go b/services/nvpair-engine-manager/llamacpp_model_unix.go new file mode 100644 index 000000000..85c8f407b --- /dev/null +++ b/services/nvpair-engine-manager/llamacpp_model_unix.go @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build !windows + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "golang.org/x/sys/unix" +) + +func openLlamaModel(cache, value string) (*os.File, error) { + root, err := filepath.Abs(cache) + if cache == "" || err != nil { + return nil, fmt.Errorf("llama.cpp model cache must be an existing directory") + } + // Resolve system aliases (e.g. /var on macOS) once. Never re-resolve the + // validated model path: doing so could bless a swapped cache parent's target. + root, err = filepath.EvalSymlinks(root) + if err != nil { + return nil, fmt.Errorf("resolve llama.cpp model cache: %w", err) + } + rel := value + if filepath.IsAbs(value) { + rel, err = filepath.Rel(root, value) + if err != nil { + return nil, fmt.Errorf("llama.cpp model must be inside the PAIR model cache") + } + } + return openLlamaModelAt(root, rel) +} + +// root is the canonical cache path, not a pathname to resolve again. Walk from +// / so a symlink swap in ANY ancestor fails rather than redirecting the open. +// Each successful open pins the next directory even if its name is then moved. +func openLlamaModelAt(root, rel string) (*os.File, error) { + parts := strings.Split(rel, string(filepath.Separator)) + if !filepath.IsAbs(root) || filepath.IsAbs(rel) || rel == "" || !strings.EqualFold(filepath.Ext(rel), ".gguf") { + return nil, fmt.Errorf("llama.cpp model must be a regular .gguf file inside the PAIR model cache") + } + for _, part := range parts { + if part == "" || part == "." || part == ".." || filepath.IsAbs(part) { + return nil, fmt.Errorf("llama.cpp model must use a cache-relative path without parent traversal") + } + } + if llamaSplitName.MatchString(rel) { + return nil, fmt.Errorf("split/multi-file GGUF models are not supported yet; select a single-file GGUF") + } + dirFlags := unix.O_RDONLY | unix.O_DIRECTORY | unix.O_NOFOLLOW | unix.O_CLOEXEC + dirfd, err := unix.Open("/", dirFlags, 0) + if err != nil { + return nil, fmt.Errorf("open llama.cpp model cache root: %w", err) + } + defer func() { _ = unix.Close(dirfd) }() + dirs := strings.Split(strings.TrimPrefix(root, string(filepath.Separator)), string(filepath.Separator)) + if root == string(filepath.Separator) { + dirs = nil + } + dirs = append(dirs, parts[:len(parts)-1]...) + for _, part := range dirs { + next, err := unix.Openat(dirfd, part, dirFlags, 0) + if err != nil { + return nil, fmt.Errorf("llama.cpp model cache directory changed or is unavailable; select an in-cache model: %w", err) + } + _ = unix.Close(dirfd) + dirfd = next + } + // Keep the parent's copy close-on-exec. ExtraFiles clears that flag only + // on the child's copy; O_NONBLOCK avoids waiting on a substituted FIFO. + fd, err := unix.Openat(dirfd, parts[len(parts)-1], unix.O_RDONLY|unix.O_NOFOLLOW|unix.O_CLOEXEC|unix.O_NONBLOCK, 0) + if err != nil { + return nil, fmt.Errorf("open llama.cpp model without symlinks: %w", err) + } + file := os.NewFile(uintptr(fd), rel) + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() { + _ = file.Close() + return nil, fmt.Errorf("llama.cpp model must be a regular .gguf file inside the PAIR model cache") + } + if err := rejectSplitGGUF(file); err != nil { + _ = file.Close() + return nil, err + } + return file, nil +} diff --git a/services/nvpair-engine-manager/llamacpp_model_unix_test.go b/services/nvpair-engine-manager/llamacpp_model_unix_test.go new file mode 100644 index 000000000..b0cd2084b --- /dev/null +++ b/services/nvpair-engine-manager/llamacpp_model_unix_test.go @@ -0,0 +1,312 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build !windows + +package main + +import ( + "encoding/binary" + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +func writeSingleGGUF(t *testing.T, path string) { + t.Helper() + data := make([]byte, 24) + copy(data, "GGUF") + binary.LittleEndian.PutUint32(data[4:], 3) + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } +} + +func TestLlamaModelSwapCannotReachOutsideCache(t *testing.T) { + for _, swapBeforeSpawn := range []bool{true, false} { + t.Run(map[bool]string{true: "before-spawn", false: "after-spawn"}[swapBeforeSpawn], func(t *testing.T) { + cache := filepath.Join(t.TempDir(), "llamacpp-models") + if err := os.Mkdir(cache, 0700); err != nil { + t.Fatal(err) + } + model := filepath.Join(cache, "model.gguf") + outside := filepath.Join(t.TempDir(), "outside.gguf") + writeSingleGGUF(t, model) + writeSingleGGUF(t, outside) + original, err := os.Stat(model) + if err != nil { + t.Fatal(err) + } + originalStat, ok := original.Sys().(*syscall.Stat_t) + if !ok { + t.Fatal("missing inode") + } + args, err := normalizeLlamaOptions([]string{"-m", model}, cache) + if err != nil { + t.Fatal(err) + } + swap := func() { + t.Helper() + if err := os.Rename(model, model+".saved"); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, model); err != nil { + t.Fatal(err) + } + } + if swapBeforeSpawn { + swap() + } + gate := filepath.Join(t.TempDir(), "go") + var mu sync.Mutex + var lines []string + proc, err := startManagedProcWithEnvironment(fakeEngineForTest(t), append([]string{"model-inode"}, args...), map[string]string{ + "LLAMA_CACHE": cache, "FAKE_MODEL_GATE": gate, "PAIR_TEST_PROCESS_ROOT": testProcessRoot, + }, false, func(_, line string) { mu.Lock(); defer mu.Unlock(); lines = append(lines, line) }) + if err != nil { + if !swapBeforeSpawn { + t.Fatal(err) + } + return // Rejecting the pre-spawn symlink is the expected safe result. + } + t.Cleanup(proc.stop) + if !swapBeforeSpawn { + swap() + } + if err := os.WriteFile(gate, nil, 0600); err != nil { + t.Fatal(err) + } + select { + case <-proc.done: + case <-time.After(10 * time.Second): + t.Fatal("fake engine did not exit") + } + mu.Lock() + defer mu.Unlock() + if len(lines) != 2 { + t.Fatalf("expected two model opens, got %v", lines) + } + for _, line := range lines { + var stat syscall.Stat_t + if err := json.Unmarshal([]byte(line), &stat); err != nil { + t.Fatal(err) + } + if stat.Dev != originalStat.Dev || stat.Ino != originalStat.Ino { + t.Fatalf("child opened outside inode %d:%d; expected verified inode %d:%d", stat.Dev, stat.Ino, originalStat.Dev, originalStat.Ino) + } + } + }) + } +} + +// Re-resolving both paths after this swap used to bless the outside inode. +func TestLlamaModelParentSwapCannotReachOutsideCache(t *testing.T) { + parent := filepath.Join(t.TempDir(), "parent") + cache := filepath.Join(parent, "llamacpp-models") + outside := t.TempDir() + for _, dir := range []string{cache, filepath.Join(outside, "llamacpp-models")} { + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + writeSingleGGUF(t, filepath.Join(dir, "model.gguf")) + } + args, err := normalizeLlamaOptions([]string{"-m", "model.gguf"}, cache) + if err != nil { + t.Fatal(err) + } + if err := os.Rename(parent, parent+".saved"); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, parent); err != nil { + t.Fatal(err) + } + proc, err := startManagedProcWithEnvironment(fakeEngineForTest(t), append([]string{"model-inode"}, args...), map[string]string{ + "LLAMA_CACHE": cache, "PAIR_TEST_PROCESS_ROOT": testProcessRoot, + }, false, nil) + if proc != nil { + proc.stop() + } + if err == nil { + t.Fatal("started with a model redirected through a swapped cache parent") + } +} + +func TestLlamaModelCanonicalParentSwapRejected(t *testing.T) { + parent := filepath.Join(t.TempDir(), "parent") + cache := filepath.Join(parent, "llamacpp-models") + if err := os.MkdirAll(cache, 0700); err != nil { + t.Fatal(err) + } + root, err := filepath.EvalSymlinks(cache) + if err != nil { + t.Fatal(err) + } + outside := t.TempDir() + if err := os.Mkdir(filepath.Join(outside, "llamacpp-models"), 0700); err != nil { + t.Fatal(err) + } + writeSingleGGUF(t, filepath.Join(outside, "llamacpp-models", "model.gguf")) + if err := os.Rename(parent, parent+".saved"); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, parent); err != nil { + t.Fatal(err) + } + file, err := openLlamaModelAt(root, "model.gguf") + if file != nil { + _ = file.Close() + } + // The directory walk must fail, before opening even the outside model. + if err == nil || !strings.Contains(err.Error(), "directory changed") { + t.Fatalf("canonical cache parent swap not rejected by directory walk: %v", err) + } +} + +func TestLlamaModelSubdirectoryLaunchClosesDescriptors(t *testing.T) { + cache := filepath.Join(t.TempDir(), "llamacpp-models") + nested := filepath.Join(cache, "nested") + if err := os.MkdirAll(nested, 0700); err != nil { + t.Fatal(err) + } + model := filepath.Join(nested, "model.gguf") + writeSingleGGUF(t, model) + original, err := os.Stat(model) + if err != nil { + t.Fatal(err) + } + originalStat, ok := original.Sys().(*syscall.Stat_t) + if !ok { + t.Fatal("missing inode") + } + args, err := normalizeLlamaOptions([]string{"-m", "nested/model.gguf"}, cache) + if err != nil { + t.Fatal(err) + } + bin := fakeEngineForTest(t) + launch := func(bin string, wantSuccess bool) { + t.Helper() + var mu sync.Mutex + var lines []string + proc, err := startManagedProcWithEnvironment(bin, append([]string{"model-inode"}, args...), map[string]string{ + "LLAMA_CACHE": cache, "PAIR_TEST_PROCESS_ROOT": testProcessRoot, + }, false, func(_, line string) { mu.Lock(); defer mu.Unlock(); lines = append(lines, line) }) + if !wantSuccess { + if proc != nil { + proc.stop() + } + if err == nil { + t.Fatal("invalid launch succeeded") + } + return + } + if err != nil { + t.Fatal(err) + } + t.Cleanup(proc.stop) + select { + case <-proc.done: + case <-time.After(10 * time.Second): + t.Fatal("fake engine did not exit") + } + mu.Lock() + defer mu.Unlock() + if len(lines) != 2 { + t.Fatalf("expected two nested model opens, got %v", lines) + } + for _, line := range lines { + var stat syscall.Stat_t + if err := json.Unmarshal([]byte(line), &stat); err != nil { + t.Fatal(err) + } + if stat.Dev != originalStat.Dev || stat.Ino != originalStat.Ino { + t.Fatal("child did not open the selected nested model") + } + } + } + countFDs := func() int { + t.Helper() + dir, err := os.Open("/dev/fd") + if err != nil { + t.Fatal(err) + } + // Descriptor entries may disappear during enumeration. Read names only: + // ReadDir can stat such an entry and fail with EBADF on macOS. + names, readErr := dir.Readdirnames(-1) + closeErr := dir.Close() + if readErr != nil { + t.Fatal(readErr) + } + if closeErr != nil { + t.Fatal(closeErr) + } + return len(names) + } + launch(bin, true) // Initialize the runtime's poller before the baseline. + before := countFDs() + for range 3 { + launch(bin, true) + launch(filepath.Join(t.TempDir(), "missing-engine"), false) + } + if err := os.Rename(nested, nested+".saved"); err != nil { + t.Fatal(err) + } + if err := os.Symlink(t.TempDir(), nested); err != nil { + t.Fatal(err) + } + launch(bin, false) // Fail after opening cache ancestors, not just at exec. + if after := countFDs(); after != before { + t.Fatalf("launch leaked descriptors: before=%d after=%d", before, after) + } +} + +func TestLlamaModelRelativeTraversalRejected(t *testing.T) { + cache := t.TempDir() + writeSingleGGUF(t, filepath.Join(cache, "model.gguf")) + for _, rel := range []string{"../model.gguf", "nested/../model.gguf", "/model.gguf", "./model.gguf"} { + file, err := openLlamaModelAt(cache, rel) + if file != nil { + _ = file.Close() + } + if err == nil { + t.Errorf("accepted non-relative or traversing model path %q", rel) + } + } +} + +func TestLlamaSplitModelRejectedAtSpawn(t *testing.T) { + cache := filepath.Join(t.TempDir(), "llamacpp-models") + if err := os.Mkdir(cache, 0700); err != nil { + t.Fatal(err) + } + model := filepath.Join(cache, "renamed.gguf") + writeSingleGGUF(t, model) + file, err := os.OpenFile(model, os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + // A renamed shard must be rejected too: split.count is GGUF uint16 (type 2). + _, _ = file.Seek(16, 0) + for _, v := range []interface{}{uint64(1), uint64(len("split.count")), []byte("split.count"), uint32(2), uint16(2)} { + if err := binary.Write(file, binary.LittleEndian, v); err != nil { + t.Fatal(err) + } + } + _ = file.Close() + // As in the production launch path, pass the canonical validated choice. + args, err := normalizeLlamaOptions([]string{"--model", model}, cache) + if err != nil { + t.Fatal(err) + } + proc, err := startManagedProcWithEnvironment(fakeEngineForTest(t), append([]string{"model-inode"}, args...), map[string]string{"LLAMA_CACHE": cache, "PAIR_TEST_PROCESS_ROOT": testProcessRoot}, false, nil) + if proc != nil { + proc.stop() + } + if err == nil || !strings.Contains(err.Error(), "split") { + t.Fatalf("split model accepted: %v", err) + } +} diff --git a/services/nvpair-engine-manager/llamacpp_model_windows.go b/services/nvpair-engine-manager/llamacpp_model_windows.go new file mode 100644 index 000000000..925e21a38 --- /dev/null +++ b/services/nvpair-engine-manager/llamacpp_model_windows.go @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" +) + +func openLlamaModel(_, _ string) (*os.File, error) { + return nil, fmt.Errorf("llama.cpp model descriptor inheritance is unsupported on Windows") +} diff --git a/services/nvpair-engine-manager/llamacpp_policy.go b/services/nvpair-engine-manager/llamacpp_policy.go index f34c6f2e3..1ef34ece4 100644 --- a/services/nvpair-engine-manager/llamacpp_policy.go +++ b/services/nvpair-engine-manager/llamacpp_policy.go @@ -27,10 +27,12 @@ type llamaOption struct { // Deliberately closed policy, audited against ggml-org/llama.cpp tag b12171: // common/arg.cpp (common_params_parser_init/common_params_parse_ex), and -// src/llama-chat.cpp (LLM_CHAT_TEMPLATES). No files, URLs or free-form programs. +// src/llama-chat.cpp (LLM_CHAT_TEMPLATES). Only contained GGUF model files; +// no URLs or free-form programs. // Unlike open server launch text, this parser also exposes host tools, agents, // tunnels, MCP and downloads. New upstream options must not become permissions. var llamaInferenceOptions = []llamaOption{ + {names: "--model -m", kind: "model"}, {names: "--ctx-size -c", kind: "int", min: 0, max: 1048576}, {names: "--batch-size -b", kind: "int", min: 1, max: 1048576}, {names: "--ubatch-size -ub", kind: "int", min: 1, max: 1048576}, @@ -61,7 +63,7 @@ var llamaInferenceOptions = []llamaOption{ {names: "--no-slots", kind: "switch", group: "slots"}, } -func normalizeLlamaOptions(args []string) ([]string, error) { +func normalizeLlamaOptions(args []string, cache string) ([]string, error) { for _, arg := range args { for _, c := range arg { if c < 32 || c > 126 { @@ -108,6 +110,12 @@ func normalizeLlamaOptions(args []string) ([]string, error) { } valid := false switch option.kind { + case "model": + path, err := llamaModelPath(cache, value) + if err != nil { + return nil, err + } + value, valid = path, true case "enum": valid = slices.Contains(strings.Fields(option.choices), value) case "gpu": @@ -155,7 +163,7 @@ func rejectLlamaConfigFiles(paths ...string) error { return nil } -func validateLlamaOverrides(rt Runtime) error { +func validateLlamaOverrides(rt Runtime, cache string) error { if !rt.EditableLaunch.llamaCPP() { return nil } @@ -163,7 +171,7 @@ func validateLlamaOverrides(rt Runtime) error { return fmt.Errorf("llama.cpp launch environment assignments are not allowed") } if rt.LaunchArgs != nil { - _, err := normalizeLlamaOptions(*rt.LaunchArgs) + _, err := normalizeLlamaOptions(*rt.LaunchArgs, cache) return err } return nil diff --git a/services/nvpair-engine-manager/llamacpp_policy_test.go b/services/nvpair-engine-manager/llamacpp_policy_test.go index 273c0e9d9..97f92145b 100644 --- a/services/nvpair-engine-manager/llamacpp_policy_test.go +++ b/services/nvpair-engine-manager/llamacpp_policy_test.go @@ -114,12 +114,12 @@ func TestLlamaAllowlistMatchesPinnedRelease(t *testing.T) { } } for _, mode := range strings.Fields("auto none mmap mlock mmap+mlock dio") { - if _, err := normalizeLlamaOptions([]string{"--load-mode", mode}); err != nil { + if _, err := normalizeLlamaOptions([]string{"--load-mode", mode}, ""); err != nil { t.Errorf("load mode %s: %v", mode, err) } } for _, rejected := range [][]string{{"--load-mode", "lazy"}, {"--mlock"}, {"--mmap"}, {"--no-mmap"}} { - if _, err := normalizeLlamaOptions(rejected); err == nil { + if _, err := normalizeLlamaOptions(rejected, ""); err == nil { t.Errorf("%v accepted", rejected) } } diff --git a/services/nvpair-engine-manager/llamacpp_test.go b/services/nvpair-engine-manager/llamacpp_test.go index 3b653848f..b0748a132 100644 --- a/services/nvpair-engine-manager/llamacpp_test.go +++ b/services/nvpair-engine-manager/llamacpp_test.go @@ -37,8 +37,8 @@ func TestLlamaManifestLifecycleOnly(t *testing.T) { if p.Install.Fetch.SHA256 != "870ef967970fa18bbd31460bc5f7929bbb5f091359d8dc0f58e20133aa4b5fcd" || !strings.HasSuffix(p.Install.Fetch.URL, "/b12171/llama-b12171-bin-macos-arm64.tar.gz") { t.Fatal("unverified release") } - if len(m.Actions) != 0 { - t.Fatal("model integration must remain deferred") + if len(m.Actions) != 1 || m.Actions["list_models"].HTTP.Path != "/v1/models" || m.Actions["list_models"].Result.Field != "id" { + t.Fatal("only read-only model inventory is supported") } if p.Runtime.Port != 8082 || p.Runtime.Bind != "127.0.0.1" || p.Runtime.Stop.WaitForExit { t.Fatal("unsafe lifecycle defaults") @@ -135,7 +135,7 @@ func TestLlamaReadinessIdentity(t *testing.T) { } } -func TestLlamaNoProxyLaunchSettings(t *testing.T) { +func TestLlamaProxyLaunchSettings(t *testing.T) { p, _ := llamaManifest(t).PlatformFor("darwin", "arm64") ex := settingsExecutor(t, true) st := settingsState(t, ex) @@ -144,16 +144,16 @@ func TestLlamaNoProxyLaunchSettings(t *testing.T) { request := settingsRequest(t, ex) request.Engine = "llamacpp" request.Settings.ServerPort = 8082 - request.Settings.ProxyPort = 0 + request.Settings.ProxyPort = 1237 request.Settings.LaunchText = "--threads 4" preview, err := ex.PreviewLaunch(request) - if err != nil || len(preview.Errors) != 0 || preview.Settings.ProxyPort != 0 { - t.Fatalf("proxy-free preview: %+v %v", preview, err) + if err != nil || len(preview.Errors) != 0 || preview.Settings.ProxyPort != 1237 { + t.Fatalf("proxy preview: %+v %v", preview, err) } - request.Settings.ProxyPort = 8081 + request.Settings.ProxyPort = 0 preview, err = ex.PreviewLaunch(request) if err == nil && len(preview.Errors) == 0 { - t.Fatal("future proxy accepted") + t.Fatal("zero proxy port accepted") } } diff --git a/services/nvpair-engine-manager/manifests/llamacpp.json b/services/nvpair-engine-manager/manifests/llamacpp.json index 746bae66b..ebf845ea9 100644 --- a/services/nvpair-engine-manager/manifests/llamacpp.json +++ b/services/nvpair-engine-manager/manifests/llamacpp.json @@ -2,6 +2,13 @@ "engine": "llamacpp", "display_name": "llama.cpp", "manifest_version": 1, + "actions": { + "list_models": { + "description": "List the model served by llama-server using its OpenAI inventory.", + "http": { "method": "GET", "path": "/v1/models" }, + "result": { "array": "data", "field": "id" } + } + }, "platforms": { "darwin/arm64": { "detect": [ diff --git a/services/nvpair-engine-manager/proc.go b/services/nvpair-engine-manager/proc.go index cd1c2375f..c84f8a7ad 100644 --- a/services/nvpair-engine-manager/proc.go +++ b/services/nvpair-engine-manager/proc.go @@ -33,7 +33,24 @@ func startManagedProc(bin string, args []string, env map[string]string, onLine f } func startManagedProcWithEnvironment(bin string, args []string, env map[string]string, inherit bool, onLine func(stream, line string)) (*managedProc, error) { + // The non-inherited environment is the controlled llama.cpp launch path. + // Pin its model immediately before spawning; never hand the child a cache + // pathname it could reopen after a symlink swap. + var files []*os.File + if !inherit { + var err error + args, files, err = prepareLlamaModelArgs(args, env["LLAMA_CACHE"]) + if err != nil { + return nil, err + } + } + defer func() { + for _, file := range files { + _ = file.Close() // parent copies, including every failed-start path + } + }() cmd := exec.Command(bin, args...) + cmd.ExtraFiles = files if len(env) > 0 || !inherit { cmd.Env = []string{} if inherit { diff --git a/services/nvpair-engine-manager/settings.go b/services/nvpair-engine-manager/settings.go index 490f0f3c8..b5aaf192e 100644 --- a/services/nvpair-engine-manager/settings.go +++ b/services/nvpair-engine-manager/settings.go @@ -17,7 +17,7 @@ import ( ) func applyLiteralLaunch(rt Runtime, command launchCommand, vars map[string]string) (launchCommand, error) { - if err := validateLlamaOverrides(rt); err != nil { + if err := validateLlamaOverrides(rt, vars["install_dir"]+"-models"); err != nil { return launchCommand{}, err } if err := rejectManagedOverrides(rt); err != nil { @@ -87,7 +87,7 @@ func applyLiteralLaunch(rt Runtime, command launchCommand, vars map[string]strin args := *rt.LaunchArgs if p.llamaCPP() { var err error - args, err = normalizeLlamaOptions(args) + args, err = normalizeLlamaOptions(args, vars["install_dir"]+"-models") if err != nil { return launchCommand{}, err } @@ -191,7 +191,7 @@ func (e *Executor) previewLaunchLocked(st *engineState, request settings.Request result := settings.Preview{Settings: request.Settings, Args: []string{}, Env: []string{}, Errors: map[string]string{}} fail := func(message string) settings.Preview { result.Errors["launchText"] = message; return result } - if (request.Engine == "llamacpp" && request.Settings.ProxyPort != 0) || (request.Engine != "llamacpp" && (request.Settings.ProxyPort < 1 || request.Settings.ProxyPort > 65535)) { + if request.Settings.ProxyPort < 1 || request.Settings.ProxyPort > 65535 { result.Errors["proxyPort"] = "Enter a proxy port from 1 to 65535." return result } @@ -239,7 +239,7 @@ func (e *Executor) previewLaunchLocked(st *engineState, request settings.Request return fail(err.Error()) } if policy.llamaCPP() { - tokens, err = normalizeLlamaOptions(tokens) + tokens, err = normalizeLlamaOptions(tokens, st.installDir+"-models") if err != nil { return fail(err.Error()) } diff --git a/services/nvpair-engine-manager/status.go b/services/nvpair-engine-manager/status.go index 4d252da24..97331c6b6 100644 --- a/services/nvpair-engine-manager/status.go +++ b/services/nvpair-engine-manager/status.go @@ -246,7 +246,7 @@ func (e *Executor) refreshPassiveStatus(engine string, st *engineState, key stat return } running, adopted, proc := st.running, st.adopted, st.proc - ready := st.plat.Runtime.Ready + ready := llamaRuntimeProbes(st.plat.Runtime, st.installDir).Ready mode := st.plat.Runtime.modeOrDefault() st.mu.Unlock() @@ -355,7 +355,7 @@ func (e *Executor) reconcilePresence(ctx context.Context, engine string, st *eng running := st.running adopted := st.adopted proc := st.proc - ready := st.plat.Runtime.Ready + ready := llamaRuntimeProbes(st.plat.Runtime, st.installDir).Ready st.mu.Unlock() // A command-mode engine needs its control CLI. A compatible HTTP endpoint diff --git a/services/nvpair-engine-manager/testdata/fakeengine/main.go b/services/nvpair-engine-manager/testdata/fakeengine/main.go index 7d65e2603..b194a5ac4 100644 --- a/services/nvpair-engine-manager/testdata/fakeengine/main.go +++ b/services/nvpair-engine-manager/testdata/fakeengine/main.go @@ -76,6 +76,11 @@ func bstr(m map[string]any, k string) string { return "" } +func filePresent(path string) bool { + _, err := os.Stat(path) + return err == nil +} + func main() { // Registration is test-only and precedes every subcommand, including // detached download children. It also follows copies of this executable. @@ -129,6 +134,30 @@ func main() { // exit (no server), standing in for a daemon's control CLI. if len(os.Args) > 1 { switch os.Args[1] { + case "model-inode": + if gate := os.Getenv("FAKE_MODEL_GATE"); gate != "" { + deadline := time.Now().Add(5 * time.Second) + for !filePresent(gate) { + if time.Now().After(deadline) { + os.Exit(2) + } + time.Sleep(time.Millisecond) + } + } + // b12171 opens the same path for metadata and again for tensors. + for i := 0; i < 2; i++ { + file, err := os.Open(os.Args[len(os.Args)-1]) + if err != nil { + os.Exit(2) + } + info, err := file.Stat() + if err != nil { + os.Exit(2) + } + _ = json.NewEncoder(os.Stdout).Encode(info.Sys()) + _ = file.Close() + } + return case "shutdown-download-fixture": if len(os.Args) < 4 { os.Exit(2) diff --git a/services/nvpair-job-scheduler/README.md b/services/nvpair-job-scheduler/README.md index 55c0a62aa..fd1132ad4 100644 --- a/services/nvpair-job-scheduler/README.md +++ b/services/nvpair-job-scheduler/README.md @@ -74,7 +74,7 @@ the order, pending counts, or pressure actually changed. ## Output -One `schedule:priority` notification per engine (`ollama`, `lmstudio`): +One `schedule:priority` notification per engine (`ollama`, `lmstudio`, `omlx`, `llamacpp`): ```json { diff --git a/services/nvpair-job-scheduler/schedule.go b/services/nvpair-job-scheduler/schedule.go index 16d939716..f88c52fbc 100644 --- a/services/nvpair-job-scheduler/schedule.go +++ b/services/nvpair-job-scheduler/schedule.go @@ -14,7 +14,7 @@ import ( // schedulerEngines is the fixed set of engine-specific output contracts. All // receive the same node-wide ranking because their work shares node resources. -var schedulerEngines = []string{"ollama", "lmstudio", "omlx"} +var schedulerEngines = []string{"ollama", "lmstudio", "omlx", "llamacpp"} // NodeRank is retained as the scheduler's public status type while the wire // definition is shared with the broker and proxies. diff --git a/services/nvpair-manual-nodes/README.md b/services/nvpair-manual-nodes/README.md index d110cf94b..f8b31e3e9 100644 --- a/services/nvpair-manual-nodes/README.md +++ b/services/nvpair-manual-nodes/README.md @@ -69,6 +69,9 @@ Emitted when a manually added node has been probed and its initial status determ "omlx_up":true, "omlx_port":1236, "omlx_models":["mlx-community/Qwen2.5-7B-Instruct-4bit"], + "llamacpp_up":true, + "llamacpp_port":8082, + "llamacpp_models":["model.gguf"], "node_info_up":true, "node_info_port":14318, "gpus":[{"name":"NVIDIA GeForce RTX 3080","utilization_percent":37}], @@ -89,7 +92,7 @@ Emitted when a manually added node has been probed and its initial status determ PAIR peers must first prove identity through pinned cluster mTLS node-info on `:14318` (or the configured `tls_port`). Matching certificate, response and broker-owned `peerMetadata` identities are required. Only authenticated metadata supplies service ports; models and engine state are then read over pinned mTLS with no HTTP or operator-CA downgrade. Completed reads recheck the exact current pin and admission-bound `peerRecordId`. -Explicitly user-added bare engines retain independent manual routes: Ollama on `:11434`, LM Studio on `:1234`, and oMLX on `:1236`. They remain labelled `identityVerification: "manual"`, never acquire peer trust, and provide no hardware scheduling telemetry. Each entry stores `kind: "pair"` or `kind: "engine"`. PAIR entries that cannot authenticate are `unverified` previews keyed `preview:`, never engine bypasses, including after worker/broker restarts or when no PAIR endpoint answers. Only an explicit `engine` choice creates a `manual:` bypass; it reads no PAIR identity, hardware, inventory or telemetry claims. The broker atomically persists kind in `configs/manual-nodes.json` and replays entries into replacement workers. Historical entries without kind are explicitly migrated on disk to `pair`: a former bare engine must be removed and re-added as **Engine only** to resume direct routing. Unknown kinds fail closed. +Explicitly user-added bare engines retain independent manual routes: Ollama on `:11434`, LM Studio on `:1234`, oMLX on `:1236`, and llama.cpp on `:8082`. They remain labelled `identityVerification: "manual"`, never acquire peer trust, and provide no hardware scheduling telemetry. Each entry stores `kind: "pair"` or `kind: "engine"`. PAIR entries that cannot authenticate are `unverified` previews keyed `preview:`, never engine bypasses, including after worker/broker restarts or when no PAIR endpoint answers. Only an explicit `engine` choice creates a `manual:` bypass; it reads no PAIR identity, hardware, inventory or telemetry claims. The broker atomically persists kind in `configs/manual-nodes.json` and replays entries into replacement workers. Historical entries without kind are explicitly migrated on disk to `pair`: a former bare engine must be removed and re-added as **Engine only** to resume direct routing. Unknown kinds fail closed. `engineNative` is the remote engine's own report of itself — the counters, the rates and the active models the node's engine-manager publishes at `GET /v1/engine-stats`. It is read over the same transport as the model list beside it, because it is the same kind of data: a peer's engine counters are cluster data, so a clustered peer is read over cluster mTLS pinned to the principal the node advertises, and there is no weaker fallback for a peer. The field is absent when the node publishes no view — an engine with no admin API, an engine that could not be reached, or a node this one cannot read. A node whose view could not be re-read keeps the view it had (the last-good rule the node-scanner follows for a discovered node). It is read on a slower cadence than the probe (once a minute), because the engine's counters move slowly and every reading is a fresh observation. @@ -210,9 +213,9 @@ Each manual node is probed every 10 seconds, with a 3-second timeout per leg, fo - **LM Studio** on port 1234: `GET /v1/models`, which doubles as the liveness check and the model list - **Node Info** on port 14318, or `tls_port`: pinned cluster mTLS hardware inventory, matching identity and authenticated service metadata (`GET /v1/node-info`). Public responses cannot establish identity or scheduling measurements. -PAIR entries use only pinned cluster mTLS node-info and authenticated engine-manager inventory. Engine entries probe Ollama, LM Studio and oMLX (`:1236`) directly and never read node-info. A node can have any combination of eligible engines, or none if unreachable. Status changes trigger `node/updated` events. Because change detection compares CPU, memory, and GPU values, a node running node-info emits a `node/updated` on most probe cycles as utilization moves. +PAIR entries use only pinned cluster mTLS node-info and authenticated engine-manager inventory. Engine entries probe Ollama, LM Studio, oMLX (`:1236`) and llama.cpp (`:8082`) directly and never read node-info. A node can have any combination of eligible engines, or none if unreachable. Status changes trigger `node/updated` events. Because change detection compares CPU, memory, and GPU values, a node running node-info emits a `node/updated` on most probe cycles as utilization moves. -Bare-engine ports are fixed (`11434`, `1234`, `1236`). PAIR engine ports come only from authenticated broker-owned service metadata; `tls_port` can override its node-info port. Unverified peers cannot supply ports or model inventory. +Bare-engine ports are fixed (`11434`, `1234`, `1236`, `8082`). PAIR engine ports come only from authenticated broker-owned service metadata; `tls_port` can override its node-info port. Unverified peers cannot supply ports or model inventory. ## Shutdown diff --git a/services/nvpair-manual-nodes/llamacpp_test.go b/services/nvpair-manual-nodes/llamacpp_test.go new file mode 100644 index 000000000..f05d3f8b1 --- /dev/null +++ b/services/nvpair-manual-nodes/llamacpp_test.go @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "net/http" + "reflect" + "testing" +) + +func TestProbeLlamaCPPReportsExactModelIDs(t *testing.T) { + manager, _, transport := newTestManager(t) + transport.set(http.MethodGet, "fixture.local:8082", "/v1/models", func(*http.Request) (*http.Response, error) { + return httpJSON(http.StatusOK, `{"data":[{"id":"model"},{"id":"model:latest"}]}`) + }) + up, models := manager.probeLlamaCPP("fixture.local", llamacppPort) + if !up || !reflect.DeepEqual(models, []string{"model", "model:latest"}) { + t.Fatalf("inventory: %v %v", up, models) + } + if up, _ := manager.probeLlamaCPP("absent.local", llamacppPort); up { + t.Fatal("absent engine reported up") + } +} diff --git a/services/nvpair-manual-nodes/manager.go b/services/nvpair-manual-nodes/manager.go index 5880523a4..7bb12fd28 100644 --- a/services/nvpair-manual-nodes/manager.go +++ b/services/nvpair-manual-nodes/manager.go @@ -199,8 +199,11 @@ type ManualNodeStatus struct { LMStudioPort int `json:"lmstudio_port"` LMStudioModels []string `json:"lmstudio_models,omitempty"` OMLXUp bool `json:"omlx_up,omitempty"` + LlamaCPPUp bool `json:"llamacpp_up,omitempty"` OMLXPort int `json:"omlx_port,omitempty"` + LlamaCPPPort int `json:"llamacpp_port,omitempty"` OMLXModels []string `json:"omlx_models,omitempty"` + LlamaCPPModels []string `json:"llamacpp_models,omitempty"` NodeInfoUp bool `json:"node_info_up"` NodeInfoPort int `json:"node_info_port"` TLSEnabled bool `json:"tls_enabled,omitempty"` @@ -724,8 +727,8 @@ func (m *Manager) probeNode(entry ManualEntry) { canProbePlaintextEngines := entry.Kind == "engine" var ( - ollamaUp, lmStudioUp, omlxUp bool - ollamaModels, lmStudioModels, omlxModels []string + ollamaUp, lmStudioUp, omlxUp, llamacppUp bool + ollamaModels, lmStudioModels, omlxModels, llamacppModels []string ) if emUp { if finalModelsByEngine != nil { @@ -741,6 +744,10 @@ func (m *Manager) probeNode(entry ManualEntry) { omlxUp = true omlxModels = omModels } + if lcModels, ok := finalModelsByEngine["llamacpp"]; ok { + llamacppUp = true + llamacppModels = lcModels + } } if canProbePlaintextEngines { if !ollamaUp { @@ -752,6 +759,9 @@ func (m *Manager) probeNode(entry ManualEntry) { if !omlxUp { omlxUp, omlxModels = m.probeOMLX(addr, omlxPort) } + if !llamacppUp { + llamacppUp, llamacppModels = m.probeLlamaCPP(addr, llamacppPort) + } } if ollamaUp && (finalModelsByEngine == nil || finalModelsByEngine["ollama"] == nil) && len(ollamaModels) > 0 { if finalModelsByEngine == nil { @@ -774,13 +784,21 @@ func (m *Manager) probeNode(entry ManualEntry) { finalModelsByEngine["omlx"] = omlxModels finalModels = mergeModels(finalModels, omlxModels) } + if llamacppUp && (finalModelsByEngine == nil || finalModelsByEngine["llamacpp"] == nil) && len(llamacppModels) > 0 { + if finalModelsByEngine == nil { + finalModelsByEngine = make(map[string][]string) + } + finalModelsByEngine["llamacpp"] = llamacppModels + finalModels = mergeModels(finalModels, llamacppModels) + } } else { if canProbePlaintextEngines { ollamaUp, ollamaModels = m.probeOllama(addr, 11434) lmStudioUp, lmStudioModels = m.probeLMStudio(addr, lmStudioPort) omlxUp, omlxModels = m.probeOMLX(addr, omlxPort) + llamacppUp, llamacppModels = m.probeLlamaCPP(addr, llamacppPort) } - finalModels = mergeModels(ollamaModels, lmStudioModels, omlxModels) + finalModels = mergeModels(ollamaModels, lmStudioModels, omlxModels, llamacppModels) finalModelsByEngine = make(map[string][]string) if len(ollamaModels) > 0 { finalModelsByEngine["ollama"] = ollamaModels @@ -791,6 +809,9 @@ func (m *Manager) probeNode(entry ManualEntry) { if len(omlxModels) > 0 { finalModelsByEngine["omlx"] = omlxModels } + if len(llamacppModels) > 0 { + finalModelsByEngine["llamacpp"] = llamacppModels + } if len(finalModelsByEngine) == 0 { finalModelsByEngine = nil } @@ -839,8 +860,11 @@ func (m *Manager) probeNode(entry ManualEntry) { LMStudioPort: lmStudioPort, LMStudioModels: lmStudioModels, OMLXUp: omlxUp, + LlamaCPPUp: llamacppUp, OMLXPort: omlxPort, + LlamaCPPPort: llamacppPort, OMLXModels: omlxModels, + LlamaCPPModels: llamacppModels, NodeInfoUp: nodeInfoUp, NodeInfoPort: nodeInfoPort, TLSEnabled: entry.TLSPort > 0, @@ -863,11 +887,13 @@ func (m *Manager) probeNode(entry ManualEntry) { newStatus.OllamaPort = info.PeerMetadata.Services[noderec.ServiceOllama] newStatus.LMStudioPort = info.PeerMetadata.Services[noderec.ServiceLMStudio] newStatus.OMLXPort = info.PeerMetadata.Services[noderec.ServiceOMLX] + newStatus.LlamaCPPPort = info.PeerMetadata.Services[noderec.ServiceLlamaCPP] newStatus.OllamaUp = newStatus.OllamaUp && newStatus.OllamaPort > 0 newStatus.LMStudioUp = newStatus.LMStudioUp && newStatus.LMStudioPort > 0 newStatus.OMLXUp = newStatus.OMLXUp && newStatus.OMLXPort > 0 + newStatus.LlamaCPPUp = newStatus.LlamaCPPUp && newStatus.LlamaCPPPort > 0 } - reachable := newStatus.OllamaUp || newStatus.LMStudioUp || newStatus.OMLXUp || newStatus.NodeInfoUp || emUp + reachable := newStatus.OllamaUp || newStatus.LMStudioUp || newStatus.OMLXUp || newStatus.LlamaCPPUp || newStatus.NodeInfoUp || emUp m.mu.Lock() tn, exists := m.nodes[id] @@ -917,13 +943,16 @@ func (m *Manager) probeNode(entry ManualEntry) { changed := prev.IdentityVerification != newStatus.IdentityVerification || prev.PeerRecordID != newStatus.PeerRecordID || prev.OllamaUp != newStatus.OllamaUp || prev.LMStudioUp != newStatus.LMStudioUp || prev.OMLXUp != newStatus.OMLXUp || + prev.LlamaCPPUp != newStatus.LlamaCPPUp || prev.OMLXPort != newStatus.OMLXPort || + prev.LlamaCPPPort != newStatus.LlamaCPPPort || prev.NodeInfoUp != newStatus.NodeInfoUp || prev.HostUUID != newStatus.HostUUID || prev.ClusterUUID != newStatus.ClusterUUID || !sliceEqual(prev.OllamaModels, newStatus.OllamaModels) || !sliceEqual(prev.LMStudioModels, newStatus.LMStudioModels) || !sliceEqual(prev.OMLXModels, newStatus.OMLXModels) || + !sliceEqual(prev.LlamaCPPModels, newStatus.LlamaCPPModels) || !sliceEqual(prev.Models, newStatus.Models) || !mapsEqual(prev.ModelsByEngine, newStatus.ModelsByEngine) || !mapsEqual(prev.LoadedByEngine, newStatus.LoadedByEngine) || @@ -994,6 +1023,7 @@ func probeFailedID(nodeID string) string { const ( lmStudioPort = 1234 omlxPort = 1236 + llamacppPort = 8082 engineManagerPort = 14322 ) @@ -1067,6 +1097,45 @@ func (m *Manager) probeLMStudio(addr string, port int) (bool, []string) { return true, models } +func (m *Manager) probeLlamaCPP(addr string, port int) (bool, []string) { + url := "http://" + net.JoinHostPort(addr, strconv.Itoa(port)) + "/v1/models" + start := time.Now() + resp, err := m.client.Get(url) + if err != nil { + slog.Debug("manual probe llama.cpp failed", + "addr", addr, "port", port, "duration_ms", time.Since(start).Milliseconds(), "err", err) + return false, nil + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + slog.Debug("manual probe llama.cpp non-OK", + "addr", addr, "port", port, "status", resp.StatusCode, + "duration_ms", time.Since(start).Milliseconds()) + return false, nil + } + var result struct { + Data []struct { + ID string `json:"id"` + } `json:"data"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, modelsResponseLimit)).Decode(&result); err != nil { + // Reachable, but the model list didn't parse — still report it up. + slog.Debug("manual probe llama.cpp up (models parse failed)", + "addr", addr, "port", port, "err", err) + return true, nil + } + models := make([]string, 0, len(result.Data)) + for _, d := range result.Data { + if d.ID != "" { + models = append(models, d.ID) + } + } + slog.Debug("manual probe llama.cpp up", + "addr", addr, "port", port, "models", len(models), + "duration_ms", time.Since(start).Milliseconds()) + return true, models +} + // probeOMLX checks oMLX's OpenAI-compatible server on addr:port. A single // GET /v1/models doubles as the liveness check and the model list. // If the target is an omlx-proxy rejecting plaintext from non-loopback, diff --git a/services/nvpair-node-scanner/README.md b/services/nvpair-node-scanner/README.md index 0fb26e707..5f60ae4e2 100644 --- a/services/nvpair-node-scanner/README.md +++ b/services/nvpair-node-scanner/README.md @@ -15,7 +15,7 @@ Concretely, the daemon: a real outage still warns when that budget ends. After the first successful read (or exhausted startup budget), failures warn immediately as before. -- **Advertises** this node's one `_nvpair-node._tcp` record, built from a registry of the local services its parent (the broker) registers — `ni`/`ol`/`lm`/`er`/`wl`/`cl`/`em`/`ec` ports plus the node identity (`uuid=`, `cluster-uuid=` when clustered, `ip=`). One record per node covers every service. +- **Advertises** this node's one `_nvpair-node._tcp` record, built from a registry of the local services its parent (the broker) registers — `ni`/`ol`/`lm`/`om`/`lc`/`er`/`wl`/`cl`/`em`/`ec` ports plus the node identity (`uuid=`, `cluster-uuid=` when clustered, `ip=`). One record per node covers every service. - **Browses** `_nvpair-node._tcp` for candidate endpoints. Verified/local records are keyed by authenticated host UUID; unverified previews have independent `preview:` record keys and an explicit `identityVerification: "unverified"` marker. A claimed UUID cannot overwrite or remove a verified record. - **Authenticates before enrichment:** remote hardware and broker-owned service metadata come from `ni` over pinned cluster mTLS, with matching certificate/response/metadata identities and an exact-current-pin check after reading. Only authenticated metadata supplies service ports. Models, per-engine attribution and loaded state then come from the authenticated `em` port over pinned mTLS. Local inventory remains an owned loopback read. Public HTTP hardware is display-only and never supplies service models or scheduler telemetry. Node-info JSON decoding reads at most 1 MiB, and model inventory decoding reads diff --git a/services/nvpair-tui/README.md b/services/nvpair-tui/README.md index 96b351dde..db07413f8 100644 --- a/services/nvpair-tui/README.md +++ b/services/nvpair-tui/README.md @@ -46,7 +46,7 @@ Tabs: | Tab | Purpose | | --- | --- | -| **Overview** | Broker version/uptime (`ping`) and all twelve workers from `workers:get-status` + full `workers:update` snapshots: state, attempts, last error. `R` restarts the selected worker after confirmation. | +| **Overview** | Broker version/uptime (`ping`) and all thirteen workers from `workers:get-status` + full `workers:update` snapshots: state, attempts, last error. `R` restarts the selected worker after confirmation. | | **Errors** | The service-error datastore (`errors:get-initial` + live `errors:update`); `c` clears the selected entry. | | **Nodes** | mDNS-discovered Ollama nodes (`discovery:subscribe` / `discovery:nodes-changed`). | | **Proxies** | Ollama, LM Studio and oMLX reverse proxies: status, cache-affinity selection rate, discovered upstreams, select a node (`enter`/`a`), set the listen port (`p`). | @@ -60,7 +60,7 @@ Tabs: Worker snapshots are refreshed on broker epoch recovery, independently of the Errors tab. The broker owns probe/retry policy; the TUI only renders its registry and relays `workers:restart {name}`. States are `running`, `degraded`, -`restarting`, `exhausted`, and `missing`. Engine-manager and the three proxies +`restarting`, `exhausted`, and `missing`. Engine-manager and the four proxies never auto-restart on a hang; use `R` if manual recovery is needed. Confirmation warns that engines may remain running or stop and inference may be interrupted. Restart initiation is not completed recovery; follow subsequent worker updates. @@ -127,9 +127,10 @@ these command-installed engines. llama.cpp is the exception: stop it first; `U` sends only `engine:install` so Go can validate and atomically exchange the PAIR-owned archive installation, keeping the previous tree on failure and preserving the sibling model cache. The engine list reports llama.cpp lifecycle -state with a note: its macOS arm64 build is not notarized, like PAIR itself, and -it does not serve requests through PAIR yet. No new proxy or model controls are -added. +state with a note: its macOS arm64 build is not notarized, like PAIR itself. +The Proxies view includes the dedicated llama.cpp worker on default port 1237, +and Overview includes it in the 13-worker registry. Model hub/download/delete +controls remain deferred. All destructive confirmations (including `u`, cluster remove/leave and manual remove) use the shared default-cancel prompt: only `y` dispatches; `n`, Esc and diff --git a/services/nvpair-tui/ui/engines.go b/services/nvpair-tui/ui/engines.go index 1ee61d012..3c004e8ef 100644 --- a/services/nvpair-tui/ui/engines.go +++ b/services/nvpair-tui/ui/engines.go @@ -624,7 +624,7 @@ func (v *enginesView) View() string { out = v.modelsView() } if v.selectedEngine() == "llamacpp" { - out += "\n" + footerStyle.Render("llama.cpp is not notarized, like PAIR itself; it does not serve requests through PAIR yet.") + out += "\n" + footerStyle.Render("llama.cpp is not notarized, like PAIR itself.") } if v.confirm.open() { out += "\n" + v.confirm.View() diff --git a/services/nvpair-tui/ui/health.go b/services/nvpair-tui/ui/health.go index f04236b55..6c502f755 100644 --- a/services/nvpair-tui/ui/health.go +++ b/services/nvpair-tui/ui/health.go @@ -212,7 +212,7 @@ func restartConsequence(worker string) string { switch worker { case "engine-manager": return "Engines PAIR started may keep running or stop; running inference may be interrupted." - case "proxy", "lmstudio-proxy", "omlx-proxy": + case "proxy", "lmstudio-proxy", "omlx-proxy", "llamacpp-proxy": return "Running inference through this proxy will be interrupted." } return "The broker restarts it under its own retry policy." diff --git a/services/nvpair-tui/ui/llamacpp_test.go b/services/nvpair-tui/ui/llamacpp_test.go index 65bd036f4..94c3aee62 100644 --- a/services/nvpair-tui/ui/llamacpp_test.go +++ b/services/nvpair-tui/ui/llamacpp_test.go @@ -14,7 +14,7 @@ func TestLlamaLifecycleNotice(t *testing.T) { v.SetSize(100, 20) v.merge(engineStatus{Engine: "llamacpp", DisplayName: "llama.cpp", Installed: true, Running: true, Port: 8082}) rendered := v.View() - for _, notice := range []string{"llama.cpp", "8082", "not notarized", "does not serve requests through PAIR yet"} { + for _, notice := range []string{"llama.cpp", "8082", "not notarized", "not notarized"} { if !strings.Contains(rendered, notice) { t.Fatalf("missing %s in %s", notice, rendered) } diff --git a/services/nvpair-tui/ui/model_test.go b/services/nvpair-tui/ui/model_test.go index bb82b0f7f..d7c3abe69 100644 --- a/services/nvpair-tui/ui/model_test.go +++ b/services/nvpair-tui/ui/model_test.go @@ -80,11 +80,11 @@ type recoveryBroker struct { } // fixtureWorkers is a full broker lifecycle snapshot in the startup order the -// registry publishes, so recovery tests can assert all twelve rows appear. +// registry publishes, so recovery tests can assert all thirteen rows appear. func fixtureWorkers() []workerStatus { names := []string{ "scanner", "errors", "settings", "engine-manager", "node-info", "proxy", - "lmstudio-proxy", "omlx-proxy", "workload-manager", "manual-nodes", + "lmstudio-proxy", "omlx-proxy", "llamacpp-proxy", "workload-manager", "manual-nodes", "cluster-manager", "scheduler", } workers := make([]workerStatus, 0, len(names)) @@ -92,7 +92,7 @@ func fixtureWorkers() []workerStatus { workers = append(workers, workerStatus{ Name: name, State: "running", - RestartOnHang: name != "engine-manager" && name != "proxy" && name != "lmstudio-proxy" && name != "omlx-proxy", + RestartOnHang: name != "engine-manager" && name != "proxy" && name != "lmstudio-proxy" && name != "omlx-proxy" && name != "llamacpp-proxy", PID: 1000, }) } @@ -164,7 +164,7 @@ func (b *recoveryBroker) serve() { switch q.Method { case "cluster:get-node-id": result = map[string]string{"nodeUuid": epoch, "nodeId": epoch + "-host", "clusterId": clusterID} - case "discovery:get-nodes", "nodes:get-initial", "nodes/list", "proxy:nodes/list", "lmstudio-proxy:nodes/list", "omlx-proxy:nodes/list": + case "discovery:get-nodes", "nodes:get-initial", "nodes/list", "proxy:nodes/list", "lmstudio-proxy:nodes/list", "omlx-proxy:nodes/list", "llamacpp-proxy:nodes/list": result = map[string]any{"nodes": []any{}} case "engine:get-installed": result = map[string]any{"engines": []any{}} @@ -281,7 +281,7 @@ func TestRecoveryReplacesEveryOwningViewAndReinstallsConnectionSubscriptions(t * e.setErrors(old) b.push(t, "B", 1, "heartbeat", map[string]any{}) l.until(t, func() bool { return l.m.hydratedEpoch == "B" }) - if len(n.nodes) != 0 || len(w.byKey) != 0 || cl.identity.NodeUUID != "B" || len(h.workers) != 12 || len(e.errs) != 0 { + if len(n.nodes) != 0 || len(w.byKey) != 0 || cl.identity.NodeUUID != "B" || len(h.workers) != 13 || len(e.errs) != 0 { t.Fatalf("stale consumers: nodes %v workloads %v identity %v workers %v errors %v", n.nodes, w.byKey, cl.identity, h.workers, e.errs) } b.mu.Lock() @@ -291,7 +291,7 @@ func TestRecoveryReplacesEveryOwningViewAndReinstallsConnectionSubscriptions(t * counts[k] = v } b.mu.Unlock() - if len(order) < 7 || order[6] != "broker:resync" { + if len(order) < 8 || order[7] != "broker:resync" { t.Fatalf("subscriptions not before resync/getters: %v", order) } for method, count := range counts { @@ -459,7 +459,7 @@ func TestRecoveryAppliesHealthyDomainsAndStaysSilentWhenOptionalGettersFail(t *t if l.m.hydratedEpoch != "A" { t.Errorf("epoch not hydrated after a successful resync with optional getter failures: %q", l.m.hydratedEpoch) } - if len(n.nodes) != 0 || len(w.byKey) != 0 || cl.identity.NodeUUID != "A" || len(h.workers) != 12 || len(e.errs) != 0 { + if len(n.nodes) != 0 || len(w.byKey) != 0 || cl.identity.NodeUUID != "A" || len(h.workers) != 13 || len(e.errs) != 0 { t.Errorf("healthy domains not applied: nodes %v workloads %v identity %v workers %v errors %v", n.nodes, w.byKey, cl.identity, h.workers, e.errs) } // A failed getter reaches its own view as that view's ordinary error result. @@ -549,14 +549,14 @@ func TestOverviewHydratesAllTwelveWorkersFromEpochRecovery(t *testing.T) { t.Fatalf("workers:get-status sent %d times, want exactly one recovery read", got) } h := l.m.views[0].(*healthView) - if len(h.workers) != 12 { - t.Fatalf("hydrated %d workers, want all 12 the broker supervises: %v", len(h.workers), h.workers) + if len(h.workers) != 13 { + t.Fatalf("hydrated %d workers, want all 13 the broker supervises: %v", len(h.workers), h.workers) } rows := h.table.Rows() - if len(rows) != 12 { - t.Fatalf("table has %d rows, want 12", len(rows)) + if len(rows) != 13 { + t.Fatalf("table has %d rows, want 13", len(rows)) } - want := []string{"scanner", "errors", "settings", "engine-manager", "node-info", "proxy", "lmstudio-proxy", "omlx-proxy", "workload-manager", "manual-nodes", "cluster-manager", "scheduler"} + want := []string{"scanner", "errors", "settings", "engine-manager", "node-info", "proxy", "lmstudio-proxy", "omlx-proxy", "llamacpp-proxy", "workload-manager", "manual-nodes", "cluster-manager", "scheduler"} for i, name := range want { if rows[i][0] != name { t.Errorf("row %d is %q, want %q in the broker's startup order", i, rows[i][0], name) diff --git a/services/nvpair-tui/ui/proxies.go b/services/nvpair-tui/ui/proxies.go index ed16b3e71..869c3852e 100644 --- a/services/nvpair-tui/ui/proxies.go +++ b/services/nvpair-tui/ui/proxies.go @@ -95,6 +95,7 @@ func newProxiesView(client *rpc.Client) *proxiesView { {label: "Ollama", prefix: "proxy", table: newTable(nil)}, {label: "LM Studio", prefix: "lmstudio-proxy", table: newTable(nil)}, {label: "oMLX", prefix: "omlx-proxy", table: newTable(nil)}, + {label: "llama.cpp", prefix: "llamacpp-proxy", table: newTable(nil)}, }, } return v @@ -216,6 +217,8 @@ func (v *proxiesView) Update(msg tea.Msg) tea.Cmd { func (v *proxiesView) handleNotification(msg *rpc.Message) tea.Cmd { idx := -1 switch { + case strings.HasPrefix(msg.Method, "llamacpp-proxy:"): + idx = 3 case strings.HasPrefix(msg.Method, "omlx-proxy:"): idx = 2 case strings.HasPrefix(msg.Method, "lmstudio-proxy:"): diff --git a/services/nvpair-tui/ui/recovery.go b/services/nvpair-tui/ui/recovery.go index 5f0e3086a..81bd5d8b5 100644 --- a/services/nvpair-tui/ui/recovery.go +++ b/services/nvpair-tui/ui/recovery.go @@ -36,7 +36,7 @@ type recoveryRead func() tea.Msg func (m Model) setupConnection() tea.Cmd { return func() tea.Msg { - methods := []string{"discovery:subscribe", "workloads:subscribe", "engine:subscribe", "proxy:subscribe", "lmstudio-proxy:subscribe", "omlx-proxy:subscribe"} + methods := []string{"discovery:subscribe", "workloads:subscribe", "engine:subscribe", "proxy:subscribe", "lmstudio-proxy:subscribe", "omlx-proxy:subscribe", "llamacpp-proxy:subscribe"} for _, method := range methods { ctx, cancel := context.WithTimeout(context.Background(), callTimeout) _, err := m.client.Call(ctx, method, nil) @@ -150,7 +150,7 @@ func (m *Model) beginRecovery() tea.Cmd { } func proxyPush(method string) bool { - return strings.HasPrefix(method, "proxy:") || strings.HasPrefix(method, "lmstudio-proxy:") || strings.HasPrefix(method, "omlx-proxy:") + return strings.HasPrefix(method, "proxy:") || strings.HasPrefix(method, "lmstudio-proxy:") || strings.HasPrefix(method, "omlx-proxy:") || strings.HasPrefix(method, "llamacpp-proxy:") } func (m Model) recoveryResultSuperseded(result tea.Msg) bool { @@ -166,7 +166,7 @@ func (m Model) recoveryResultSuperseded(result tea.Msg) bool { if idx < 0 { return false } - prefixes := []string{"proxy:", "lmstudio-proxy:", "omlx-proxy:"} + prefixes := []string{"proxy:", "lmstudio-proxy:", "omlx-proxy:", "llamacpp-proxy:"} for _, push := range m.recoveryPushes { if strings.HasPrefix(push.Msg.Method, prefixes[idx]) { return true diff --git a/services/nvpair-ui-broker/ENGINE_SETTINGS.md b/services/nvpair-ui-broker/ENGINE_SETTINGS.md index 11a566e51..e12fad483 100644 --- a/services/nvpair-ui-broker/ENGINE_SETTINGS.md +++ b/services/nvpair-ui-broker/ENGINE_SETTINGS.md @@ -8,7 +8,7 @@ SPDX-License-Identifier: Apache-2.0 The broker coordinates validation, private persistence, engine stop/restart, proxy rebind and service advertisement as one revisioned operation. Desktop clients relay commands and render reported state; they do not implement this -transaction. The fork retains its three separate proxy workers and each +transaction. The fork retains its four separate proxy workers and each worker's existing `set-port` RPC and `ready` port notification. ## Client requests @@ -61,13 +61,13 @@ PAIR-owned proxy port can change. Go rejects server-port, argument and environment edits with a message directing the user to the oMLX app. A proxy rebind never stops or restarts that app, including adopted running instances. -For llama.cpp, the same revision/recovery protocol manages the server port and -launch text without a proxy worker: `proxyPort` and `effectiveProxyPort` are -**zero**, the client hides the proxy-port field, and nonzero proxy ports are -rejected. No service is advertised and no model routing is enabled. A missing -proxy is not an editing error for this engine. Configured engine ports, existing -service reservations and live listener collisions still apply. Readiness uses -`/props` router identity. Load profiles remain LM Studio-only. See the +For llama.cpp, the same revision/recovery protocol manages server port, proxy +port and allowlisted launch text. The default server/proxy ports are 8082/1237; +the dedicated `llamacpp-proxy` uses service `lc` and the same convergence and +bounded port-write rules as the other proxies. The settings panel edits both +ports. `--model`/`-m` selects a contained regular cached GGUF, validated in Go; +readiness uses `/props` router identity without a model, or the exact canonical +`model_path` with one. Load profiles remain LM Studio-only. See the [b12171 launch audit and installation limits](../nvpair-engine-manager/LLAMACPP.md). ## Per-model LM Studio load options diff --git a/services/nvpair-ui-broker/README.md b/services/nvpair-ui-broker/README.md index 5a8925328..e203e4f67 100644 --- a/services/nvpair-ui-broker/README.md +++ b/services/nvpair-ui-broker/README.md @@ -12,7 +12,7 @@ the broker from the same installation directory. The broker is the parent process and canonical backend entry point: it supervises the worker subprocesses on the UI's behalf, speaking JSON-RPC over stdio. -The broker supervises **twelve** worker subprocesses, so a client gets the whole +The broker supervises **thirteen** worker subprocesses, so a client gets the whole backend behind one endpoint. Each is spawned at startup and relayed under its own namespace: @@ -23,6 +23,7 @@ namespace: | `ollama-proxy` | Ollama-compatible inference proxy and router | `proxy:*` | | `lmstudio-proxy` | The LM Studio counterpart, supervised identically | `lmstudio-proxy:*` | | `omlx-proxy` | OpenAI-compatible oMLX proxy; broker-negotiated listen port | `omlx-proxy:*` | +| `llamacpp-proxy` | llama.cpp routing proxy, default 1237 | `llamacpp-proxy:*` | | `nvpair-engine-manager` | Local engine and model control plane; also serves `GET /v1/models` to peers | `engine:*` | | `nvpair-cluster-manager` | Node identity, trusted-node store, PIN pairing | `cluster:*`, `nodes:*` | | `nvpair-workload-manager` | Cluster workload relay between this node and peers | `workloads:*` | @@ -42,9 +43,9 @@ lifecycle, and relay rules. These responsibilities live in the broker itself rather than in a worker: -- **Engine advertising.** The broker polls local Ollama, LM Studio, and oMLX - availability and registers each running engine's port (`ol` / `lm` / `om`) - with discovery, carrying all three in this host's single `_nvpair-node` record. The +- **Engine advertising.** The broker polls local Ollama, LM Studio, oMLX and llama.cpp + availability and registers each running engine's port (`ol` / `lm` / `om` / `lc`) + with discovery, carrying all four in this host's single `_nvpair-node` record. The model list is not part of that record — it is served over HTTP by `nvpair-engine-manager` on the `em` service and fetched by a peer's daemon during discovery enrichment. @@ -79,9 +80,9 @@ The broker probes `worker/health` (`{"ok":true}`) every 30 s with a 5 s deadline a successful probe restores `running`. After three failures, scanner, errors, settings, node-info, workload-manager, manual-nodes, cluster-manager, and scheduler are stopped through the existing bounded escalation and restarted. Engine-manager -and all three proxies only degrade on a hang: restarting them automatically can +and all four proxies only degrade on a hang: restarting them automatically can orphan engines or interrupt inference. They still accept explicit restarts. -Desktop Settings → Service → Workers shows all twelve workers and +Desktop Settings → Service → Workers shows all thirteen workers and "Not responding — restart manually" for these degraded rows; restarting engine-manager or a proxy asks for confirmation. TUI Overview uses the same registry and offers `R` with confirmation for any selected worker. @@ -107,11 +108,12 @@ The [node-owned settings API](ENGINE_SETTINGS.md) exposes `engine:get-settings`, `engine:preview-settings` and `engine:apply-settings` for local engines and verified paired devices. It validates and journals the whole change before coordinating the engine lifecycle and the selected one of the -three existing proxies. The broker, not the desktop, owns revisioning, port +four proxies. The broker, not the desktop, owns revisioning, port reservations and recovery. oMLX's app launch remains unchanged; its settings -permit only the PAIR proxy port. llama.cpp exposes the server port and launch -text with proxy port zero: no additional worker, service advertisement or routing -is enabled. Settings recovery and collision checks still apply. +permit only the PAIR proxy port. llama.cpp exposes server and proxy ports and +allowlisted launch text, including a contained cached GGUF via `--model`/`-m`. +Its dedicated worker has the same settings convergence, bounded port writer, +health supervision and manual hang-restart policy as the other proxies. ## Communication @@ -126,6 +128,7 @@ Bidirectional newline-delimited JSON-RPC 2.0 — same conventions as every other | `--node-info-path ` | `./nvpair-node-info[.exe]` in the CWD | Explicit path to the `nvpair-node-info` binary the broker should spawn. When omitted and no default sibling exists, the broker runs without the local inventory server (non-fatal); when set to an invalid path, the broker exits with an error | | `--proxy-path ` | `./ollama-proxy[.exe]` in the CWD | Explicit path to the `ollama-proxy` binary the broker spawns for the local Ollama reverse proxy. Same optional semantics as `--node-info-path`: an absent default sibling means no local proxy (non-fatal); an invalid explicit path exits with an error | | `--lmstudio-proxy-path ` | `./lmstudio-proxy[.exe]` in the CWD | Explicit path to the `lmstudio-proxy` binary the broker spawns for the local LM Studio reverse proxy. Same optional semantics as `--proxy-path` | +| `--llamacpp-proxy-path ` | `./llamacpp-proxy` | Dedicated llama.cpp proxy worker | | `--omlx-proxy-path ` | `./omlx-proxy[.exe]` in the CWD | Explicit path to the oMLX proxy. Same optional semantics as `--proxy-path`; its port is negotiated to avoid LM Studio/reservation collisions. | | `--workload-manager-path ` | `./nvpair-workload-manager[.exe]` in the CWD | Explicit path to the `nvpair-workload-manager` binary the broker spawns for the cluster workload relay. Same optional semantics as `--node-info-path`: an absent default sibling means no workload relay (non-fatal); an invalid explicit path exits with an error | | `--errors-path ` | `./nvpair-errors[.exe]` in the CWD | Explicit path to the `nvpair-errors` binary the broker spawns (with `--peer-sync`) for the service-error pipeline. Same optional semantics as `--node-info-path`: an absent default sibling means the error pipeline is disabled — producers' errors are dropped (non-fatal); an invalid explicit path exits with an error | @@ -148,7 +151,15 @@ listed above over stdio before emitting `app:ready`. Scanner initial spawn is fatal; auxiliary workers do not gate `app:ready` on readiness. The proxies announce their bound ports asynchronously. oMLX's standalone default is `1234`; the broker can move it to a free fallback starting at `1236` when LM Studio reserves `1234`. -Use the reported port, not a hardcoded endpoint. +All four proxies share fallback exclusions: every configured engine-server port +(including stopped engines and the proxy's own engine), **other engines'** +default, persisted/accepted and live proxy ports, the Ollama alias, and +fixed/registered node-service ports. A proxy does not exclude its own proxy +ports; a free saved port is tried before searching for an alternate. Caller +exclusions (such as a failed bind) and all engine-server reservations still win. +Inventory comes from `engine:configured-ports`, not a live status probe. Settings +locks are released before worker calls; port writes retain the per-engine +port-operation serialiser. Use the reported port, not a hardcoded endpoint. **`nvpair-node-scanner`** (the consolidated discovery daemon) is spawned first. It pushes `discovery:node-discovered`, `discovery:node-updated`, and `discovery:node-removed` notifications into the broker, which maintains them in an in-memory map keyed by `id`. Clients query that map via `discovery:get-nodes` and — once they've opted in via `discovery:subscribe` — receive a `discovery:nodes-changed` notification on every store mutation. The raw `discovery:node-*` notifications are never forwarded as-is. The scanner polls healthy node-info endpoints on a staggered two-second cadence, backs consecutive remote failures off to a 30-second cap, and emits compact `discovery:node-telemetry` observations containing maximum GPU utilization, validity, and age; these remain internal to broker scheduling. Those polls carry an `X-NVPAIR-Poll` intent the broker owns: `telemetry/set-scheduling-demand` tells the scanner (and `nvpair-manual-nodes`, which polls node-info on the same terms) whether any workload anywhere — queued, running, local, or a peer's — still needs fresh telemetry. While it does, the polls renew each node's five-second scheduling lease; when the last workload reaches a terminal state the broker pushes `{active:false}` and every poll becomes a background read that renews no lease, so each node falls back to its ten-second idle cadence. Both workers start inactive and are re-told on every respawn, so a restart cannot pin the fleet to a faster cadence by itself. The broker registers this node's local service ports (`ni`/`er`/`wl`/`cl`/`em`, plus `ol`/`lm`/`om` from the engine pollers) with the daemon over the same link, so the daemon can advertise them all in one `_nvpair-node` record. @@ -190,7 +201,7 @@ Shared lifecycle for all workers: - Path resolution: the explicit `--scanner-path` / `--node-info-path` / `--proxy-path` / `--workload-manager-path` / `--cluster-manager-path` (and the analogous flags for lmstudio-proxy / errors / engine-manager / manual-nodes / settings) if set, otherwise the same-named `./` (with `.exe` on Windows) in the broker's current working directory. No PATH fallback — a clear "not found" beats a surprising stale binary. A missing scanner is fatal (it's the broker's core job); every other worker is optional. - Log level: the broker's currently resolved level is forwarded at spawn time via `--log-level `. Runtime `log/set-level` requests are fanned out to every running child over its stdin (see `log/set-level` below). - Hidden console window on Windows (`CREATE_NO_WINDOW`). -- Shutdown: on signal, peer EOF, or `shutdown`, the broker joins all three proxies concurrently to close ingress, requests `engine:prepare-shutdown`, then closes worker stdin and joins started workers concurrently with bounded grace/terminate/kill escalation. Bounded process teardown is not proof of orphan-free engine cleanup; engine-manager remains ineligible for automatic hang restart. +- Shutdown: on signal, peer EOF, or `shutdown`, the broker joins all four proxies concurrently to close ingress, requests `engine:prepare-shutdown`, then closes worker stdin and joins started workers concurrently with bounded grace/terminate/kill escalation. Bounded process teardown is not proof of orphan-free engine cleanup; engine-manager remains ineligible for automatic hang restart. - **Auto-restart with crash surfacing** for every supervised worker (see [Supervision & recovery](#supervision--recovery)): a crash is reported as `supervisor:subprocess-crashed:` and the worker is restarted with backoff, clearing the entry once it's healthy again and leaving it up if the restart budget is exhausted. ## JSON-RPC Surface @@ -212,7 +223,7 @@ Shared lifecycle for all workers: `exhausted`, or `missing`), `attempts`, optional `pid` and `lastError`, `restartOnHang`, and `since` (RFC 3339 timestamp of the last state transition). Missing status names the expected binary path in `lastError`; it never applies -to scanner. Snapshots always list all twelve names in startup order: +to scanner. Snapshots always list all thirteen names in startup order: `scanner`, `errors`, `settings`, `engine-manager`, `node-info`, `proxy`, `lmstudio-proxy`, `omlx-proxy`, `workload-manager`, `manual-nodes`, `cluster-manager`, `scheduler`. Clients should fetch this snapshot after epoch changes; diff --git a/services/nvpair-ui-broker/advertiser.go b/services/nvpair-ui-broker/advertiser.go index 75f86d0b2..ec2cee440 100644 --- a/services/nvpair-ui-broker/advertiser.go +++ b/services/nvpair-ui-broker/advertiser.go @@ -24,6 +24,7 @@ const ( defaultOllamaPort = 11434 defaultLMStudioPort = 1234 defaultOMLXPort = 1235 + defaultLlamaCPPPort = 8082 // engineManagerHTTPPort is the fixed LAN port the broker tells // nvpair-engine-manager to serve its HTTP surface (/v1/models) on, and the port @@ -354,6 +355,54 @@ func (b *Broker) reconcileAdvertiseOMLX(client *http.Client) { } } +// runAutoAdvertiseLlamaCPP is the llama.cpp counterpart of runAutoAdvertise: it polls +// the local llama.cpp server and reconciles this node's om service registration +// against it, so an llama.cpp host appears on the cluster. +func (b *Broker) runAutoAdvertiseLlamaCPP(ctx context.Context) { + client := &http.Client{Timeout: 2 * time.Second} + ticker := time.NewTicker(autoAdvertiseInterval) + defer ticker.Stop() + + b.reconcileAdvertiseLlamaCPP(client) + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + b.reconcileAdvertiseLlamaCPP(client) + } + } +} + +// reconcileAdvertiseLlamaCPP brings this node's om registration into line with +// the local llama.cpp server: it advertises the promoted proxy port (never the engine) +// and hands the engine's loopback port to the llama.cpp proxy via node/set-local-backend. +func (b *Broker) reconcileAdvertiseLlamaCPP(client *http.Client) { + b.engineConfigMu.Lock() + defer b.engineConfigMu.Unlock() + if b.settingsAvailabilityBlockedLocked("llamacpp") { + return + } + if r := b.engineSettings["llamacpp"]; r != nil && r.Snapshot.Phase == "applying" { + return + } + enginePort, probe := b.localEnginePort("llamacpp", defaultLlamaCPPPort) + proxyPort := b.llamacppProxyListenPort() + if proxyPort != 0 && enginePort == proxyPort { + enginePort = 0 + probe = false + } + up := probe && proxyPort != 0 && enginePort != proxyPort && checkLMStudioHealth(client, enginePort) + if up { + b.registerService(noderec.RegisterParams{Service: noderec.ServiceLlamaCPP, Port: proxyPort}) + b.setProxyLocalBackend(b.getLlamaCPPProxy(), "llamacpp", enginePort, true) + } else { + b.unregisterService(noderec.ServiceLlamaCPP) + b.setProxyLocalBackend(b.getLlamaCPPProxy(), "llamacpp", enginePort, false) + } +} + // checkOMLXHealth reports whether a local oMLX server is answering on // the given port (native unauthenticated /health endpoint or /v1/models). func checkOMLXHealth(client *http.Client, port int) bool { diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go index 9afdaf5ee..e933a75c7 100644 --- a/services/nvpair-ui-broker/broker.go +++ b/services/nvpair-ui-broker/broker.go @@ -204,6 +204,7 @@ type Broker struct { ollamaExplicitSettings atomic.Bool lmstudioExplicitSettings atomic.Bool omlxExplicitSettings atomic.Bool + llamacppExplicitSettings atomic.Bool workerStatusMu sync.Mutex workerEntries []*workerEntry workerHealthInterval time.Duration @@ -221,6 +222,7 @@ type Broker struct { proxyPath string lmstudioProxyPath string omlxProxyPath string + llamacppProxyPath string workloadMgrPath string errorsPath string engineMgrPath string @@ -236,7 +238,7 @@ type Broker struct { telemetryReconcileMu sync.Mutex telemetryDesired telemetryDesiredConfig telemetryGeneration uint64 - telemetryApplications [3]telemetryApplication + telemetryApplications [4]telemetryApplication telemetryMu sync.Mutex telemetryStarting atomic.Bool telemetryCancelStart context.CancelFunc @@ -271,8 +273,11 @@ type Broker struct { lmstudioPortReadyOnce sync.Once lmstudioReadyMu sync.Mutex omlxProxyStartupPort atomic.Int32 + llamacppProxyStartupPort atomic.Int32 omlxProxyGeneration atomic.Uint64 + llamacppProxyGeneration atomic.Uint64 omlxProxyPublishedGeneration atomic.Uint64 + llamacppProxyPublishedGeneration atomic.Uint64 store *discoveryStore telemetry *telemetryCache // relayDir is the discovery directory, fed by the promoted daemon's @@ -301,6 +306,7 @@ type Broker struct { proxy *proxyProcess lmstudioProxy *proxyProcess omlxProxy *proxyProcess + llamacppProxy *proxyProcess workloadMgr *workloadManagerProcess errorsProc *errorsProcess engineMgr *rpcWorker @@ -317,6 +323,7 @@ type Broker struct { proxySup *supervisor lmstudioProxySup *supervisor omlxProxySup *supervisor + llamacppProxySup *supervisor workloadMgrSup *supervisor errorsSup *supervisor engineMgrSup *supervisor @@ -343,6 +350,7 @@ type Broker struct { proxySubscribed bool lmstudioProxySubscribed bool omlxProxySubscribed bool + llamacppProxySubscribed bool // workloadsMu guards workloadsSubscribed. The workloads:* stream is // opt-in too: emitWorkloadEvent (called on the proxy reader goroutine @@ -478,6 +486,7 @@ type workerPaths struct { proxy string lmstudioProxy string omlxProxy string + llamacppProxy string workloadMgr string errors string engineMgr string @@ -525,6 +534,7 @@ func NewBroker(codec *Codec, paths workerPaths) *Broker { proxyPath: paths.proxy, lmstudioProxyPath: paths.lmstudioProxy, omlxProxyPath: paths.omlxProxy, + llamacppProxyPath: paths.llamacppProxy, workloadMgrPath: paths.workloadMgr, errorsPath: paths.errors, engineMgrPath: paths.engineMgr, @@ -1703,8 +1713,8 @@ func (b *Broker) survivingAliasLocked(key string) (manualNodeStatusEntry, bool) if status.NodeInfoUp { rank = 1 } - if status.OllamaUp || status.LMStudioUp || status.OMLXUp || - len(status.OllamaModels)+len(status.LMStudioModels)+len(status.OMLXModels)+len(status.ModelsByEngine) > 0 { + if status.OllamaUp || status.LMStudioUp || status.OMLXUp || status.LlamaCPPUp || + len(status.OllamaModels)+len(status.LMStudioModels)+len(status.OMLXModels)+len(status.LlamaCPPModels)+len(status.ModelsByEngine) > 0 { rank = 2 } if rank > bestRank || (rank == bestRank && id < best) { @@ -1883,6 +1893,8 @@ func (b *Broker) proxyForEngine(engine string) *proxyProcess { return b.getLMStudioProxy() case "omlx": return b.getOMLXProxy() + case "llamacpp": + return b.getLlamaCPPProxy() default: return nil } @@ -2108,6 +2120,9 @@ func (b *Broker) Serve(ctx context.Context) error { b.prepareOMLXProxyPort() } b.startWorker("omlx-proxy") + b.prepareLlamaCPPProxyPort() + b.startWorker("llamacpp-proxy") + go b.runAutoAdvertiseLlamaCPP(ctx) // Ownership gates remain closed through retries and open only on readiness // or terminal exhaustion/missing, never just a transient spawn failure. @@ -2406,7 +2421,7 @@ func (b *Broker) routeProxyWorkload(method string, params json.RawMessage) { // to the cluster, but never into the catalog — it is a snapshot of a running // request, not a state transition. // -// All three proxies publish both kinds, so the routing is shared: a frame +// All four proxies publish both kinds, so the routing is shared: a frame // that only the Ollama proxy's path recognised would leave the LM Studio and // oMLX jobs without a live view. func (b *Broker) routeProxyWorkloadMethod(method string, params json.RawMessage) bool { @@ -2612,6 +2627,11 @@ func (b *Broker) forwardLogLevel(level string) { slog.Warn("failed to forward log/set-level to omlx-proxy", "err", err) } } + if p := b.getLlamaCPPProxy(); p != nil { + if err := p.SetLogLevel(level); err != nil { + slog.Warn("failed to forward log/set-level to llamacpp-proxy", "err", err) + } + } if wm := b.getWorkloadMgr(); wm != nil { if err := wm.SetLogLevel(level); err != nil { slog.Warn("failed to forward log/set-level to workload-manager", "err", err) @@ -3390,6 +3410,47 @@ func (b *Broker) handleMessage(msg *Message) { log.Printf("failed to respond to omlx-proxy:unsubscribe: %v", err) } + case "llamacpp-proxy:set-port": + go b.handleSettingsPortRPC(msg, "llamacpp") + + case "llamacpp-proxy:get-status": + var result ProxyStatusResult + if p := b.getLlamaCPPProxy(); p != nil { + ready, port := p.Status() + result.Ready = ready + result.Port = port + result.Requests, result.CacheAffinityHits = p.routingStatus() + } + if err := b.codec.Respond(msg.ID, result); err != nil { + log.Printf("failed to respond to llamacpp-proxy:get-status: %v", err) + } + + case "llamacpp-proxy:subscribe": + b.proxyMu.Lock() + wasSubscribed := b.llamacppProxySubscribed + b.llamacppProxySubscribed = true + b.proxyMu.Unlock() + if err := b.codec.Respond(msg.ID, SubscriptionResult{Subscribed: true}); err != nil { + log.Printf("failed to respond to llamacpp-proxy:subscribe: %v", err) + } + if !wasSubscribed { + if p := b.getLlamaCPPProxy(); p != nil { + if rp := p.ReadyParams(); rp != nil { + if err := b.codec.Notify("llamacpp-proxy:ready", rp); err != nil { + slog.Warn("emit baseline llamacpp-proxy:ready failed", "err", err) + } + } + } + } + + case "llamacpp-proxy:unsubscribe": + b.proxyMu.Lock() + b.llamacppProxySubscribed = false + b.proxyMu.Unlock() + if err := b.codec.Respond(msg.ID, SubscriptionResult{Subscribed: false}); err != nil { + log.Printf("failed to respond to llamacpp-proxy:unsubscribe: %v", err) + } + case "workloads:subscribe": b.workloadsMu.Lock() b.workloadsSubscribed = true @@ -3506,6 +3567,10 @@ func (b *Broker) handleMessage(msg *Message) { b.relayToOMLXProxy(msg) return } + if strings.HasPrefix(msg.Method, "llamacpp-proxy:") { + b.relayToLlamaCPPProxy(msg) + return + } if strings.HasPrefix(msg.Method, "lmstudio-proxy:") { b.relayToLMStudioProxy(msg) return diff --git a/services/nvpair-ui-broker/engine_settings_contract_test.go b/services/nvpair-ui-broker/engine_settings_contract_test.go index 43b54937e..41cd1a8bc 100644 --- a/services/nvpair-ui-broker/engine_settings_contract_test.go +++ b/services/nvpair-ui-broker/engine_settings_contract_test.go @@ -16,11 +16,11 @@ import ( ) func TestSettingsRebindOnlySelectedProxy(t *testing.T) { - for _, engine := range []string{"ollama", "lmstudio", "omlx"} { + for _, engine := range []string{"ollama", "lmstudio", "omlx", "llamacpp"} { t.Run(engine, func(t *testing.T) { h := newSettingsHarness(t) before := map[string]int{} - for _, name := range []string{"ollama", "lmstudio", "omlx"} { + for _, name := range []string{"ollama", "lmstudio", "omlx", "llamacpp"} { _, before[name] = h.b.settingsProxy(name).Status() } listener, err := net.Listen("tcp", "127.0.0.1:0") @@ -45,8 +45,8 @@ func TestSettingsRebindOnlySelectedProxy(t *testing.T) { } } -func TestSettingsReservesAllThreeProxies(t *testing.T) { - for _, engine := range []string{"lmstudio", "omlx"} { +func TestSettingsReservesAllFourProxies(t *testing.T) { + for _, engine := range []string{"lmstudio", "omlx", "llamacpp"} { t.Run(engine, func(t *testing.T) { h := newSettingsHarness(t) request := h.request(t) @@ -59,6 +59,8 @@ func TestSettingsReservesAllThreeProxies(t *testing.T) { h.b.engineSettings[engine] = &engineSettingsRecord{Explicit: true, Snapshot: settings.Snapshot{Settings: settings.Config{ProxyPort: port}}} if engine == "omlx" { h.b.setOMLXProxy(nil) + } else if engine == "llamacpp" { + h.b.setLlamaCPPProxy(nil) } else { h.b.setLMStudioProxy(nil) } diff --git a/services/nvpair-ui-broker/enginesettings.go b/services/nvpair-ui-broker/enginesettings.go index 4f6399af9..bb81d5877 100644 --- a/services/nvpair-ui-broker/enginesettings.go +++ b/services/nvpair-ui-broker/enginesettings.go @@ -181,6 +181,9 @@ func (b *Broker) settingsProxy(engine string) *proxyProcess { if engine == "omlx" { return b.getOMLXProxy() } + if engine == "llamacpp" { + return b.getLlamaCPPProxy() + } return nil } @@ -196,7 +199,7 @@ func (b *Broker) settingsSnapshotLocked(ctx context.Context, engine string) (set return settings.Snapshot{}, err } proxyPort := 0 - ready := engine == "llamacpp" // lifecycle-only; no proxy exists yet + ready := false if proxy := b.settingsProxy(engine); proxy != nil { ready, proxyPort = proxy.Status() } @@ -290,10 +293,7 @@ func (b *Broker) sendSettingsPublications() { } func (b *Broker) validateSettingsPortsLocked(ctx context.Context, engine string, config settings.Config, current settings.Snapshot) error { - if engine == "llamacpp" && config.ProxyPort != 0 { - return fmt.Errorf("llama.cpp has no PAIR proxy yet; proxy port must be zero") - } - if config.ServerPort < 1 || config.ServerPort > 65535 || (engine != "llamacpp" && (config.ProxyPort < 1 || config.ProxyPort > 65535)) { + if config.ServerPort < 1 || config.ServerPort > 65535 || (config.ProxyPort < 1 || config.ProxyPort > 65535) { return fmt.Errorf("ports must be between 1 and 65535") } if config.ServerPort == config.ProxyPort { @@ -328,7 +328,7 @@ func (b *Broker) validateSettingsPortsLocked(ctx context.Context, engine string, return fmt.Errorf("a selected port is reserved by another configured engine") } } - for _, other := range []string{"ollama", "lmstudio", "omlx"} { + for _, other := range []string{"ollama", "lmstudio", "omlx", "llamacpp"} { if other == engine { continue } @@ -775,12 +775,6 @@ func (b *Broker) rebindSettingsProxy(ctx context.Context, engine string, port in } func (b *Broker) rebindSettingsProxyOn(ctx context.Context, p *proxyProcess, engine string, port int) error { - if engine == "llamacpp" { - if p != nil || port != 0 { - return fmt.Errorf("llama.cpp has no PAIR proxy yet") - } - return ctx.Err() - } if p == nil { return fmt.Errorf("proxy unavailable") } @@ -841,6 +835,8 @@ func engineSettingsService(engine string) noderec.ServiceKey { return noderec.ServiceLMStudio case "omlx": return noderec.ServiceOMLX + case "llamacpp": + return noderec.ServiceLlamaCPP default: return "" } @@ -854,6 +850,8 @@ func (b *Broker) setExplicitSettings(engine string, value bool) { b.lmstudioExplicitSettings.Store(value) case "omlx": b.omlxExplicitSettings.Store(value) + case "llamacpp": + b.llamacppExplicitSettings.Store(value) } } @@ -875,5 +873,7 @@ func (b *Broker) settingsStartupPort(engine string, port int) { b.lmstudioProxyStartupPort.Store(int32(port)) case "omlx": b.omlxProxyStartupPort.Store(int32(port)) + case "llamacpp": + b.llamacppProxyStartupPort.Store(int32(port)) } } diff --git a/services/nvpair-ui-broker/enginesettings_test.go b/services/nvpair-ui-broker/enginesettings_test.go index 562324837..9a5062ee5 100644 --- a/services/nvpair-ui-broker/enginesettings_test.go +++ b/services/nvpair-ui-broker/enginesettings_test.go @@ -40,7 +40,7 @@ type settingsHarness struct { func newSettingsHarness(t *testing.T) *settingsHarness { t.Helper() - ports := make([]int, 4) + ports := make([]int, 5) listeners := []net.Listener{} for i := range ports { ln, err := net.Listen("tcp", "127.0.0.1:0") @@ -56,7 +56,7 @@ func newSettingsHarness(t *testing.T) *settingsHarness { h := &settingsHarness{b: &Broker{codec: NewCodec(&bytes.Buffer{}), nodeID: "target", clusterDir: filepath.Join(t.TempDir(), "cluster")}, notifications: make(chan *Message, 100)} worker, codec := newTestRPCWorkerPipe(t) h.b.setEngineMgr(worker) - for i, engine := range []string{"ollama", "lmstudio", "omlx"} { + for i, engine := range []string{"ollama", "lmstudio", "omlx", "llamacpp"} { proxyWorker, proxyCodec := newTestRPCWorkerPipe(t) proxy := &proxyProcess{peer: proxyWorker.peer, ready: true, port: ports[i+1]} switch engine { @@ -66,6 +66,8 @@ func newSettingsHarness(t *testing.T) *settingsHarness { h.b.setLMStudioProxy(proxy) case "omlx": h.b.setOMLXProxy(proxy) + case "llamacpp": + h.b.setLlamaCPPProxy(proxy) } go func() { for { diff --git a/services/nvpair-ui-broker/llamacpp_settings_test.go b/services/nvpair-ui-broker/llamacpp_settings_test.go index 395f8cbb0..0a8a77138 100644 --- a/services/nvpair-ui-broker/llamacpp_settings_test.go +++ b/services/nvpair-ui-broker/llamacpp_settings_test.go @@ -13,15 +13,15 @@ import ( "nvpair-shared/noderec" ) -func TestLlamaSettingsWithoutProxy(t *testing.T) { +func TestLlamaSettingsWithProxy(t *testing.T) { h := newSettingsHarness(t) ctx := context.Background() snapshot, err := h.b.getEngineSettings(ctx, settings.Request{Engine: "llamacpp"}, "") if err != nil { t.Fatal(err) } - if !snapshot.Editable || snapshot.Settings.ProxyPort != 0 || snapshot.EffectiveProxyPort != 0 || h.b.settingsProxy("llamacpp") != nil { - t.Fatalf("llama.cpp requires a proxy: %+v", snapshot) + if !snapshot.Editable || snapshot.Settings.ProxyPort == 0 || snapshot.EffectiveProxyPort == 0 || h.b.settingsProxy("llamacpp") == nil { + t.Fatalf("llama.cpp proxy unavailable: %+v", snapshot) } request := settings.Request{Engine: "llamacpp", ExpectedRevision: snapshot.Revision, RequestID: settingsID(), Settings: snapshot.Settings} request.Settings.LaunchText = "--threads 4" @@ -34,11 +34,11 @@ func TestLlamaSettingsWithoutProxy(t *testing.T) { t.Fatalf("apply: %+v %v", receipt, err) } after, err := h.b.getEngineSettings(ctx, settings.Request{Engine: "llamacpp"}, "") - if err != nil || after.Settings.LaunchText != "--threads 4" || !after.Running || after.EffectiveProxyPort != 0 { + if err != nil || after.Settings.LaunchText != "--threads 4" || !after.Running || after.EffectiveProxyPort != snapshot.EffectiveProxyPort { t.Fatalf("result: %+v %v", after, err) } - if engineSettingsService("llamacpp") != "" { - t.Fatal("lifecycle-only engine maps to a routed service") + if engineSettingsService("llamacpp") != noderec.ServiceLlamaCPP { + t.Fatal("missing llama.cpp service") } // In particular a settings change must not unregister another engine's service. if engineSettingsService("omlx") != noderec.ServiceOMLX { @@ -61,14 +61,14 @@ func TestLlamaSettingsPortReservations(t *testing.T) { t.Fatal(err) } request := settings.Request{Engine: "llamacpp", ExpectedRevision: s.Revision, Settings: s.Settings} - for _, port := range []int{1, 8081} { + for _, port := range []int{0, 65536} { request.Settings.ProxyPort = port p, err := h.b.previewEngineSettings(ctx, request, "") if err == nil && len(p.Errors) == 0 { - t.Fatal("proxy port accepted before proxy integration") + t.Fatal("invalid proxy port accepted") } } - request.Settings.ProxyPort = 0 + request.Settings.ProxyPort = s.Settings.ProxyPort h.otherEnginePort.Store(int32(request.Settings.ServerPort)) p, err := h.b.previewEngineSettings(ctx, request, "") if err == nil && !strings.Contains(p.Errors["ports"], "configured engine") { @@ -87,7 +87,7 @@ func TestLlamaSettingsPortReservations(t *testing.T) { } } -func TestLlamaSettingsRecoveryWithoutProxy(t *testing.T) { +func TestLlamaSettingsRecoveryWithProxy(t *testing.T) { h := newSettingsHarness(t) ctx := context.Background() s, err := h.b.getEngineSettings(ctx, settings.Request{Engine: "llamacpp"}, "") @@ -105,7 +105,7 @@ func TestLlamaSettingsRecoveryWithoutProxy(t *testing.T) { if excluded := h.b.recoverEngineSettings(); len(excluded) != 0 { t.Fatalf("recovery blocked: %v", excluded) } - if r.Snapshot.Phase != "succeeded" || r.Snapshot.Settings.ProxyPort != 0 { + if r.Snapshot.Phase != "succeeded" || r.Snapshot.Settings.ProxyPort != s.Settings.ProxyPort { t.Fatalf("bad recovery: %+v", r.Snapshot) } } diff --git a/services/nvpair-ui-broker/llamacppproxy.go b/services/nvpair-ui-broker/llamacppproxy.go new file mode 100644 index 000000000..7e2e95f96 --- /dev/null +++ b/services/nvpair-ui-broker/llamacppproxy.go @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "log/slog" + "net/http" + "strings" + + "nvpair-shared/applog" + "nvpair-shared/noderec" +) + +// llamacppproxy.go is the broker's llama.cpp counterpart to its ollama-proxy and +// lmstudio-proxy wiring. llamacpp-proxy speaks the exact same JSON-RPC control plane +// as ollama-proxy/lmstudio-proxy (a "ready" port notification, +// node/add-manual/remove-manual, nodes/list, node/select, the workload:* lifecycle +// stream), so it reuses the proxyProcess client type; only the namespace differs +// — the broker relays it under llamacpp-proxy: instead of proxy: or lmstudio-proxy:. + +func (b *Broker) setLlamaCPPProxy(p *proxyProcess) { + b.workersMu.Lock() + b.llamacppProxy = p + b.workersMu.Unlock() +} + +func (b *Broker) getLlamaCPPProxy() *proxyProcess { + b.workersMu.Lock() + defer b.workersMu.Unlock() + return b.llamacppProxy +} + +func (b *Broker) llamacppProxyListenPort() int { + p := b.getLlamaCPPProxy() + if p == nil { + return 0 + } + ready, port := p.Status() + if !ready { + return 0 + } + return port +} + +func (b *Broker) configureLlamaCPPProxySupervisorCallbacks(sup *supervisor) { + sup.onCrash, sup.onRecovered = b.supervisedWorkerCallbacks("llamacpp-proxy", func() { b.setLlamaCPPProxy(nil) }) + sup.onExhausted = func(attempt int) { + slog.Warn("llamacpp-proxy is terminally unavailable", "attempt", attempt) + } +} + +func (b *Broker) llamacppProxyArgs() []string { + var args []string + if port := int(b.llamacppProxyStartupPort.Load()); port != 0 { + args = []string{"--port", fmt.Sprintf("%d", port), "--ignore-persisted-port"} + } + return append(args, b.clusterDirArgs()...) +} + +func (b *Broker) spawnLlamaCPPProxy() (supervisedHandle, error) { + finish := b.beginPortOperation("llamacpp") + defer finish() + generation := b.llamacppProxyGeneration.Add(1) + pp, err := startProxy( + "llamacpp-proxy", + b.llamacppProxyPath, + applog.LevelString(), + b.relayDir, + func(method string, params json.RawMessage) { + b.forwardLlamaCPPProxyNotificationForGeneration(generation, method, params) + }, + b.llamacppProxyArgs()..., + ) + if err != nil { + return nil, err + } + b.setLlamaCPPProxy(pp) + if ready, _ := pp.Status(); ready { + go b.reconcileTelemetry() + } + b.llamacppProxyPublishedGeneration.Store(generation) + if ready, port := pp.Status(); ready && port > 0 { + go b.reconcileLlamaCPPProxyPortOnReadyForGeneration(generation, port) + } + slog.Info("llamacpp-proxy started", "path", b.llamacppProxyPath, "pid", pp.cmd.Process.Pid) + return pp, nil +} + +func (b *Broker) forwardLlamaCPPProxyNotification(method string, params json.RawMessage) { + b.forwardLlamaCPPProxyNotificationForGeneration(b.llamacppProxyGeneration.Load(), method, params) +} + +func (b *Broker) forwardLlamaCPPProxyNotificationForGeneration(generation uint64, method string, params json.RawMessage) { + if b.llamacppProxyGeneration.Load() != generation { + return + } + if b.dispatchErrorsNotif("llamacpp-proxy", method, params) { + return + } + if method == "error" { + var ep struct { + Code string `json:"code"` + Port int `json:"port"` + } + if json.Unmarshal(params, &ep) == nil && ep.Code == "bind-failed" && !b.llamacppExplicitSettings.Load() { + go b.recoverProxyBindFailure("llamacpp", generation, ep.Port) + } + } + if b.routeProxyWorkloadMethod(method, params) { + return + } + if method == noderec.NotifyNodeActivity { + b.routeNodeActivity(params) + return + } + if method == "ready" { + go b.reconcileTelemetry() + var rp proxyReadyParams + if err := json.Unmarshal(params, &rp); err == nil && rp.Port > 0 { + go b.reconcileLlamaCPPProxyPortOnReadyForGeneration(generation, rp.Port) + } + } + b.proxyMu.Lock() + subscribed := b.llamacppProxySubscribed + b.proxyMu.Unlock() + if !subscribed { + return + } + if err := b.codec.Notify("llamacpp-proxy:"+method, params); err != nil { + slog.Warn("forward llamacpp-proxy notification failed", "method", method, "err", err) + } +} + +func (b *Broker) reconcileLlamaCPPProxyPortOnReadyForGeneration(generation uint64, boundPort int) { + finish := b.beginPortOperation("llamacpp") + defer finish() + if b.llamacppProxyGeneration.Load() != generation { + return + } + b.repushPriority("llamacpp") + b.reconcileAdvertiseLlamaCPP(http.DefaultClient) +} + +// Prepare accepted settings before spawning; implicit defaults may move to an +// unoccupied port, while explicit settings never silently change. +func (b *Broker) prepareLlamaCPPProxyPort() { + finish := b.beginPortOperation("llamacpp") + defer finish() + if b.prepareExplicitEngineSettings("llamacpp") || b.llamacppProxyStartupPort.Load() != 0 { + return + } + b.setLlamaCPPProxyFallback() +} + +func (b *Broker) setLlamaCPPProxyFallback(excluded ...int) int { + port := b.nextProxyFallbackPort("llamacpp", 1237, excluded, tcpPortAvailable) + b.llamacppProxyStartupPort.Store(int32(port)) + return port +} + +func (b *Broker) relayToLlamaCPPProxy(msg *Message) { + method := strings.TrimPrefix(msg.Method, "llamacpp-proxy:") + if method == "shutdown" { + if err := b.codec.RespondError(msg.ID, -32601, "llamacpp-proxy:shutdown is not allowed; the broker owns the proxy lifecycle"); err != nil { + log.Printf("failed to respond to llamacpp-proxy:shutdown: %v", err) + } + return + } + + p := b.getLlamaCPPProxy() + if p == nil { + if err := b.codec.RespondError(msg.ID, -32000, "llamacpp-proxy not available"); err != nil { + log.Printf("failed to respond to %s: %v", msg.Method, err) + } + return + } + + result, rpcErr, err := p.Call(context.Background(), method, msg.Params) + switch { + case err != nil: + if err := b.codec.RespondError(msg.ID, -32000, fmt.Sprintf("llamacpp-proxy call failed: %v", err)); err != nil { + log.Printf("failed to respond to %s: %v", msg.Method, err) + } + case rpcErr != nil: + if err := b.codec.RespondError(msg.ID, rpcErr.Code, rpcErr.Message); err != nil { + log.Printf("failed to relay llamacpp-proxy error for %s: %v", msg.Method, err) + } + default: + if err := b.codec.Respond(msg.ID, result); err != nil { + log.Printf("failed to relay llamacpp-proxy result for %s: %v", msg.Method, err) + } + } +} diff --git a/services/nvpair-ui-broker/llamacppproxy_test.go b/services/nvpair-ui-broker/llamacppproxy_test.go new file mode 100644 index 000000000..ce22678ae --- /dev/null +++ b/services/nvpair-ui-broker/llamacppproxy_test.go @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "io" + "net/http" + "strings" + "testing" + "time" + + "nvpair-shared/noderec" + "nvpair-ui-broker/relay" +) + +type llamaHealthTransport struct { + status int + t *testing.T +} + +func (f *llamaHealthTransport) RoundTrip(r *http.Request) (*http.Response, error) { + if r.URL.Port() != "8082" || r.URL.Path != "/v1/models" { + f.t.Errorf("health target: %s", r.URL) + } + return &http.Response{StatusCode: f.status, Body: io.NopCloser(strings.NewReader(`{"data":[{"id":"model.gguf"}]}`)), Header: make(http.Header)}, nil +} + +func TestLlamaCPPAdvertisesProxyAndLoopbackEngine(t *testing.T) { + worker, codec := newTestRPCWorkerPipe(t) + backends := make(chan proxyLocalBackend, 2) + go func() { + for { + msg, err := codec.Read() + if err != nil { + return + } + if msg.Method == "node/set-local-backend" { + var backend proxyLocalBackend + _ = json.Unmarshal(msg.Params, &backend) + backends <- backend + } + if msg.IsRequest() { + _ = codec.Respond(msg.ID, nil) + } + } + }() + b := &Broker{regCache: relay.NewRegistrationCache()} + b.setLlamaCPPProxy(&proxyProcess{peer: worker.peer, ready: true, port: 1237}) + transport := &llamaHealthTransport{status: http.StatusOK, t: t} + b.reconcileAdvertiseLlamaCPP(&http.Client{Transport: transport}) + records := b.regCache.Snapshot() + if len(records) != 1 || records[0].Service != noderec.ServiceLlamaCPP || records[0].Port != 1237 { + t.Fatalf("advertisement: %+v", records) + } + select { + case backend := <-backends: + if backend.Engine != "llamacpp" || backend.Host != "127.0.0.1" || backend.Port != 8082 || !backend.Healthy { + t.Fatalf("backend: %+v", backend) + } + case <-time.After(time.Second): + t.Fatal("backend not delivered") + } + transport.status = http.StatusServiceUnavailable + b.reconcileAdvertiseLlamaCPP(&http.Client{Transport: transport}) + if len(b.regCache.Snapshot()) != 0 { + t.Fatal("unhealthy service remained advertised") + } + select { + case backend := <-backends: + if backend.Healthy { + t.Fatal("unhealthy engine retained") + } + case <-time.After(time.Second): + t.Fatal("clear not delivered") + } +} + +func TestLlamaCPPProxyPathResolution(t *testing.T) { + if _, err := resolveLlamaCPPProxyPath("/nonexistent/pair-test/llamacpp-proxy"); err == nil { + t.Fatal("invalid explicit path accepted") + } +} diff --git a/services/nvpair-ui-broker/lmstudioport.go b/services/nvpair-ui-broker/lmstudioport.go index 4ad93bea0..0c4b43fb4 100644 --- a/services/nvpair-ui-broker/lmstudioport.go +++ b/services/nvpair-ui-broker/lmstudioport.go @@ -125,9 +125,6 @@ func (b *Broker) reportLMStudioPortOwnershipBlocked(reason string) { } func (b *Broker) setLMStudioProxyFallback(excludedPorts ...int) int { - if aliasPort := b.currentOllamaHostAlias().Port; aliasPort > 0 { - excludedPorts = append(excludedPorts, aliasPort) - } if backend := int(b.lmstudioBackendPort.Load()); backend > 0 { excludedPorts = append(excludedPorts, backend) } else { @@ -136,7 +133,7 @@ func (b *Broker) setLMStudioProxyFallback(excludedPorts ...int) int { // free proxy port. excludedPorts = append(excludedPorts, managedLMStudioFacadePort, managedLMStudioBackendStart) } - fallback := nextAvailablePortExcluding(managedLMStudioBackendStart, excludedPorts, tcpPortAvailable) + fallback := b.nextProxyFallbackPort("lmstudio", managedLMStudioBackendStart, excludedPorts, tcpPortAvailable) b.lmstudioProxyStartupPort.Store(int32(fallback)) return fallback } diff --git a/services/nvpair-ui-broker/lmstudioport_test.go b/services/nvpair-ui-broker/lmstudioport_test.go index d845438ab..bd1c2c8c5 100644 --- a/services/nvpair-ui-broker/lmstudioport_test.go +++ b/services/nvpair-ui-broker/lmstudioport_test.go @@ -865,7 +865,7 @@ func TestUnmanagedLMStudioCollisionRebindsBeforeStatusProbe(t *testing.T) { } }() go func() { - msg, err := engineCodec.Read() + msg, err := readPastFallbackInventory(engineCodec, 12400) if err != nil { return } @@ -930,7 +930,7 @@ func TestUnknownLMStudioBackendMovesProxyBeforeStatusProbe(t *testing.T) { } }() go func() { - msg, err := engineCodec.Read() + msg, err := readPastFallbackInventory(engineCodec, managedLMStudioBackendStart) if err != nil { return } @@ -1074,7 +1074,7 @@ func TestEngineManagerRespawnReconcilesUnknownCustomBackendBeforeRestore(t *test order <- status.Method _ = newEngineCodec.Respond(status.ID, ollamaPortStatus{Port: 12400}) - restore, err := newEngineCodec.Read() + restore, err := readPastFallbackInventory(newEngineCodec, 12400) if err == nil { order <- restore.Method } diff --git a/services/nvpair-ui-broker/main.go b/services/nvpair-ui-broker/main.go index 7db6caeeb..42bce5e83 100644 --- a/services/nvpair-ui-broker/main.go +++ b/services/nvpair-ui-broker/main.go @@ -28,6 +28,7 @@ func main() { proxyPath := flag.String("proxy-path", "", "path to ollama-proxy binary (default: ./ollama-proxy in the current working directory)") lmstudioProxyPath := flag.String("lmstudio-proxy-path", "", "path to lmstudio-proxy binary (default: ./lmstudio-proxy in the current working directory)") omlxProxyPath := flag.String("omlx-proxy-path", "", "path to omlx-proxy binary (default: ./omlx-proxy in the current working directory)") + llamacppProxyPath := flag.String("llamacpp-proxy-path", "", "path to llamacpp-proxy binary (default: ./llamacpp-proxy in the current working directory)") workloadMgrPath := flag.String("workload-manager-path", "", "path to nvpair-workload-manager binary (default: ./nvpair-workload-manager in the current working directory)") errorsPath := flag.String("errors-path", "", "path to nvpair-errors binary (default: ./nvpair-errors in the current working directory)") engineMgrPath := flag.String("engine-manager-path", "", "path to nvpair-engine-manager binary (default: ./nvpair-engine-manager in the current working directory)") @@ -157,6 +158,15 @@ func main() { resolvedOMLXProxy = "" } + resolvedLlamaCPPProxy, err := resolveLlamaCPPProxyPath(*llamacppProxyPath) + if err != nil { + if *llamacppProxyPath != "" { + fatalf("llamacpp-proxy binary: %v", err) + } + slog.Warn("llamacpp-proxy binary not found; broker will run without local llama.cpp proxy", "err", err) + resolvedLlamaCPPProxy = "" + } + // nvpair-workload-manager is auxiliary too, resolved with the same rules: // an explicit --workload-manager-path that doesn't exist is a loud // operator mistake (fatal), but an absent default sibling just means @@ -277,6 +287,7 @@ func main() { proxy: resolvedProxy, lmstudioProxy: resolvedLMStudioProxy, omlxProxy: resolvedOMLXProxy, + llamacppProxy: resolvedLlamaCPPProxy, workloadMgr: resolvedWorkloadMgr, errors: resolvedErrors, engineMgr: resolvedEngineMgr, @@ -353,6 +364,10 @@ func resolveOMLXProxyPath(override string) (string, error) { return resolveSiblingBinary(override, "omlx-proxy", "--omlx-proxy-path") } +func resolveLlamaCPPProxyPath(override string) (string, error) { + return resolveSiblingBinary(override, "llamacpp-proxy", "--llamacpp-proxy-path") +} + // resolveWorkloadManagerPath mirrors resolveProxyPath for the // nvpair-workload-manager binary the broker supervises. Like node-info and the // proxy its result is optional at the call site: a not-found default sibling diff --git a/services/nvpair-ui-broker/managedport.go b/services/nvpair-ui-broker/managedport.go index 13cb5e909..bd5de6e05 100644 --- a/services/nvpair-ui-broker/managedport.go +++ b/services/nvpair-ui-broker/managedport.go @@ -177,7 +177,12 @@ func (b *Broker) reconcileCurrentOllamaPorts(boundPort int) { })) b.ollamaBackendPort.Store(int32(status.Port)) if plan.Blocked != "" { - b.blockManagedOllamaFacade(plan.Blocked, status.Port) + if !b.ollamaPortIOUnlockedLocked(func() { b.setOllamaProxyFallback(status.Port) }) { + b.engineConfigMu.Unlock() + continue + } + b.recordBlockedOllamaFacade(plan.Blocked) + b.markOllamaPortReady() } else { b.managedOllamaBackend.Store(int32(plan.BackendPort)) } diff --git a/services/nvpair-ui-broker/manualnodes.go b/services/nvpair-ui-broker/manualnodes.go index 651ba7299..d7df35d85 100644 --- a/services/nvpair-ui-broker/manualnodes.go +++ b/services/nvpair-ui-broker/manualnodes.go @@ -41,8 +41,11 @@ type manualNodeStatus struct { LMStudioPort int `json:"lmstudio_port"` LMStudioModels []string `json:"lmstudio_models,omitempty"` OMLXUp bool `json:"omlx_up"` + LlamaCPPUp bool `json:"llamacpp_up"` OMLXPort int `json:"omlx_port"` + LlamaCPPPort int `json:"llamacpp_port"` OMLXModels []string `json:"omlx_models,omitempty"` + LlamaCPPModels []string `json:"llamacpp_models,omitempty"` NodeInfoUp bool `json:"node_info_up"` NodeInfoPort int `json:"node_info_port"` GPUs []GPUInfo `json:"gpus"` @@ -491,7 +494,7 @@ func manualToEnriched(s manualNodeStatus) EnrichedNode { } models := s.Models if len(models) == 0 { - models = mergeModels(s.OllamaModels, s.LMStudioModels, s.OMLXModels) + models = mergeModels(s.OllamaModels, s.LMStudioModels, s.OMLXModels, s.LlamaCPPModels) } modelsByEngine := s.ModelsByEngine if len(modelsByEngine) == 0 { @@ -619,6 +622,19 @@ func (b *Broker) manualToDirectoryNode(s manualNodeStatus, key string) noderec.D } services[noderec.ServiceOMLX] = noderec.ServiceStatus{Port: port} } + llamacppUp := s.LlamaCPPUp + if !llamacppUp && modelsByEngine != nil { + if _, ok := modelsByEngine["llamacpp"]; ok { + llamacppUp = true + } + } + if llamacppUp { + port := s.LlamaCPPPort + if port == 0 { + port = 8082 + } + services[noderec.ServiceLlamaCPP] = noderec.ServiceStatus{Port: port} + } ips := []string{} if s.Address != "" { @@ -627,7 +643,7 @@ func (b *Broker) manualToDirectoryNode(s manualNodeStatus, key string) noderec.D models := s.Models if len(models) == 0 { - models = mergeModels(s.OllamaModels, s.LMStudioModels, s.OMLXModels) + models = mergeModels(s.OllamaModels, s.LMStudioModels, s.OMLXModels, s.LlamaCPPModels) } verification := "unverified" @@ -701,6 +717,9 @@ func manualModelsByEngine(s manualNodeStatus) map[string][]string { if len(s.OMLXModels) > 0 { byEngine["omlx"] = s.OMLXModels } + if len(s.LlamaCPPModels) > 0 { + byEngine["llamacpp"] = s.LlamaCPPModels + } if len(byEngine) == 0 { return nil } @@ -787,6 +806,15 @@ func (b *Broker) bridgeManualNode(s manualNodeStatus, key string) { omlxPort = 1236 } b.bridgeToProxy(b.getOMLXProxy(), "omlx", s, key, omlxUp, omlxPort, omlxModels) + llamacppModels := s.LlamaCPPModels + if len(llamacppModels) == 0 && s.ModelsByEngine != nil { + llamacppModels = s.ModelsByEngine["llamacpp"] + } + llamacppPort := s.LlamaCPPPort + if llamacppPort == 0 { + llamacppPort = 8082 + } + b.bridgeToProxy(b.getLlamaCPPProxy(), "llamacpp", s, key, s.LlamaCPPUp || len(llamacppModels) > 0, llamacppPort, llamacppModels) } // bridgeToProxy adds the node to p when its engine is reachable, or removes it @@ -824,6 +852,7 @@ func (b *Broker) removeManualNodeFromProxies(id string) { b.callProxyManual(b.getProxy(), "ollama", "node/remove-manual", map[string]string{"id": id}, id) b.callProxyManual(b.getLMStudioProxy(), "lmstudio", "node/remove-manual", map[string]string{"id": id}, id) b.callProxyManual(b.getOMLXProxy(), "omlx", "node/remove-manual", map[string]string{"id": id}, id) + b.callProxyManual(b.getLlamaCPPProxy(), "llamacpp", "node/remove-manual", map[string]string{"id": id}, id) } // callProxyManual issues a best-effort node/add-manual|remove-manual to a diff --git a/services/nvpair-ui-broker/ollamahost.go b/services/nvpair-ui-broker/ollamahost.go index 68a5d4c59..9a893be9d 100644 --- a/services/nvpair-ui-broker/ollamahost.go +++ b/services/nvpair-ui-broker/ollamahost.go @@ -326,20 +326,49 @@ func (b *Broker) configuredEnginePorts() (map[int]string, error) { // configuredLMStudioProxyPort mirrors lmstudio-proxy's small persisted-port // contract so the Ollama alias cannot win its bind race. Invalid/missing state // means the proxy will use its default. -func configuredLMStudioProxyPort() int { - path, err := appdir.Path(lmstudioProxyPortFile) - if err != nil { +func configuredLMStudioProxyPort() int { return configuredProxyPort("lmstudio") } + +func configuredProxyPort(engine string) int { + if port := persistedProxyPort(engine); port != 0 { + return port + } + switch engine { + case "lmstudio": return managedLMStudioFacadePort + case "omlx": + return 1236 + case "llamacpp": + return 1237 + default: + return managedOllamaFacadePort + } +} + +// Return only a saved choice, not a standalone default: oMLX shares its +// standalone :1234 default with LM Studio, but uses :1236 under the broker. +func persistedProxyPort(engine string) int { + file := "proxy-port.json" + switch engine { + case "lmstudio": + file = lmstudioProxyPortFile + case "omlx": + file = "omlx-proxy-port.json" + case "llamacpp": + file = "llamacpp-proxy-port.json" + } + path, err := appdir.Path(file) + if err != nil { + return 0 } data, err := os.ReadFile(path) if err != nil { - return managedLMStudioFacadePort + return 0 } var stored struct { Port int `json:"port"` } if json.Unmarshal(data, &stored) != nil || stored.Port < 1 || stored.Port > 65535 { - return managedLMStudioFacadePort + return 0 } return stored.Port } diff --git a/services/nvpair-ui-broker/ollamahost_test.go b/services/nvpair-ui-broker/ollamahost_test.go index 9350ec6a3..088b91a91 100644 --- a/services/nvpair-ui-broker/ollamahost_test.go +++ b/services/nvpair-ui-broker/ollamahost_test.go @@ -192,7 +192,7 @@ func brokerWithEngineInventory(t *testing.T, ports map[string]int) *Broker { return } _ = codec.Respond(request.ID, map[string]any{"port": ports[params.Engine]}) - case "engine:get-installed": + case "engine:get-installed", "engine:configured-ports": engines := make([]map[string]any, 0, len(ports)) for engine, port := range ports { engines = append(engines, map[string]any{"engine": engine, "port": port}) diff --git a/services/nvpair-ui-broker/omlxproxy.go b/services/nvpair-ui-broker/omlxproxy.go index f5bf5a787..b13d021a5 100644 --- a/services/nvpair-ui-broker/omlxproxy.go +++ b/services/nvpair-ui-broker/omlxproxy.go @@ -191,17 +191,7 @@ func (b *Broker) setOMLXProxyFallback(excludedPorts ...int) int { } func (b *Broker) setOMLXProxyFallbackWithPortCheck(portAvailable func(int) bool, excludedPorts ...int) int { - if aliasPort := b.currentOllamaHostAlias().Port; aliasPort > 0 { - excludedPorts = append(excludedPorts, aliasPort) - } - if lmPort := b.lmstudioProxyListenPort(); lmPort > 0 { - excludedPorts = append(excludedPorts, lmPort) - } - if lmStPort := int(b.lmstudioProxyStartupPort.Load()); lmStPort > 0 { - excludedPorts = append(excludedPorts, lmStPort) - } - excludedPorts = append(excludedPorts, 1234, 1235) - fallback := nextAvailablePortExcluding(1236, excludedPorts, portAvailable) + fallback := b.nextProxyFallbackPort("omlx", 1236, excludedPorts, portAvailable) b.omlxProxyStartupPort.Store(int32(fallback)) return fallback } diff --git a/services/nvpair-ui-broker/peer_inventory.go b/services/nvpair-ui-broker/peer_inventory.go index 9dd20d606..d55a244ad 100644 --- a/services/nvpair-ui-broker/peer_inventory.go +++ b/services/nvpair-ui-broker/peer_inventory.go @@ -46,7 +46,8 @@ func (b *Broker) normalizeManualStatus(s manualNodeStatus) manualNodeStatus { s.OllamaUp = false s.LMStudioUp = false s.OMLXUp = false - s.Models, s.OllamaModels, s.LMStudioModels, s.OMLXModels = nil, nil, nil, nil + s.LlamaCPPUp = false + s.Models, s.OllamaModels, s.LMStudioModels, s.OMLXModels, s.LlamaCPPModels = nil, nil, nil, nil, nil s.ModelsByEngine, s.LoadedByEngine = nil, nil s.GPUs, s.CPU, s.Memory = nil, nil, nil s.EngineNative = nil diff --git a/services/nvpair-ui-broker/portoperation.go b/services/nvpair-ui-broker/portoperation.go index 39c02b2b4..7d0ec89ab 100644 --- a/services/nvpair-ui-broker/portoperation.go +++ b/services/nvpair-ui-broker/portoperation.go @@ -96,6 +96,8 @@ func (b *Broker) proxyStartupPort(engine string) int { return int(b.lmstudioProxyStartupPort.Load()) case "omlx": return int(b.omlxProxyStartupPort.Load()) + case "llamacpp": + return int(b.llamacppProxyStartupPort.Load()) } return 0 } diff --git a/services/nvpair-ui-broker/portoperation_test.go b/services/nvpair-ui-broker/portoperation_test.go index 7fc5028dc..dcadd9736 100644 --- a/services/nvpair-ui-broker/portoperation_test.go +++ b/services/nvpair-ui-broker/portoperation_test.go @@ -83,7 +83,7 @@ func TestPortOperationConvergenceFailureBlocksStart(t *testing.T) { } func TestPortOperationCorrectsStaleReadyToAcceptedPort(t *testing.T) { - for _, engine := range []string{"ollama", "lmstudio", "omlx"} { + for _, engine := range []string{"ollama", "lmstudio", "omlx", "llamacpp"} { t.Run(engine, func(t *testing.T) { h := newSettingsHarness(t) request := h.request(t) @@ -108,6 +108,8 @@ func TestPortOperationCorrectsStaleReadyToAcceptedPort(t *testing.T) { h.b.reconcileProxyPortOnReady(11434) case "lmstudio": h.b.reconcileLMStudioProxyPortOnReady(11434) + case "llamacpp": + h.b.reconcileLlamaCPPProxyPortOnReadyForGeneration(0, 11434) case "omlx": h.b.reconcileOMLXProxyPortOnReadyForGeneration(0, 11434) } diff --git a/services/nvpair-ui-broker/proxyfallback.go b/services/nvpair-ui-broker/proxyfallback.go new file mode 100644 index 000000000..f35851e1e --- /dev/null +++ b/services/nvpair-ui-broker/proxyfallback.go @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "log/slog" + "slices" +) + +// Called under the target engine's port-operation serialiser. Snapshot local +// settings under their short lock, then release it BEFORE any worker call. +// Stopped engines reserve their configured ports just as running ones do. +func (b *Broker) proxyFallbackExclusions(engine string, excluded []int) []int { + // Engine-server defaults are never proxy candidates, even for their owner. + excluded = append(excluded, 11435, 1235, 8082, + engineManagerHTTPPort, engineControlPort, nodeInfoHTTPPort, errorsHTTPPort, workloadHTTPPort, clusterManagerHTTPPort) + if port := b.currentOllamaHostAlias().Port; port > 0 { + excluded = append(excluded, port) + } + b.engineConfigMu.Lock() + for other, record := range b.engineSettings { + if record != nil { + excluded = append(excluded, record.Snapshot.Settings.ServerPort, record.Previous.ServerPort) + if other != engine { + excluded = append(excluded, record.Snapshot.Settings.ProxyPort, record.Previous.ProxyPort) + } + } + } + b.engineConfigMu.Unlock() + for other, defaultPort := range map[string]int{"ollama": 11434, "lmstudio": 1234, "omlx": 1236, "llamacpp": 1237} { + if other == engine { + continue + } + excluded = append(excluded, defaultPort, b.proxyStartupPort(other), configuredProxyPort(other)) + if proxy := b.settingsProxy(other); proxy != nil { + _, port := proxy.Status() + excluded = append(excluded, port) + } + } + if b.regCache != nil { + for _, service := range b.regCache.Snapshot() { + if service.Service != engineSettingsService(engine) { + excluded = append(excluded, service.Port) + } + } + } + if b.getEngineMgr() != nil { + var configured struct { + Engines []struct { + Port int `json:"port"` + } `json:"engines"` + } + ctx, cancel := context.WithTimeout(context.Background(), rpcWorkerCallTimeout) + err := b.settingsWorkerCall(ctx, "engine:configured-ports", nil, &configured) + cancel() + if err != nil { + slog.Warn("cannot verify configured engine ports for proxy fallback", "engine", engine, "err", err) + // No safe candidate is proven. Do not probe arbitrary ports on a + // failed inventory read; the next supervised attempt can retry. + return nil + } + for _, configuredEngine := range configured.Engines { + excluded = append(excluded, configuredEngine.Port) + } + } + return excluded +} + +func (b *Broker) nextProxyFallbackPort(engine string, start int, excluded []int, available func(int) bool) int { + ports := b.proxyFallbackExclusions(engine, excluded) + if ports == nil { + return 0 + } + // Startup overrides must not discard a usable saved choice. A bind-failure + // caller explicitly excludes the failed port, so recovery still moves on. + if saved := persistedProxyPort(engine); saved > 0 && !slices.Contains(ports, saved) && available(saved) { + return saved + } + port := nextAvailablePortExcluding(start, ports, available) + if port > 0 { + slog.Info("selected proxy fallback port", "engine", engine, "port", port) + } + return port +} diff --git a/services/nvpair-ui-broker/proxyfallback_test.go b/services/nvpair-ui-broker/proxyfallback_test.go new file mode 100644 index 000000000..a81adde19 --- /dev/null +++ b/services/nvpair-ui-broker/proxyfallback_test.go @@ -0,0 +1,240 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "os" + "path/filepath" + "slices" + "testing" + + "nvpair-shared/appdir" + settings "nvpair-shared/enginesettings" + "nvpair-shared/noderec" + "nvpair-ui-broker/relay" +) + +func TestProxyFallbackAllowsOwnDefaultAndSavedPorts(t *testing.T) { + isolateOllamaHostTestConfig(t) + for _, tc := range []struct { + engine string + port int + }{ + {"ollama", 11434}, {"lmstudio", 1234}, {"omlx", 1236}, {"llamacpp", 1237}, + } { + t.Run(tc.engine, func(t *testing.T) { + b := &Broker{engineSettings: map[string]*engineSettingsRecord{ + tc.engine: {Snapshot: settings.Snapshot{Settings: settings.Config{ServerPort: 12410, ProxyPort: tc.port}}, Previous: settings.Config{ServerPort: 12412, ProxyPort: 12413}}, + }, regCache: relay.NewRegistrationCache()} + b.settingsStartupPort(tc.engine, 12414) + b.regCache.Register(noderec.RegisterParams{Service: engineSettingsService(tc.engine), Port: 12415}) + // A generation can still report its previous port during recovery. + proxy := &proxyProcess{ready: true, port: 12416} + switch tc.engine { + case "ollama": + b.setProxy(proxy) + case "lmstudio": + b.setLMStudioProxy(proxy) + case "omlx": + b.setOMLXProxy(proxy) + case "llamacpp": + b.setLlamaCPPProxy(proxy) + } + ports := b.proxyFallbackExclusions(tc.engine, []int{12417}) + for _, own := range []int{tc.port, 12413, 12414, 12415, 12416} { + if slices.Contains(ports, own) { + t.Errorf("own port %d excluded: %v", own, ports) + } + if got := b.nextProxyFallbackPort(tc.engine, own, nil, func(int) bool { return true }); got != own { + t.Errorf("selected %d, want own port %d", got, own) + } + } + for _, reserved := range []int{11435, 1235, 8082, 12410, 12412, 12417} { + if !slices.Contains(ports, reserved) { + t.Errorf("reserved port %d allowed", reserved) + } + } + for _, other := range []int{11434, 1234, 1236, 1237} { + if other != tc.port && !slices.Contains(ports, other) { + t.Errorf("other proxy default %d allowed", other) + } + } + if got := b.nextProxyFallbackPort(tc.engine, tc.port, []int{tc.port}, func(int) bool { return true }); got == tc.port { + t.Fatal("caller exclusion ignored for own port") + } + }) + } +} + +func TestProxyFallbackPrefersPersistedPort(t *testing.T) { + for engine, file := range map[string]string{ + "ollama": "proxy-port.json", "lmstudio": "lmstudio-proxy-port.json", + "omlx": "omlx-proxy-port.json", "llamacpp": "llamacpp-proxy-port.json", + } { + t.Run(engine, func(t *testing.T) { + isolateOllamaHostTestConfig(t) + path, err := appdir.Path(file) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(`{"port":12420}`), 0600); err != nil { + t.Fatal(err) + } + b := &Broker{} + free := func(int) bool { return true } + if got := b.nextProxyFallbackPort(engine, 1236, nil, free); got != 12420 { + t.Fatalf("discarded free saved port: got %d, want 12420", got) + } + if got := b.nextProxyFallbackPort(engine, 1236, []int{12420}, free); got == 12420 { + t.Fatal("bind-failure exclusion ignored for saved port") + } + if got := b.nextProxyFallbackPort(engine, 1236, nil, func(port int) bool { return port != 12420 }); got == 12420 { + t.Fatal("occupied saved port reused") + } + if engine == "omlx" { + b.lmstudioProxyPath = "/fixture/lmstudio-proxy" + b.prepareOMLXProxyPortWithPortCheck(free) + if got := int(b.omlxProxyStartupPort.Load()); got != 12420 { + t.Fatalf("oMLX startup port = %d", got) + } + } + if engine == "llamacpp" { + b.prepareLlamaCPPProxyPort() + if got := int(b.llamacppProxyStartupPort.Load()); got != 12420 { + t.Fatalf("llama.cpp startup port = %d", got) + } + } + }) + } +} + +func TestOMLXStartupPreservesSavedPortAndRevision(t *testing.T) { + isolateOllamaHostTestConfig(t) + h := newSettingsHarness(t) + b := h.b + proxy := b.getOMLXProxy() + proxy.port = 1236 + before, err := b.getEngineSettings(context.Background(), settings.Request{Engine: "omlx"}, "") + if err != nil { + t.Fatal(err) + } + b.setOMLXProxy(nil) + b.getLMStudioProxy().port = 1234 + b.getLlamaCPPProxy().port = 1237 + b.lmstudioProxyPath = "/fixture/lmstudio-proxy" + b.prepareOMLXProxyPortWithPortCheck(func(port int) bool { return port != 1234 }) + port := int(b.omlxProxyStartupPort.Load()) + if port != 1236 { + t.Errorf("startup port = %d, want saved 1236", port) + } + proxy.port = port + b.setOMLXProxy(proxy) + after, err := b.getEngineSettings(context.Background(), settings.Request{Engine: "omlx"}, "") + if err != nil { + t.Fatal(err) + } + if after.Revision != before.Revision || after.Settings.ProxyPort != before.Settings.ProxyPort { + t.Fatalf("startup changed saved settings: before %+v; after %+v", before, after) + } +} + +// Inventory enumeration precedes a fallback bind, but never performs an engine +// status probe against the proxy's not-yet-vacated port. +func readPastFallbackInventory(codec *Codec, port int) (*Message, error) { + for { + msg, err := codec.Read() + if err != nil || msg.Method != "engine:configured-ports" { + return msg, err + } + if err := codec.Respond(msg.ID, map[string]any{"engines": []any{map[string]any{"engine": "lmstudio", "port": port}}}); err != nil { + return nil, err + } + } +} + +func TestProxyFallbackExcludesProxyAndNodePorts(t *testing.T) { + isolateOllamaHostTestConfig(t) + b := &Broker{regCache: relay.NewRegistrationCache(), engineSettings: map[string]*engineSettingsRecord{ + "omlx": {Snapshot: settings.Snapshot{Settings: settings.Config{ServerPort: 12401, ProxyPort: 12402}}}, + }} + b.regCache.Register(noderec.RegisterParams{Service: noderec.ServiceNodeInfo, Port: 12403}) + b.omlxProxyStartupPort.Store(12404) + b.setLMStudioProxy(&proxyProcess{ready: true, port: 12405}) + for _, name := range []string{"proxy-port.json", "lmstudio-proxy-port.json", "omlx-proxy-port.json"} { + path, err := appdir.Path(name) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(`{"port":12406}`), 0600); err != nil { + t.Fatal(err) + } + } + ports := b.proxyFallbackExclusions("llamacpp", nil) + for _, port := range []int{12401, 12402, 12403, 12404, 12405, 12406, engineManagerHTTPPort, engineControlPort, nodeInfoHTTPPort, errorsHTTPPort, workloadHTTPPort, clusterManagerHTTPPort} { + if !slices.Contains(ports, port) { + t.Errorf("port %d not excluded", port) + } + } +} + +func TestProxyFallbackInventoryReleasesSettingsLock(t *testing.T) { + worker, codec := newTestRPCWorkerPipe(t) + b := &Broker{} + b.setEngineMgr(worker) + unlocked := make(chan bool, 1) + go func() { + msg, err := codec.Read() + if err != nil { + return + } + ok := b.engineConfigMu.TryLock() + if ok { + b.engineConfigMu.Unlock() + } + unlocked <- ok + _ = codec.Respond(msg.ID, map[string]any{"engines": []any{}}) + }() + finish := b.beginPortOperation("llamacpp") + b.setLlamaCPPProxyFallback() + finish() + if !<-unlocked { + t.Fatal("settings lock held during inventory worker call") + } +} + +func TestProxyFallbackReservesStoppedEnginePorts(t *testing.T) { + for _, tc := range []struct { + engine string + port int + choose func(*Broker) int + }{ + {"llamacpp", 1237, func(b *Broker) int { return b.setLlamaCPPProxyFallback() }}, + {"ollama", 11435, func(b *Broker) int { return b.setOllamaProxyFallback() }}, + {"lmstudio", 1235, func(b *Broker) int { return b.setLMStudioProxyFallback() }}, + {"omlx", 1236, func(b *Broker) int { return b.setOMLXProxyFallback() }}, + } { + t.Run(tc.engine, func(t *testing.T) { + // The static inventory includes every engine, including stopped ones. + // The llama.cpp case reproduces stopped oMLX configured on 1237. + other := "omlx" + if tc.engine == other { + other = "llamacpp" + } + b := brokerWithEngineInventory(t, map[string]int{other: tc.port, tc.engine: tc.port + 1, "custom": tc.port + 2}) + finish := b.beginPortOperation(tc.engine) + port := tc.choose(b) + finish() + if port <= 0 || port == tc.port || port == tc.port+1 || port == tc.port+2 { + t.Fatalf("fallback %d stole a stopped engine's configured port (%d..%d)", port, tc.port, tc.port+2) + } + }) + } +} diff --git a/services/nvpair-ui-broker/proxyport.go b/services/nvpair-ui-broker/proxyport.go index aeb712528..492d07cbc 100644 --- a/services/nvpair-ui-broker/proxyport.go +++ b/services/nvpair-ui-broker/proxyport.go @@ -127,16 +127,21 @@ func (b *Broker) blockManagedOllamaFacade(reason string, excludedPorts ...int) i // The live move owner releases the gate only after cancellation and rollback // have joined. Other startup failures can use blockManagedOllamaFacade above. func (b *Broker) blockManagedOllamaFacadePending(reason string, excludedPorts ...int) int { - b.managedOllamaFacade.Store(false) - b.managedOllamaBackend.Store(0) if backend := int(b.ollamaBackendPort.Load()); backend > 0 { excludedPorts = append(excludedPorts, backend) } fallback := b.setOllamaProxyFallback(excludedPorts...) - b.reportPortOwnershipBlocked(reason) + b.recordBlockedOllamaFacade(reason) return fallback } +// Commit only after an unlocked selection's settings revalidation succeeds. +func (b *Broker) recordBlockedOllamaFacade(reason string) { + b.managedOllamaFacade.Store(false) + b.managedOllamaBackend.Store(0) + b.reportPortOwnershipBlocked(reason) +} + func (b *Broker) markOllamaPortReady() { if b.ollamaPortReady != nil { b.ollamaPortReadyOnce.Do(func() { close(b.ollamaPortReady) }) @@ -144,10 +149,7 @@ func (b *Broker) markOllamaPortReady() { } func (b *Broker) setOllamaProxyFallback(excludedPorts ...int) int { - if aliasPort := b.currentOllamaHostAlias().Port; aliasPort > 0 { - excludedPorts = append(excludedPorts, aliasPort) - } - fallback := nextAvailablePortExcluding(managedOllamaBackendStart, excludedPorts, tcpPortAvailable) + fallback := b.nextProxyFallbackPort("ollama", managedOllamaBackendStart, excludedPorts, tcpPortAvailable) b.ollamaProxyStartupPort.Store(int32(fallback)) return fallback } @@ -377,7 +379,11 @@ func (b *Broker) reconcileProxyPortOnReadyInOperation(boundPort int) { return } if err != nil || rpcErr != nil { - b.blockManagedOllamaFacade("the proxy could not bind the compatibility port") + if !call(func() { b.setOllamaProxyFallback(int(b.ollamaBackendPort.Load())) }) { + return + } + b.recordBlockedOllamaFacade("the proxy could not bind the compatibility port") + b.markOllamaPortReady() slog.Warn("failed to bind managed Ollama facade", "err", err, "rpcErr", rpcErr) } return @@ -414,7 +420,11 @@ func (b *Broker) reconcileProxyPortOnReadyInOperation(boundPort int) { // The RPC may have applied and then lost its response. Keep the // fallback off both candidate ports, vacate :11434 first, then // best-effort restore the original stopped-engine configuration. - fallback := b.blockManagedOllamaFacadePending("the stopped Ollama backend could not be moved", backend) + var fallback int + if !call(func() { fallback = b.setOllamaProxyFallback(backend, int(b.ollamaBackendPort.Load())) }) { + return + } + b.recordBlockedOllamaFacade("the stopped Ollama backend could not be moved") vacated := false if fallback != 0 { body, _ := json.Marshal(map[string]int{"port": fallback}) diff --git a/services/nvpair-ui-broker/proxyport_write.go b/services/nvpair-ui-broker/proxyport_write.go index f68006ac6..de397ef07 100644 --- a/services/nvpair-ui-broker/proxyport_write.go +++ b/services/nvpair-ui-broker/proxyport_write.go @@ -106,5 +106,9 @@ func (b *Broker) recoverProxyBindFailure(engine string, generation uint64, port if generation == b.omlxProxyGeneration.Load() { b.setOMLXProxyFallback(port) } + case "llamacpp": + if generation == b.llamacppProxyGeneration.Load() { + b.setLlamaCPPProxyFallback(port) + } } } diff --git a/services/nvpair-ui-broker/stopwait.go b/services/nvpair-ui-broker/stopwait.go index 0c71d97df..a8f6576f8 100644 --- a/services/nvpair-ui-broker/stopwait.go +++ b/services/nvpair-ui-broker/stopwait.go @@ -58,7 +58,7 @@ const ( // // which is the 15s the desktop parent and nvpair-tui both allow. Everything // inside the first line — the three proxy joins, the engine StopAll wait, and - // all twelve joins — draws on this one budget, so no step can be lengthened + // all thirteen joins — draws on this one budget, so no step can be lengthened // without the others giving way. It must also exceed the largest single worker // grace, or that worker could never be granted it. teardownBudget = 10 * time.Second diff --git a/services/nvpair-ui-broker/telemetry_reconcile.go b/services/nvpair-ui-broker/telemetry_reconcile.go index 209805d02..b15631361 100644 --- a/services/nvpair-ui-broker/telemetry_reconcile.go +++ b/services/nvpair-ui-broker/telemetry_reconcile.go @@ -37,12 +37,12 @@ type telemetryApplication struct { state workerTelemetryState } -var telemetryProxyNames = [...]string{"proxy", "lmstudio-proxy", "omlx-proxy"} +var telemetryProxyNames = [...]string{"proxy", "lmstudio-proxy", "omlx-proxy", "llamacpp-proxy"} -func (b *Broker) telemetryProxies() [3]*proxyProcess { +func (b *Broker) telemetryProxies() [4]*proxyProcess { b.workersMu.Lock() defer b.workersMu.Unlock() - return [3]*proxyProcess{b.proxy, b.lmstudioProxy, b.omlxProxy} + return [4]*proxyProcess{b.proxy, b.lmstudioProxy, b.omlxProxy, b.llamacppProxy} } // desiredTelemetryLocked supplies the fail-closed policy before settings is ready. @@ -155,7 +155,7 @@ func (b *Broker) reconcileTelemetry() { } } -func (b *Broker) telemetrySnapshot() (telemetryDesiredConfig, []workerTelemetryState, [3]*proxyProcess) { +func (b *Broker) telemetrySnapshot() (telemetryDesiredConfig, []workerTelemetryState, [4]*proxyProcess) { b.telemetryConfigMu.Lock() defer b.telemetryConfigMu.Unlock() proxies := b.telemetryProxies() @@ -180,7 +180,7 @@ func (b *Broker) telemetrySnapshot() (telemetryDesiredConfig, []workerTelemetryS return b.desiredTelemetryLocked(), states, proxies } -func (b *Broker) queryTelemetryHealth(ctx context.Context, states []workerTelemetryState, proxies [3]*proxyProcess) { +func (b *Broker) queryTelemetryHealth(ctx context.Context, states []workerTelemetryState, proxies [4]*proxyProcess) { for i, p := range proxies { if p == nil { continue diff --git a/services/nvpair-ui-broker/telemetry_reconcile_test.go b/services/nvpair-ui-broker/telemetry_reconcile_test.go index ed43d7a56..ea460bb8b 100644 --- a/services/nvpair-ui-broker/telemetry_reconcile_test.go +++ b/services/nvpair-ui-broker/telemetry_reconcile_test.go @@ -87,13 +87,14 @@ func receiveTelemetryConfig(t *testing.T, ch <-chan telemetryDesiredConfig) tele func TestReconcileAppliesDesiredToEveryProxy(t *testing.T) { b := &Broker{} - seen := make(chan telemetryDesiredConfig, 3) + seen := make(chan telemetryDesiredConfig, 4) b.setProxy(newTelemetryProxyPeer(t, acceptTelemetryConfig(t, seen))) b.setLMStudioProxy(newTelemetryProxyPeer(t, acceptTelemetryConfig(t, seen))) b.setOMLXProxy(newTelemetryProxyPeer(t, acceptTelemetryConfig(t, seen))) + b.setLlamaCPPProxy(newTelemetryProxyPeer(t, acceptTelemetryConfig(t, seen))) b.cacheDesiredTelemetry(telemetryConfigFixture(7)) b.reconcileTelemetry() - for range 3 { + for range 4 { got := receiveTelemetryConfig(t, seen) if got.Revision != 7 || !got.Enabled || got.CollectorPort != 54317 || !got.RecordPayloads { t.Errorf("configure = %+v", got) @@ -107,7 +108,7 @@ func TestReconcileAppliesDesiredToEveryProxy(t *testing.T) { } func TestProxyRestartGetsLatestConfig(t *testing.T) { - for _, name := range []string{"proxy", "lmstudio-proxy", "omlx-proxy"} { + for _, name := range []string{"proxy", "lmstudio-proxy", "omlx-proxy", "llamacpp-proxy"} { t.Run(name, func(t *testing.T) { b := &Broker{} install, notify, index := b.setProxy, b.forwardProxyNotification, 0 @@ -117,6 +118,9 @@ func TestProxyRestartGetsLatestConfig(t *testing.T) { if name == "omlx-proxy" { install, notify, index = b.setOMLXProxy, b.forwardOMLXProxyNotification, 2 } + if name == "llamacpp-proxy" { + install, notify, index = b.setLlamaCPPProxy, b.forwardLlamaCPPProxyNotification, 3 + } b.cacheDesiredTelemetry(telemetryConfigFixture(2)) seen := make(chan telemetryDesiredConfig, 2) install(newTelemetryProxyPeer(t, acceptTelemetryConfig(t, seen))) @@ -265,6 +269,7 @@ func TestStatusReportsDesiredAndEffective(t *testing.T) { {"name": "proxy", "appliedRevision": float64(12), "state": "applied", "error": "", "lastExportOk": "2026-10-18T01:02:03Z", "lastExportError": "delivery failed", "lastExportErrorAt": "2026-10-18T01:02:04Z"}, {"name": "lmstudio-proxy", "appliedRevision": float64(0), "state": "unavailable", "error": "", "lastExportOk": nil, "lastExportError": nil, "lastExportErrorAt": nil}, {"name": "omlx-proxy", "appliedRevision": float64(0), "state": "unavailable", "error": "", "lastExportOk": nil, "lastExportError": nil, "lastExportErrorAt": nil}, + {"name": "llamacpp-proxy", "appliedRevision": float64(0), "state": "unavailable", "error": "", "lastExportOk": nil, "lastExportError": nil, "lastExportErrorAt": nil}, } if !reflect.DeepEqual(workers, expected) { t.Errorf("workers = %+v, want %+v", workers, expected) @@ -364,7 +369,7 @@ func TestSettingsNotificationWithoutWorkerCannotEnableTracing(t *testing.T) { } func TestSettingsLossKeepsLastLoadedConfigForRunningAndRestartedProxies(t *testing.T) { - for _, name := range []string{"proxy", "lmstudio-proxy", "omlx-proxy"} { + for _, name := range []string{"proxy", "lmstudio-proxy", "omlx-proxy", "llamacpp-proxy"} { t.Run(name, func(t *testing.T) { b := &Broker{} install, notify, index := b.setProxy, b.forwardProxyNotification, 0 @@ -374,6 +379,9 @@ func TestSettingsLossKeepsLastLoadedConfigForRunningAndRestartedProxies(t *testi if name == "omlx-proxy" { install, notify, index = b.setOMLXProxy, b.forwardOMLXProxyNotification, 2 } + if name == "llamacpp-proxy" { + install, notify, index = b.setLlamaCPPProxy, b.forwardLlamaCPPProxyNotification, 3 + } worker, codec := newTestRPCWorkerPipe(t) b.setSettings(worker) go func() { diff --git a/services/nvpair-ui-broker/verified_inventory_test.go b/services/nvpair-ui-broker/verified_inventory_test.go index c2ceb817e..bb8aef3e7 100644 --- a/services/nvpair-ui-broker/verified_inventory_test.go +++ b/services/nvpair-ui-broker/verified_inventory_test.go @@ -27,7 +27,7 @@ func inventoryTestBroker(t *testing.T) *Broker { return b } func verifiedDirectoryForTest(b *Broker) noderec.DirectoryNode { - return noderec.DirectoryNode{HostUUID: "peer", ClusterUUID: "peer", Name: "peer", IP: "192.0.2.2", Trusted: true, IdentityVerification: "verified", PeerRecordID: b.mesh.PeerRecordID("peer"), Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceNodeInfo: {Port: 14318}, noderec.ServiceOllama: {Port: 11434}, noderec.ServiceLMStudio: {Port: 1234}, noderec.ServiceOMLX: {Port: 1236}, noderec.ServiceEngineControl: {Port: 14323}}, Models: []string{"model"}} + return noderec.DirectoryNode{HostUUID: "peer", ClusterUUID: "peer", Name: "peer", IP: "192.0.2.2", Trusted: true, IdentityVerification: "verified", PeerRecordID: b.mesh.PeerRecordID("peer"), Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceNodeInfo: {Port: 14318}, noderec.ServiceOllama: {Port: 11434}, noderec.ServiceLMStudio: {Port: 1234}, noderec.ServiceOMLX: {Port: 1236}, noderec.ServiceLlamaCPP: {Port: 1237}, noderec.ServiceEngineControl: {Port: 14323}}, Models: []string{"model"}} } func TestClaimedManualPrincipalIsNotTrustedOrScheduled(t *testing.T) { @@ -42,7 +42,7 @@ func TestClaimedManualPrincipalIsNotTrustedOrScheduled(t *testing.T) { if got := b.schedulerNodes(); len(got) != 0 { t.Fatalf("unproven inventory in scheduler: %+v", got) } - for _, service := range []noderec.ServiceKey{noderec.ServiceOllama, noderec.ServiceLMStudio, noderec.ServiceOMLX, noderec.ServiceEngineControl} { + for _, service := range []noderec.ServiceKey{noderec.ServiceOllama, noderec.ServiceLMStudio, noderec.ServiceOMLX, noderec.ServiceLlamaCPP, noderec.ServiceEngineControl} { if got := b.relayDir.Snapshot(service); len(got) != 0 { t.Fatalf("unproven %s target: %+v", service, got) } diff --git a/services/nvpair-ui-broker/workers.go b/services/nvpair-ui-broker/workers.go index 20d3fe0c1..8b911aac3 100644 --- a/services/nvpair-ui-broker/workers.go +++ b/services/nvpair-ui-broker/workers.go @@ -46,6 +46,7 @@ func (b *Broker) initWorkerRegistry() { {"proxy", "ollama-proxy", &b.proxyPath, &b.proxySup, b.spawnProxy, func() { b.setProxy(nil) }}, {"lmstudio-proxy", "lmstudio-proxy", &b.lmstudioProxyPath, &b.lmstudioProxySup, b.spawnLMStudioProxy, func() { b.setLMStudioProxy(nil) }}, {"omlx-proxy", "omlx-proxy", &b.omlxProxyPath, &b.omlxProxySup, b.spawnOMLXProxy, func() { b.setOMLXProxy(nil) }}, + {"llamacpp-proxy", "llamacpp-proxy", &b.llamacppProxyPath, &b.llamacppProxySup, b.spawnLlamaCPPProxy, func() { b.setLlamaCPPProxy(nil) }}, {"workload-manager", "nvpair-workload-manager", &b.workloadMgrPath, &b.workloadMgrSup, b.spawnWorkloadManager, func() { b.setWorkloadMgr(nil) }}, {"manual-nodes", "nvpair-manual-nodes", &b.manualNodesPath, &b.manualNodesSup, b.spawnManualNodes, b.clearManualNodesState}, {"cluster-manager", "nvpair-cluster-manager", &b.clusterMgrPath, &b.clusterMgrSup, b.spawnClusterManager, func() { b.setClusterMgr(nil) }}, @@ -64,7 +65,7 @@ func (b *Broker) initWorkerRegistry() { if err != nil { expected = binary } - eligible := spec.name != "engine-manager" && spec.name != "proxy" && spec.name != "lmstudio-proxy" && spec.name != "omlx-proxy" + eligible := spec.name != "engine-manager" && spec.name != "proxy" && spec.name != "lmstudio-proxy" && spec.name != "omlx-proxy" && spec.name != "llamacpp-proxy" status := WorkerStatus{Name: spec.name, State: WorkerRestarting, RestartOnHang: eligible, Since: time.Now()} if *spec.path == "" && spec.name != "scanner" { status.State = WorkerMissing @@ -81,6 +82,7 @@ func (b *Broker) initWorkerRegistry() { b.errorsSup.onRecovered = func() { slog.Info("nvpair-errors recovered") } b.configureLMStudioProxySupervisorCallbacks(b.lmstudioProxySup) b.configureOMLXProxySupervisorCallbacks(b.omlxProxySup) + b.configureLlamaCPPProxySupervisorCallbacks(b.llamacppProxySup) b.proxySup.onExhausted = func(int) { b.finishOllamaProxyTerminal() } } @@ -148,13 +150,14 @@ func (b *Broker) stopWorkers() { // so the deferred stopWorkers call can repeat this after the ordinary path. func (b *Broker) stopInferenceProxies() { var proxies []*supervisor - for _, sup := range []*supervisor{b.omlxProxySup, b.lmstudioProxySup, b.proxySup} { + for _, sup := range []*supervisor{b.llamacppProxySup, b.omlxProxySup, b.lmstudioProxySup, b.proxySup} { if sup != nil && b.workerActive(sup.name) { proxies = append(proxies, sup) } } stopSupervisors(proxies) b.setOMLXProxy(nil) + b.setLlamaCPPProxy(nil) b.setLMStudioProxy(nil) b.setProxy(nil) } @@ -315,6 +318,10 @@ func (b *Broker) probeWorker(ctx context.Context, name string) error { if b.omlxProxy != nil { health = b.omlxProxy.Health } + case "llamacpp-proxy": + if b.llamacppProxy != nil { + health = b.llamacppProxy.Health + } case "workload-manager": if b.workloadMgr != nil { health = b.workloadMgr.Health diff --git a/services/nvpair-ui-broker/workers_test.go b/services/nvpair-ui-broker/workers_test.go index ee4fe66fe..c5a122282 100644 --- a/services/nvpair-ui-broker/workers_test.go +++ b/services/nvpair-ui-broker/workers_test.go @@ -23,7 +23,7 @@ import ( func TestWorkersSnapshotOrderAndEligibility(t *testing.T) { b := NewBroker(NewCodec(writeOnlyReadWriter{&bytes.Buffer{}}), workerPaths{}) - want := []string{"scanner", "errors", "settings", "engine-manager", "node-info", "proxy", "lmstudio-proxy", "omlx-proxy", "workload-manager", "manual-nodes", "cluster-manager", "scheduler"} + want := []string{"scanner", "errors", "settings", "engine-manager", "node-info", "proxy", "lmstudio-proxy", "omlx-proxy", "llamacpp-proxy", "workload-manager", "manual-nodes", "cluster-manager", "scheduler"} got := b.workerSnapshot().Workers if len(got) != len(want) { t.Fatalf("got %d workers", len(got)) @@ -32,7 +32,7 @@ func TestWorkersSnapshotOrderAndEligibility(t *testing.T) { if got[i].Name != name { t.Fatalf("worker %d = %s, want %s", i, got[i].Name, name) } - eligible := name != "engine-manager" && name != "proxy" && name != "lmstudio-proxy" && name != "omlx-proxy" + eligible := name != "engine-manager" && name != "proxy" && name != "lmstudio-proxy" && name != "omlx-proxy" && name != "llamacpp-proxy" policy := b.workerEntries[i].sup.hang.withDefaults() if policy.interval != 30*time.Second || policy.deadline != 5*time.Second || policy.degradeAfter != 2 || policy.restartAfter != 3 { t.Fatalf("policy = %+v", policy) @@ -46,11 +46,11 @@ func TestWorkersSnapshotOrderAndEligibility(t *testing.T) { func TestWorkersMissingRestartAndNotifications(t *testing.T) { var out bytes.Buffer b := NewBroker(NewCodec(writeOnlyReadWriter{&out}), workerPaths{}) - entry := b.workerEntries[9] // manual-nodes + entry := b.workerEntries[10] // manual-nodes entry.expectedPath = filepath.Join(t.TempDir(), "nvpair-manual-nodes") h := newFakeHandle() entry.sup.spawn = func() (supervisedHandle, error) { return h, nil } - before := b.workerSnapshot().Workers[9] + before := b.workerSnapshot().Workers[10] if before.State != WorkerMissing || !strings.Contains(before.LastError, "nvpair-manual-nodes") { t.Fatalf("missing: %+v", before) } @@ -66,7 +66,7 @@ func TestWorkersMissingRestartAndNotifications(t *testing.T) { } defer entry.sup.Stop() deadline := time.After(time.Second) - for b.workerSnapshot().Workers[9].State != WorkerRunning { + for b.workerSnapshot().Workers[10].State != WorkerRunning { select { case <-deadline: t.Fatal("never running") @@ -90,10 +90,10 @@ func TestWorkersMissingRestartAndNotifications(t *testing.T) { if err := json.Unmarshal(msg.Params, &snapshot); err != nil { t.Fatal(err) } - if len(snapshot.Workers) != 12 { + if len(snapshot.Workers) != 13 { t.Fatal("not a full snapshot") } - states = append(states, snapshot.Workers[9].State) + states = append(states, snapshot.Workers[10].State) } if len(states) != 2 || states[0] != WorkerRestarting || states[1] != WorkerRunning { t.Fatalf("updates = %v", states) @@ -156,7 +156,7 @@ func TestWorkersRPCSurface(t *testing.T) { if err := json.Unmarshal(msg.Result, &result); err != nil { t.Fatal(err) } - if len(result.Workers) != 12 { + if len(result.Workers) != 13 { t.Fatalf("get-status: %+v", msg) } } @@ -204,7 +204,7 @@ func TestWorkersRestartDoesNotWaitForStop(t *testing.T) { func TestWorkersMissingRestartDoesNotWaitForSpawn(t *testing.T) { b := NewBroker(NewCodec(writeOnlyReadWriter{io.Discard}), workerPaths{}) - entry := b.workerEntries[9] + entry := b.workerEntries[10] entry.expectedPath = filepath.Join(t.TempDir(), "nvpair-manual-nodes") if err := os.WriteFile(entry.expectedPath, []byte("fixture"), 0700); err != nil { t.Fatal(err) @@ -265,7 +265,7 @@ func TestWorkersCrashReason(t *testing.T) { b := NewBroker(NewCodec(writeOnlyReadWriter{io.Discard}), workerPaths{}) var sink bytes.Buffer b.setErrors(&errorsProcess{peer: NewPeer(NewCodec(writeOnlyReadWriter{&sink}))}) - status := b.workerSnapshot().Workers[11] + status := b.workerSnapshot().Workers[12] status.State, status.LastError = WorkerRestarting, reason b.updateWorker(status) crash, _ := b.crashReporter("scheduler") @@ -299,6 +299,7 @@ func TestWorkersProxyReadyDoesNotClearCrashError(t *testing.T) { {"proxy", (*Broker).forwardProxyNotification}, {"lmstudio-proxy", (*Broker).forwardLMStudioProxyNotification}, {"omlx-proxy", (*Broker).forwardOMLXProxyNotification}, + {"llamacpp-proxy", (*Broker).forwardLlamaCPPProxyNotification}, } { t.Run(tc.name, func(t *testing.T) { b := NewBroker(NewCodec(writeOnlyReadWriter{io.Discard}), workerPaths{}) diff --git a/services/nvpair-ui-broker/workload_preview_test.go b/services/nvpair-ui-broker/workload_preview_test.go index ed1d418c3..fa6f01dbe 100644 --- a/services/nvpair-ui-broker/workload_preview_test.go +++ b/services/nvpair-ui-broker/workload_preview_test.go @@ -16,7 +16,7 @@ import ( // Rotating the pin before discovery publishes its renewed proof must not // discard the local proxy's authoritative report about an already routed job. func TestLocalWorkloadNotLostDuringPeerCertificateRenewal(t *testing.T) { - for _, engine := range []string{"ollama", "lmstudio", "omlx"} { + for _, engine := range []string{"ollama", "lmstudio", "omlx", "llamacpp"} { for _, departure := range []string{"renewed-then-lost", "trust-withdrawal", "scanner-retirement"} { t.Run(engine+"/"+departure, func(t *testing.T) { b := inventoryTestBroker(t) diff --git a/services/readme.md b/services/readme.md index 676dea214..317574e88 100644 --- a/services/readme.md +++ b/services/readme.md @@ -11,7 +11,7 @@ local network: each node advertises itself over mDNS as one consolidated node offers and where to reach them. What a discovered node can actually serve is a separate question, answered after -discovery. A node may run [Ollama](https://ollama.com/), LM Studio, oMLX, any +discovery. A node may run [Ollama](https://ollama.com/), LM Studio, oMLX, llama.cpp, any supported combination, or none. Model inventory is fetched from engine-manager rather than crammed into mDNS TXT records. The bundled oMLX engine manifest supports macOS arm64 only and has no model pull/delete actions. @@ -31,7 +31,8 @@ never intercepted. The oMLX proxy defaults to `1234` standalone. Under the broker, when LM Studio occupies or reserves `1234`, oMLX is assigned a free fallback starting at `1236`, excluding reserved ports. Use reported proxy status rather than assuming its -endpoint. Model inventory is served on `14322`; node-info on `14318` serves +endpoint. llama.cpp uses its own proxy on **1237**, forwarding to the managed +engine on **127.0.0.1:8082**; its discovery service is `lc`. Model inventory is served on `14322`; node-info on `14318` serves hardware, utilization, and identity telemetry, not the model list. PAIR ships a graphical UI alongside these services. The UI launches @@ -64,7 +65,7 @@ change's runtime validation covers Darwin and Linux only. ## Architecture -This tree builds 14 service executables: `nvpair-ui-broker`, 12 supervised +This tree builds 15 service executables: `nvpair-ui-broker`, 13 supervised workers, and `nvpair-tui`. The broker attempts to start available workers at startup. Standalone, only the scanner is required; missing default auxiliary binaries and initial auxiliary spawn failures are non-fatal. Invalid explicit @@ -80,14 +81,14 @@ Each worker answers `worker/health` with `{"ok":true}`. The broker probes every interval. Two consecutive failures report `degraded`; a successful probe restores `running` and clears the failure count. Three failures restart scanner, errors, settings, node-info, workload-manager, manual-nodes, cluster-manager, and scheduler. -Engine-manager and the three proxies never auto-restart on a hang: restart them +Engine-manager and the four proxies never auto-restart on a hang: restart them explicitly if needed. This checks control responsiveness, not inference readiness. -`workers:get-status` and full-snapshot `workers:update` expose all twelve workers; +`workers:get-status` and full-snapshot `workers:update` expose all thirteen workers; `workers:restart` requests a restart or retry even after budget exhaustion. Desktop renders the registry under Settings → Service → Workers (including "Not responding — restart manually" for ineligible hung workers). TUI Overview -shows all twelve and offers `R` with confirmation. Worker probing is separate +shows all thirteen and offers `R` with confirmation. Worker probing is separate from the broker's 15 s heartbeat and clients' 45 s inbound-silence handling. Cross-process hang tests are Unix-only; Windows hang behavior remains unproven. @@ -102,6 +103,7 @@ Windows named pipe, with relay protocol v2 notification envelopes. See | `ollama-proxy` | Ollama-compatible HTTP reverse proxy. Routes only to advertised model owners, with owner failover and scheduler priorities. | | `lmstudio-proxy` | LM Studio counterpart to `ollama-proxy`, forwarding OpenAI-compatible inference routes with equivalent owner-only routing and failover behavior. | | `omlx-proxy` | OpenAI-compatible routing proxy for oMLX, with owner-only routing, failover, and cache-affinity hints. | +| `llamacpp-proxy` | llama.cpp routing proxy on 1237, with exact IDs, model-list aggregation and native Messages passthrough. | | `nvpair-node-info` | Local HTTP service on `:14318` exposing GPU, CPU, and memory inventory at `/v1/node-info`. | | `nvpair-node-scanner` | Consolidated discovery daemon. Advertises and browses `_nvpair-node._tcp`, maintains the node directory, and enriches peers with hardware and model information over HTTP. | | `nvpair-manual-nodes` | Manages user-added nodes that don't appear via mDNS; probes them every 10 s, delivering compact scheduling telemetry every ~2 s. | @@ -135,6 +137,7 @@ nvpair-ui-broker/ Parent service / JSON-RPC API surface ollama-proxy/ Ollama-compatible routing proxy lmstudio-proxy/ OpenAI-compatible routing proxy for LM Studio omlx-proxy/ OpenAI-compatible routing proxy for oMLX +llamacpp-proxy/ OpenAI-compatible routing proxy for llama.cpp nvpair-node-info/ Local GPU-inventory HTTP service nvpair-node-scanner/ Consolidated _nvpair-node._tcp discovery daemon nvpair-manual-nodes/ Manual-node manager @@ -149,8 +152,8 @@ shared/ Shared Go module (nvpair-shared/…) eap-noob/ EAP-NOOB implementation used by cluster pairing tests/ Cross-process integration tests (separate go.mod) versions.json Single source of truth for every component version -build.bat Builds all fourteen service executables (Windows) -build.sh Builds all fourteen service executables (Linux/macOS) +build.bat Builds all fifteen service executables (Windows) +build.sh Builds all fifteen service executables (Linux/macOS) VERSIONING.md SemVer rules and version-bump workflow ``` @@ -180,7 +183,7 @@ On Linux and macOS: ./build.sh ``` -Both scripts read `versions.json`, build all fourteen service executables with +Both scripts read `versions.json`, build all fifteen service executables with `-X main.Version=…` ldflags, and stage them together in `services/build/bin/`. Do **not** build individual components by hand without also copying their binaries into `build/bin/`: the broker will silently keep using the older binary there. @@ -284,7 +287,7 @@ cd shared go test ./... ``` -**Every one of the fourteen service executables has tests**, as do `shared/` and +**Every one of the fifteen service executables has tests**, as do `shared/` and `eap-noob/`. Depth varies with how much behaviour a component carries: `nvpair-engine-manager` and `nvpair-cluster-manager` have the largest suites, while a component with one test file may still hold twenty test functions in it. diff --git a/services/shared/noderec/llamacpp_test.go b/services/shared/noderec/llamacpp_test.go new file mode 100644 index 000000000..63629ca1b --- /dev/null +++ b/services/shared/noderec/llamacpp_test.go @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package noderec + +import ( + "reflect" + "testing" +) + +func TestLlamaCPPServiceRoundTrip(t *testing.T) { + record := ParseTXT([]string{"v=1", "uuid=fixture", "lc=1237", "lm=1234"}) + if port, ok := record.Port(ServiceLlamaCPP); !ok || port != 1237 { + t.Fatalf("llama.cpp service: %+v", record) + } + if !reflect.DeepEqual(ParseTXT(record.TXT()), record) { + t.Fatal("service lost during TXT round trip") + } + node := DirectoryNode{Models: []string{"same", "model.gguf"}, ModelsByEngine: map[string][]string{"lmstudio": {"same"}, "llamacpp": {"same", "model.gguf"}}} + if !reflect.DeepEqual(node.EngineModels("llamacpp"), []string{"same", "model.gguf"}) || !reflect.DeepEqual(node.EngineModels("lmstudio"), []string{"same"}) { + t.Fatal("engine inventories merged") + } +} diff --git a/services/shared/noderec/noderec.go b/services/shared/noderec/noderec.go index 52cad03ce..e2baa1e39 100644 --- a/services/shared/noderec/noderec.go +++ b/services/shared/noderec/noderec.go @@ -92,6 +92,7 @@ const ( ServiceOllama ServiceKey = "ol" ServiceLMStudio ServiceKey = "lm" ServiceOMLX ServiceKey = "om" + ServiceLlamaCPP ServiceKey = "lc" ServiceErrors ServiceKey = "er" ServiceWorkload ServiceKey = "wl" ServiceCluster ServiceKey = "cl" @@ -109,7 +110,7 @@ const ( // serviceKeyOrder is the deterministic emit order for service ports in TXT. var serviceKeyOrder = []ServiceKey{ - ServiceNodeInfo, ServiceOllama, ServiceLMStudio, ServiceOMLX, + ServiceNodeInfo, ServiceOllama, ServiceLMStudio, ServiceOMLX, ServiceLlamaCPP, ServiceErrors, ServiceWorkload, ServiceCluster, ServiceEngineManager, ServiceEngineControl, } diff --git a/services/shared/noderec/noderec_test.go b/services/shared/noderec/noderec_test.go index 3ec64112b..cbd9187f9 100644 --- a/services/shared/noderec/noderec_test.go +++ b/services/shared/noderec/noderec_test.go @@ -171,6 +171,7 @@ func TestTransportPolicy(t *testing.T) { {ServiceOllama, TransportPlain, false, false}, {ServiceLMStudio, TransportPlain, false, false}, {ServiceOMLX, TransportPlain, false, false}, + {ServiceLlamaCPP, TransportPlain, false, false}, {ServiceEngineManager, TransportPlain, false, false}, {ServiceErrors, TransportMTLSWhenClustered, true, false}, {ServiceWorkload, TransportMTLSWhenClustered, true, false}, diff --git a/services/tests/llamacpp_interop_test.go b/services/tests/llamacpp_interop_test.go new file mode 100644 index 000000000..b1628062c --- /dev/null +++ b/services/tests/llamacpp_interop_test.go @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestBrokerLlamaCPPProxyRoutesOpenAI(t *testing.T) { + const model = "fixture-model.gguf" + const stream = "data: {\"choices\":[{\"delta\":{\"content\":\"fixture\"}}]}\n\ndata: [DONE]\n\n" + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/models": + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":[{"id":"fixture-model.gguf"}]}`) + case "/v1/chat/completions": + var body struct { + Model string `json:"model"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Model != model { + t.Errorf("wrong routed model: %q %v", body.Model, err) + } + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, stream) + default: + t.Errorf("unexpected upstream path %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + defer engine.Close() + stdin, msgs, stderr, cleanup := startBrokerWith(t, "--llamacpp-proxy-path", llamacppProxyBin) + defer cleanup() + go func() { + for range stderr { + } + }() + waitForMethod(t, msgs, "app:ready", 10*time.Second) + port := waitRoutingProxyReady(t, stdin, msgs, "llamacpp-proxy") + writeRawFrame(t, stdin, `{"jsonrpc":"2.0","id":50001,"method":"workers:get-status"}`) + inventory := waitForResponseID(t, msgs, 50001, 5*time.Second) + var status struct { + Workers []struct { + Name string `json:"name"` + RestartOnHang bool `json:"restartOnHang"` + } `json:"workers"` + } + if err := json.Unmarshal(inventory.Result, &status); err != nil || len(status.Workers) != 13 { + t.Fatalf("worker inventory: %s %v", inventory.Result, err) + } + found := false + for _, worker := range status.Workers { + if worker.Name == "llamacpp-proxy" { + found = true + if worker.RestartOnHang { + t.Fatal("proxy must require manual hang restart") + } + } + } + if !found { + t.Fatal("new worker missing") + } + callBrokerRPC(t, stdin, msgs, 50002, "llamacpp-proxy:node/add-manual", map[string]any{"id": "fixture", "host": "127.0.0.1", "addresses": []string{"127.0.0.1"}, "port": portOfURL(t, engine.URL), "models": []string{model}}) + client := &http.Client{Timeout: 5 * time.Second} + defer client.CloseIdleConnections() + response, err := client.Post(fmt.Sprintf("http://127.0.0.1:%d/v1/chat/completions", port), "application/json", bytes.NewBufferString(`{"model":"fixture-model.gguf","messages":[],"stream":true}`)) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(response.Body) + _ = response.Body.Close() + if err != nil || response.StatusCode != http.StatusOK || string(body) != stream { + t.Fatalf("stream response: status=%d body=%q error=%v", response.StatusCode, body, err) + } + response, err = client.Get(fmt.Sprintf("http://127.0.0.1:%d/v1/models", port)) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + var models struct { + Data []struct { + ID string `json:"id"` + } `json:"data"` + } + if err := json.NewDecoder(response.Body).Decode(&models); err != nil || response.StatusCode != http.StatusOK || len(models.Data) != 1 || models.Data[0].ID != model { + t.Fatalf("inventory: %+v %v", models, err) + } +} diff --git a/services/tests/main_test.go b/services/tests/main_test.go index 7b5f76e2d..d6523fc0c 100644 --- a/services/tests/main_test.go +++ b/services/tests/main_test.go @@ -23,6 +23,7 @@ var ( proxyBin string lmstudioProxyBin string omlxProxyBin string + llamacppProxyBin string errorsBin string nodeInfoBin string scannerBin string @@ -49,6 +50,7 @@ func TestMain(m *testing.M) { proxyBin = filepath.Join(tmpDir, "ollama-proxy"+ext) lmstudioProxyBin = filepath.Join(tmpDir, "lmstudio-proxy"+ext) omlxProxyBin = filepath.Join(tmpDir, "omlx-proxy"+ext) + llamacppProxyBin = filepath.Join(tmpDir, "llamacpp-proxy"+ext) errorsBin = filepath.Join(tmpDir, "nvpair-errors"+ext) nodeInfoBin = filepath.Join(tmpDir, "nvpair-node-info"+ext) scannerBin = filepath.Join(tmpDir, "nvpair-node-scanner"+ext) @@ -80,6 +82,12 @@ func TestMain(m *testing.M) { log.Fatalf("build omlx-proxy: %v", err) } + log.Println("building llamacpp-proxy...") + if err := goBuild(filepath.Join("..", "llamacpp-proxy"), llamacppProxyBin); err != nil { + os.RemoveAll(tmpDir) + log.Fatalf("build llamacpp-proxy: %v", err) + } + log.Println("building nvpair-errors...") if err := goBuild(filepath.Join("..", "nvpair-errors"), errorsBin); err != nil { os.RemoveAll(tmpDir) diff --git a/services/tests/model_routing_interop_test.go b/services/tests/model_routing_interop_test.go index 5255fb089..8297ce54c 100644 --- a/services/tests/model_routing_interop_test.go +++ b/services/tests/model_routing_interop_test.go @@ -45,7 +45,7 @@ func newRoutingUpstream(t *testing.T, status int) *routingUpstream { // workers' replies must not supply another proxy's port. func waitRoutingProxyReady(t *testing.T, stdin io.Writer, msgs <-chan jsonrpc.Message, prefix string) int { t.Helper() - id := map[string]int{"proxy": 10000, "lmstudio-proxy": 20000, "omlx-proxy": 30000}[prefix] + id := map[string]int{"proxy": 10000, "lmstudio-proxy": 20000, "omlx-proxy": 30000, "llamacpp-proxy": 40000}[prefix] send := func() { writeRawFrame(t, stdin, fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":%q}`, id, prefix+":get-status")) } @@ -92,6 +92,7 @@ func TestStrictModelRoutingAcrossProcesses(t *testing.T) { "--proxy-path", proxyBin, "--lmstudio-proxy-path", lmstudioProxyBin, "--omlx-proxy-path", omlxProxyBin, + "--llamacpp-proxy-path", llamacppProxyBin, ) t.Cleanup(cleanup) go func() { @@ -103,6 +104,7 @@ func TestStrictModelRoutingAcrossProcesses(t *testing.T) { ollamaPort := waitRoutingProxyReady(t, stdin, msgs, "proxy") lmstudioPort := waitRoutingProxyReady(t, stdin, msgs, "lmstudio-proxy") omlxPort := waitRoutingProxyReady(t, stdin, msgs, "omlx-proxy") + llamacppPort := waitRoutingProxyReady(t, stdin, msgs, "llamacpp-proxy") type proxyCase struct { name string @@ -116,6 +118,8 @@ func TestStrictModelRoutingAcrossProcesses(t *testing.T) { {name: "lmstudio", rpcPrefix: "lmstudio-proxy", path: "/v1/chat/completions", port: lmstudioPort}, {name: "lmstudio-anthropic", rpcPrefix: "lmstudio-proxy", path: "/v1/messages", port: lmstudioPort}, {name: "omlx-anthropic", rpcPrefix: "omlx-proxy", path: "/v1/messages", port: omlxPort}, + {name: "llamacpp", rpcPrefix: "llamacpp-proxy", path: "/v1/chat/completions", port: llamacppPort}, + {name: "llamacpp-anthropic", rpcPrefix: "llamacpp-proxy", path: "/v1/messages", port: llamacppPort}, } client := &http.Client{Timeout: 5 * time.Second} t.Cleanup(client.CloseIdleConnections) diff --git a/services/tests/proxy_startup_ports_test.go b/services/tests/proxy_startup_ports_test.go new file mode 100644 index 000000000..ae7fec83b --- /dev/null +++ b/services/tests/proxy_startup_ports_test.go @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "encoding/json" + "fmt" + "net" + "os" + "path/filepath" + "runtime" + "testing" + "time" +) + +func TestBrokerPreservesFourProxyStartupPorts(t *testing.T) { + for _, port := range []int{11434, 1234, 1236, 1237} { + if portBusy(port) { + t.Fatalf("startup regression requires free proxy port %d", port) + } + } + root, cluster := t.TempDir(), t.TempDir() + base := root + if runtime.GOOS == "darwin" { + base = filepath.Join(root, "Library", "Application Support") + } + store := filepath.Join(base, "Nvidia Corporation", "Personal AI Router") + if err := os.MkdirAll(store, 0700); err != nil { + t.Fatal(err) + } + // Model an upgrade: the component's saved port and the broker's baseline + // agree on oMLX :1236. This is not an explicit settings Apply. + for file, port := range map[string]int{ + "proxy-port.json": 11434, "lmstudio-proxy-port.json": 1234, + "omlx-proxy-port.json": 1236, "llamacpp-proxy-port.json": 1237, + } { + if err := os.WriteFile(filepath.Join(store, file), []byte(fmt.Sprintf(`{"port":%d}`, port)), 0600); err != nil { + t.Fatal(err) + } + } + journal := []byte(`{"omlx":{"snapshot":{"engine":"omlx","revision":1,"appliedRevision":1,"phase":"idle","settings":{"serverPort":1235,"proxyPort":1236,"launchText":""}},"explicit":false}}`) + if err := os.WriteFile(filepath.Join(cluster, "engine-settings-operations.json"), journal, 0600); err != nil { + t.Fatal(err) + } + stdin, msgs, stderr, cleanup := startBrokerWithDirsAndEnv(t, root, cluster, []string{"OLLAMA_HOST="}, + "--settings-path", nodeSettingsBin, "--engine-manager-path", engineMgrBin, + "--proxy-path", proxyBin, "--lmstudio-proxy-path", lmstudioProxyBin, + "--omlx-proxy-path", omlxProxyBin, "--llamacpp-proxy-path", llamacppProxyBin, + ) + defer cleanup() + go func() { + for range stderr { + } + }() + waitForMethod(t, msgs, "app:ready", 15*time.Second) + for _, proxy := range []struct { + prefix string + port int + }{ + {"proxy", 11434}, {"lmstudio-proxy", 1234}, {"omlx-proxy", 1236}, {"llamacpp-proxy", 1237}, + } { + if got := waitRoutingProxyReady(t, stdin, msgs, proxy.prefix); got != proxy.port { + t.Fatalf("%s started on %d, want %d", proxy.prefix, got, proxy.port) + } + conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", proxy.port), time.Second) + if err != nil { + t.Fatalf("%s did not listen: %v", proxy.prefix, err) + } + _ = conn.Close() + } + data, err := os.ReadFile(filepath.Join(store, "omlx-proxy-port.json")) + if err != nil { + t.Fatal(err) + } + var saved struct { + Port int `json:"port"` + } + if err := json.Unmarshal(data, &saved); err != nil || saved.Port != 1236 { + t.Fatalf("startup changed saved oMLX proxy port: %s (%v)", data, err) + } +} diff --git a/services/versions.json b/services/versions.json index 181d07ab0..b4e49e16e 100644 --- a/services/versions.json +++ b/services/versions.json @@ -6,16 +6,17 @@ "ollama-proxy": "1.5.1", "lmstudio-proxy": "1.5.1", "nvpair-node-info": "1.4.1", - "nvpair-node-scanner": "1.3.1", - "nvpair-manual-nodes": "1.3.1", + "nvpair-node-scanner": "1.4.0", + "nvpair-manual-nodes": "1.4.0", "nvpair-workload-manager": "1.3.2", "nvpair-errors": "1.1.2", "nvpair-node-settings": "3.2.1", "nvpair-ui-broker": "2.6.0", "nvpair-engine-manager": "1.5.0", "nvpair-cluster-manager": "2.3.0", - "nvpair-job-scheduler": "1.1.1", + "nvpair-job-scheduler": "1.2.0", "nvpair-tui": "1.4.0", - "omlx-proxy": "1.5.1" + "omlx-proxy": "1.5.1", + "llamacpp-proxy": "1.0.0" } }