From c87a5299b9b29d2adaa5ebba1c5b7aa8bc6bae8a Mon Sep 17 00:00:00 2001 From: Seth Schneider Date: Wed, 21 Oct 2026 08:33:59 -0700 Subject: [PATCH 1/6] Add ModelScope adapter for llama.cpp downloads Translate ModelScope metadata and file requests into the Hugging Face API shape expected by the pinned llama.cpp downloader. Keep GGUF selection, caching, progress and cancellation in the router. Review guide (1 of 4; review the series oldest first): - Start with services/nvpair-engine-manager/modelscope.go, then modelscope_test.go. Later commits select and connect this adapter. - Check refs/tree/resolve translation, immutable file revisions (including abbreviated repository commits), snapshot bounds, path/hash validation, and rejection of incomplete 3,000-entry listings. - Follow GET/HEAD streaming through CDN redirects: Range headers and cancellation must survive; Hugging Face Authorization/Cookie headers must stay inside the loopback adapter; interrupted transfers must abort. Key coverage: TestModelScopeMetadata_PreservesSnapshotAfterBranchChanges, TestModelScopeMetadata_UsesRepositoryCommitWhenFileRevisionIsAbsent, and TestModelScopeDownload_PreservesRangesWithoutForwardingCredentials. From services/nvpair-engine-manager: go test ./... -run '^TestModelScope(Metadata|Download)_' -count=1 Validation uses local HTTP fixtures plus live public metadata/HEAD checks. A complete live llama.cpp model-weight download was not exercised. Signed-off-by: Seth Schneider --- services/nvpair-engine-manager/modelscope.go | 196 ++++++++++++++++++ .../nvpair-engine-manager/modelscope_test.go | 145 +++++++++++++ 2 files changed, 341 insertions(+) create mode 100644 services/nvpair-engine-manager/modelscope.go create mode 100644 services/nvpair-engine-manager/modelscope_test.go diff --git a/services/nvpair-engine-manager/modelscope.go b/services/nvpair-engine-manager/modelscope.go new file mode 100644 index 000000000..31c9b5fb8 --- /dev/null +++ b/services/nvpair-engine-manager/modelscope.go @@ -0,0 +1,196 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "net/url" + "sort" + "strings" + "sync" +) + +const modelScopeURL = "https://modelscope.cn" + +var errModelScopeListingTooLarge = errors.New("ModelScope repository exceeds the complete file-list limit") + +type modelScopeFile struct { + Path string + Type string + Revision string + SHA256 string `json:"Sha256"` + Size int64 +} + +func validModelScopePath(path string) bool { + return fs.ValidPath(path) && !strings.ContainsAny(path, "\\:\x00") +} + +func modelScopeHash(value string, size int) bool { + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == size +} + +func fetchModelScopeFiles(ctx context.Context, client *http.Client, endpoint, repo, revision string) ([]modelScopeFile, error) { + ctx, cancel := context.WithTimeout(ctx, catalogHTTPTimeout) + defer cancel() + query := url.Values{"Revision": {revision}, "Recursive": {"true"}} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint+"/api/v1/models/"+repo+"/repo/files?"+query.Encode(), nil) + if err != nil { + return nil, fmt.Errorf("build ModelScope metadata request: %w", withoutURL(err)) + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("ModelScope metadata request failed: %w", withoutURL(err)) + } + defer resp.Body.Close() + var result struct { + Success bool + Data struct { + Files []modelScopeFile + LatestCommitter struct { + ShortID string `json:"ShortId"` + } + } + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("ModelScope metadata returned HTTP %d", resp.StatusCode) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, maxCatalogBody+1)) + if err != nil || len(data) > maxCatalogBody || json.Unmarshal(data, &result) != nil || !result.Success || result.Data.Files == nil { + return nil, fmt.Errorf("invalid ModelScope file metadata") + } + if len(result.Data.Files) >= 3000 { + return nil, errModelScopeListingTooLarge + } + files := make([]modelScopeFile, 0, len(result.Data.Files)) + for _, file := range result.Data.Files { + if file.Type != "blob" { + continue + } + if file.Revision == "" { + file.Revision = result.Data.LatestCommitter.ShortID + } + if !validModelScopePath(file.Path) || (!modelScopeHash(file.Revision, 20) && !modelScopeHash(file.Revision, 4)) || !modelScopeHash(file.SHA256, 32) || file.Size < 0 { + return nil, fmt.Errorf("invalid ModelScope file metadata") + } + files = append(files, file) + } + sort.Slice(files, func(i, j int) bool { return files[i].Path < files[j].Path }) + return files, nil +} + +// The pinned llama.cpp downloader expects Hugging Face metadata. Translate +// metadata and stream files, leaving downloads, caching and progress to it. +func newModelScopeHandler(client *http.Client, endpoint string) http.Handler { + var mu sync.Mutex + snapshots := make(map[string][]modelScopeFile) + var order []string + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + parts := strings.Split(strings.TrimPrefix(r.URL.Path, "/"), "/") + metadata := len(parts) >= 5 && parts[0] == "api" && parts[1] == "models" + if metadata { + parts = parts[2:] + } + if len(parts) < 3 || !llamaCPPRepoPattern.MatchString(parts[0]+"/"+parts[1]) { + http.NotFound(w, r) + return + } + repo := parts[0] + "/" + parts[1] + if metadata && len(parts) == 3 && parts[2] == "refs" { + files, err := fetchModelScopeFiles(r.Context(), client, endpoint, repo, "master") + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + // ModelScope supplies per-file or abbreviated repository revisions. + // Identify the exact file set with the full-width snapshot ID llama.cpp + // expects, so a moving branch cannot change an accepted download. + data, _ := json.Marshal(files) + digest := sha1.Sum(append([]byte("modelscope\x00"), data...)) + commit := hex.EncodeToString(digest[:]) + key := repo + "/" + commit + mu.Lock() + if _, exists := snapshots[key]; !exists { + snapshots[key] = files + order = append(order, key) + if len(order) > 64 { + delete(snapshots, order[0]) + order = order[1:] + } + } + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"branches": []map[string]string{{"name": "main", "targetCommit": commit}}}) + return + } + if len(parts) < 4 || !modelScopeHash(parts[3], 20) { + http.NotFound(w, r) + return + } + mu.Lock() + files, exists := snapshots[repo+"/"+parts[3]] + mu.Unlock() + if !exists { + http.NotFound(w, r) + return + } + if metadata && len(parts) == 4 && parts[2] == "tree" { + rows := make([]map[string]any, 0, len(files)) + for _, file := range files { + rows = append(rows, map[string]any{"type": "file", "path": file.Path, "oid": file.SHA256, "size": file.Size}) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(rows) + return + } + path := strings.Join(parts[4:], "/") + if !metadata && parts[2] == "resolve" && validModelScopePath(path) { + for _, file := range files { + if file.Path == path { + query := url.Values{"Revision": {file.Revision}, "FilePath": {path}} + req, err := http.NewRequestWithContext(r.Context(), r.Method, endpoint+"/api/v1/models/"+repo+"/repo?"+query.Encode(), nil) + if err != nil { + http.Error(w, "invalid ModelScope download request", http.StatusBadGateway) + return + } + for _, header := range []string{"Range", "If-Range", "If-None-Match", "If-Modified-Since"} { + if value := r.Header.Get(header); value != "" { + req.Header.Set(header, value) + } + } + resp, err := client.Do(req) + if err != nil { + http.Error(w, "ModelScope download request failed", http.StatusBadGateway) + return + } + defer resp.Body.Close() + for _, header := range []string{"Content-Length", "Content-Type", "ETag", "Accept-Ranges", "Content-Range", "Last-Modified"} { + if value := resp.Header.Get(header); value != "" { + w.Header().Set(header, value) + } + } + w.WriteHeader(resp.StatusCode) + if _, err := io.Copy(w, resp.Body); err != nil { + panic(http.ErrAbortHandler) + } + return + } + } + } + http.NotFound(w, r) + }) +} diff --git a/services/nvpair-engine-manager/modelscope_test.go b/services/nvpair-engine-manager/modelscope_test.go new file mode 100644 index 000000000..09439c0ea --- /dev/null +++ b/services/nvpair-engine-manager/modelscope_test.go @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const modelScopeTestRevision = "0123456789abcdef0123456789abcdef01234567" +const modelScopeTestSHA256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func modelScopeTestCommit(t *testing.T, handler http.Handler) string { + t.Helper() + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/models/owner/repo/refs", nil)) + require.Equal(t, http.StatusOK, response.Code) + var result struct { + Branches []struct{ Name, TargetCommit string } + } + require.NoError(t, json.Unmarshal(response.Body.Bytes(), &result)) + require.Len(t, result.Branches, 1) + assert.Equal(t, "main", result.Branches[0].Name) + require.Len(t, result.Branches[0].TargetCommit, 40) + return result.Branches[0].TargetCommit +} + +func TestModelScopeMetadata_PreservesSnapshotAfterBranchChanges(t *testing.T) { + var changed atomic.Bool + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/v1/models/owner/repo/repo/files", r.URL.Path) + assert.Equal(t, "master", r.URL.Query().Get("Revision")) + assert.Equal(t, "true", r.URL.Query().Get("Recursive")) + file := modelScopeFile{Path: "nested/model-Q4_K_M.gguf", Type: "blob", Revision: modelScopeTestRevision, SHA256: modelScopeTestSHA256, Size: 100} + if changed.Load() { + file.SHA256 = strings.Repeat("a", 64) + } + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{"Success": true, "Data": map[string]any{"Files": []modelScopeFile{file}}})) + })) + t.Cleanup(upstream.Close) + handler := newModelScopeHandler(upstream.Client(), upstream.URL) + commit := modelScopeTestCommit(t, handler) + changed.Store(true) + assert.NotEqual(t, commit, modelScopeTestCommit(t, handler)) + + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/models/owner/repo/tree/"+commit+"?recursive=true", nil)) + require.Equal(t, http.StatusOK, response.Code) + var files []struct { + Type, Path, OID string + Size int64 + } + require.NoError(t, json.Unmarshal(response.Body.Bytes(), &files)) + require.Len(t, files, 1) + assert.Equal(t, "file", files[0].Type) + assert.Equal(t, "nested/model-Q4_K_M.gguf", files[0].Path) + assert.Equal(t, modelScopeTestSHA256, files[0].OID) + assert.Equal(t, int64(100), files[0].Size) +} + +func TestModelScopeDownload_PreservesRangesWithoutForwardingCredentials(t *testing.T) { + cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("Authorization")) + assert.Empty(t, r.Header.Get("Cookie")) + assert.Equal(t, "bytes=10-19", r.Header.Get("Range")) + w.Header().Set("Content-Range", "bytes 10-19/100") + w.Header().Set("ETag", "model-etag") + w.WriteHeader(http.StatusPartialContent) + _, err := fmt.Fprint(w, "0123456789") + assert.NoError(t, err) + })) + t.Cleanup(cdn.Close) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("Authorization")) + assert.Empty(t, r.Header.Get("Cookie")) + if strings.HasSuffix(r.URL.Path, "/files") { + _, err := fmt.Fprintf(w, `{"Success":true,"Data":{"Files":[{"Path":"nested/model-Q4_K_M.gguf","Type":"blob","Revision":%q,"Sha256":%q,"Size":100}]}}`, modelScopeTestRevision, modelScopeTestSHA256) + assert.NoError(t, err) + return + } + assert.Equal(t, modelScopeTestRevision, r.URL.Query().Get("Revision")) + assert.Equal(t, "nested/model-Q4_K_M.gguf", r.URL.Query().Get("FilePath")) + http.Redirect(w, r, cdn.URL, http.StatusTemporaryRedirect) + })) + t.Cleanup(upstream.Close) + handler := newModelScopeHandler(upstream.Client(), upstream.URL) + commit := modelScopeTestCommit(t, handler) + request := httptest.NewRequest(http.MethodGet, "/owner/repo/resolve/"+commit+"/nested/model-Q4_K_M.gguf", nil) + request.Header.Set("Range", "bytes=10-19") + request.Header.Set("Authorization", "Bearer hf-test-secret") + request.Header.Set("Cookie", "session=test-secret") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + + assert.Equal(t, http.StatusPartialContent, response.Code) + assert.Equal(t, "bytes 10-19/100", response.Header().Get("Content-Range")) + assert.Equal(t, "model-etag", response.Header().Get("ETag")) + assert.Equal(t, "0123456789", response.Body.String()) +} + +func TestModelScopeMetadata_RejectsUnsafeFilePaths(t *testing.T) { + for _, path := range []string{"../model.gguf", "nested/../../model.gguf", `C:\model.gguf`, "/model.gguf"} { + t.Run(path, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{"Success": true, "Data": map[string]any{"Files": []modelScopeFile{{Path: path, Type: "blob", Revision: modelScopeTestRevision, SHA256: modelScopeTestSHA256}}}})) + })) + t.Cleanup(upstream.Close) + response := httptest.NewRecorder() + newModelScopeHandler(upstream.Client(), upstream.URL).ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/models/owner/repo/refs", nil)) + assert.Equal(t, http.StatusBadGateway, response.Code) + }) + } +} + +func TestModelScopeMetadata_RejectsUpstreamFailure(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(upstream.Close) + response := httptest.NewRecorder() + newModelScopeHandler(upstream.Client(), upstream.URL).ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/models/owner/repo/refs", nil)) + assert.Equal(t, http.StatusBadGateway, response.Code) +} + +func TestModelScopeMetadata_UsesRepositoryCommitWhenFileRevisionIsAbsent(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, err := fmt.Fprintf(w, `{"Success":true,"Data":{"LatestCommitter":{"ShortId":"f659fb25"},"Files":[{"Path":"model-Q4_K_M.gguf","Type":"blob","Sha256":%q,"Size":100}]}}`, modelScopeTestSHA256) + assert.NoError(t, err) + })) + t.Cleanup(upstream.Close) + files, err := fetchModelScopeFiles(context.Background(), upstream.Client(), upstream.URL, "owner/repo", "master") + require.NoError(t, err) + require.Len(t, files, 1) + assert.Equal(t, "f659fb25", files[0].Revision) +} From c277c9287837ce50965269c4bf06f42674a0b0a1 Mon Sep 17 00:00:00 2001 From: Seth Schneider Date: Wed, 21 Oct 2026 08:45:54 -0700 Subject: [PATCH 2/6] Persist llama.cpp model download source in engine settings Carry the node-owned modelSource choice through the existing settings protocol, persist it as runtime.model_source, and select MODEL_ENDPOINT at engine start. Review guide (2 of 4): - Start with services/shared/enginesettings/settings.go, then engine-manager settings.go and setport.go, followed by broker enginesettings.go. Trace the field through snapshot, preview, apply, journal and worker reload. - Check the Hugging Face default, llama.cpp-only validation, preservation by other settings consumers, and source-only restart decisions. A stopped engine must stay stopped and previews must remain read-only. - Inspect engine-manager lifecycle.go and modelsource.go: create the loopback adapter only during a ModelScope start, close it on shutdown, and prevent a saved MODEL_ENDPOINT with different Windows casing from overriding the choice. Key coverage: TestModelSource_*, the broker's TestSettingsCoordinator_PreservesModelSourceAcrossWorkerAndJournal, and TestEngineManager_RelaysSettingsRebindAndReloadsWorker (real JSON-RPC workers with temporary configuration and an isolated port). From services/nvpair-engine-manager: go test ./... -run '^TestModelSource_|TestEngineManager_RelaysSettings' -count=1 From services/nvpair-ui-broker: go test ./... -run '^TestSettingsCoordinator_PreservesModelSource' -count=1 Signed-off-by: Seth Schneider --- services/nvpair-engine-manager/MANIFEST.md | 4 +- services/nvpair-engine-manager/e2e_test.go | 24 ++-- services/nvpair-engine-manager/executor.go | 8 +- services/nvpair-engine-manager/lifecycle.go | 23 ++++ .../manifest.schema.json | 3 + services/nvpair-engine-manager/modelsource.go | 41 +++++++ .../nvpair-engine-manager/modelsource_test.go | 112 ++++++++++++++++++ services/nvpair-engine-manager/registry.go | 5 + services/nvpair-engine-manager/setport.go | 7 +- services/nvpair-engine-manager/settings.go | 22 +++- .../settings_permissions_test.go | 2 +- .../testdata/fakeengine/main.go | 2 +- services/nvpair-ui-broker/ENGINE_SETTINGS.md | 6 +- services/nvpair-ui-broker/enginesettings.go | 2 +- .../nvpair-ui-broker/enginesettings_test.go | 32 ++++- services/shared/enginesettings/settings.go | 4 +- 16 files changed, 274 insertions(+), 23 deletions(-) create mode 100644 services/nvpair-engine-manager/modelsource.go create mode 100644 services/nvpair-engine-manager/modelsource_test.go diff --git a/services/nvpair-engine-manager/MANIFEST.md b/services/nvpair-engine-manager/MANIFEST.md index 5c9b2d3b0..20f3449b0 100644 --- a/services/nvpair-engine-manager/MANIFEST.md +++ b/services/nvpair-engine-manager/MANIFEST.md @@ -110,7 +110,9 @@ is validated again before execution, including settings saved by earlier version The editor saves literal extra arguments in host-platform `runtime.launch_args` and explicit environment assignments in -`runtime.launch_env`, after trusted template expansion. Empty arrays clear the +`runtime.launch_env`, after trusted template expansion. llama.cpp's selected +download provider is saved in host-platform `runtime.model_source` (`huggingface` +or `modelscope`). Empty arrays clear the saved arguments or environment assignments. See [LAUNCH_TEXT.md](LAUNCH_TEXT.md) for validation and [the broker protocol](../nvpair-ui-broker/ENGINE_SETTINGS.md) for application and recovery. Editing `args`/`start` directly remains trusted manifest authoring. diff --git a/services/nvpair-engine-manager/e2e_test.go b/services/nvpair-engine-manager/e2e_test.go index 3ecbf36a4..8f5c09c4c 100644 --- a/services/nvpair-engine-manager/e2e_test.go +++ b/services/nvpair-engine-manager/e2e_test.go @@ -366,19 +366,21 @@ func waitNotify(t *testing.T, frames <-chan frame, method string, timeout time.D // servicing unrelated messages; literal settings survive a fresh worker. func TestEngineManager_RelaysSettingsRebindAndReloadsWorker(t *testing.T) { cfg, home := t.TempDir(), t.TempDir() - fixture := settingsExecutor(t, false) - manifest, _ := fixture.reg.Get("fake") + engine := "llamacpp" + port, err := freePort() + require.NoError(t, err) for _, dir := range []string{filepath.Join(cfg, "Nvidia Corporation", "Personal AI Router", "engines"), filepath.Join(home, "Library", "Application Support", "Nvidia Corporation", "Personal AI Router", "engines")} { - writeE2EManifest(t, dir, manifest) + require.NoError(t, os.MkdirAll(dir, 0o700)) + require.NoError(t, writeJSONAtomic(filepath.Join(dir, engine+".json"), map[string]any{"engine": engine, "runtime": map[string]int{"port": port}})) } manager := startE2EManager(t, cfg, home) send(t, manager.stdin, 1, "engine:get-installed", nil) waitResult(t, manager.frames, "1", 5*time.Second) - send(t, manager.stdin, 2, "engine:get-launch", map[string]string{"engine": "fake"}) + send(t, manager.stdin, 2, "engine:get-launch", map[string]string{"engine": engine}) var launch settings.LaunchState require.NoError(t, json.Unmarshal(waitResult(t, manager.frames, "2", 5*time.Second), &launch)) - desired := settings.Config{ServerPort: launch.ServerPort, ProxyPort: 54301, LaunchText: `OLLAMA_ORIGINS="http://localhost" PAIR_TEST_LITERAL="$HOME {port}" ` + launch.LaunchText} - send(t, manager.stdin, 3, "engine:configure-launch", settings.Configure{Engine: "fake", Settings: desired, OperationID: "operation"}) + desired := settings.Config{ServerPort: launch.ServerPort, ProxyPort: 54301, LaunchText: `OLLAMA_ORIGINS="http://localhost" PAIR_TEST_LITERAL="$HOME {port}" ` + launch.LaunchText, ModelSource: "modelscope"} + send(t, manager.stdin, 3, "engine:configure-launch", settings.Configure{Engine: engine, Settings: desired, OperationID: "operation"}) var relay settings.Relay timer := time.NewTimer(5 * time.Second) defer timer.Stop() @@ -386,6 +388,9 @@ func TestEngineManager_RelaysSettingsRebindAndReloadsWorker(t *testing.T) { for waiting { select { case f := <-manager.frames: + if string(f.ID) == "3" { + require.Contains(t, []string{"", "null"}, string(f.Error), "configure failed before rebind") + } if f.Method == "engine:settings-request" { require.NoError(t, json.Unmarshal(f.Params, &relay)) waiting = false @@ -397,18 +402,21 @@ func TestEngineManager_RelaysSettingsRebindAndReloadsWorker(t *testing.T) { require.Equal(t, "rebind", relay.Method, "bad relay (%v)", relay) require.Equal(t, "operation", relay.Request.RequestID, "bad relay (%v)", relay) require.Equal(t, 54301, relay.Request.Settings.ProxyPort, "bad relay (%v)", relay) - send(t, manager.stdin, 4, "engine:describe", map[string]string{"engine": "fake"}) + require.Equal(t, "modelscope", relay.Request.Settings.ModelSource) + send(t, manager.stdin, 4, "engine:describe", map[string]string{"engine": engine}) waitResult(t, manager.frames, "4", 5*time.Second) notify(t, manager.stdin, "engine:settings-reply", settingsReply{ID: relay.ID, Result: json.RawMessage(`{}`)}) require.NoError(t, json.Unmarshal(waitResult(t, manager.frames, "3", 5*time.Second), &launch), "configure result") require.False(t, launch.Running, "configure result (%v)", launch) + require.Equal(t, "modelscope", launch.ModelSource) saved := launch.LaunchText manager.stop(t) restored := startE2EManager(t, cfg, home) send(t, restored.stdin, 1, "engine:get-installed", nil) waitResult(t, restored.frames, "1", 5*time.Second) - send(t, restored.stdin, 2, "engine:get-launch", map[string]string{"engine": "fake"}) + send(t, restored.stdin, 2, "engine:get-launch", map[string]string{"engine": engine}) require.NoError(t, json.Unmarshal(waitResult(t, restored.frames, "2", 5*time.Second), &launch), "worker reload lost literal settings") require.Equal(t, saved, launch.LaunchText, "worker reload lost literal settings (%v)", launch) + require.Equal(t, "modelscope", launch.ModelSource) restored.stop(t) } diff --git a/services/nvpair-engine-manager/executor.go b/services/nvpair-engine-manager/executor.go index f71584570..db71609a5 100644 --- a/services/nvpair-engine-manager/executor.go +++ b/services/nvpair-engine-manager/executor.go @@ -126,9 +126,11 @@ type Executor struct { reservedPort atomic.Int32 // StopAll is terminal for an Executor; the gate closes its start/snapshot race. - shuttingDown atomic.Bool - mu sync.Mutex - engines map[string]*engineState + shuttingDown atomic.Bool + mu sync.Mutex + engines map[string]*engineState + modelScope *http.Server + modelScopeURL string } func NewExecutor(reg *Registry, reporter *Reporter, emit func(string, any), baseDir string) *Executor { diff --git a/services/nvpair-engine-manager/lifecycle.go b/services/nvpair-engine-manager/lifecycle.go index e1b2c53c2..5c7ee0610 100644 --- a/services/nvpair-engine-manager/lifecycle.go +++ b/services/nvpair-engine-manager/lifecycle.go @@ -240,6 +240,24 @@ func (e *Executor) bringUpProcess(ctx context.Context, st *engineState, engine s if err := validateEffectiveLaunch(rt, launch, vars["host"], vars["port"]); err != nil { return err } + if engine == "llamacpp" { + endpoint := "https://huggingface.co" + if modelSource(rt) == "modelscope" { + endpoint, err = e.modelScopeEndpoint() + if err != nil { + return err + } + } + if launch.Env == nil { + launch.Env = map[string]string{} + } + for key := range launch.Env { + if environmentKey(key) == environmentKey("MODEL_ENDPOINT") { + delete(launch.Env, key) + } + } + launch.Env["MODEL_ENDPOINT"] = endpoint + } diagnostics := newStartupOutput(launchDiagnosticArgs(rt), launch.Env) defer diagnostics.close() @@ -789,6 +807,11 @@ func (e *Executor) StopAll() { }(st, n) } wg.Wait() + e.mu.Lock() + if e.modelScope != nil { + _ = e.modelScope.Close() + } + e.mu.Unlock() } func (e *Executor) runHealth(ctx context.Context, st *engineState, engine string, port int, gen int64) { diff --git a/services/nvpair-engine-manager/manifest.schema.json b/services/nvpair-engine-manager/manifest.schema.json index 713b01664..50eaa3b27 100644 --- a/services/nvpair-engine-manager/manifest.schema.json +++ b/services/nvpair-engine-manager/manifest.schema.json @@ -23,6 +23,9 @@ "runtime": { "type": "object", "properties": { + "model_source": { + "enum": ["huggingface", "modelscope"] + }, "editable_launch": { "$ref": "#/definitions/editableLaunch" } diff --git a/services/nvpair-engine-manager/modelsource.go b/services/nvpair-engine-manager/modelsource.go new file mode 100644 index 000000000..2aad253f7 --- /dev/null +++ b/services/nvpair-engine-manager/modelsource.go @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "net" + "net/http" + "time" +) + +func modelSource(rt Runtime) string { + if rt.ModelSource == "" { + return "huggingface" + } + return rt.ModelSource +} + +// Only an actual engine start creates a listener; settings previews stay read-only. +func (e *Executor) modelScopeEndpoint() (string, error) { + e.mu.Lock() + defer e.mu.Unlock() + if e.shuttingDown.Load() { + return "", fmt.Errorf("engine manager is shutting down") + } + if e.modelScope == nil { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return "", fmt.Errorf("start ModelScope adapter: %w", err) + } + e.modelScope = &http.Server{ + Handler: newModelScopeHandler(e.client, modelScopeURL), + ReadHeaderTimeout: 5 * time.Second, + IdleTimeout: time.Minute, + } + e.modelScopeURL = "http://" + listener.Addr().String() + go func() { _ = e.modelScope.Serve(listener) }() + } + return e.modelScopeURL, nil +} diff --git a/services/nvpair-engine-manager/modelsource_test.go b/services/nvpair-engine-manager/modelsource_test.go new file mode 100644 index 000000000..62ba27876 --- /dev/null +++ b/services/nvpair-engine-manager/modelsource_test.go @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + settings "nvpair-shared/enginesettings" +) + +func llamaSourceExecutor(t *testing.T) (*Executor, settings.Request) { + t.Helper() + manifest := testEngineManifest(fakeEngineBin) + manifest.Engine = "llamacpp" + key := runtime.GOOS + "/" + runtime.GOARCH + platform := manifest.Platforms[key] + port, err := freePort() + require.NoError(t, err) + platform.Runtime.Port = port + platform.Runtime.Args = []string{"--port", "{port}"} + platform.Runtime.EditableLaunch = &EditableLaunch{Controls: []LaunchControl{{Value: "{server.port}", Flags: []string{"--port"}}}} + manifest.Platforms[key] = platform + executor := newTestExecutor(t, manifest) + executor.overrideDir = t.TempDir() + snapshot, err := executor.LaunchSettings("llamacpp") + require.NoError(t, err) + return executor, settings.Request{Engine: "llamacpp", Settings: settings.Config{ServerPort: snapshot.ServerPort, ProxyPort: 54322, LaunchText: snapshot.LaunchText}} +} + +func TestModelSource_DefaultsToHuggingFace(t *testing.T) { + executor, request := llamaSourceExecutor(t) + preview, err := executor.PreviewLaunch(request) + require.NoError(t, err) + assert.Empty(t, preview.Errors) + assert.Equal(t, "huggingface", preview.Settings.ModelSource) + assert.Nil(t, executor.modelScope, "preview must not open a listener") +} + +func TestModelSource_RejectsUnsupportedSelection(t *testing.T) { + executor, request := llamaSourceExecutor(t) + request.Settings.ModelSource = "https://example.com" + preview, err := executor.PreviewLaunch(request) + require.NoError(t, err) + assert.Contains(t, preview.Errors, "modelSource") +} + +func TestModelSource_RequestsRestartForSourceOnlyEdit(t *testing.T) { + executor, request := llamaSourceExecutor(t) + state, err := executor.state("llamacpp") + require.NoError(t, err) + state.running = true + request.Settings.ModelSource = "modelscope" + preview, err := executor.PreviewLaunch(request) + require.NoError(t, err) + require.Empty(t, preview.Errors) + assert.True(t, preview.Restart) + assert.Equal(t, request.Settings.LaunchText, preview.Settings.LaunchText) +} + +func TestModelSource_PersistsSelectionWithoutStartingStoppedEngine(t *testing.T) { + executor, request := llamaSourceExecutor(t) + request.Settings.ModelSource = "modelscope" + snapshot, err := executor.ConfigureLaunch(context.Background(), settings.Configure{Engine: request.Engine, Settings: request.Settings}, func() error { return nil }) + require.NoError(t, err) + assert.Equal(t, "modelscope", snapshot.ModelSource) + assert.False(t, snapshot.Running) + assert.Nil(t, executor.modelScope) + data, err := os.ReadFile(filepath.Join(executor.overrideDir, "llamacpp.json")) + require.NoError(t, err) + var saved Manifest + require.NoError(t, json.Unmarshal(data, &saved)) + assert.Equal(t, "modelscope", saved.Platforms[runtime.GOOS+"/"+runtime.GOARCH].Runtime.ModelSource) +} + +func TestModelSource_LaunchesWithSelectedEndpoint(t *testing.T) { + for _, source := range []string{"huggingface", "modelscope"} { + t.Run(source, func(t *testing.T) { + executor, _ := llamaSourceExecutor(t) + t.Cleanup(executor.StopAll) + state, err := executor.state("llamacpp") + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "environment.json") + state.plat.Runtime.Env["PAIR_TEST_ENV_FILE"] = path + key := "MODEL_ENDPOINT" + if runtime.GOOS == "windows" { + key = "model_endpoint" + } + state.plat.Runtime.LaunchEnv = []string{key + "=https://unselected.invalid"} + state.plat.Runtime.ModelSource = source + require.NoError(t, executor.Start(context.Background(), "llamacpp")) + data, err := os.ReadFile(path) + require.NoError(t, err) + var environment map[string]string + require.NoError(t, json.Unmarshal(data, &environment)) + if source == "modelscope" { + assert.Equal(t, executor.modelScopeURL, environment["MODEL_ENDPOINT"]) + require.NotNil(t, executor.modelScope) + } else { + assert.Equal(t, "https://huggingface.co", environment["MODEL_ENDPOINT"]) + assert.Nil(t, executor.modelScope) + } + }) + } +} diff --git a/services/nvpair-engine-manager/registry.go b/services/nvpair-engine-manager/registry.go index 3b3278871..68ae98a8f 100644 --- a/services/nvpair-engine-manager/registry.go +++ b/services/nvpair-engine-manager/registry.go @@ -142,6 +142,8 @@ type InstallArtifact struct { // (e.g. LM Studio's `lms`); liveness = the readiness/health probe, // and Stop.Cmd brings it down. type Runtime struct { + // ModelSource selects llama.cpp's model catalogue and download provider. + ModelSource string `json:"model_source,omitempty"` // EditableLaunch declares which startup arguments and environment controls // PAIR can safely expose through engine settings. EditableLaunch *EditableLaunch `json:"editable_launch,omitempty"` @@ -728,6 +730,9 @@ func (m *Manifest) Validate() error { } func (p *Platform) validate(key string) error { + if source := p.Runtime.ModelSource; source != "" && source != "huggingface" && source != "modelscope" { + return fmt.Errorf("platform %q: invalid model_source", key) + } if policy := p.Runtime.EditableLaunch; policy != nil { if policy.StartIndex < 0 || (p.Runtime.modeOrDefault() == "command" && policy.StartIndex >= len(p.Runtime.Start)) { return fmt.Errorf("platform %q: invalid editable launch definition", key) diff --git a/services/nvpair-engine-manager/setport.go b/services/nvpair-engine-manager/setport.go index 66f0b2837..68dd88765 100644 --- a/services/nvpair-engine-manager/setport.go +++ b/services/nvpair-engine-manager/setport.go @@ -101,13 +101,13 @@ func (e *Executor) SetPort(ctx context.Context, engine string, port int) (Engine // A host-platform port is updated too, when present, because platform values // take precedence over shared runtime defaults at load time. func (e *Executor) persistPort(engine string, port int) error { - return e.persistRuntimeConfig(engine, port, nil, nil) + return e.persistRuntimeConfig(engine, port, nil, nil, nil) } // persistRuntimeConfig writes the port override plus, when non-nil, the launch // argument and environment overrides. A nil slice pointer leaves that override // exactly as it is on disk; an empty slice clears it to "declared, but empty". -func (e *Executor) persistRuntimeConfig(engine string, port int, args, environment *[]string) error { +func (e *Executor) persistRuntimeConfig(engine string, port int, args, environment *[]string, modelSource *string) error { if e.overrideDir == "" { return fmt.Errorf("no config directory available to persist the port") } @@ -184,6 +184,9 @@ func (e *Executor) persistRuntimeConfig(engine string, port int, args, environme if environment != nil { hostRuntime["launch_env"] = append([]string{}, (*environment)...) } + if modelSource != nil && *modelSource != "" { + hostRuntime["model_source"] = *modelSource + } setOverrideObject(platform, "runtime", hostRuntime) setOverrideObject(platforms, host, platform) setOverrideObject(m, "platforms", platforms) diff --git a/services/nvpair-engine-manager/settings.go b/services/nvpair-engine-manager/settings.go index 568915107..689272607 100644 --- a/services/nvpair-engine-manager/settings.go +++ b/services/nvpair-engine-manager/settings.go @@ -131,6 +131,9 @@ func (e *Executor) launchStateLocked(engine string, st *engineState) settings.La result := settings.LaunchState{Engine: engine, ServerPort: st.plat.Runtime.Port, EffectivePort: st.port, Running: st.running, Adopted: st.adopted, Format: launchTextFormat} st.mu.Unlock() command, err := launchForState(st, result.ServerPort) + if engine == "llamacpp" { + result.ModelSource = modelSource(st.plat.Runtime) + } if err == nil { result.LaunchText, err = command.argumentText(st.plat.Runtime.EditableLaunch) } @@ -246,6 +249,18 @@ func (e *Executor) PreviewLaunch(p settings.Request) (settings.Preview, error) { func (e *Executor) previewLaunchLocked(st *engineState, request settings.Request) settings.Preview { result := settings.Preview{Settings: request.Settings, Args: []string{}, Env: []string{}, Errors: map[string]string{}} fail := func(message string) settings.Preview { result.Errors["launchText"] = message; return result } + if request.Engine == "llamacpp" { + if result.Settings.ModelSource == "" { + result.Settings.ModelSource = "huggingface" + } + if result.Settings.ModelSource != "huggingface" && result.Settings.ModelSource != "modelscope" { + result.Errors["modelSource"] = "Select Hugging Face or ModelScope." + return result + } + } else if result.Settings.ModelSource != "" { + result.Errors["modelSource"] = "Model download source is only supported for llama.cpp." + return result + } if request.Settings.ServerPort < 1 || request.Settings.ServerPort > 65535 { result.Errors["serverPort"] = "Enter a server port from 1 to 65535." return result @@ -381,7 +396,7 @@ func (e *Executor) previewLaunchLocked(st *engineState, request settings.Request return fail(err.Error()) } current := e.launchStateLocked(request.Engine, st) - result.Restart = current.Running && (current.ServerPort != result.Settings.ServerPort || current.LaunchText != result.Settings.LaunchText) + result.Restart = current.Running && (current.ServerPort != result.Settings.ServerPort || current.LaunchText != result.Settings.LaunchText || current.ModelSource != result.Settings.ModelSource) if !current.Editable { return fail(current.Reason) } @@ -411,7 +426,7 @@ func (e *Executor) ConfigureLaunch(ctx context.Context, p settings.Configure, re e.reporter.clear(startFailedID(p.Engine)) e.reporter.clear(exitedID(p.Engine)) old := e.launchStateLocked(p.Engine, st) - changed := old.ServerPort != preview.Settings.ServerPort || old.LaunchText != preview.Settings.LaunchText + changed := old.ServerPort != preview.Settings.ServerPort || old.LaunchText != preview.Settings.LaunchText || old.ModelSource != preview.Settings.ModelSource // Stop before writing the override file. A stop failure returns with the // engine still on its old launch, so persisting first would leave disk on // the new one and the next start would silently adopt a configuration the @@ -422,7 +437,7 @@ func (e *Executor) ConfigureLaunch(ctx context.Context, p settings.Configure, re } } if changed || !reflect.DeepEqual(st.plat.Runtime.LaunchArgs, preview.Args) || !reflect.DeepEqual(st.plat.Runtime.LaunchEnv, preview.Env) { - if err := e.persistRuntimeConfig(p.Engine, preview.Settings.ServerPort, &preview.Args, &preview.Env); err != nil { + if err := e.persistRuntimeConfig(p.Engine, preview.Settings.ServerPort, &preview.Args, &preview.Env, &preview.Settings.ModelSource); err != nil { return e.launchStateLocked(p.Engine, st), err } } @@ -430,6 +445,7 @@ func (e *Executor) ConfigureLaunch(ctx context.Context, p settings.Configure, re st.plat.Runtime.Port = preview.Settings.ServerPort st.plat.Runtime.LaunchArgs = preview.Args st.plat.Runtime.LaunchEnv = preview.Env + st.plat.Runtime.ModelSource = preview.Settings.ModelSource st.port = preview.Settings.ServerPort st.mu.Unlock() if err := rebind(); err != nil { diff --git a/services/nvpair-engine-manager/settings_permissions_test.go b/services/nvpair-engine-manager/settings_permissions_test.go index cb798e926..257340fc4 100644 --- a/services/nvpair-engine-manager/settings_permissions_test.go +++ b/services/nvpair-engine-manager/settings_permissions_test.go @@ -21,7 +21,7 @@ func TestSettingsOverride_RestrictsExistingPermissions(t *testing.T) { path := filepath.Join(e.overrideDir, "fake.json") require.NoError(t, os.WriteFile(path, []byte(`{"engine":"fake"}`), 0644)) environment := []string{"PAIR_TEST=private"} - require.NoError(t, e.persistRuntimeConfig("fake", 54321, nil, &environment)) + require.NoError(t, e.persistRuntimeConfig("fake", 54321, nil, &environment, nil)) for path, want := range map[string]os.FileMode{e.overrideDir: 0700, path: 0600} { info, err := os.Stat(path) require.NoError(t, err) diff --git a/services/nvpair-engine-manager/testdata/fakeengine/main.go b/services/nvpair-engine-manager/testdata/fakeengine/main.go index 9f12ba821..2172d2170 100644 --- a/services/nvpair-engine-manager/testdata/fakeengine/main.go +++ b/services/nvpair-engine-manager/testdata/fakeengine/main.go @@ -73,7 +73,7 @@ func bstr(m map[string]any, k string) string { func main() { if path := os.Getenv("PAIR_TEST_ENV_FILE"); path != "" { values := map[string]string{} - for _, key := range []string{"OLLAMA_ORIGINS", "PAIR_TEST_LITERAL", "PAIR_TEST_EMPTY", "OLLAMA_HOST"} { + for _, key := range []string{"OLLAMA_ORIGINS", "PAIR_TEST_LITERAL", "PAIR_TEST_EMPTY", "OLLAMA_HOST", "MODEL_ENDPOINT"} { values[key] = os.Getenv(key) } data, _ := json.Marshal(values) diff --git a/services/nvpair-ui-broker/ENGINE_SETTINGS.md b/services/nvpair-ui-broker/ENGINE_SETTINGS.md index 5d5f5728e..7eb50c285 100644 --- a/services/nvpair-ui-broker/ENGINE_SETTINGS.md +++ b/services/nvpair-ui-broker/ENGINE_SETTINGS.md @@ -17,7 +17,11 @@ propagation is not part of this API. | `engine:apply-settings` | Preview request plus `requestId` | `{revision, phase}` acknowledgement | `engine` is `ollama`, `lmstudio`, or `llamacpp`. `settings` contains all three -fields: `serverPort`, `proxyPort`, `launchText`. The last field contains +fields: `serverPort`, `proxyPort`, `launchText`. llama.cpp additionally exposes +`modelSource` (`huggingface` or `modelscope`, default `huggingface`); other engines +omit it. The selection is persisted and relayed with the rest of the settings. +Changing it restarts a running llama.cpp router and changes both its catalogue +and downloads. `launchText` contains arguments and leading environment assignments, without the executable or startup subcommand. The argument grammar is [`pair-arguments-v1`](../nvpair-engine-manager/LAUNCH_TEXT.md). Preview does not diff --git a/services/nvpair-ui-broker/enginesettings.go b/services/nvpair-ui-broker/enginesettings.go index de6d2bbe5..d6d358b0d 100644 --- a/services/nvpair-ui-broker/enginesettings.go +++ b/services/nvpair-ui-broker/enginesettings.go @@ -201,7 +201,7 @@ func (b *Broker) settingsSnapshotLocked(ctx context.Context, engine string) (set if !ready && record != nil { configuredProxy = record.Snapshot.Settings.ProxyPort } - current := settings.Config{ServerPort: launch.ServerPort, ProxyPort: configuredProxy, LaunchText: launch.LaunchText} + current := settings.Config{ServerPort: launch.ServerPort, ProxyPort: configuredProxy, LaunchText: launch.LaunchText, ModelSource: launch.ModelSource} if record == nil { record = &engineSettingsRecord{Snapshot: settings.Snapshot{NodeID: b.nodeID, Engine: engine, Revision: 1, AppliedRevision: 1, Epoch: b.engineSettingsEpoch, Phase: "idle", Settings: current}, Receipts: make(map[string]settingsReceipt)} b.engineSettings[engine] = record diff --git a/services/nvpair-ui-broker/enginesettings_test.go b/services/nvpair-ui-broker/enginesettings_test.go index e276c579c..63490fdb9 100644 --- a/services/nvpair-ui-broker/enginesettings_test.go +++ b/services/nvpair-ui-broker/enginesettings_test.go @@ -99,6 +99,9 @@ func newSettingsHarnessForEngine(t *testing.T, engine string) *settingsHarness { } }() launch := settings.LaunchState{Engine: engine, ServerPort: ports[0], EffectivePort: ports[0], LaunchText: "--fixture-option", Running: true, Editable: true, Format: "pair-arguments-v1"} + if engine == "llamacpp" { + launch.ModelSource = "huggingface" + } go func() { for { msg, err := codec.Read() @@ -124,7 +127,7 @@ func newSettingsHarnessForEngine(t *testing.T, engine string) *settingsHarness { p.Settings.LaunchText = strings.TrimPrefix(p.Settings.LaunchText, "managed serve ") } h.launchMu.Lock() - restart := launch.Running && (p.Settings.ServerPort != launch.ServerPort || p.Settings.LaunchText != launch.LaunchText) + restart := launch.Running && (p.Settings.ServerPort != launch.ServerPort || p.Settings.LaunchText != launch.LaunchText || p.Settings.ModelSource != launch.ModelSource) h.launchMu.Unlock() assert.NoError(t, codec.Respond(msg.ID, settings.Preview{Settings: p.Settings, Restart: restart})) case "engine:configure-launch": @@ -153,6 +156,7 @@ func newSettingsHarnessForEngine(t *testing.T, engine string) *settingsHarness { h.launchMu.Lock() launch.ServerPort = p.Settings.ServerPort launch.LaunchText = p.Settings.LaunchText + launch.ModelSource = p.Settings.ModelSource launch.Running = false h.launchMu.Unlock() if err := h.b.rebindSettingsProxy(p.Engine, p.Settings.ProxyPort); err != nil { @@ -191,6 +195,32 @@ func (h *settingsHarness) request(t *testing.T) settings.Request { return settings.Request{Engine: "ollama", ExpectedRevision: s.Revision, RequestID: settingsID(), Settings: s.Settings} } +func TestSettingsCoordinator_PreservesModelSourceAcrossWorkerAndJournal(t *testing.T) { + h := newSettingsHarnessForEngine(t, "llamacpp") + target := settings.Request{Engine: "llamacpp"} + before, err := h.b.getEngineSettings(context.Background(), target, "") + require.NoError(t, err) + require.Equal(t, "huggingface", before.Settings.ModelSource) + request := settings.Request{Engine: target.Engine, ExpectedRevision: before.Revision, RequestID: settingsID(), Settings: before.Settings} + request.Settings.ModelSource = "modelscope" + preview, err := h.b.previewEngineSettings(context.Background(), request, "") + require.NoError(t, err) + assert.True(t, preview.Restart) + receipt, err := h.b.applyEngineSettings(context.Background(), request, "") + require.NoError(t, err) + require.Equal(t, "succeeded", receipt.Phase) + after, err := h.b.getEngineSettings(context.Background(), target, "") + require.NoError(t, err) + assert.Equal(t, "modelscope", after.Settings.ModelSource) + journal, err := h.b.engineSettingsPath() + require.NoError(t, err) + data, err := os.ReadFile(journal) + require.NoError(t, err) + var saved map[string]engineSettingsRecord + require.NoError(t, json.Unmarshal(data, &saved)) + assert.Equal(t, "modelscope", saved["llamacpp"].Snapshot.Settings.ModelSource) +} + func TestSettingsCoordinator_TracksRevisionsDeduplicatesAndReportsFailures(t *testing.T) { h := newSettingsHarness(t) p := h.request(t) diff --git a/services/shared/enginesettings/settings.go b/services/shared/enginesettings/settings.go index 7de762c1b..a967c3254 100644 --- a/services/shared/enginesettings/settings.go +++ b/services/shared/enginesettings/settings.go @@ -9,7 +9,8 @@ type Config struct { ProxyPort int `json:"proxyPort"` // Arguments and leading environment assignments; excludes the executable // and manifest-owned startup subcommand. Snapshot.Format identifies syntax. - LaunchText string `json:"launchText"` + LaunchText string `json:"launchText"` + ModelSource string `json:"modelSource,omitempty"` } type Request struct { @@ -70,6 +71,7 @@ type LaunchState struct { ServerPort int `json:"serverPort"` EffectivePort int `json:"effectivePort"` LaunchText string `json:"launchText"` + ModelSource string `json:"modelSource,omitempty"` Running bool `json:"running"` Adopted bool `json:"adopted"` Editable bool `json:"editable"` From 4c41334574da79144cedfbaca350ce6fdf2b4caf Mon Sep 17 00:00:00 2001 From: Seth Schneider Date: Wed, 21 Oct 2026 08:46:18 -0700 Subject: [PATCH 3/6] Browse and search llama.cpp models from the selected source Use the saved source for llama.cpp catalogue requests and reuse the existing public, generative-model and Q4_K_M GGUF normalization. Review guide (3 of 4): - Start with services/nvpair-engine-manager/modelscope_catalog.go, then the source selection in catalog.go and manager.go. - Check the official GET listing parameters, bounded candidate/file requests, existing GGUF/quantization filters, and ModelScope model-page links. Repositories with truncated 3,000-entry file lists must not block valid ones. - Follow provider-prefixed cache keys through a source switch: identical queries must not reuse another provider's results. Errors must omit search text. Reading the selected source must not wait for an engine lifecycle operation. Key coverage: TestModelScopeCatalog_* and TestCatalog_KeepsSourceCachesSeparateWhenSelectionChanges. From services/nvpair-engine-manager: go test ./... -run '^TestModelScopeCatalog_|^TestCatalog_KeepsSourceCachesSeparate' -count=1 No new dependency or desktop download implementation is introduced. Signed-off-by: Seth Schneider --- services/nvpair-engine-manager/README.md | 7 + services/nvpair-engine-manager/catalog.go | 49 ++++--- services/nvpair-engine-manager/manager.go | 9 ++ .../modelscope_catalog.go | 106 +++++++++++++++ .../modelscope_catalog_test.go | 124 ++++++++++++++++++ services/nvpair-engine-manager/spec.md | 15 +++ 6 files changed, 295 insertions(+), 15 deletions(-) create mode 100644 services/nvpair-engine-manager/modelscope_catalog.go create mode 100644 services/nvpair-engine-manager/modelscope_catalog_test.go diff --git a/services/nvpair-engine-manager/README.md b/services/nvpair-engine-manager/README.md index ced4bd404..e536dc324 100644 --- a/services/nvpair-engine-manager/README.md +++ b/services/nvpair-engine-manager/README.md @@ -22,6 +22,13 @@ The `llamacpp` manifest runs `llama-server` in router mode on loopback port uses the router's native `DELETE /models` cache operation and does not restart the router. `LLAMA_CACHE` points at the manifest's `models_dir`, `~/.llamacpp`, which no removal this service performs will delete. +The llama.cpp `modelSource` setting selects Hugging Face (default) or ModelScope +for both its catalogue and downloads. The manager translates ModelScope metadata +through a loopback-only adapter and sets the router's `MODEL_ENDPOINT`; llama.cpp +still owns GGUF selection, caching, progress and cancellation. ModelScope serves +public, ungated models without forwarding Hugging Face credentials. +Repositories that reach ModelScope's 3,000-entry file-list limit are excluded +rather than offered with incomplete GGUF shard metadata. Readiness requires `/props` to report `role:"router"`, so model-selection arguments that switch `llama-server` to single-model mode and incompatible listeners already occupying the port are rejected rather than adopted. diff --git a/services/nvpair-engine-manager/catalog.go b/services/nvpair-engine-manager/catalog.go index ed52b1199..933147f7c 100644 --- a/services/nvpair-engine-manager/catalog.go +++ b/services/nvpair-engine-manager/catalog.go @@ -29,8 +29,7 @@ import ( // holds, but not what its upstream offers. // // It lives here rather than in a client so both the desktop app and the terminal -// interface see the same catalogue. Each engine has exactly one curated source, -// and they are deliberately different in kind: +// interface see the same catalogue. Sources differ by engine: // // - Ollama has no public library API, only a client-rendered web page. A live // scrape was fragile and broke whenever the markup changed, so the list is a @@ -39,10 +38,10 @@ import ( // - LM Studio's catalogue *is* a Hugging Face org (`lmstudio-community`), whose // repo ids are exactly the strings `lms get` accepts. That has a real API, so // it is fetched live and cached. -// - llama.cpp downloads any GGUF repo on Hugging Face as `repo:quantization`. -// The browse list is the GGUF repos of a few approved publishers, fetched -// live and cached; a query searches all of Hugging Face, each query cached -// on its own. +// - llama.cpp downloads GGUF repos from its selected Hugging Face or ModelScope +// source as `repo:quantization`. +// Hugging Face browses approved publishers; ModelScope browses popular +// repositories. Both fetch live, with each source and search cached separately. // catalogSourceKind distinguishes how an engine's catalogue is obtained. type catalogSourceKind int @@ -244,7 +243,9 @@ type catalogService struct { // baseURL overrides the listing endpoint in tests. Empty means the real one; // the caching and coalescing around the fetch is the part worth testing, and // it cannot be exercised against the live API. - baseURL string + baseURL string + modelScopeEndpoint string + llamaSource func() (string, error) } // listingCache holds one fetched catalogue. It serves from memory while the @@ -326,6 +327,21 @@ func (c *catalogService) Catalog(ctx context.Context, engine, platform, arch, qu if err != nil { return CatalogResult{}, err } + if c.llamaSource != nil { + source, err := c.llamaSource() + if err != nil { + return CatalogResult{}, err + } + if source == "modelscope" { + models, fetchedAt, err := c.catalogSearch("modelscope:"+strings.ToLower(query), func(ctx context.Context) ([]CatalogModel, error) { + return c.fetchModelScope(ctx, query) + }).get(ctx) + if err != nil { + return CatalogResult{}, err + } + return CatalogResult{Models: models, Source: modelScopeURL + "/models", Platform: platform, Arch: arch, FetchedAt: fetchedAt.UTC().Format(time.RFC3339), Searchable: true, Query: query}, nil + } + } models, fetchedAt, err := c.llamaCPPCatalog(ctx, query) if err != nil { return CatalogResult{}, err @@ -579,7 +595,16 @@ func (c *catalogService) llamaCPPCatalog(ctx context.Context, query string) ([]C // llamaCPPSearch is the cache for one query, created on first use. Queries // differing only in case share one. func (c *catalogService) llamaCPPSearch(query string) *listingCache { - key := strings.ToLower(query) + return c.catalogSearch("huggingface:"+strings.ToLower(query), func(ctx context.Context) ([]CatalogModel, error) { + rows, err := c.fetchHF(ctx, llamaCPPListing("search", query)) + if err != nil { + return nil, err + } + return normalizeLlamaCPPRows(rows), nil + }) +} + +func (c *catalogService) catalogSearch(key string, fetch func(context.Context) ([]CatalogModel, error)) *listingCache { c.searchMu.Lock() defer c.searchMu.Unlock() search, ok := c.llamaSearches[key] @@ -587,13 +612,7 @@ func (c *catalogService) llamaCPPSearch(query string) *listingCache { search = &listingCache{ label: "llama.cpp search", emptyOK: true, - fetch: func(ctx context.Context) ([]CatalogModel, error) { - rows, err := c.fetchHF(ctx, llamaCPPListing("search", query)) - if err != nil { - return nil, err - } - return normalizeLlamaCPPRows(rows), nil - }, + fetch: fetch, } c.llamaSearches[key] = search } diff --git a/services/nvpair-engine-manager/manager.go b/services/nvpair-engine-manager/manager.go index af41f9ddb..2e66a8537 100644 --- a/services/nvpair-engine-manager/manager.go +++ b/services/nvpair-engine-manager/manager.go @@ -98,6 +98,15 @@ func NewManager(codec *Codec, exec *Executor, mesh *clustertrust.Mesh) *Manager cancel: func() {}, } m.settingsRelay.send = codec.Notify + m.catalog.llamaSource = func() (string, error) { + state, err := exec.state("llamacpp") + if err != nil { + return "", err + } + state.mu.Lock() + defer state.mu.Unlock() + return modelSource(state.plat.Runtime), nil + } exec.settingsParent = m.settingsRelay.call return m } diff --git a/services/nvpair-engine-manager/modelscope_catalog.go b/services/nvpair-engine-manager/modelscope_catalog.go new file mode 100644 index 000000000..08646418b --- /dev/null +++ b/services/nvpair-engine-manager/modelscope_catalog.go @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" +) + +// ModelScope lists files separately, so keep the page small enough to check +// every repository's offered quantization within the catalog timeout. +const modelScopeCatalogLimit = 10 + +func (c *catalogService) fetchModelScope(ctx context.Context, query string) ([]CatalogModel, error) { + ctx, cancel := context.WithTimeout(ctx, catalogHTTPTimeout) + defer cancel() + endpoint := c.modelScopeEndpoint + if endpoint == "" { + endpoint = modelScopeURL + } + q := url.Values{ + "search": {query}, "sort": {"downloads"}, "page_number": {"1"}, + "page_size": {fmt.Sprint(modelScopeCatalogLimit)}, "filter.library": {"gguf"}, + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint+"/openapi/v1/models?"+q.Encode(), nil) + if err != nil { + return nil, fmt.Errorf("build ModelScope catalog request: %w", withoutURL(err)) + } + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "PAIR/1.0") + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("ModelScope catalog request failed: %w", withoutURL(err)) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("ModelScope catalog returned HTTP %d", resp.StatusCode) + } + var result struct { + Success bool `json:"success"` + Data struct { + Models []struct { + hfModelRow + Tasks []string `json:"tasks"` + LastModified string `json:"last_modified"` + CreatedAt string `json:"created_at"` + } `json:"models"` + } `json:"data"` + } + data, err := io.ReadAll(io.LimitReader(resp.Body, maxCatalogBody+1)) + if err != nil || len(data) > maxCatalogBody || json.Unmarshal(data, &result) != nil || !result.Success || result.Data.Models == nil || len(result.Data.Models) > modelScopeCatalogLimit { + return nil, fmt.Errorf("invalid ModelScope catalog response") + } + rows := make([]hfModelRow, len(result.Data.Models)) + errs := make([]error, len(rows)) + var wg sync.WaitGroup + for i, entry := range result.Data.Models { + row := entry.hfModelRow + row.LastModified, row.CreatedAt = entry.LastModified, entry.CreatedAt + for j, tag := range row.Tags { + if _, value, ok := strings.Cut(tag, ":"); ok { + row.Tags[j] = value + } + } + if len(entry.Tasks) > 0 { + row.PipelineTag = entry.Tasks[0] + } + if row.Private || string(row.Gated) != "false" || !llamaCPPRepoPattern.MatchString(row.ID) || !hasTag(row.Tags, "gguf") || !isGenerative(row) { + continue + } + wg.Add(1) + go func() { + defer wg.Done() + files, err := fetchModelScopeFiles(ctx, c.client, endpoint, row.ID, "master") + if errors.Is(err, errModelScopeListingTooLarge) { + return + } + if err != nil { + errs[i] = err + return + } + for _, file := range files { + row.Siblings = append(row.Siblings, hfSibling{RFilename: file.Path}) + } + rows[i] = row + }() + } + wg.Wait() + if err := errors.Join(errs...); err != nil { + return nil, err + } + models := normalizeLlamaCPPRows(rows) + for i := range models { + repo, _, _ := strings.Cut(models[i].ID, ":") + models[i].URL = modelScopeURL + "/models/" + repo + } + return models, nil +} diff --git a/services/nvpair-engine-manager/modelscope_catalog_test.go b/services/nvpair-engine-manager/modelscope_catalog_test.go new file mode 100644 index 000000000..cdd5ee881 --- /dev/null +++ b/services/nvpair-engine-manager/modelscope_catalog_test.go @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestModelScopeCatalog_ReturnsOnlyOpenGenerativeQ4Models(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/openapi/v1/models" { + assert.Equal(t, "Qwen", r.URL.Query().Get("search")) + assert.Equal(t, "gguf", r.URL.Query().Get("filter.library")) + assert.Equal(t, "10", r.URL.Query().Get("page_size")) + _, err := w.Write([]byte(`{"success":true,"data":{"models":[ + {"id":"owner/chat","gated":false,"tags":["library:gguf"],"tasks":["text-generation"],"last_modified":"2026-01-01T00:00:00Z","downloads":12}, + {"id":"owner/q8","gated":false,"tags":["library:gguf"],"tasks":["text-generation"],"last_modified":"2026-01-01T00:00:00Z"}, + {"id":"owner/large","gated":false,"tags":["library:gguf"],"tasks":["text-generation"],"last_modified":"2026-01-01T00:00:00Z"}, + {"id":"owner/embedding","gated":false,"tags":["library:gguf"],"tasks":["sentence-embedding"]}, + {"id":"owner/private","private":true,"gated":false,"tags":["library:gguf"],"tasks":["text-generation"]}, + {"id":"owner/gated","gated":true,"tags":["library:gguf"],"tasks":["text-generation"]} + ]}}`)) + assert.NoError(t, err) + return + } + assert.Equal(t, "master", r.URL.Query().Get("Revision")) + assert.Equal(t, "true", r.URL.Query().Get("Recursive")) + file := "model-Q4_K_M-00001-of-00002.gguf" + switch r.URL.Path { + case "/api/v1/models/owner/chat/repo/files": + case "/api/v1/models/owner/q8/repo/files": + file = "model-Q8_0.gguf" + case "/api/v1/models/owner/large/repo/files": + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{"Success": true, "Data": map[string]any{"Files": make([]modelScopeFile, 3000)}})) + return + default: + t.Errorf("unexpected request: %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "Success": true, + "Data": map[string]any{"Files": []modelScopeFile{{ + Path: file, Type: "blob", Revision: "0123456789012345678901234567890123456789", + SHA256: "0123456789012345678901234567890123456789012345678901234567890123", Size: 42, + }}}, + })) + })) + defer srv.Close() + c := newCatalogService() + c.modelScopeEndpoint = srv.URL + + models, err := c.fetchModelScope(context.Background(), "Qwen") + require.NoError(t, err) + require.Len(t, models, 1) + assert.Equal(t, "owner/chat:Q4_K_M", models[0].ID) + assert.Equal(t, "https://modelscope.cn/models/owner/chat", models[0].URL) + assert.Equal(t, 12, models[0].Downloads) +} + +func TestModelScopeCatalog_AcceptsAnEmptySearchResult(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, err := w.Write([]byte(`{"success":true,"data":{"models":[]}}`)) + assert.NoError(t, err) + })) + defer srv.Close() + c := newCatalogService() + c.modelScopeEndpoint = srv.URL + models, err := c.fetchModelScope(context.Background(), "absent") + require.NoError(t, err) + assert.Empty(t, models) +} + +func TestCatalog_KeepsSourceCachesSeparateWhenSelectionChanges(t *testing.T) { + var huggingFaceCalls, modelScopeCalls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/openapi/") { + modelScopeCalls.Add(1) + _, err := w.Write([]byte(`{"success":true,"data":{"models":[]}}`)) + assert.NoError(t, err) + } else { + huggingFaceCalls.Add(1) + _, err := w.Write([]byte(`[]`)) + assert.NoError(t, err) + } + })) + t.Cleanup(server.Close) + catalog := newCatalogService() + catalog.baseURL, catalog.modelScopeEndpoint = server.URL, server.URL + selected := "huggingface" + catalog.llamaSource = func() (string, error) { return selected, nil } + for _, source := range []string{"huggingface", "modelscope", "huggingface", "modelscope"} { + selected = source + result, err := catalog.Catalog(context.Background(), "llamacpp", "", "", "absent") + require.NoError(t, err) + assert.Contains(t, result.Source, source) + } + assert.Equal(t, int32(1), huggingFaceCalls.Load()) + assert.Equal(t, int32(1), modelScopeCalls.Load()) +} + +func TestModelScopeCatalog_DoesNotRevealSearchTextInErrors(t *testing.T) { + srv := httptest.NewServer(http.NotFoundHandler()) + srv.Close() + c := newCatalogService() + c.modelScopeEndpoint = srv.URL + _, err := c.fetchModelScope(context.Background(), "private model name") + require.Error(t, err) + var urlErr *url.Error + assert.NotErrorAs(t, err, &urlErr) + assert.NotContains(t, err.Error(), "private model name") +} diff --git a/services/nvpair-engine-manager/spec.md b/services/nvpair-engine-manager/spec.md index c18f172f3..9ea410b0d 100644 --- a/services/nvpair-engine-manager/spec.md +++ b/services/nvpair-engine-manager/spec.md @@ -67,6 +67,21 @@ extensibility story for an open-source product. - **Edge case — checksum mismatch**: install fails before `run`, is reported, and never executes an unverified payload. - **Edge case — readiness timeout / crash**: failed readiness returns to Stopped; a later crash flips health and is reported. +### llama.cpp model sources + +The node-owned engine settings `modelSource` field selects `huggingface` (default) +or `modelscope` for llama.cpp catalogue queries and downloads. It is saved as +host-platform `runtime.model_source` and included in launch snapshots. Changing +the source uses the existing preview/apply, revision, restart and paired-device +settings protocol; stopped engines stay stopped. Other engines omit the field. + +ModelScope's public GGUF catalogue is normalized through the same quantization +and generative-model filters as Hugging Face. A loopback-only adapter translates +ModelScope metadata and streams file requests without forwarding Hugging Face +credentials. The pinned llama.cpp router remains responsible for GGUF selection, +cache layout, split files, progress and cancellation. Its `MODEL_ENDPOINT` is +managed from the selected source; no TypeScript download implementation is used. + ## 4. Open Questions / Risks - **`engine:*` method stability**: the Broker forwards the namespace verbatim; evolve it additively and preserve `engine:get-installed` as the installed-engine query. - **`severity` / `action` enums**: current wire values are `info|warning|error` and `dismiss|retry|none`; coordinate any future shared-enum expansion without changing existing values. From eef90e53b12ed250a19a9ecd60ed0b8bad10c8bd Mon Sep 17 00:00:00 2001 From: Seth Schneider Date: Wed, 21 Oct 2026 08:46:19 -0700 Subject: [PATCH 4/6] Add model download source dropdown to llama.cpp settings Expose Hugging Face (default) and ModelScope in the existing llama.cpp Settings editor, using the normal draft, preview, Apply and restart flow. Review guide (4 of 4): - Start with desktop/src/ui/components/BackendRow/EngineSettingsSection.tsx, then shared/types/engine-settings.ts and electron/service-bridge/engine-settings.ts. - Check llama.cpp-only visibility, capability/disabled states, source-only dirty detection, stale-draft protection, enum validation, and preservation of the normalized source in preview/apply payloads. - Read desktop/tests/modular/engine-settings-bridge.test.ts for payload coverage. These tests do not exercise dropdown interaction. - Manually choose each source, Apply, confirm a running-engine restart when prompted, reopen Settings, and check the saved value and model browser. Also check that applying to a stopped engine leaves it stopped. docs/engine-settings.mdx describes the user flow. Validation for the series: Windows x64 desktop and all bundled executables built; all desktop gates passed (311 tests, two Unix-only skips); broker tests passed. The engine-manager suite passed with TestRemoveTreePreserving_ReportsUndeletableEntriesAndRemovesOthers excluded: that permission test fails identically on untouched develop on this PC. Feature tests passed again after the final ModelScope metadata correction. Signed-off-by: Seth Schneider --- .../service-bridge/engine-settings.ts | 6 +- desktop/src/shared/types/engine-settings.ts | 1 + .../BackendRow/EngineSettingsSection.tsx | 46 ++++++++++-- .../modular/engine-settings-bridge.test.ts | 74 +++++++++++++++++++ docs/engine-settings.mdx | 15 ++++ 5 files changed, 135 insertions(+), 7 deletions(-) diff --git a/desktop/src/electron/service-bridge/engine-settings.ts b/desktop/src/electron/service-bridge/engine-settings.ts index bc74bf61e..313f62c58 100644 --- a/desktop/src/electron/service-bridge/engine-settings.ts +++ b/desktop/src/electron/service-bridge/engine-settings.ts @@ -39,10 +39,14 @@ function isReceiptPhase(value: JsonValue | undefined): value is EngineSettingsRe } function config(value: JsonValue | undefined): EngineSettingsConfig { const row = object(value) + const modelSource = row.modelSource + if (modelSource !== undefined && modelSource !== 'huggingface' && modelSource !== 'modelscope') + throw new Error('Invalid model download source.') return { serverPort: number(row.serverPort), proxyPort: number(row.proxyPort), - launchText: text(row.launchText) + launchText: text(row.launchText), + ...(modelSource ? { modelSource } : {}) } } export function parseEngineSettings(value: JsonValue | undefined): EngineSettingsSnapshot { diff --git a/desktop/src/shared/types/engine-settings.ts b/desktop/src/shared/types/engine-settings.ts index 02311a343..34c875832 100644 --- a/desktop/src/shared/types/engine-settings.ts +++ b/desktop/src/shared/types/engine-settings.ts @@ -6,6 +6,7 @@ export interface EngineSettingsConfig { proxyPort: number /** Arguments and leading environment assignments, without an executable/subcommand. */ launchText: string + modelSource?: 'huggingface' | 'modelscope' } import type { EngineManagerName } from '@/shared/types/engines' diff --git a/desktop/src/ui/components/BackendRow/EngineSettingsSection.tsx b/desktop/src/ui/components/BackendRow/EngineSettingsSection.tsx index bc038359c..6be4dc281 100644 --- a/desktop/src/ui/components/BackendRow/EngineSettingsSection.tsx +++ b/desktop/src/ui/components/BackendRow/EngineSettingsSection.tsx @@ -10,6 +10,7 @@ import { ModalDialog, ModalHeading, ModalRoot, + Select, Stack, Text, TextArea, @@ -36,14 +37,19 @@ interface Draft { serverPort: string proxyPort: string launchText: string + modelSource: NonNullable } const asDraft = (config: EngineSettingsConfig): Draft => ({ ...config, serverPort: String(config.serverPort), - proxyPort: String(config.proxyPort) + proxyPort: String(config.proxyPort), + modelSource: config.modelSource ?? 'huggingface' }) const sameDraft = (a: Draft, b: Draft): boolean => - a.serverPort === b.serverPort && a.proxyPort === b.proxyPort && a.launchText === b.launchText + a.serverPort === b.serverPort && + a.proxyPort === b.proxyPort && + a.launchText === b.launchText && + a.modelSource === b.modelSource export function EngineSettingsSection({ nodeId, @@ -61,7 +67,12 @@ export function EngineSettingsSection({ const connected = useConnectionStore(state => state.connected) const isLocalNode = useConnectionStore(state => state.selfId === nodeId) const [base, setBase] = useState() - const [draft, setDraft] = useState({ serverPort: '', proxyPort: '', launchText: '' }) + const [draft, setDraft] = useState({ + serverPort: '', + proxyPort: '', + launchText: '', + modelSource: 'huggingface' + }) const [errorDialog, setErrorDialog] = useState('') const [confirmation, setConfirmation] = useState() const [checking, setChecking] = useState(false) @@ -109,7 +120,7 @@ export function EngineSettingsSection({ } }, [snapshot, base, dirty]) - function change(field: keyof Draft, value: string): void { + function change(field: K, value: Draft[K]): void { generation.current++ if (field === 'serverPort') serverPortEdited.current = true setDraft(current => ({ ...current, [field]: value })) @@ -142,7 +153,12 @@ export function EngineSettingsSection({ const request: EngineSettingsRequest = { ...target, expectedRevision: base.revision, - settings: { serverPort, proxyPort, launchText: draft.launchText }, + settings: { + serverPort, + proxyPort, + launchText: draft.launchText, + ...(engineType === 'llama-cpp' ? { modelSource: draft.modelSource } : {}) + }, resolution: useServerPort ? 'server' : 'launch' } try { @@ -254,6 +270,24 @@ export function EngineSettingsSection({ /> + {engineType === 'llama-cpp' && ( + +