From e599b4eff82511407f5148489a26fa99f523d511 Mon Sep 17 00:00:00 2001 From: Hrt-Htk Date: Thu, 27 Aug 2026 13:36:24 +0200 Subject: [PATCH 1/3] feat(models): rename MTP 27B preset to Qwen3.8 with baked-in draft head qwen3.8-27b-q4-mtp now points at models/Qwen3.8-27B-UD-Q4_K_XL.gguf (draft head baked in, no separate .mtp file) and the qwen38 F16 mmproj. 224k ctx cap kept; re-test 262k with model_acceptance.py --full-ctx before raising. #18 --- proxy_config.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/proxy_config.py b/proxy_config.py index e260bbf..f9b1c33 100644 --- a/proxy_config.py +++ b/proxy_config.py @@ -58,14 +58,15 @@ def contexts(self, default: list[int]) -> tuple[int, ...] | list[int]: chat_template_file=ROOT / "chat_template_sharp.jinja", ), ModelChoice( - "Qwen3.6-27B Q4 MTP", - "qwen3.6-27b-q4-mtp", - ROOT / "models" / "Qwen3.6-27B-UD-Q4_K_XL.mtp.gguf", - ROOT / "models" / "_aux" / "mmproj-27b-BF16.gguf", + "Qwen3.8-27B Q4 MTP", + "qwen3.8-27b-q4-mtp", + ROOT / "models" / "Qwen3.8-27B-UD-Q4_K_XL.gguf", + ROOT / "models" / "_aux" / "mmproj-qwen38-27b-F16.gguf", spec_mtp=True, mmproj_offload=False, - # MTP draft buffers add ~1.7 GB, so full 262k leaves only ~300 MiB — - # too tight for the prefill spike. 224k is the safe max (~1 GB headroom). + # Inherited 224k cap from the 3.6-27B MTP preset (draft buffers add ~1.7 GB). + # 3.8 only keeps a KV cache on 16/64 layers, so 262k may now fit — re-test + # with `model_acceptance.py --full-ctx` before raising this. ctx_choices=(229376,), ), ] From f592c0c0bf900880c04a5864a9adc2dda45d3125 Mon Sep 17 00:00:00 2001 From: Hrt-Htk Date: Thu, 27 Aug 2026 13:36:30 +0200 Subject: [PATCH 2/3] chore(docs): remove cloudflared references, document ZBOX entry point Cloudflared no longer runs on this box; the ZBOX (htk-ZBOX-PI336) Caddy is the entry point forwarding to :8001. Old config archived at C:\Users\HTK\.cloudflared.disabled. - CLAUDE.md/README.md: new topology diagram, drop ai.example.com framing, ZBOX entry-point section - Fix stale tunnel references in code comments (proxy_base, embed_proxy, router_manager, proxy_request_handlers, log_paths.ps1) - Trim bloat: CLAUDE.md MTP bullet shrunken to a pointer (details live in proxy_config.py/README), preset-ID bullet no longer hardcodes the model list, README stale model/ctx examples fixed #20 --- CLAUDE.md | 19 ++++++++++--------- README.md | 38 +++++++++++++++++++++----------------- embed_proxy.py | 4 ++-- log_paths.ps1 | 2 +- proxy_base.py | 12 ++++++------ proxy_request_handlers.py | 2 +- router_manager.py | 2 +- 7 files changed, 42 insertions(+), 37 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 5df7d7f..67c4337 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,16 +1,17 @@ # Repo Lay of the Land Two independent Python proxies wrap two `llama-server.exe` router instances. -Cloudflared exposes them under a single hostname; `proxy.py` reverse-proxies -`/embedding/*` to the embed stack on :8003. +`proxy.py` listens on `0.0.0.0:8001` and reverse-proxies `/embedding/*` to the +embed stack on :8003. No tunnel runs on this box — the ZBOX's Caddy is the +entry point and forwards to :8001. ``` -ai.example.com/chat/* → :8001 proxy.py → :8002 router (chat) -ai.example.com/embedding/* → :8001 proxy.py → :8003 embed_proxy.py → :8004 router (embeddings) +:8001 proxy.py → :8002 router (chat) +:8001 /embedding/* → :8003 embed_proxy.py → :8004 router (embeddings) ``` -Bare `ai.example.com/v1/...` at the root also still hits the chat router -(backwards compat); the `/chat` prefix is the preferred public alias. +Bare `/v1/...` at the root also still hits the chat router +(backwards compat); the `/chat` prefix is the preferred alias. ## Code @@ -37,7 +38,7 @@ Bare `ai.example.com/v1/...` at the root also still hits the chat router ## External - `.venv/` — project virtualenv. Python at `.venv\Scripts\python.exe`. Only deps used are `aiohttp` (proxies) and the stdlib. -- Cloudflared tunnel configured separately at `C:\Users\HTK\.cloudflared\config.yml`. +- ZBOX (`htk-ZBOX-PI336`, LAN 192.168.178.43 / Tailscale 100.80.201.44) — entry point; its Caddy forwards to :8001 on this box. The old cloudflared config is archived at `C:\Users\HTK\.cloudflared.disabled`. ## Logs @@ -47,9 +48,9 @@ All under `logs//`. Daily-rotated, bucketed by ISO week. Two pairs (`proxy - **Router stays up across model loads/unloads** — that's the whole point. Don't kill the router process on idle; only call `/models/unload`. - **`--models-max 1` per router** — loading a different model evicts the current one. The two routers don't share state, so chat and embedder coexist fine. -- **MTP (built-in speculative decoding) is available on the 27B** as a *separate, additional* model `qwen3.6-27b-q4-mtp` (label "Qwen3.6-27B Q4 MTP", weights `models/Qwen3.6-27B-UD-Q4_K_XL.mtp.gguf`). Enabled per-model via `spec_mtp=True` on its `ModelChoice`, which makes `_model_preset_section` emit `spec-type=draft-mtp`, `spec-draft-n-max=2`, `spec-draft-p-min=0.0`. Requires the b9209+ router binary (PR ggml-org/llama.cpp#22673). The original non-MTP `qwen3.6-27b-q4` and both 35B models are unchanged — only one model loads at a time (`--models-max 1`), so the extra preset costs no VRAM unless selected. +- **MTP (built-in speculative decoding)** — `spec_mtp=True` on a `ModelChoice` enables it (emits `spec-type=draft-mtp` etc. in the preset; see `proxy_config.py` for the current 27B setup). Requires the b9209+ router binary and a build supporting the model's GGUF arch. Only one model loads at a time, so the extra preset costs no VRAM unless selected. - **API key** comes from `$env:LLAMA_API_KEY` with a hardcoded fallback in both proxies. The proxies inject `Authorization: Bearer …` if the client omits it. -- **Preset IDs are the API model names.** Chat: `qwen3.6-35b-q3-32k`, `…-q4-128k`, `qwen3.6-27b-q4-mtp` (MTP), etc. Embed: `qwen3-embedding-4b-8k`. Clients pick via the `model` field in the request body. +- **Preset IDs are the API model names** — `-k`, generated from `MODELS` × `CTX_CHOICES` in `proxy_config.py` (don't hardcode the list here; it rots). Clients pick via the `model` field in the request body. - **KV cache is quantized** — `cache-type-k = q4_0`, `cache-type-v = q4_0` in all presets. - **Custom chat template** — router uses `--jinja --chat-template-file chat_template.jinja --chat-template-kwargs '{"preserve_thinking":true}'`. diff --git a/README.md b/README.md index c08f33d..2337c4e 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,19 @@ # llama.cpp Wake-on-Demand Proxy -Two `llama-server.exe` router instances behind small Python proxies, exposed over a -cloudflared tunnel under a single hostname. `proxy.py` reverse-proxies `/embedding/*` -to the embed stack. Each proxy loads its model on first request and unloads after -10 minutes of inactivity — the router process stays up so the tunnel never breaks. +Two `llama-server.exe` router instances behind small Python proxies. `proxy.py` +listens on `0.0.0.0:8001` and reverse-proxies `/embedding/*` to the embed stack. +No tunnel runs on this box — the ZBOX's Caddy is the entry point and forwards +to :8001. Each proxy loads its model on first request and unloads after +10 minutes of inactivity — the router process stays up so the front-end +connection never breaks. ``` -ai.example.com/chat/* → :8001 proxy.py → :8002 router (chat) -ai.example.com/embedding/* → :8001 proxy.py → :8003 embed_proxy.py → :8004 router (embeddings) +:8001 proxy.py → :8002 router (chat) +:8001 /embedding/* → :8003 embed_proxy.py → :8004 router (embeddings) ``` -Bare `ai.example.com/v1/...` at the root also still hits the chat router -(backwards compat); the `/chat` prefix is the preferred public alias. +Bare `/v1/...` at the root also still hits the chat router +(backwards compat); the `/chat` prefix is the preferred alias. Both stacks run side-by-side as independent processes; both models can be loaded concurrently. @@ -67,20 +69,22 @@ H:\llama.cpp\watchdog-embed.ps1 # embed stack on :8003 H:\llama.cpp\create-scheduler-tasks.ps1 # run once as Administrator ``` -**Public tunnel** — `cloudflared` is set up separately (outside this repo). Install `cloudflared.exe`, configure your tunnel in `~/.cloudflared/config.yml`, then run it however you prefer. +**Entry point** — the ZBOX (`htk-ZBOX-PI336`, LAN 192.168.178.43 / Tailscale +100.80.201.44) runs Caddy, which forwards to `:8001` on this box. No tunnel +runs on this machine. Headless chat variant (skip the interactive picker — sets the *fallback* model -when a client request omits the `model` field; all combos are still routable): +when a client request omits the `model` field; all presets are still routable): ```powershell -H:\llama.cpp\watchdog.ps1 --model "Qwen3.6-35B-A3B Q3" --ctx-size 32768 +H:\llama.cpp\watchdog.ps1 --model "Nail-Qwen3.6-35B-A3B Q4" ``` ## Endpoints -- **Chat:** `https://ai.example.com/chat/v1/chat/completions` — model field selects preset (e.g. `qwen3.6-35b-q3-32k`). -- **Embeddings:** `https://ai.example.com/embedding/v1/embeddings` — model `qwen3-embedding-4b-8k`. -- **Re-ranking:** `https://ai.example.com/embedding/v1/rerank` — same embed router. +- **Chat:** `/chat/v1/chat/completions` (via the ZBOX entry point) — model field selects preset (e.g. `nail-35b-a3b-q4-262k`). +- **Embeddings:** `/embedding/v1/embeddings` — model `qwen3-embedding-4b-8k`. +- **Re-ranking:** `/embedding/v1/rerank` — same embed router. - Both require `Authorization: Bearer `. Set via `$env:LLAMA_API_KEY` (overrides the hardcoded fallback in the proxy code). `/health` on either port reports proxy + router state and which model is currently loaded. @@ -89,18 +93,18 @@ H:\llama.cpp\watchdog.ps1 --model "Qwen3.6-35B-A3B Q3" --ctx-size 32768 1. Watchdog starts the proxy. Proxy spawns the router with `--no-models-autoload` and `--models-max 1`. GPU is idle, router is healthy. 2. First chat/embedding request arrives. Proxy POSTs `/models/load` to the router, polls `/v1/models` until `status.value == "loaded"`, then forwards. -3. Idle watchdog (inside each proxy) checks every 30s. After 10 minutes of no activity it POSTs `/models/unload`. VRAM frees; router stays running; tunnel socket stays alive. +3. Idle watchdog (inside each proxy) checks every 30s. After 10 minutes of no activity it POSTs `/models/unload`. VRAM frees; router stays running; front-end connection stays alive. 4. Next request reloads on demand. Cold-load latency: ~15–20s for the 35B chat model on a 3090 Ti; ~2–3s for the 4B embedder. ## Adding a model -Chat side — append a `ModelChoice(...)` to `MODELS` in `proxy.py`. Every entry is automatically crossed with `CTX_CHOICES` (32k, 64k, 96k, 128k) to produce one preset per (model × ctx) pair. No INI editing. +Chat side — append a `ModelChoice(...)` to `MODELS` in `proxy_config.py`. Every entry is crossed with `CTX_CHOICES` (currently a single 262k) to produce one preset per (model × ctx) pair; a model can pin its own `ctx_choices` to override. No INI editing. Each preset uses KV cache quantization (`cache-type-k = q4_0`, `cache-type-v = q4_0`), flash attention, and a custom Jinja chat template (`chat_template.jinja` with `preserve_thinking` kwarg). -**MTP (speculative decoding)** — set `spec_mtp=True` on a `ModelChoice` to enable built-in MTP speculative decoding. This emits `spec-type=draft-mtp`, `spec-draft-n-max=2`, `spec-draft-p-min=0.0` in the preset. Requires the b9209+ router binary (PR ggml-org/llama.cpp#22673). The 27B model has an MTP variant (`qwen3.6-27b-q4-mtp`) that uses `models/Qwen3.6-27B-UD-Q4_K_XL.mtp.gguf`. +**MTP (speculative decoding)** — set `spec_mtp=True` on a `ModelChoice` to enable built-in MTP speculative decoding. This emits `spec-type=draft-mtp`, `spec-draft-n-max=2`, `spec-draft-p-min=0.0` in the preset. Requires the b9209+ router binary (PR ggml-org/llama.cpp#22673). The 27B model (`qwen3.8-27b-q4-mtp`) uses `models/Qwen3.8-27B-UD-Q4_K_XL.gguf`, which ships the MTP draft head baked in. Embed side — `embed_proxy.py` is hardcoded to a single model (`MODEL_FILE`, `MODEL_ID`, `CTX_SIZE` constants at the top). Change those if you swap the embedder. diff --git a/embed_proxy.py b/embed_proxy.py index 71c9cc8..951e197 100644 --- a/embed_proxy.py +++ b/embed_proxy.py @@ -29,13 +29,13 @@ # Single embedding model. Wrapped in the same router-mode pattern as # proxy.py so the embedder can be unloaded after idle while the router -# stays up — cloudflared keeps a stable origin. +# stays up — the front-end (ZBOX Caddy) keeps a stable origin. MODEL_FILE = ROOT / "models" / "Qwen3-Embedding-4B-Q8_0.gguf" MODEL_ID = "qwen3-embedding-4b-8k" CTX_SIZE = 8192 PROXY_HOST = "::" -PROXY_PORT = 8003 # cloudflared origin (embed.example.com) +PROXY_PORT = 8003 # internal; clients reach it via proxy.py /embedding/* SERVER_HOST = "127.0.0.1" SERVER_PORT = 8004 # internal router IDLE_TIMEOUT = 600 diff --git a/log_paths.ps1 b/log_paths.ps1 index 20df169..e1428bc 100644 --- a/log_paths.ps1 +++ b/log_paths.ps1 @@ -1,4 +1,4 @@ -# Shared log-path helpers for PowerShell wrappers (watchdogs + tunnel runner). +# Shared log-path helpers for PowerShell wrappers (watchdogs). # Mirrors log_paths.py: writes into logs// buckets (local Europe/Zurich # date), and prunes any week folder older than the two most recent. # diff --git a/proxy_base.py b/proxy_base.py index eff63fd..87445d4 100644 --- a/proxy_base.py +++ b/proxy_base.py @@ -153,14 +153,14 @@ def filter_response_headers(headers) -> dict[str, str]: # Reverse-proxy hops allowed to set forwarding headers. We listen on # 0.0.0.0:8001, so a direct LAN client could spoof X-Forwarded-For / -# CF-Connecting-IP; only trust those headers when the TCP peer is Caddy (LAN -# front) or cloudflared (loopback). Override the Caddy IP via $TRUSTED_PROXY. +# CF-Connecting-IP; only trust those headers when the TCP peer is Caddy (ZBOX +# front) or loopback. Override the Caddy IP via $TRUSTED_PROXY. TRUSTED_PROXIES = {"127.0.0.1", "::1", os.environ.get("TRUSTED_PROXY", "192.168.178.43")} def client_ip(request: web.Request) -> str: """Real client IP. When the request arrives from a trusted reverse proxy - (Caddy on the LAN, or cloudflared on loopback) we read the forwarded client + (Caddy on the ZBOX, or a loopback tool) we read the forwarded client out of CF-Connecting-IP / X-Forwarded-For; otherwise we report the raw TCP peer. Untrusted peers can't spoof their way to a fake IP.""" peer = request.remote or "-" @@ -185,7 +185,7 @@ def _format_a(request, response, time): return client_ip(request) -# Paths reachable without an API key. /health is the cloudflared/uptime probe. +# Paths reachable without an API key. /health is the uptime probe. PUBLIC_PATHS = {"/health"} @@ -193,8 +193,8 @@ def _format_a(request, response, time): async def auth_middleware(request: web.Request, handler): """Reject any request that doesn't carry the configured API key. - This proxy is the internet-facing origin for the Cloudflare tunnel, so - it — not the localhost-only llama-server behind it — is where client + This proxy is the front-facing origin (the ZBOX's Caddy forwards to it), + so it — not the localhost-only llama-server behind it — is where client authentication has to happen. filter_request_headers() still injects the key on the *upstream* hop so the backend keeps trusting only this proxy. """ diff --git a/proxy_request_handlers.py b/proxy_request_handlers.py index 89c058f..d8edc35 100644 --- a/proxy_request_handlers.py +++ b/proxy_request_handlers.py @@ -301,7 +301,7 @@ async def _do_forward() -> web.StreamResponse: except DeadWorkerError: raise # propagate to retry loop except ClientGone: - # Downstream client (cloudflared / end client) hung up before we finished + # Downstream client (front proxy / end client) hung up before we finished # sending the response — raised when downstream.prepare() hit a closing # transport. Same benign disconnect the write loops already handle, just # earlier. Not a proxy fault, so log calmly and return 499. diff --git a/router_manager.py b/router_manager.py index f39e6b3..f7e0de8 100644 --- a/router_manager.py +++ b/router_manager.py @@ -216,7 +216,7 @@ class ChatRouterManager(RouterManager): The router process starts with the proxy and dies with it. Models are loaded on first request and unloaded by the idle watchdog — the router - itself stays up so the cloudflared tunnel never breaks. + itself stays up so the front-end connection never breaks. """ LOAD_TIMEOUT = 300 From 007d6ae33dbdcb14e0ecf29acf0643a4b98b6a93 Mon Sep 17 00:00:00 2001 From: Hrt-Htk Date: Thu, 27 Aug 2026 13:37:30 +0200 Subject: [PATCH 3/3] chore: gitignore tmp/ scratch dir (model-acceptance/research output) #20 --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7e23010..c852430 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ # Temp and cache .tmp/ +tmp/ __pycache__/ *.pyc *.pyo