Validate official vLLM Router with srt-slurm

.github/workflows/run-sweep.yml

@@ -249,7 +249,7 @@ jobs:
                       --model 'claude-fable-5'
                       --max-turns 8
                       --allowedTools "Read,Glob,Grep,Bash(git diff:*)"
-                      --json-schema '{"type":"object","properties":{"criteria":{"type":"array","items":{"type":"string","enum":["multi-node","agentic","eval-only","fp4","mtp","eagle","eagle3","sglang","vllm","dynamo-sglang","dynamo-vllm","glm5","glm5.1","kimik2.5","kimik3","dsv4","minimaxm3","qwen3.5","dsr1","checklist-complete","patchwork"]},"uniqueItems":true},"reason":{"type":"string"}},"required":["criteria","reason"]}'
+                      --json-schema '{"type":"object","properties":{"criteria":{"type":"array","items":{"type":"string","enum":["multi-node","agentic","eval-only","fp4","mtp","eagle","eagle3","sglang","vllm","sgl-router","vllm-router","dynamo-sglang","dynamo-vllm","glm5","glm5.1","kimik2.5","kimik3","dsv4","minimaxm3","qwen3.5","dsr1","checklist-complete","patchwork"]},"uniqueItems":true},"reason":{"type":"string"}},"required":["criteria","reason"]}'
                   prompt: |
                       Inspect this Actions run's change range.
 
@@ -264,7 +264,7 @@ jobs:
                       - eval-only: evaluation without throughput measurement
                       - fp4: FP4 precision
                       - mtp, eagle, eagle3: speculative decoding method
-                      - sglang, vllm, dynamo-vllm: runtime framework
+                      - sglang, vllm, sgl-router, vllm-router, dynamo-vllm: runtime framework
                       - model criterion: matching configured model family
                       - checklist-complete: PR checklist is satisfied
                       - patchwork: modified upstream engine or runtime source

benchmarks/multi_node/agentic_srt.sh

@@ -32,7 +32,14 @@ install_agentic_deps
 wait_for_agentic_servers_idle() {
     local timeout_seconds="${AIPERF_DRAIN_TIMEOUT_SECONDS:-1800}"
     local poll_seconds="${AIPERF_DRAIN_POLL_SECONDS:-10}"
-    local frontend_metrics_url="http://localhost:${PORT}/metrics"
+    local frontend_metrics_url=""
+
+    # Dynamo exposes its own active-request gauge. Native static routers do
+    # not promise a metrics endpoint, so their drain signal comes entirely
+    # from the explicit logical worker endpoints supplied by srt-slurm.
+    case "$FRAMEWORK" in
+        dynamo-*) frontend_metrics_url="http://localhost:${PORT}/metrics" ;;
+    esac
 
     "$AIPERF_PYTHON" - \
         "$timeout_seconds" \
@@ -70,13 +77,19 @@ def metric_sum(metrics: str, name: str) -> float:
 
 while time.monotonic() < deadline:
     try:
-        frontend_metrics = fetch_metrics(frontend_url)
-        frontend_active = metric_sum(frontend_metrics, "dynamo_frontend_active_requests")
+        frontend_active = 0.0
+        if frontend_url:
+            frontend_metrics = fetch_metrics(frontend_url)
+            frontend_active = metric_sum(
+                frontend_metrics, "dynamo_frontend_active_requests"
+            )
         worker_active = 0.0
         for worker_url in worker_urls:
             worker_metrics = fetch_metrics(worker_url)
             worker_active += metric_sum(worker_metrics, "vllm:num_requests_running")
             worker_active += metric_sum(worker_metrics, "vllm:num_requests_waiting")
+            worker_active += metric_sum(worker_metrics, "sglang:num_requests_running")
+            worker_active += metric_sum(worker_metrics, "sglang:num_requests_waiting")
         print(
             f"Agentic drain status: frontend_active={frontend_active:g} "
             f"worker_running_or_waiting={worker_active:g}",

benchmarks/multi_node/srt-slurm-recipes/configs/native-sgl-router-deps.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+python3 -m pip install --no-cache-dir --upgrade "sglang-router==0.3.2"
+python3 -c 'import sglang_router; print(sglang_router.__file__)'

benchmarks/multi_node/srt-slurm-recipes/configs/native-sgl-router-pd-fix-deps.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Temporary exact-source pin for the upstream SGLang Model Gateway one-token
+# P/D response fix. The shared wheel cache and flock make the source build
+# happen once even though srt-slurm runs the recipe setup on every role.
+SGLANG_REPOSITORY="https://github.com/cquil11/sglang.git"
+SGLANG_REF="fix/pd-one-token-prefill-response"
+SGLANG_COMMIT="9072e0dc5aaf1962b4e2c6f1a51094356b5d3324"
+WHEEL_CACHE_ROOT="/router_wheels/sglang-router-${SGLANG_COMMIT}"
+LOCK_FILE="${WHEEL_CACHE_ROOT}.lock"
+
+mkdir -p "${WHEEL_CACHE_ROOT}"
+exec 9>"${LOCK_FILE}"
+flock -w 3600 9
+
+shopt -s nullglob
+router_wheels=("${WHEEL_CACHE_ROOT}"/sglang_router-*.whl)
+if (( ${#router_wheels[@]} == 0 )); then
+    build_dir=$(mktemp -d /tmp/sglang-router-build.XXXXXX)
+    trap 'rm -rf "${build_dir}"' EXIT
+
+    git clone --filter=blob:none --no-checkout --single-branch \
+        --branch "${SGLANG_REF}" "${SGLANG_REPOSITORY}" "${build_dir}"
+    git -C "${build_dir}" sparse-checkout init --cone
+    git -C "${build_dir}" sparse-checkout set sgl-model-gateway
+    git -C "${build_dir}" checkout --detach "${SGLANG_COMMIT}"
+    if [[ "$(git -C "${build_dir}" rev-parse HEAD)" != "${SGLANG_COMMIT}" ]]; then
+        echo "ERROR: SGLang checkout does not match ${SGLANG_COMMIT}" >&2
+        exit 1
+    fi
+
+    python3 -m pip install --break-system-packages --no-cache-dir --upgrade \
+        "maturin>=1.0,<2.0"
+    (
+        cd "${build_dir}/sgl-model-gateway/bindings/python"
+        CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS:-16}" \
+            maturin build --release --out "${WHEEL_CACHE_ROOT}"
+    )
+    router_wheels=("${WHEEL_CACHE_ROOT}"/sglang_router-*.whl)
+fi
+
+if (( ${#router_wheels[@]} != 1 )); then
+    echo "ERROR: expected exactly one cached SGLang Router wheel, found ${#router_wheels[@]}" >&2
+    exit 1
+fi
+
+# Install the released package first to resolve its Python dependencies, then
+# replace only the native package with the wheel built from the exact fix SHA.
+python3 -m pip install --break-system-packages --no-cache-dir --upgrade \
+    "sglang-router==0.3.2"
+python3 -m pip install --break-system-packages --no-cache-dir \
+    --force-reinstall --no-deps "${router_wheels[0]}"
+python3 -c 'import sglang_router; print(sglang_router.__file__)'
+echo "Installed SGLang Router from ${SGLANG_COMMIT}"

benchmarks/multi_node/srt-slurm-recipes/configs/native-vllm-router-deps.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+python3 -m pip install --no-cache-dir --upgrade "vllm-router==0.1.15"
+command -v vllm-router >/dev/null
+python3 - <<'PY'
+from importlib.metadata import version
+
+assert version("vllm-router") == "0.1.15"
+print(f"vllm-router {version('vllm-router')}")
+PY

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b200-2xdep8-mtp-sgl-router-agentic.yaml

@@ -0,0 +1,96 @@
+name: "dsv4-b200-sgl-router-agg-2xdep8-mtp-agentic"
+
+model:
+  path: "deepseek-v4-pro"
+  container: "lmsysorg/sglang:v0.5.16-cu130"
+  precision: "fp4"
+
+setup_script: native-sgl-router-deps.sh
+
+slurm:
+  time_limit: "4:00:00"
+
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+
+resources:
+  gpu_type: "b200"
+  gpus_per_node: 8
+  agg_nodes: 2
+  agg_workers: 2
+  gpus_per_agg: 8
+
+infra:
+  etcd_nats_dedicated_node: false
+
+frontend:
+  type: sglang
+  enable_multiple_frontends: false
+  env:
+    PIP_BREAK_SYSTEM_PACKAGES: "1"
+
+backend:
+  type: sglang
+  aggregated_environment:
+    PIP_BREAK_SYSTEM_PACKAGES: "1"
+    PYTHONUNBUFFERED: "1"
+    SGLANG_DEFAULT_THINKING: "1"
+    SGLANG_DSV4_REASONING_EFFORT: "high"
+    SGLANG_SIMULATE_ACC_LEN: "2.99"
+    SGLANG_SIMULATE_ACC_METHOD: "match-expected"
+    SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token"
+    SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1"
+    SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
+    SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1"
+    SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1"
+    SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
+    SGLANG_OPT_USE_JIT_NORM: "1"
+    SGLANG_OPT_USE_TOPK_V2: "True"
+    NCCL_CUMEM_ENABLE: "1"
+    NCCL_NVLS_ENABLE: "1"
+  sglang_config:
+    aggregated:
+      served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
+      enable-metrics: true
+      enable-cache-report: true
+      trust-remote-code: true
+      tool-call-parser: "deepseekv4"
+      stream-interval: 10
+      watchdog-timeout: 1800
+      mem-fraction-static: 0.90
+      chunked-prefill-size: 8192
+      moe-runner-backend: "flashinfer_mxfp4"
+      disable-flashinfer-autotune: true
+      max-running-requests: 256
+      cuda-graph-max-bs: 256
+      scheduler-recv-interval: 30
+      dp-size: 8
+      tp-size: 8
+      ep-size: 8
+      enable-dp-attention: true
+      enable-dp-lm-head: true
+      speculative-algorithm: "EAGLE"
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+
+sbatch_directives:
+  cpus-per-task: "144"
+  mem: "0"
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp-sgl-router.yaml

@@ -0,0 +1,152 @@
+name: "glm5.2-h200-sgl-router-disagg-2p2d-mtp-agentic"
+
+# Native SGL Router validation of the established GLM-5.2 H200 Mooncake
+# topology. This deliberately changes only the request plane: model, worker
+# layout, KV transport, MTP, and AgentX behavior match the Dynamo baseline.
+
+model:
+  path: "hf:zai-org/GLM-5.2-FP8"
+  container: "lmsysorg/sglang:v0.5.16-cu130"
+  precision: "fp8"
+
+identity:
+  model:
+    repo: "zai-org/GLM-5.2-FP8"
+    revision: "ba978f7d347eaf65d22f1a86833408afdb953541"
+  container:
+    image: "lmsysorg/sglang:v0.5.16-cu130"
+  frameworks:
+    sglang: "0.5.16"
+
+setup_script: native-sgl-router-pd-fix-deps.sh
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+
+resources:
+  gpu_type: h200
+  gpus_per_node: 8
+  prefill_nodes: 2
+  decode_nodes: 2
+  prefill_workers: 2
+  decode_workers: 2
+  gpus_per_prefill: 8
+  gpus_per_decode: 8
+
+infra:
+  etcd_nats_dedicated_node: false
+
+frontend:
+  type: sglang
+  enable_multiple_frontends: false
+  env:
+    PIP_BREAK_SYSTEM_PACKAGES: "1"
+
+backend:
+  type: sglang
+  prefill_environment:
+    HF_HOME: "/hf_hub_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    PIP_BREAK_SYSTEM_PACKAGES: "1"
+    PYTHONUNBUFFERED: "1"
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900"
+    SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
+    SGLANG_OPT_USE_TOPK_V2: "1"
+    SGLANG_TIMEOUT_KEEP_ALIVE: "900"
+  decode_environment:
+    HF_HOME: "/hf_hub_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    PIP_BREAK_SYSTEM_PACKAGES: "1"
+    PYTHONUNBUFFERED: "1"
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900"
+    SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
+    SGLANG_OPT_USE_TOPK_V2: "1"
+    SGLANG_TIMEOUT_KEEP_ALIVE: "900"
+    SGLANG_SIMULATE_ACC_LEN: "2.99"
+    SGLANG_SIMULATE_ACC_METHOD: "match-expected"
+    SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token"
+
+  sglang_config:
+    prefill:
+      host: 0.0.0.0
+      model-path: /model/
+      served-model-name: zai-org/GLM-5.2-FP8
+      trust-remote-code: true
+      tool-call-parser: glm47
+      reasoning-parser: glm45
+      tp-size: 8
+      ep-size: 1
+      attn-cp-size: 8
+      enable-prefill-cp: true
+      cp-strategy: interleave
+      enable-dsa-cache-layer-split: true
+      disaggregation-transfer-backend: mooncake
+      disaggregation-mode: prefill
+      kv-cache-dtype: fp8_e4m3
+      context-length: 1048576
+      max-total-tokens: 1048576
+      chunked-prefill-size: 32768
+      mem-fraction-static: 0.85
+      max-running-requests: 32
+      speculative-algorithm: EAGLE
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      watchdog-timeout: 86400
+      stream-interval: 60
+      enable-metrics: true
+      enable-cache-report: true
+
+    decode:
+      host: 0.0.0.0
+      model-path: /model/
+      served-model-name: zai-org/GLM-5.2-FP8
+      trust-remote-code: true
+      tool-call-parser: glm47
+      reasoning-parser: glm45
+      tp-size: 8
+      dp-size: 8
+      ep-size: 1
+      enable-dp-attention: true
+      disaggregation-transfer-backend: mooncake
+      disaggregation-mode: decode
+      kv-cache-dtype: fp8_e4m3
+      dsa-decode-backend: flashmla_kv
+      context-length: 1048576
+      max-total-tokens: 1048576
+      mem-fraction-static: 0.85
+      max-running-requests: 200
+      page-size: 64
+      disable-radix-cache: true
+      speculative-algorithm: EAGLE
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      watchdog-timeout: 86400
+      stream-interval: 60
+      enable-metrics: true
+      enable-cache-report: true
+
+sbatch_directives:
+  mem: "0"
+
+srun_options:
+  mem: "0"
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"