From 10e66391bc515724d6b28ca7441d3f9567b7156d Mon Sep 17 00:00:00 2001
From: ApostaC <yihua98@uchicago.edu>
Date: Fri, 10 Jul 2026 09:44:42 -0700
Subject: [PATCH 01/10] Add LMCache configs for dsv4 vllm b200/b300

Signed-off-by: ApostaC <yihua98@uchicago.edu>
---
 .../single_node/agentic/dsv4_fp4_b200_vllm.sh | 86 ++++++++++++++++++-
 .../single_node/agentic/dsv4_fp4_b300_vllm.sh | 86 ++++++++++++++++++-
 configs/nvidia-master.yaml                    | 39 +++++++++
 perf-changelog.yaml                           |  9 ++
 4 files changed, 216 insertions(+), 4 deletions(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
index d723754ac8..718a7e3f4a 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
@@ -21,7 +21,7 @@ set -x
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
 #
-# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake.
+# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake or lmcache.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -101,15 +101,20 @@ export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768
 SERVER_LOG="$RESULT_DIR/server.log"
 ROUTER_LOG="$RESULT_DIR/router.log"
 MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log"
+LMCACHE_SERVER_LOG="$RESULT_DIR/lmcache_server.log"
 mkdir -p "$RESULT_DIR"
 
 SERVER_PID=""
 ROUTER_PID=""
 MOONCAKE_MASTER_PID=""
+LMCACHE_SERVER_PID=""
 
 OFFLOAD_ARGS=()
 
-if require_agentic_kv_offload_backend mooncake; then
+if agentic_kv_offload_enabled; then
+    case "$KV_OFFLOAD_BACKEND" in
+    mooncake)
+        require_agentic_kv_offload_backend mooncake
         # Embedded mode contributes one segment per GPU rank to a shared
         # distributed store, so pre-divide the aggregate host-memory budget.
         PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / GPU_COUNT))
@@ -169,6 +174,83 @@ EOF
             --kv-transfer-config
             '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
         )
+        ;;
+    lmcache)
+        require_agentic_kv_offload_backend lmcache
+        # The LMCache MP server owns the host-DRAM KV pool as one shared
+        # tier; vLLM ranks attach via LMCacheMPConnector, so the aggregate
+        # host budget is passed through undivided (unlike Mooncake's
+        # per-rank segments). Follows the LMCache DeepSeek-V4 recipe
+        # (docs.lmcache.ai/recipes/deepseek_v4_flash.html); LMCache handles
+        # DSV4's Sparse-MLA hybrid KV geometries automatically.
+        LMCACHE_VERSION=0.5.1
+        agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION"
+        python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null
+
+        LMCACHE_HOST=127.0.0.1
+        LMCACHE_PORT=$((PORT + 12000))
+        LMCACHE_HTTP_PORT=$((PORT + 13000))
+        # LMCacheMPConnector concatenates lmcache.mp.host and port into the
+        # ZMQ endpoint. Bind the server to a raw host, but pass the connector
+        # a ZMQ-style host string.
+        LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST"
+        # The pool grows lazily from the initial allocation, so the full
+        # --l1-size-gb budget is not pinned at startup.
+        LMCACHE_L1_INIT_SIZE_GB=20
+        LMCACHE_MQ_TIMEOUT=300
+        # Identical prefixes must hash to identical cache keys across DP ranks.
+        export PYTHONHASHSEED=0
+
+        echo "Starting LMCache MP server on port $LMCACHE_PORT..."
+        # One GPU-side transfer worker avoids concurrent-GPU-transfer stalls
+        # under heavy async-load pressure; CPU-side workers stay at 8.
+        lmcache server \
+            --host "$LMCACHE_HOST" \
+            --port "$LMCACHE_PORT" \
+            --http-host "$LMCACHE_HOST" \
+            --http-port "$LMCACHE_HTTP_PORT" \
+            --l1-size-gb "$TOTAL_CPU_DRAM_GB" \
+            --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \
+            --max-gpu-workers 1 \
+            --max-cpu-workers 8 \
+            --chunk-size 1024 \
+            --l1-align-bytes 16384 \
+            --eviction-trigger-watermark 0.85 \
+            --eviction-ratio 0.10 \
+            --eviction-policy LRU \
+            > "$LMCACHE_SERVER_LOG" 2>&1 &
+        LMCACHE_SERVER_PID=$!
+        LMCACHE_READY=0
+        for _ in $(seq 1 60); do
+            if ! kill -0 "$LMCACHE_SERVER_PID" 2>/dev/null; then
+                echo "LMCache server died during startup." >&2
+                cat "$LMCACHE_SERVER_LOG" >&2
+                exit 1
+            fi
+            if curl --output /dev/null --silent --fail \
+                "http://127.0.0.1:$LMCACHE_HTTP_PORT/healthcheck"; then
+                LMCACHE_READY=1
+                break
+            fi
+            sleep 2
+        done
+        if [ "$LMCACHE_READY" -ne 1 ]; then
+            echo "LMCache server did not become healthy in time." >&2
+            cat "$LMCACHE_SERVER_LOG" >&2
+            exit 1
+        fi
+
+        unset VLLM_USE_SIMPLE_KV_OFFLOAD
+        OFFLOAD_ARGS=(
+            --kv-transfer-config
+            "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":$LMCACHE_MQ_TIMEOUT}}"
+        )
+        ;;
+    *)
+        echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected one of: mooncake, lmcache)" >&2
+        exit 1
+        ;;
+    esac
 fi
 
 PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
index 74dc2129be..c2846060d5 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
@@ -20,7 +20,7 @@ set -x
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
 #
-# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake.
+# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake or lmcache.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -105,14 +105,19 @@ export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768
 SERVER_LOG="$RESULT_DIR/server.log"
 ROUTER_LOG="$RESULT_DIR/router.log"
 MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log"
+LMCACHE_SERVER_LOG="$RESULT_DIR/lmcache_server.log"
 mkdir -p "$RESULT_DIR"
 
 SERVER_PID=""
 ROUTER_PID=""
 MOONCAKE_MASTER_PID=""
+LMCACHE_SERVER_PID=""
 
 OFFLOAD_ARGS=()
-if require_agentic_kv_offload_backend mooncake; then
+if agentic_kv_offload_enabled; then
+    case "$KV_OFFLOAD_BACKEND" in
+    mooncake)
+        require_agentic_kv_offload_backend mooncake
         # Mooncake embedded mode contributes one global segment per GPU rank to
         # a shared distributed store. Pre-divide the aggregate host budget
         # across those rank-contributed segments.
@@ -171,6 +176,83 @@ EOF
             --kv-transfer-config
             '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
         )
+        ;;
+    lmcache)
+        require_agentic_kv_offload_backend lmcache
+        # The LMCache MP server owns the host-DRAM KV pool as one shared
+        # tier; vLLM ranks attach via LMCacheMPConnector, so the aggregate
+        # host budget is passed through undivided (unlike Mooncake's
+        # per-rank segments). Follows the LMCache DeepSeek-V4 recipe
+        # (docs.lmcache.ai/recipes/deepseek_v4_flash.html); LMCache handles
+        # DSV4's Sparse-MLA hybrid KV geometries automatically.
+        LMCACHE_VERSION=0.5.1
+        agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION"
+        python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null
+
+        LMCACHE_HOST=127.0.0.1
+        LMCACHE_PORT=$((PORT + 12000))
+        LMCACHE_HTTP_PORT=$((PORT + 13000))
+        # LMCacheMPConnector concatenates lmcache.mp.host and port into the
+        # ZMQ endpoint. Bind the server to a raw host, but pass the connector
+        # a ZMQ-style host string.
+        LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST"
+        # The pool grows lazily from the initial allocation, so the full
+        # --l1-size-gb budget is not pinned at startup.
+        LMCACHE_L1_INIT_SIZE_GB=20
+        LMCACHE_MQ_TIMEOUT=300
+        # Identical prefixes must hash to identical cache keys across DP ranks.
+        export PYTHONHASHSEED=0
+
+        echo "Starting LMCache MP server on port $LMCACHE_PORT..."
+        # One GPU-side transfer worker avoids concurrent-GPU-transfer stalls
+        # under heavy async-load pressure; CPU-side workers stay at 8.
+        lmcache server \
+            --host "$LMCACHE_HOST" \
+            --port "$LMCACHE_PORT" \
+            --http-host "$LMCACHE_HOST" \
+            --http-port "$LMCACHE_HTTP_PORT" \
+            --l1-size-gb "$TOTAL_CPU_DRAM_GB" \
+            --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \
+            --max-gpu-workers 1 \
+            --max-cpu-workers 8 \
+            --chunk-size 1024 \
+            --l1-align-bytes 16384 \
+            --eviction-trigger-watermark 0.85 \
+            --eviction-ratio 0.10 \
+            --eviction-policy LRU \
+            > "$LMCACHE_SERVER_LOG" 2>&1 &
+        LMCACHE_SERVER_PID=$!
+        LMCACHE_READY=0
+        for _ in $(seq 1 60); do
+            if ! kill -0 "$LMCACHE_SERVER_PID" 2>/dev/null; then
+                echo "LMCache server died during startup." >&2
+                cat "$LMCACHE_SERVER_LOG" >&2
+                exit 1
+            fi
+            if curl --output /dev/null --silent --fail \
+                "http://127.0.0.1:$LMCACHE_HTTP_PORT/healthcheck"; then
+                LMCACHE_READY=1
+                break
+            fi
+            sleep 2
+        done
+        if [ "$LMCACHE_READY" -ne 1 ]; then
+            echo "LMCache server did not become healthy in time." >&2
+            cat "$LMCACHE_SERVER_LOG" >&2
+            exit 1
+        fi
+
+        unset VLLM_USE_SIMPLE_KV_OFFLOAD
+        OFFLOAD_ARGS=(
+            --kv-transfer-config
+            "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":$LMCACHE_MQ_TIMEOUT}}"
+        )
+        ;;
+    *)
+        echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected one of: mooncake, lmcache)" >&2
+        exit 1
+        ;;
+    esac
 fi
 
 PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 6be8e006f9..85f9be5e57 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -1782,6 +1782,25 @@ dsv4-fp4-b200-vllm-agentic:
       # Retain the external-cache transition and peak-throughput region.
       - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [16, 38, 44, 56, 64, 66, 68] }
 
+# LMCache CPU-offload arm of dsv4-fp4-b200-vllm-agentic, split into its own
+# config so it can be triggered/tested independently of the Mooncake curve.
+# Points mirror the Mooncake offload points; the GPU-cache (kv-offloading:
+# none) baselines live in the parent config only.
+dsv4-fp4-b200-vllm-agentic-lmcache:
+  image: vllm/vllm-openai:v0.23.0
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4
+  runner: cluster:b200-dgxc
+  precision: fp4
+  framework: vllm
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.80
+      search-space:
+      - { tp: 8, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [8, 10, 16] }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [16, 38, 44, 56, 64, 66, 68] }
+
 dsv4-fp4-b200-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
   model: deepseek-ai/DeepSeek-V4-Pro
@@ -3229,6 +3248,26 @@ dsv4-fp4-b300-vllm-agentic:
       # TP8 DEP retains representative low, peak, and transition points.
       - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none,  conc-list: [52, 72, 100, 128, 144] }
 
+# LMCache CPU-offload arm of dsv4-fp4-b300-vllm-agentic, split into its own
+# config so it can be triggered/tested independently of the Mooncake curve.
+# Points mirror the Mooncake offload points; the GPU-cache (kv-offloading:
+# none) baselines live in the parent config only.
+dsv4-fp4-b300-vllm-agentic-lmcache:
+  image: vllm/vllm-openai:v0.23.0
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4
+  runner: cluster:b300-nv
+  precision: fp4
+  framework: vllm
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.80
+      search-space:
+      - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [16, 18, 20, 24] }
+      - { tp: 8, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [52] }
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [32] }
+
 dsv4-fp4-b300-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
   model: deepseek-ai/DeepSeek-V4-Pro
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index be28208ab4..76f70a7dce 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4716,3 +4716,12 @@
     - "Clean the export envs"
     - "Enable two batch overlap"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093
+
+- config-keys:
+    - dsv4-fp4-b200-vllm-agentic-lmcache
+    - dsv4-fp4-b300-vllm-agentic-lmcache
+  description:
+    - "Add LMCache 0.5.1 DRAM KV-offload configs (kv-offload-backend: lmcache) as standalone sections alongside the Mooncake parents"
+    - "LMCache MP server + LMCacheMPConnector with one shared host-DRAM pool sized to the aggregate TOTAL_CPU_DRAM_GB budget"
+    - "LMCache points mirror the Mooncake offload points: B300 TP4 conc 16-24, TP8 conc 52, TP4-DEP4 conc 32; B200 TP8 conc 8-16, TP8-DEP8 conc 16-68"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX

From 73137f194dd8457a590f2782a5d354053f04447c Mon Sep 17 00:00:00 2001
From: ApostaC <yihua98@uchicago.edu>
Date: Fri, 10 Jul 2026 09:50:27 -0700
Subject: [PATCH 02/10] update perf changelog

Signed-off-by: ApostaC <yihua98@uchicago.edu>
---
 perf-changelog.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 76f70a7dce..3ffc8a4203 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4724,4 +4724,4 @@
     - "Add LMCache 0.5.1 DRAM KV-offload configs (kv-offload-backend: lmcache) as standalone sections alongside the Mooncake parents"
     - "LMCache MP server + LMCacheMPConnector with one shared host-DRAM pool sized to the aggregate TOTAL_CPU_DRAM_GB budget"
     - "LMCache points mirror the Mooncake offload points: B300 TP4 conc 16-24, TP8 conc 52, TP4-DEP4 conc 32; B200 TP8 conc 8-16, TP8-DEP8 conc 16-68"
-  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2153

From 4244f91fcdabfbf26513c9249f36dc2096393439 Mon Sep 17 00:00:00 2001
From: ApostaC <yihua98@uchicago.edu>
Date: Fri, 10 Jul 2026 12:04:20 -0700
Subject: [PATCH 03/10] lmcache: derate L1 pool to 75%, move to vLLM v0.24.0,
 defer B300 section

Signed-off-by: ApostaC <yihua98@uchicago.edu>
---
 .../single_node/agentic/dsv4_fp4_b200_vllm.sh | 19 +++++++++++---
 .../single_node/agentic/dsv4_fp4_b300_vllm.sh | 19 +++++++++++---
 configs/nvidia-master.yaml                    | 25 +++----------------
 perf-changelog.yaml                           | 10 +++++---
 4 files changed, 39 insertions(+), 34 deletions(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
index 718a7e3f4a..e644b9fa79 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
@@ -70,8 +70,12 @@ export AIPERF_AGENTIC_CACHE_WARMUP_DURATION=600
 # vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings,
 # which fails DSV4 indexer compilation with mlir_global_dtors(..., data).
 # Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved.
-agentic_pip_install --quiet --force-reinstall --no-deps \
-    'nvidia-cutlass-dsl-libs-cu13==4.5.2'
+# The v0.24 images ship fixed bindings; reinstalling there would downgrade.
+VLLM_INSTALLED_VERSION=$(python3 -c "from importlib.metadata import version; print(version('vllm'))")
+if [ "$(printf '%s\n' "$VLLM_INSTALLED_VERSION" 0.24.0 | sort -V | head -n1)" != "0.24.0" ]; then
+    agentic_pip_install --quiet --force-reinstall --no-deps \
+        'nvidia-cutlass-dsl-libs-cu13==4.5.2'
+fi
 
 # vllm-project/router expands the one HTTP backend into one logical worker per
 # DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's
@@ -194,8 +198,15 @@ EOF
         # ZMQ endpoint. Bind the server to a raw host, but pass the connector
         # a ZMQ-style host string.
         LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST"
+        # Pool target derated to 75% of the aggregate budget: pinned host
+        # memory is unswappable and also consumes GPU-side mapping
+        # resources, so leave headroom for vLLM host buffers and the OS.
+        # Full-budget targets OOM-killed the node (host OOM-killer or
+        # cudaErrorMemoryAllocation) as the cache filled past ~2 TB during
+        # PR #2153 bring-up.
+        LMCACHE_L1_SIZE_GB=$((TOTAL_CPU_DRAM_GB * 3 / 4))
         # The pool grows lazily from the initial allocation, so the full
-        # --l1-size-gb budget is not pinned at startup.
+        # --l1-size-gb target is not pinned at startup.
         LMCACHE_L1_INIT_SIZE_GB=20
         LMCACHE_MQ_TIMEOUT=300
         # Identical prefixes must hash to identical cache keys across DP ranks.
@@ -209,7 +220,7 @@ EOF
             --port "$LMCACHE_PORT" \
             --http-host "$LMCACHE_HOST" \
             --http-port "$LMCACHE_HTTP_PORT" \
-            --l1-size-gb "$TOTAL_CPU_DRAM_GB" \
+            --l1-size-gb "$LMCACHE_L1_SIZE_GB" \
             --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \
             --max-gpu-workers 1 \
             --max-cpu-workers 8 \
diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
index c2846060d5..9aced4c74d 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
@@ -73,8 +73,12 @@ export AIPERF_AGENTIC_CACHE_WARMUP_DURATION=600
 # vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings,
 # which fails DSV4 indexer compilation with mlir_global_dtors(..., data).
 # Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved.
-agentic_pip_install --quiet --force-reinstall --no-deps \
-    'nvidia-cutlass-dsl-libs-cu13==4.5.2'
+# The v0.24 images ship fixed bindings; reinstalling there would downgrade.
+VLLM_INSTALLED_VERSION=$(python3 -c "from importlib.metadata import version; print(version('vllm'))")
+if [ "$(printf '%s\n' "$VLLM_INSTALLED_VERSION" 0.24.0 | sort -V | head -n1)" != "0.24.0" ]; then
+    agentic_pip_install --quiet --force-reinstall --no-deps \
+        'nvidia-cutlass-dsl-libs-cu13==4.5.2'
+fi
 
 # vllm-project/router expands the one HTTP backend into one logical worker per
 # DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's
@@ -196,8 +200,15 @@ EOF
         # ZMQ endpoint. Bind the server to a raw host, but pass the connector
         # a ZMQ-style host string.
         LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST"
+        # Pool target derated to 75% of the aggregate budget: pinned host
+        # memory is unswappable and also consumes GPU-side mapping
+        # resources, so leave headroom for vLLM host buffers and the OS.
+        # Full-budget targets OOM-killed the node (host OOM-killer or
+        # cudaErrorMemoryAllocation) as the cache filled past ~2 TB during
+        # PR #2153 bring-up.
+        LMCACHE_L1_SIZE_GB=$((TOTAL_CPU_DRAM_GB * 3 / 4))
         # The pool grows lazily from the initial allocation, so the full
-        # --l1-size-gb budget is not pinned at startup.
+        # --l1-size-gb target is not pinned at startup.
         LMCACHE_L1_INIT_SIZE_GB=20
         LMCACHE_MQ_TIMEOUT=300
         # Identical prefixes must hash to identical cache keys across DP ranks.
@@ -211,7 +222,7 @@ EOF
             --port "$LMCACHE_PORT" \
             --http-host "$LMCACHE_HOST" \
             --http-port "$LMCACHE_HTTP_PORT" \
-            --l1-size-gb "$TOTAL_CPU_DRAM_GB" \
+            --l1-size-gb "$LMCACHE_L1_SIZE_GB" \
             --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \
             --max-gpu-workers 1 \
             --max-cpu-workers 8 \
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 85f9be5e57..5a53b17709 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -1785,9 +1785,10 @@ dsv4-fp4-b200-vllm-agentic:
 # LMCache CPU-offload arm of dsv4-fp4-b200-vllm-agentic, split into its own
 # config so it can be triggered/tested independently of the Mooncake curve.
 # Points mirror the Mooncake offload points; the GPU-cache (kv-offloading:
-# none) baselines live in the parent config only.
+# none) baselines live in the parent config only. Runs the v0.24.0 image
+# (the LMCache-0.5.x-validated pairing) while the parent stays on v0.23.0.
 dsv4-fp4-b200-vllm-agentic-lmcache:
-  image: vllm/vllm-openai:v0.23.0
+  image: vllm/vllm-openai:v0.24.0
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: cluster:b200-dgxc
@@ -3248,26 +3249,6 @@ dsv4-fp4-b300-vllm-agentic:
       # TP8 DEP retains representative low, peak, and transition points.
       - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none,  conc-list: [52, 72, 100, 128, 144] }
 
-# LMCache CPU-offload arm of dsv4-fp4-b300-vllm-agentic, split into its own
-# config so it can be triggered/tested independently of the Mooncake curve.
-# Points mirror the Mooncake offload points; the GPU-cache (kv-offloading:
-# none) baselines live in the parent config only.
-dsv4-fp4-b300-vllm-agentic-lmcache:
-  image: vllm/vllm-openai:v0.23.0
-  model: deepseek-ai/DeepSeek-V4-Pro
-  model-prefix: dsv4
-  runner: cluster:b300-nv
-  precision: fp4
-  framework: vllm
-  multinode: false
-  scenarios:
-    agentic-coding:
-    - dram-utilization: 0.80
-      search-space:
-      - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [16, 18, 20, 24] }
-      - { tp: 8, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [52] }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [32] }
-
 dsv4-fp4-b300-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
   model: deepseek-ai/DeepSeek-V4-Pro
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 79c8bd07b2..a7e631118f 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4726,9 +4726,11 @@
 
 - config-keys:
     - dsv4-fp4-b200-vllm-agentic-lmcache
-    - dsv4-fp4-b300-vllm-agentic-lmcache
+  scenario-type:
+    - agentic-coding
   description:
-    - "Add LMCache 0.5.1 DRAM KV-offload configs (kv-offload-backend: lmcache) as standalone sections alongside the Mooncake parents"
-    - "LMCache MP server + LMCacheMPConnector with one shared host-DRAM pool sized to the aggregate TOTAL_CPU_DRAM_GB budget"
-    - "LMCache points mirror the Mooncake offload points: B300 TP4 conc 16-24, TP8 conc 52, TP4-DEP4 conc 32; B200 TP8 conc 8-16, TP8-DEP8 conc 16-68"
+    - "Add LMCache 0.5.1 DRAM KV-offload config (kv-offload-backend: lmcache) as a standalone section alongside the Mooncake parent (B300 section deferred)"
+    - "Image vllm/vllm-openai:v0.24.0 for the LMCache section (parent stays on v0.23.0); CUTLASS DSL cu13 reinstall gated to vLLM < 0.24"
+    - "LMCache MP server + LMCacheMPConnector; L1 pool derated to 75% of TOTAL_CPU_DRAM_GB after full-budget pinned pools OOM-killed nodes in bring-up"
+    - "LMCache points mirror the Mooncake offload points: B200 TP8 conc 8-16, TP8-DEP8 conc 16-68"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2153

From 380ee5a38303fdba588eb9dfb225b46181fa734e Mon Sep 17 00:00:00 2001
From: ApostaC <yihua98@uchicago.edu>
Date: Fri, 10 Jul 2026 15:37:33 -0700
Subject: [PATCH 04/10] fixing the comments and picking optimizations from
 #2138

Signed-off-by: ApostaC <yihua98@uchicago.edu>
---
 .../single_node/agentic/dsv4_fp4_b200_vllm.sh | 27 +++++---
 .../single_node/agentic/dsv4_fp4_b300_vllm.sh | 27 +++++---
 .../single_node/agentic/patch_vllm_pr45406.py | 63 +++++++++++++++++++
 3 files changed, 101 insertions(+), 16 deletions(-)
 create mode 100755 benchmarks/single_node/agentic/patch_vllm_pr45406.py

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
index e644b9fa79..068f10100b 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
@@ -70,12 +70,8 @@ export AIPERF_AGENTIC_CACHE_WARMUP_DURATION=600
 # vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings,
 # which fails DSV4 indexer compilation with mlir_global_dtors(..., data).
 # Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved.
-# The v0.24 images ship fixed bindings; reinstalling there would downgrade.
-VLLM_INSTALLED_VERSION=$(python3 -c "from importlib.metadata import version; print(version('vllm'))")
-if [ "$(printf '%s\n' "$VLLM_INSTALLED_VERSION" 0.24.0 | sort -V | head -n1)" != "0.24.0" ]; then
-    agentic_pip_install --quiet --force-reinstall --no-deps \
-        'nvidia-cutlass-dsl-libs-cu13==4.5.2'
-fi
+agentic_pip_install --quiet --force-reinstall --no-deps \
+    'nvidia-cutlass-dsl-libs-cu13==4.5.2'
 
 # vllm-project/router expands the one HTTP backend into one logical worker per
 # DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's
@@ -127,6 +123,7 @@ if agentic_kv_offload_enabled; then
         agentic_pip_install --quiet --no-cache-dir --no-deps \
             --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION"
         python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null
+        python3 "$(dirname "$0")/patch_vllm_pr45406.py"
 
         MOONCAKE_MASTER_PORT=$((PORT + 12000))
         MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json"
@@ -190,6 +187,11 @@ EOF
         LMCACHE_VERSION=0.5.1
         agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION"
         python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null
+        # Async KV loads park requests in WAITING_FOR_REMOTE_KVS holding
+        # blocks; without the PR #45406 scheduler fix the waiting-queue scan
+        # can freeze permanently once nothing is running (hung warmup
+        # stragglers in PR #2153 bring-up).
+        python3 "$(dirname "$0")/patch_vllm_pr45406.py"
 
         LMCACHE_HOST=127.0.0.1
         LMCACHE_PORT=$((PORT + 12000))
@@ -229,6 +231,7 @@ EOF
             --eviction-trigger-watermark 0.85 \
             --eviction-ratio 0.10 \
             --eviction-policy LRU \
+            --supported-transfer-mode lmcache_driven \
             > "$LMCACHE_SERVER_LOG" 2>&1 &
         LMCACHE_SERVER_PID=$!
         LMCACHE_READY=0
@@ -271,7 +274,10 @@ fi
 
 EP_ARGS=()
 if [ "$EP_SIZE" -gt 1 ]; then
-    EP_ARGS=(--enable-expert-parallel)
+    EP_ARGS=(
+        --enable-expert-parallel
+        --moe-backend deep_gemm_mega_moe
+    )
 fi
 
 # AgentX concurrency counts live session trees, not individual requests.
@@ -295,15 +301,20 @@ VLLM_CMD=(
     "${PARALLEL_ARGS[@]}"
     "${VLLM_CP_ARGS[@]}"
     "${EP_ARGS[@]}"
-    --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'
+    --prefill-schedule-interval 8
+    --numa-bind
+    --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
+    --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}'
     --attention_config.use_fp4_indexer_cache=True
     --tokenizer-mode deepseek_v4
     --tool-call-parser deepseek_v4
     --enable-auto-tool-choice
     --reasoning-parser deepseek_v4
+    --no-enable-flashinfer-autotune
     --enable-prefix-caching
     --no-disable-hybrid-kv-cache-manager
     --max-num-seqs "$MAX_NUM_SEQS"
+    --disable-uvicorn-access-log
     "${OFFLOAD_ARGS[@]}"
 )
 printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"
diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
index 9aced4c74d..79c6a0591a 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
@@ -73,12 +73,8 @@ export AIPERF_AGENTIC_CACHE_WARMUP_DURATION=600
 # vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings,
 # which fails DSV4 indexer compilation with mlir_global_dtors(..., data).
 # Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved.
-# The v0.24 images ship fixed bindings; reinstalling there would downgrade.
-VLLM_INSTALLED_VERSION=$(python3 -c "from importlib.metadata import version; print(version('vllm'))")
-if [ "$(printf '%s\n' "$VLLM_INSTALLED_VERSION" 0.24.0 | sort -V | head -n1)" != "0.24.0" ]; then
-    agentic_pip_install --quiet --force-reinstall --no-deps \
-        'nvidia-cutlass-dsl-libs-cu13==4.5.2'
-fi
+agentic_pip_install --quiet --force-reinstall --no-deps \
+    'nvidia-cutlass-dsl-libs-cu13==4.5.2'
 
 # vllm-project/router expands the one HTTP backend into one logical worker per
 # DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's
@@ -131,6 +127,7 @@ if agentic_kv_offload_enabled; then
         agentic_pip_install --quiet --no-cache-dir --no-deps \
             --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION"
         python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null
+        python3 "$(dirname "$0")/patch_vllm_pr45406.py"
 
         MOONCAKE_MASTER_PORT=$((PORT + 12000))
         MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json"
@@ -192,6 +189,11 @@ EOF
         LMCACHE_VERSION=0.5.1
         agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION"
         python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null
+        # Async KV loads park requests in WAITING_FOR_REMOTE_KVS holding
+        # blocks; without the PR #45406 scheduler fix the waiting-queue scan
+        # can freeze permanently once nothing is running (hung warmup
+        # stragglers in PR #2153 bring-up).
+        python3 "$(dirname "$0")/patch_vllm_pr45406.py"
 
         LMCACHE_HOST=127.0.0.1
         LMCACHE_PORT=$((PORT + 12000))
@@ -231,6 +233,7 @@ EOF
             --eviction-trigger-watermark 0.85 \
             --eviction-ratio 0.10 \
             --eviction-policy LRU \
+            --supported-transfer-mode lmcache_driven \
             > "$LMCACHE_SERVER_LOG" 2>&1 &
         LMCACHE_SERVER_PID=$!
         LMCACHE_READY=0
@@ -273,7 +276,10 @@ fi
 
 EP_ARGS=()
 if [ "$EP_SIZE" -gt 1 ]; then
-    EP_ARGS=(--enable-expert-parallel)
+    EP_ARGS=(
+        --enable-expert-parallel
+        --moe-backend deep_gemm_mega_moe
+    )
 fi
 
 # AgentX concurrency counts live session trees, not individual requests.
@@ -298,15 +304,20 @@ vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \
 "${PARALLEL_ARGS[@]}" \
 "${VLLM_CP_ARGS[@]}" \
 "${EP_ARGS[@]}" \
---compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \
+--prefill-schedule-interval 8 \
+--numa-bind \
+--compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' \
+--attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}' \
 --attention_config.use_fp4_indexer_cache=True \
 --tokenizer-mode deepseek_v4 \
 --tool-call-parser deepseek_v4 \
 --enable-auto-tool-choice \
 --reasoning-parser deepseek_v4 \
+--no-enable-flashinfer-autotune \
 --enable-prefix-caching \
 --no-disable-hybrid-kv-cache-manager \
 --max-num-seqs "$MAX_NUM_SEQS" \
+--disable-uvicorn-access-log \
 "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 &
 SERVER_PID=$!
 echo "Server PID: $SERVER_PID"
diff --git a/benchmarks/single_node/agentic/patch_vllm_pr45406.py b/benchmarks/single_node/agentic/patch_vllm_pr45406.py
new file mode 100755
index 0000000000..4b41b60c30
--- /dev/null
+++ b/benchmarks/single_node/agentic/patch_vllm_pr45406.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Backport the scheduler fix from vllm-project/vllm PR #45406."""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+
+PATCH_MARKER = "See https://github.com/vllm-project/vllm/issues/45388"
+
+
+def main() -> None:
+    spec = importlib.util.find_spec("vllm")
+    if spec is None or not spec.submodule_search_locations:
+        raise RuntimeError("Could not locate the installed vllm package")
+
+    package_root = Path(next(iter(spec.submodule_search_locations)))
+    scheduler = package_root / "v1" / "core" / "sched" / "scheduler.py"
+    text = scheduler.read_text()
+    if PATCH_MARKER in text:
+        print(f"vLLM PR #45406 backport already present in {scheduler}")
+        return
+
+    old = """                    if request.has_encoder_inputs:
+                        self.encoder_cache_manager.free(request)
+                    break
+
+                # KVTransfer: the connector uses this info to determine
+"""
+    new = """                    if request.has_encoder_inputs:
+                        self.encoder_cache_manager.free(request)
+                    if self.running:
+                        # Running requests will free blocks when they
+                        # complete; stop here to preserve queue-order
+                        # admission.
+                        break
+                    # Nothing is running, so no future event frees blocks and
+                    # stopping at this request would freeze this state
+                    # permanently. Requests behind this one may hold blocks
+                    # while parked (async KV loads in WAITING_FOR_REMOTE_KVS)
+                    # and are only promoted when this traversal reaches them.
+                    # Keep scanning so they can be promoted, scheduled, and
+                    # eventually free the blocks this request needs.
+                    # See https://github.com/vllm-project/vllm/issues/45388
+                    request_queue.pop_request()
+                    step_skipped_waiting.prepend_request(request)
+                    continue
+
+                # KVTransfer: the connector uses this info to determine
+"""
+    if text.count(old) != 1:
+        raise RuntimeError(
+            f"Expected exactly one vLLM PR #45406 patch target in {scheduler}"
+        )
+
+    scheduler.write_text(text.replace(old, new, 1))
+    compile(scheduler.read_text(), str(scheduler), "exec")
+    print(f"Applied vLLM PR #45406 backport to {scheduler}")
+
+
+if __name__ == "__main__":
+    main()

From d9209ddce981bfcc079ccafc335f655ba09d4f57 Mon Sep 17 00:00:00 2001
From: ApostaC <yihua98@uchicago.edu>
Date: Fri, 10 Jul 2026 17:58:31 -0700
Subject: [PATCH 05/10] revert to 0.24.0

Signed-off-by: ApostaC <yihua98@uchicago.edu>
---
 benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh | 12 ++----------
 benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh | 12 ++----------
 perf-changelog.yaml                                  |  2 +-
 3 files changed, 5 insertions(+), 21 deletions(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
index 068f10100b..8598de93f3 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
@@ -274,10 +274,7 @@ fi
 
 EP_ARGS=()
 if [ "$EP_SIZE" -gt 1 ]; then
-    EP_ARGS=(
-        --enable-expert-parallel
-        --moe-backend deep_gemm_mega_moe
-    )
+    EP_ARGS=(--enable-expert-parallel)
 fi
 
 # AgentX concurrency counts live session trees, not individual requests.
@@ -301,20 +298,15 @@ VLLM_CMD=(
     "${PARALLEL_ARGS[@]}"
     "${VLLM_CP_ARGS[@]}"
     "${EP_ARGS[@]}"
-    --prefill-schedule-interval 8
-    --numa-bind
-    --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
-    --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}'
+    --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'
     --attention_config.use_fp4_indexer_cache=True
     --tokenizer-mode deepseek_v4
     --tool-call-parser deepseek_v4
     --enable-auto-tool-choice
     --reasoning-parser deepseek_v4
-    --no-enable-flashinfer-autotune
     --enable-prefix-caching
     --no-disable-hybrid-kv-cache-manager
     --max-num-seqs "$MAX_NUM_SEQS"
-    --disable-uvicorn-access-log
     "${OFFLOAD_ARGS[@]}"
 )
 printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"
diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
index 79c6a0591a..701d29adc5 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
@@ -276,10 +276,7 @@ fi
 
 EP_ARGS=()
 if [ "$EP_SIZE" -gt 1 ]; then
-    EP_ARGS=(
-        --enable-expert-parallel
-        --moe-backend deep_gemm_mega_moe
-    )
+    EP_ARGS=(--enable-expert-parallel)
 fi
 
 # AgentX concurrency counts live session trees, not individual requests.
@@ -304,20 +301,15 @@ vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \
 "${PARALLEL_ARGS[@]}" \
 "${VLLM_CP_ARGS[@]}" \
 "${EP_ARGS[@]}" \
---prefill-schedule-interval 8 \
---numa-bind \
---compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' \
---attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}' \
+--compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \
 --attention_config.use_fp4_indexer_cache=True \
 --tokenizer-mode deepseek_v4 \
 --tool-call-parser deepseek_v4 \
 --enable-auto-tool-choice \
 --reasoning-parser deepseek_v4 \
---no-enable-flashinfer-autotune \
 --enable-prefix-caching \
 --no-disable-hybrid-kv-cache-manager \
 --max-num-seqs "$MAX_NUM_SEQS" \
---disable-uvicorn-access-log \
 "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 &
 SERVER_PID=$!
 echo "Server PID: $SERVER_PID"
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index a5144018f3..4bef8a1709 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4725,6 +4725,6 @@
     - "Add LMCache 0.5.1 DRAM KV-offload config (kv-offload-backend: lmcache) as a standalone section alongside the Mooncake parent (B300 section deferred)"
     - "Image vllm/vllm-openai:v0.24.0 for the LMCache section (parent stays on v0.23.0)"
     - "LMCache MP server (lmcache_driven transfer mode) + LMCacheMPConnector; L1 pool derated to 75% of TOTAL_CPU_DRAM_GB after full-budget pinned pools OOM-killed nodes in bring-up"
-    - "Adopt the vLLM perf flags from #2138 (Mega-MoE, FULL_DECODE_ONLY cudagraphs, sparse MLA attention, NUMA binding) and the vLLM PR #45406 scheduler-freeze backport for async KV loads (waiver: docs/waiver/2153.md)"
+    - "Apply the vLLM PR #45406 scheduler-freeze backport for async KV loads (waiver: docs/waiver/2153.md); serving flags stay on the stock v0.23-era recipe"
     - "LMCache points mirror the Mooncake offload points: B200 TP8 conc 8-16, TP8-DEP8 conc 16-68"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2153

From fa1a16c8bf07497d2dba7393754963adcaae0096 Mon Sep 17 00:00:00 2001
From: ApostaC <yihua98@uchicago.edu>
Date: Sat, 11 Jul 2026 11:44:10 -0700
Subject: [PATCH 06/10] update vllm commands

Signed-off-by: ApostaC <yihua98@uchicago.edu>
---
 .../single_node/agentic/dsv4_fp4_b200_vllm.sh | 56 +++++++++++++----
 .../single_node/agentic/dsv4_fp4_b300_vllm.sh |  6 --
 .../single_node/agentic/patch_vllm_pr45406.py | 63 -------------------
 configs/nvidia-master.yaml                    |  6 +-
 docs/waiver/2153.md                           | 54 ----------------
 perf-changelog.yaml                           |  5 +-
 6 files changed, 49 insertions(+), 141 deletions(-)
 delete mode 100755 benchmarks/single_node/agentic/patch_vllm_pr45406.py
 delete mode 100644 docs/waiver/2153.md

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
index 8598de93f3..86cccb1948 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
@@ -109,6 +109,17 @@ ROUTER_PID=""
 MOONCAKE_MASTER_PID=""
 LMCACHE_SERVER_PID=""
 
+# AgentX concurrency counts live session trees, not individual requests.
+# Subagent fan-out can push instantaneous request concurrency above CONC, so
+# leave 2x headroom rather than clipping those bursts at the scheduler.
+MAX_NUM_SEQS=$((2 * CONC))
+
+# Serving-flag overlays: the lmcache arm replaces these with its tuned
+# recipe; Mooncake and GPU-cache baselines keep the stock flags.
+VLLM_COMPILATION_CONFIG='{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'
+VLLM_TUNING_ARGS=()
+EP_TUNING_ARGS=()
+
 OFFLOAD_ARGS=()
 
 if agentic_kv_offload_enabled; then
@@ -123,7 +134,6 @@ if agentic_kv_offload_enabled; then
         agentic_pip_install --quiet --no-cache-dir --no-deps \
             --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION"
         python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null
-        python3 "$(dirname "$0")/patch_vllm_pr45406.py"
 
         MOONCAKE_MASTER_PORT=$((PORT + 12000))
         MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json"
@@ -187,11 +197,6 @@ EOF
         LMCACHE_VERSION=0.5.1
         agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION"
         python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null
-        # Async KV loads park requests in WAITING_FOR_REMOTE_KVS holding
-        # blocks; without the PR #45406 scheduler fix the waiting-queue scan
-        # can freeze permanently once nothing is running (hung warmup
-        # stragglers in PR #2153 bring-up).
-        python3 "$(dirname "$0")/patch_vllm_pr45406.py"
 
         LMCACHE_HOST=127.0.0.1
         LMCACHE_PORT=$((PORT + 12000))
@@ -214,6 +219,35 @@ EOF
         # Identical prefixes must hash to identical cache keys across DP ranks.
         export PYTHONHASHSEED=0
 
+        # Tuned DeepSeek-V4 serving recipe for this section's image:
+        # decode-only CUDA graphs, sparse MLA with prefill query
+        # quantization, AMXF4 Mega-MoE, NUMA binding, cumem allocator,
+        # and fastsafetensors weight loading.
+        VLLM_COMPILATION_CONFIG='{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
+        VLLM_TUNING_ARGS=(
+            --gpu-memory-utilization 0.85
+            --numa-bind
+            --enable-cumem-allocator
+            --prefill-schedule-interval 8
+            --max-cudagraph-capture-size "$MAX_NUM_SEQS"
+            --load-format fastsafetensors
+            --no-enable-flashinfer-autotune
+            --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}'
+            --disable-uvicorn-access-log
+        )
+        EP_TUNING_ARGS=(
+            --moe-backend deep_gemm_amxf4_mega_moe
+            --enable-ep-weight-filter
+        )
+        export VLLM_USE_V2_MODEL_RUNNER=1
+        export VLLM_USE_RUST_FRONTEND=1
+        export VLLM_DEEPSEEK_V4_USE_MEGA_MOE=1
+        export VLLM_USE_FLASHINFER_MOE_FP8=0
+        export VLLM_DEEP_GEMM_WARMUP=skip
+        export VLLM_DSV4_MEGA_FP8_COMBINE=1
+        export VLLM_RPC_TIMEOUT=600000
+        export TILELANG_CLEANUP_TEMP_FILES=1
+
         echo "Starting LMCache MP server on port $LMCACHE_PORT..."
         # One GPU-side transfer worker avoids concurrent-GPU-transfer stalls
         # under heavy async-load pressure; CPU-side workers stay at 8.
@@ -274,14 +308,9 @@ fi
 
 EP_ARGS=()
 if [ "$EP_SIZE" -gt 1 ]; then
-    EP_ARGS=(--enable-expert-parallel)
+    EP_ARGS=(--enable-expert-parallel "${EP_TUNING_ARGS[@]}")
 fi
 
-# AgentX concurrency counts live session trees, not individual requests.
-# Subagent fan-out can push instantaneous request concurrency above CONC, so
-# leave 2x headroom rather than clipping those bursts at the scheduler.
-MAX_NUM_SEQS=$((2 * CONC))
-
 echo "Starting vllm server..."
 export TORCH_CUDA_ARCH_LIST="10.0"
 export PYTHONNOUSERSITE=1
@@ -298,7 +327,8 @@ VLLM_CMD=(
     "${PARALLEL_ARGS[@]}"
     "${VLLM_CP_ARGS[@]}"
     "${EP_ARGS[@]}"
-    --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'
+    --compilation-config "$VLLM_COMPILATION_CONFIG"
+    "${VLLM_TUNING_ARGS[@]}"
     --attention_config.use_fp4_indexer_cache=True
     --tokenizer-mode deepseek_v4
     --tool-call-parser deepseek_v4
diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
index 701d29adc5..d82751baab 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
@@ -127,7 +127,6 @@ if agentic_kv_offload_enabled; then
         agentic_pip_install --quiet --no-cache-dir --no-deps \
             --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION"
         python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null
-        python3 "$(dirname "$0")/patch_vllm_pr45406.py"
 
         MOONCAKE_MASTER_PORT=$((PORT + 12000))
         MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json"
@@ -189,11 +188,6 @@ EOF
         LMCACHE_VERSION=0.5.1
         agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION"
         python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null
-        # Async KV loads park requests in WAITING_FOR_REMOTE_KVS holding
-        # blocks; without the PR #45406 scheduler fix the waiting-queue scan
-        # can freeze permanently once nothing is running (hung warmup
-        # stragglers in PR #2153 bring-up).
-        python3 "$(dirname "$0")/patch_vllm_pr45406.py"
 
         LMCACHE_HOST=127.0.0.1
         LMCACHE_PORT=$((PORT + 12000))
diff --git a/benchmarks/single_node/agentic/patch_vllm_pr45406.py b/benchmarks/single_node/agentic/patch_vllm_pr45406.py
deleted file mode 100755
index 4b41b60c30..0000000000
--- a/benchmarks/single_node/agentic/patch_vllm_pr45406.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env python3
-"""Backport the scheduler fix from vllm-project/vllm PR #45406."""
-
-from __future__ import annotations
-
-import importlib.util
-from pathlib import Path
-
-
-PATCH_MARKER = "See https://github.com/vllm-project/vllm/issues/45388"
-
-
-def main() -> None:
-    spec = importlib.util.find_spec("vllm")
-    if spec is None or not spec.submodule_search_locations:
-        raise RuntimeError("Could not locate the installed vllm package")
-
-    package_root = Path(next(iter(spec.submodule_search_locations)))
-    scheduler = package_root / "v1" / "core" / "sched" / "scheduler.py"
-    text = scheduler.read_text()
-    if PATCH_MARKER in text:
-        print(f"vLLM PR #45406 backport already present in {scheduler}")
-        return
-
-    old = """                    if request.has_encoder_inputs:
-                        self.encoder_cache_manager.free(request)
-                    break
-
-                # KVTransfer: the connector uses this info to determine
-"""
-    new = """                    if request.has_encoder_inputs:
-                        self.encoder_cache_manager.free(request)
-                    if self.running:
-                        # Running requests will free blocks when they
-                        # complete; stop here to preserve queue-order
-                        # admission.
-                        break
-                    # Nothing is running, so no future event frees blocks and
-                    # stopping at this request would freeze this state
-                    # permanently. Requests behind this one may hold blocks
-                    # while parked (async KV loads in WAITING_FOR_REMOTE_KVS)
-                    # and are only promoted when this traversal reaches them.
-                    # Keep scanning so they can be promoted, scheduled, and
-                    # eventually free the blocks this request needs.
-                    # See https://github.com/vllm-project/vllm/issues/45388
-                    request_queue.pop_request()
-                    step_skipped_waiting.prepend_request(request)
-                    continue
-
-                # KVTransfer: the connector uses this info to determine
-"""
-    if text.count(old) != 1:
-        raise RuntimeError(
-            f"Expected exactly one vLLM PR #45406 patch target in {scheduler}"
-        )
-
-    scheduler.write_text(text.replace(old, new, 1))
-    compile(scheduler.read_text(), str(scheduler), "exec")
-    print(f"Applied vLLM PR #45406 backport to {scheduler}")
-
-
-if __name__ == "__main__":
-    main()
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 5a53b17709..454b84b749 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -1785,10 +1785,10 @@ dsv4-fp4-b200-vllm-agentic:
 # LMCache CPU-offload arm of dsv4-fp4-b200-vllm-agentic, split into its own
 # config so it can be triggered/tested independently of the Mooncake curve.
 # Points mirror the Mooncake offload points; the GPU-cache (kv-offloading:
-# none) baselines live in the parent config only. Runs the v0.24.0 image
-# (the LMCache-0.5.x-validated pairing) while the parent stays on v0.23.0.
+# none) baselines live in the parent config only. Runs a tuned vLLM build
+# while the parent stays on the stock v0.23.0 image.
 dsv4-fp4-b200-vllm-agentic-lmcache:
-  image: vllm/vllm-openai:v0.24.0
+  image: inferactinc/public:sprint-agentx-fast-x86_64-cu13.0.1-6a1e7bf
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: cluster:b200-dgxc
diff --git a/docs/waiver/2153.md b/docs/waiver/2153.md
deleted file mode 100644
index 5adcdabde8..0000000000
--- a/docs/waiver/2153.md
+++ /dev/null
@@ -1,54 +0,0 @@
-# Inference-engine patch waiver — PR #2153
-
-## What is patched
-
-`benchmarks/single_node/agentic/patch_vllm_pr45406.py` rewrites
-`vllm/v1/core/sched/scheduler.py` inside the pinned `vllm/vllm-openai:v0.24.0`
-image before the server starts. It is invoked only from the `lmcache` arm of
-`benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh` and
-`dsv4_fp4_b300_vllm.sh`, so Mooncake and GPU-cache baseline runs execute the
-image as shipped. The patch is a verbatim backport of upstream
-vllm-project/vllm PR #45406 and is idempotent (it no-ops when the fix is
-already present and fails loudly if the target code has drifted).
-
-## Why the unmodified image cannot run this benchmark
-
-The `dsv4-fp4-b200-vllm-agentic-lmcache` config uses `LMCacheMPConnector`
-with asynchronous KV loads. Requests park in `WAITING_FOR_REMOTE_KVS` while
-holding KV blocks. In v0.24.0 the scheduler's waiting-queue traversal stops
-at the first request that cannot allocate blocks even when nothing is
-running; parked requests behind it are never promoted, so the queue freezes
-permanently. In PR #2153 bring-up this reproduced as hung agentic warmup
-stragglers (e.g. 35 in-flight requests never returning at DEP conc 68),
-aborting the benchmark with `warmup_failure`. The deadlock is upstream issue
-vllm-project/vllm#45388; no released image up to and including v0.24.0
-contains the fix.
-
-## Upstream reference
-
-- Issue: https://github.com/vllm-project/vllm/issues/45388
-- Fix: https://github.com/vllm-project/vllm/pull/45406 (merged upstream)
-
-## Removal plan
-
-Remove `patch_vllm_pr45406.py` and its invocation when the LMCache section's
-pinned image first includes upstream PR #45406 (the first vLLM release or
-nightly after v0.24.0 that ships it). Because the patch script detects an
-already-fixed scheduler and no-ops, an image bump cannot silently double-apply
-it; the removal is a plain deletion in the same PR as that image bump.
-
-## 中文说明
-
-**补丁内容**：`patch_vllm_pr45406.py` 在服务启动前修改固定镜像
-`vllm/vllm-openai:v0.24.0` 中的 `vllm/v1/core/sched/scheduler.py`，为上游
-vllm-project/vllm PR #45406 的逐字回移（backport），仅在 LMCache 分支中调用；
-Mooncake 与 GPU 缓存基线均按原始镜像运行。
-
-**为何原始镜像无法运行本基准测试**：LMCacheMPConnector 的异步 KV 加载会让请求
-在 `WAITING_FOR_REMOTE_KVS` 状态下持有 KV block。v0.24.0 的调度器在等待队列
-遍历中遇到无法分配 block 的请求即停止，即使当前没有运行中的请求，其后被挂起
-的请求永远不会被调度，队列永久冻结（上游 issue #45388）。在 PR #2153 调试中
-表现为 agentic 预热请求挂起、基准测试以 warmup_failure 中止。
-
-**移除计划**：当 LMCache 配置的固定镜像包含上游 PR #45406 后，在镜像升级的
-同一 PR 中删除该补丁脚本及其调用。补丁脚本具备幂等检测，不会重复应用。
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 4bef8a1709..4f725521ab 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4723,8 +4723,9 @@
     - agentic-coding
   description:
     - "Add LMCache 0.5.1 DRAM KV-offload config (kv-offload-backend: lmcache) as a standalone section alongside the Mooncake parent (B300 section deferred)"
-    - "Image vllm/vllm-openai:v0.24.0 for the LMCache section (parent stays on v0.23.0)"
+    - "Image inferactinc/public:sprint-agentx-fast-x86_64-cu13.0.1-6a1e7bf for the LMCache section (parent stays on v0.23.0)"
     - "LMCache MP server (lmcache_driven transfer mode) + LMCacheMPConnector; L1 pool derated to 75% of TOTAL_CPU_DRAM_GB after full-budget pinned pools OOM-killed nodes in bring-up"
-    - "Apply the vLLM PR #45406 scheduler-freeze backport for async KV loads (waiver: docs/waiver/2153.md); serving flags stay on the stock v0.23-era recipe"
+    - "Adopt a tuned DeepSeek-V4 serving recipe for the LMCache arm (AMXF4 Mega-MoE, FULL_DECODE_ONLY cudagraphs, sparse MLA with prefill query quantization, NUMA binding, cumem allocator, fastsafetensors, v2 model runner + rust frontend)"
+    - "The image ships the vLLM PR #45406 scheduler fix for async KV loads, so no engine patch is applied"
     - "LMCache points mirror the Mooncake offload points: B200 TP8 conc 8-16, TP8-DEP8 conc 16-68"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2153

From ef9db5e94f05dac7a7d418df5210c151d44c5ec9 Mon Sep 17 00:00:00 2001
From: ApostaC <yihua98@uchicago.edu>
Date: Sat, 11 Jul 2026 14:06:27 -0700
Subject: [PATCH 07/10] update configs

Signed-off-by: ApostaC <yihua98@uchicago.edu>
---
 .../single_node/agentic/dsv4_fp4_b200_vllm.sh       | 13 +++++++++----
 perf-changelog.yaml                                 |  2 +-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
index 86cccb1948..d7da89fdc5 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
@@ -221,23 +221,28 @@ EOF
 
         # Tuned DeepSeek-V4 serving recipe for this section's image:
         # decode-only CUDA graphs, sparse MLA with prefill query
-        # quantization, AMXF4 Mega-MoE, NUMA binding, cumem allocator,
-        # and fastsafetensors weight loading.
+        # quantization, AMXF4 Mega-MoE, and NUMA binding.
+        # --enable-cumem-allocator is deliberately absent: LMCacheMPConnector
+        # exports the KV cache to the LMCache server through legacy CUDA IPC
+        # handles, and cuMem/VMM allocations cannot be exported that way
+        # (register_kv_caches fails with cudaErrorInvalidValue).
         VLLM_COMPILATION_CONFIG='{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
         VLLM_TUNING_ARGS=(
             --gpu-memory-utilization 0.85
             --numa-bind
-            --enable-cumem-allocator
             --prefill-schedule-interval 8
             --max-cudagraph-capture-size "$MAX_NUM_SEQS"
-            --load-format fastsafetensors
             --no-enable-flashinfer-autotune
             --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}'
             --disable-uvicorn-access-log
         )
+        # fastsafetensors stages checkpoint shards on-GPU while loading.
+        # Without --enable-ep-weight-filter every rank stages the full
+        # expert weights and OOMs during load, so it is EP-only here.
         EP_TUNING_ARGS=(
             --moe-backend deep_gemm_amxf4_mega_moe
             --enable-ep-weight-filter
+            --load-format fastsafetensors
         )
         export VLLM_USE_V2_MODEL_RUNNER=1
         export VLLM_USE_RUST_FRONTEND=1
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 4f725521ab..203d85da3b 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4725,7 +4725,7 @@
     - "Add LMCache 0.5.1 DRAM KV-offload config (kv-offload-backend: lmcache) as a standalone section alongside the Mooncake parent (B300 section deferred)"
     - "Image inferactinc/public:sprint-agentx-fast-x86_64-cu13.0.1-6a1e7bf for the LMCache section (parent stays on v0.23.0)"
     - "LMCache MP server (lmcache_driven transfer mode) + LMCacheMPConnector; L1 pool derated to 75% of TOTAL_CPU_DRAM_GB after full-budget pinned pools OOM-killed nodes in bring-up"
-    - "Adopt a tuned DeepSeek-V4 serving recipe for the LMCache arm (AMXF4 Mega-MoE, FULL_DECODE_ONLY cudagraphs, sparse MLA with prefill query quantization, NUMA binding, cumem allocator, fastsafetensors, v2 model runner + rust frontend)"
+    - "Adopt a tuned DeepSeek-V4 serving recipe for the LMCache arm (AMXF4 Mega-MoE, FULL_DECODE_ONLY cudagraphs, sparse MLA with prefill query quantization, NUMA binding, v2 model runner + rust frontend; fastsafetensors on EP points only — non-EP ranks OOM staging full expert weights; no cumem allocator — cuMem/VMM memory cannot be CUDA-IPC-shared with the LMCache server)"
     - "The image ships the vLLM PR #45406 scheduler fix for async KV loads, so no engine patch is applied"
     - "LMCache points mirror the Mooncake offload points: B200 TP8 conc 8-16, TP8-DEP8 conc 16-68"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2153

From 74395002ec87026e587b8fcb1972d3429f1093c5 Mon Sep 17 00:00:00 2001
From: ApostaC <yihua98@uchicago.edu>
Date: Sat, 11 Jul 2026 15:34:30 -0700
Subject: [PATCH 08/10] update gpu memory util

Signed-off-by: ApostaC <yihua98@uchicago.edu>
---
 benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
index d7da89fdc5..9d920324be 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
@@ -228,7 +228,6 @@ EOF
         # (register_kv_caches fails with cudaErrorInvalidValue).
         VLLM_COMPILATION_CONFIG='{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
         VLLM_TUNING_ARGS=(
-            --gpu-memory-utilization 0.85
             --numa-bind
             --prefill-schedule-interval 8
             --max-cudagraph-capture-size "$MAX_NUM_SEQS"

From b7507027ffe042ba0c8020e31038f252466ba7ee Mon Sep 17 00:00:00 2001
From: ApostaC <yihua98@uchicago.edu>
Date: Sat, 11 Jul 2026 18:13:27 -0700
Subject: [PATCH 09/10] add gpu memory util for EP

Signed-off-by: ApostaC <yihua98@uchicago.edu>
---
 benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
index 9d920324be..a9fdaf4b59 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
@@ -238,10 +238,16 @@ EOF
         # fastsafetensors stages checkpoint shards on-GPU while loading.
         # Without --enable-ep-weight-filter every rank stages the full
         # expert weights and OOMs during load, so it is EP-only here.
+        # The 0.85 GPU memory utilization is also EP-only: the Mega-MoE
+        # path JIT-loads DeepGEMM kernels at runtime, and those driver
+        # allocations live outside the vLLM pool (at the default
+        # utilization they fail with CUDA_ERROR_OUT_OF_MEMORY). The
+        # non-EP points keep the default for a larger KV cache pool.
         EP_TUNING_ARGS=(
             --moe-backend deep_gemm_amxf4_mega_moe
             --enable-ep-weight-filter
             --load-format fastsafetensors
+            --gpu-memory-utilization 0.85
         )
         export VLLM_USE_V2_MODEL_RUNNER=1
         export VLLM_USE_RUST_FRONTEND=1

From 0b8646f8ae0a592f3e6adb9a1a4ae783ea6126d3 Mon Sep 17 00:00:00 2001
From: ApostaC <yihua98@uchicago.edu>
Date: Sat, 11 Jul 2026 20:16:11 -0700
Subject: [PATCH 10/10] move to 0.88 gpu mem util

Signed-off-by: ApostaC <yihua98@uchicago.edu>
---
 benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
index a9fdaf4b59..eb2b5eb437 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
@@ -238,16 +238,18 @@ EOF
         # fastsafetensors stages checkpoint shards on-GPU while loading.
         # Without --enable-ep-weight-filter every rank stages the full
         # expert weights and OOMs during load, so it is EP-only here.
-        # The 0.85 GPU memory utilization is also EP-only: the Mega-MoE
-        # path JIT-loads DeepGEMM kernels at runtime, and those driver
-        # allocations live outside the vLLM pool (at the default
-        # utilization they fail with CUDA_ERROR_OUT_OF_MEMORY). The
+        # The reduced GPU memory utilization is also EP-only: the Mega-MoE
+        # path JIT-loads DeepGEMM/TileLang kernels at runtime, and those
+        # driver allocations live outside the vLLM pool (at the default
+        # utilization they fail with CUDA_ERROR_OUT_OF_MEMORY). 0.85 is
+        # stable but costs KV-cache headroom at high concurrency; 0.88
+        # aims to keep the JIT headroom while recovering throughput. The
         # non-EP points keep the default for a larger KV cache pool.
         EP_TUNING_ARGS=(
             --moe-backend deep_gemm_amxf4_mega_moe
             --enable-ep-weight-filter
             --load-format fastsafetensors
-            --gpu-memory-utilization 0.85
+            --gpu-memory-utilization 0.88
         )
         export VLLM_USE_V2_MODEL_RUNNER=1
         export VLLM_USE_RUST_FRONTEND=1