diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh index d723754ac..eb2b5eb43 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh @@ -21,7 +21,7 @@ set -x # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake. +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake or lmcache. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -101,15 +101,31 @@ export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 SERVER_LOG="$RESULT_DIR/server.log" ROUTER_LOG="$RESULT_DIR/router.log" MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +LMCACHE_SERVER_LOG="$RESULT_DIR/lmcache_server.log" mkdir -p "$RESULT_DIR" SERVER_PID="" ROUTER_PID="" MOONCAKE_MASTER_PID="" +LMCACHE_SERVER_PID="" + +# AgentX concurrency counts live session trees, not individual requests. +# Subagent fan-out can push instantaneous request concurrency above CONC, so +# leave 2x headroom rather than clipping those bursts at the scheduler. +MAX_NUM_SEQS=$((2 * CONC)) + +# Serving-flag overlays: the lmcache arm replaces these with its tuned +# recipe; Mooncake and GPU-cache baselines keep the stock flags. +VLLM_COMPILATION_CONFIG='{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' +VLLM_TUNING_ARGS=() +EP_TUNING_ARGS=() OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend mooncake; then +if agentic_kv_offload_enabled; then + case "$KV_OFFLOAD_BACKEND" in + mooncake) + require_agentic_kv_offload_backend mooncake # Embedded mode contributes one segment per GPU rank to a shared # distributed store, so pre-divide the aggregate host-memory budget. PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / GPU_COUNT)) @@ -169,6 +185,132 @@ EOF --kv-transfer-config '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' ) + ;; + lmcache) + require_agentic_kv_offload_backend lmcache + # The LMCache MP server owns the host-DRAM KV pool as one shared + # tier; vLLM ranks attach via LMCacheMPConnector, so the aggregate + # host budget is passed through undivided (unlike Mooncake's + # per-rank segments). Follows the LMCache DeepSeek-V4 recipe + # (docs.lmcache.ai/recipes/deepseek_v4_flash.html); LMCache handles + # DSV4's Sparse-MLA hybrid KV geometries automatically. + LMCACHE_VERSION=0.5.1 + agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION" + python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + + LMCACHE_HOST=127.0.0.1 + LMCACHE_PORT=$((PORT + 12000)) + LMCACHE_HTTP_PORT=$((PORT + 13000)) + # LMCacheMPConnector concatenates lmcache.mp.host and port into the + # ZMQ endpoint. Bind the server to a raw host, but pass the connector + # a ZMQ-style host string. + LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST" + # Pool target derated to 75% of the aggregate budget: pinned host + # memory is unswappable and also consumes GPU-side mapping + # resources, so leave headroom for vLLM host buffers and the OS. + # Full-budget targets OOM-killed the node (host OOM-killer or + # cudaErrorMemoryAllocation) as the cache filled past ~2 TB during + # PR #2153 bring-up. + LMCACHE_L1_SIZE_GB=$((TOTAL_CPU_DRAM_GB * 3 / 4)) + # The pool grows lazily from the initial allocation, so the full + # --l1-size-gb target is not pinned at startup. + LMCACHE_L1_INIT_SIZE_GB=20 + LMCACHE_MQ_TIMEOUT=300 + # Identical prefixes must hash to identical cache keys across DP ranks. + export PYTHONHASHSEED=0 + + # Tuned DeepSeek-V4 serving recipe for this section's image: + # decode-only CUDA graphs, sparse MLA with prefill query + # quantization, AMXF4 Mega-MoE, and NUMA binding. + # --enable-cumem-allocator is deliberately absent: LMCacheMPConnector + # exports the KV cache to the LMCache server through legacy CUDA IPC + # handles, and cuMem/VMM allocations cannot be exported that way + # (register_kv_caches fails with cudaErrorInvalidValue). + VLLM_COMPILATION_CONFIG='{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + VLLM_TUNING_ARGS=( + --numa-bind + --prefill-schedule-interval 8 + --max-cudagraph-capture-size "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}' + --disable-uvicorn-access-log + ) + # fastsafetensors stages checkpoint shards on-GPU while loading. + # Without --enable-ep-weight-filter every rank stages the full + # expert weights and OOMs during load, so it is EP-only here. + # The reduced GPU memory utilization is also EP-only: the Mega-MoE + # path JIT-loads DeepGEMM/TileLang kernels at runtime, and those + # driver allocations live outside the vLLM pool (at the default + # utilization they fail with CUDA_ERROR_OUT_OF_MEMORY). 0.85 is + # stable but costs KV-cache headroom at high concurrency; 0.88 + # aims to keep the JIT headroom while recovering throughput. The + # non-EP points keep the default for a larger KV cache pool. + EP_TUNING_ARGS=( + --moe-backend deep_gemm_amxf4_mega_moe + --enable-ep-weight-filter + --load-format fastsafetensors + --gpu-memory-utilization 0.88 + ) + export VLLM_USE_V2_MODEL_RUNNER=1 + export VLLM_USE_RUST_FRONTEND=1 + export VLLM_DEEPSEEK_V4_USE_MEGA_MOE=1 + export VLLM_USE_FLASHINFER_MOE_FP8=0 + export VLLM_DEEP_GEMM_WARMUP=skip + export VLLM_DSV4_MEGA_FP8_COMBINE=1 + export VLLM_RPC_TIMEOUT=600000 + export TILELANG_CLEANUP_TEMP_FILES=1 + + echo "Starting LMCache MP server on port $LMCACHE_PORT..." + # One GPU-side transfer worker avoids concurrent-GPU-transfer stalls + # under heavy async-load pressure; CPU-side workers stay at 8. + lmcache server \ + --host "$LMCACHE_HOST" \ + --port "$LMCACHE_PORT" \ + --http-host "$LMCACHE_HOST" \ + --http-port "$LMCACHE_HTTP_PORT" \ + --l1-size-gb "$LMCACHE_L1_SIZE_GB" \ + --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \ + --max-gpu-workers 1 \ + --max-cpu-workers 8 \ + --chunk-size 1024 \ + --l1-align-bytes 16384 \ + --eviction-trigger-watermark 0.85 \ + --eviction-ratio 0.10 \ + --eviction-policy LRU \ + --supported-transfer-mode lmcache_driven \ + > "$LMCACHE_SERVER_LOG" 2>&1 & + LMCACHE_SERVER_PID=$! + LMCACHE_READY=0 + for _ in $(seq 1 60); do + if ! kill -0 "$LMCACHE_SERVER_PID" 2>/dev/null; then + echo "LMCache server died during startup." >&2 + cat "$LMCACHE_SERVER_LOG" >&2 + exit 1 + fi + if curl --output /dev/null --silent --fail \ + "http://127.0.0.1:$LMCACHE_HTTP_PORT/healthcheck"; then + LMCACHE_READY=1 + break + fi + sleep 2 + done + if [ "$LMCACHE_READY" -ne 1 ]; then + echo "LMCache server did not become healthy in time." >&2 + cat "$LMCACHE_SERVER_LOG" >&2 + exit 1 + fi + + unset VLLM_USE_SIMPLE_KV_OFFLOAD + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":$LMCACHE_MQ_TIMEOUT}}" + ) + ;; + *) + echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected one of: mooncake, lmcache)" >&2 + exit 1 + ;; + esac fi PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) @@ -178,14 +320,9 @@ fi EP_ARGS=() if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) + EP_ARGS=(--enable-expert-parallel "${EP_TUNING_ARGS[@]}") fi -# AgentX concurrency counts live session trees, not individual requests. -# Subagent fan-out can push instantaneous request concurrency above CONC, so -# leave 2x headroom rather than clipping those bursts at the scheduler. -MAX_NUM_SEQS=$((2 * CONC)) - echo "Starting vllm server..." export TORCH_CUDA_ARCH_LIST="10.0" export PYTHONNOUSERSITE=1 @@ -202,7 +339,8 @@ VLLM_CMD=( "${PARALLEL_ARGS[@]}" "${VLLM_CP_ARGS[@]}" "${EP_ARGS[@]}" - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + --compilation-config "$VLLM_COMPILATION_CONFIG" + "${VLLM_TUNING_ARGS[@]}" --attention_config.use_fp4_indexer_cache=True --tokenizer-mode deepseek_v4 --tool-call-parser deepseek_v4 diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh index 74dc2129b..d82751baa 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh @@ -20,7 +20,7 @@ set -x # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake. +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake or lmcache. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -105,14 +105,19 @@ export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 SERVER_LOG="$RESULT_DIR/server.log" ROUTER_LOG="$RESULT_DIR/router.log" MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +LMCACHE_SERVER_LOG="$RESULT_DIR/lmcache_server.log" mkdir -p "$RESULT_DIR" SERVER_PID="" ROUTER_PID="" MOONCAKE_MASTER_PID="" +LMCACHE_SERVER_PID="" OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend mooncake; then +if agentic_kv_offload_enabled; then + case "$KV_OFFLOAD_BACKEND" in + mooncake) + require_agentic_kv_offload_backend mooncake # Mooncake embedded mode contributes one global segment per GPU rank to # a shared distributed store. Pre-divide the aggregate host budget # across those rank-contributed segments. @@ -171,6 +176,91 @@ EOF --kv-transfer-config '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' ) + ;; + lmcache) + require_agentic_kv_offload_backend lmcache + # The LMCache MP server owns the host-DRAM KV pool as one shared + # tier; vLLM ranks attach via LMCacheMPConnector, so the aggregate + # host budget is passed through undivided (unlike Mooncake's + # per-rank segments). Follows the LMCache DeepSeek-V4 recipe + # (docs.lmcache.ai/recipes/deepseek_v4_flash.html); LMCache handles + # DSV4's Sparse-MLA hybrid KV geometries automatically. + LMCACHE_VERSION=0.5.1 + agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION" + python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + + LMCACHE_HOST=127.0.0.1 + LMCACHE_PORT=$((PORT + 12000)) + LMCACHE_HTTP_PORT=$((PORT + 13000)) + # LMCacheMPConnector concatenates lmcache.mp.host and port into the + # ZMQ endpoint. Bind the server to a raw host, but pass the connector + # a ZMQ-style host string. + LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST" + # Pool target derated to 75% of the aggregate budget: pinned host + # memory is unswappable and also consumes GPU-side mapping + # resources, so leave headroom for vLLM host buffers and the OS. + # Full-budget targets OOM-killed the node (host OOM-killer or + # cudaErrorMemoryAllocation) as the cache filled past ~2 TB during + # PR #2153 bring-up. + LMCACHE_L1_SIZE_GB=$((TOTAL_CPU_DRAM_GB * 3 / 4)) + # The pool grows lazily from the initial allocation, so the full + # --l1-size-gb target is not pinned at startup. + LMCACHE_L1_INIT_SIZE_GB=20 + LMCACHE_MQ_TIMEOUT=300 + # Identical prefixes must hash to identical cache keys across DP ranks. + export PYTHONHASHSEED=0 + + echo "Starting LMCache MP server on port $LMCACHE_PORT..." + # One GPU-side transfer worker avoids concurrent-GPU-transfer stalls + # under heavy async-load pressure; CPU-side workers stay at 8. + lmcache server \ + --host "$LMCACHE_HOST" \ + --port "$LMCACHE_PORT" \ + --http-host "$LMCACHE_HOST" \ + --http-port "$LMCACHE_HTTP_PORT" \ + --l1-size-gb "$LMCACHE_L1_SIZE_GB" \ + --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \ + --max-gpu-workers 1 \ + --max-cpu-workers 8 \ + --chunk-size 1024 \ + --l1-align-bytes 16384 \ + --eviction-trigger-watermark 0.85 \ + --eviction-ratio 0.10 \ + --eviction-policy LRU \ + --supported-transfer-mode lmcache_driven \ + > "$LMCACHE_SERVER_LOG" 2>&1 & + LMCACHE_SERVER_PID=$! + LMCACHE_READY=0 + for _ in $(seq 1 60); do + if ! kill -0 "$LMCACHE_SERVER_PID" 2>/dev/null; then + echo "LMCache server died during startup." >&2 + cat "$LMCACHE_SERVER_LOG" >&2 + exit 1 + fi + if curl --output /dev/null --silent --fail \ + "http://127.0.0.1:$LMCACHE_HTTP_PORT/healthcheck"; then + LMCACHE_READY=1 + break + fi + sleep 2 + done + if [ "$LMCACHE_READY" -ne 1 ]; then + echo "LMCache server did not become healthy in time." >&2 + cat "$LMCACHE_SERVER_LOG" >&2 + exit 1 + fi + + unset VLLM_USE_SIMPLE_KV_OFFLOAD + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":$LMCACHE_MQ_TIMEOUT}}" + ) + ;; + *) + echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected one of: mooncake, lmcache)" >&2 + exit 1 + ;; + esac fi PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6be8e006f..454b84b74 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1782,6 +1782,26 @@ dsv4-fp4-b200-vllm-agentic: # Retain the external-cache transition and peak-throughput region. - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [16, 38, 44, 56, 64, 66, 68] } +# LMCache CPU-offload arm of dsv4-fp4-b200-vllm-agentic, split into its own +# config so it can be triggered/tested independently of the Mooncake curve. +# Points mirror the Mooncake offload points; the GPU-cache (kv-offloading: +# none) baselines live in the parent config only. Runs a tuned vLLM build +# while the parent stays on the stock v0.23.0 image. +dsv4-fp4-b200-vllm-agentic-lmcache: + image: inferactinc/public:sprint-agentx-fast-x86_64-cu13.0.1-6a1e7bf + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:b200-dgxc + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [8, 10, 16] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [16, 38, 44, 56, 64, 66, 68] } + dsv4-fp4-b200-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index be28208ab..203d85da3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4716,3 +4716,16 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - dsv4-fp4-b200-vllm-agentic-lmcache + scenario-type: + - agentic-coding + description: + - "Add LMCache 0.5.1 DRAM KV-offload config (kv-offload-backend: lmcache) as a standalone section alongside the Mooncake parent (B300 section deferred)" + - "Image inferactinc/public:sprint-agentx-fast-x86_64-cu13.0.1-6a1e7bf for the LMCache section (parent stays on v0.23.0)" + - "LMCache MP server (lmcache_driven transfer mode) + LMCacheMPConnector; L1 pool derated to 75% of TOTAL_CPU_DRAM_GB after full-budget pinned pools OOM-killed nodes in bring-up" + - "Adopt a tuned DeepSeek-V4 serving recipe for the LMCache arm (AMXF4 Mega-MoE, FULL_DECODE_ONLY cudagraphs, sparse MLA with prefill query quantization, NUMA binding, v2 model runner + rust frontend; fastsafetensors on EP points only — non-EP ranks OOM staging full expert weights; no cumem allocator — cuMem/VMM memory cannot be CUDA-IPC-shared with the LMCache server)" + - "The image ships the vLLM PR #45406 scheduler fix for async KV loads, so no engine patch is applied" + - "LMCache points mirror the Mooncake offload points: B200 TP8 conc 8-16, TP8-DEP8 conc 16-68" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2153