diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh index 4321644d8..2174f69fd 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh @@ -99,13 +99,20 @@ fi PARALLEL_ARGS=(--tp "$TP") METRICS_ARGS=(--enable-metrics) CHUNKED_PREFILL_SIZE=8192 +# DEP recipe splits by concurrency. The high-throughput tail (conc >= 54) uses +# the SGLang cookbook recipe (chunked 65536 + prefill delayer + tighter mem/SWA +# + cuda-graph-max-bs-decode 544, plus the 8192 tokens/rank cap that chunked +# 65536 needs to stay on the DeepGEMM MoE path). conc < 54 keeps the +# conservative recipe (chunked 32768) that avoids the pre-54 throughput cliff. +DEP_HIGH_CONC=false +[ "$DP_ATTENTION" = "true" ] && [ "$CONC" -ge 54 ] && DEP_HIGH_CONC=true +DEP_EXTRA_ARGS=() if [ "$DP_ATTENTION" = "true" ]; then DEEPEP_CONFIG='{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' export SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1 export SGLANG_OPT_FIX_HASH_MEGA_MOE=1 export SGLANG_OPT_USE_FAST_MASK_EP=1 export SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096 export SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0 PARALLEL_ARGS+=( @@ -120,7 +127,14 @@ if [ "$DP_ATTENTION" = "true" ]; then --moe-a2a-backend deepep --deepep-config "$DEEPEP_CONFIG" ) - CHUNKED_PREFILL_SIZE=32768 + if [ "$DEP_HIGH_CONC" = "true" ]; then + export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8192 + CHUNKED_PREFILL_SIZE=65536 + DEP_EXTRA_ARGS=(--enable-prefill-delayer) + else + export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096 + CHUNKED_PREFILL_SIZE=32768 + fi else PARALLEL_ARGS+=( --moe-runner-backend flashinfer_mxfp4 @@ -134,13 +148,32 @@ MODEL_ARGS=() # DeepGEMM's DSv4 indexer needs a multi-GiB temporary allocation at long # contexts. Leave the same HBM headroom used by the B300 recipe so a nearly # full GPU KV cache does not OOM while HiCache is spilling to host memory. -MEM_FRACTION_STATIC=0.88 +# The low-latency TP-only path (conc <= 16) runs GPU-only with no HiCache +# spill, so it can take a larger static fraction for more KV headroom. +if [ "$DP_ATTENTION" = "true" ]; then + if [ "$DEP_HIGH_CONC" = "true" ]; then + MEM_FRACTION_STATIC=0.85 + else + MEM_FRACTION_STATIC=0.88 + fi +else + MEM_FRACTION_STATIC=0.90 +fi # AgentX concurrency counts live session trees, not individual requests. # Allow subagent fan-out to exceed CONC without clipping request bursts. MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS=$CONC -[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 +# The cookbook DEP tail captures a large decode graph (batch 544); other paths +# scale the unified cuda-graph batch with CONC (capped at 64). +if [ "$DEP_HIGH_CONC" = "true" ]; then + CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 544) +else + CUDA_GRAPH_MAX_BS=$CONC + [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 + CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS") +fi +SWA_FULL_TOKENS_RATIO=0.1 +[ "$DEP_HIGH_CONC" = "true" ] && SWA_FULL_TOKENS_RATIO=0.075 export PYTHONNOUSERSITE=1 export TORCH_CUDA_ARCH_LIST=10.0 @@ -175,10 +208,11 @@ SGLANG_CMD=( --trust-remote-code "${PARALLEL_ARGS[@]}" --mem-fraction-static "$MEM_FRACTION_STATIC" - --swa-full-tokens-ratio 0.1 + --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + "${CUDA_GRAPH_ARGS[@]}" --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" + "${DEP_EXTRA_ARGS[@]}" --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --chat-template "$SCRIPT_DIR/../chat_templates/deepseek_v4_thinking.jinja" diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh index 5599ca471..8b451c792 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh @@ -105,7 +105,28 @@ PARALLEL_ARGS=(--tp "$TP") METRICS_ARGS=(--enable-metrics) MEM_FRACTION_STATIC=0.88 CHUNKED_PREFILL_SIZE=8192 +SWA_FULL_TOKENS_RATIO=0.1 +# Default (non-DP) attention path: compressed attention, shared-expert fusion off. +MODEL_ARGS=( + --attention-backend compressed + --page-size 256 + --disable-shared-experts-fusion +) +CUDA_GRAPH_ARGS=() +EXTRA_ARGS=() if [ "$DP_ATTENTION" = "true" ]; then + # DP-attention high-throughput agentic recipe (SGLang cookbook: DSV4-Pro FP4 + # B300, high-throughput, single-node): MegaMoE DeepGEMM MoE (DeepEP dispatch + + # fused shared experts + kernel autotune), tuned for conc128. + # Measured DEP8 conc128 vs the previous flashinfer_mxfp4 path: + # 24,466 -> 33,220 tok/s/gpu (surpasses the vLLM reference of 28,962). + export SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1 + export SGLANG_OPT_FIX_HASH_MEGA_MOE=1 + export SGLANG_OPT_USE_FAST_MASK_EP=1 + export SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1 + export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 + export SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0 PARALLEL_ARGS+=( --dp "$TP" --tokenizer-worker-num "$TP" @@ -115,11 +136,43 @@ if [ "$DP_ATTENTION" = "true" ]; then --stream-interval 20 --dist-init-addr "127.0.0.1:$((PORT + 2000))" --ep-size "$EP_SIZE" + --moe-a2a-backend megamoe + ) + MEM_FRACTION_STATIC=0.85 + SWA_FULL_TOKENS_RATIO=0.075 + # MegaMoE requires the default (non-compressed) attention path and fused shared experts. + MODEL_ARGS=() + EXTRA_ARGS=(--enable-prefill-delayer) + # chunked-prefill is divided by dp_size (=TP) under DP-attention; keep the + # effective chunk at 8192. Size the decode graph to served concurrency: + # DEP8 (dp 8) serves conc up to 512, DEP4 (dp 4) tops out near 128. + if [ "$TP" = "8" ]; then + CHUNKED_PREFILL_SIZE=65536 + CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 544) + else + # DEP4 (TP4/EP4) shards the model across only 4 GPUs, so per-GPU weights + # are ~2x DEP8 (233 GB loaded, ~32 GB free). At the DEP8-tuned 0.835 the + # KV pool cannot allocate (profiler floor ~0.879). Set 0.93: ~12 GB KV + # for the no-offload conc points, while keeping ~18 GB of activation/ + # CUDA-graph headroom for the MegaMoE workspace (DEP4's footprint is + # lighter than DEP8's: decode graph 128 vs 544, same 8192 eff chunk). + MEM_FRACTION_STATIC=0.93 + CHUNKED_PREFILL_SIZE=32768 + CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 128) + fi +elif { [ "$TP" = "8" ] || [ "$TP" = "4" ]; } && [ "${CONC:-999}" -le 16 ]; then + # TP-only low-latency (TP4 or TP8, non-DP, conc <= 16): SGLang cookbook + # DSV4-Pro FP4 B300 low-latency single-node recipe (mirrors the fixed_seq_len + # MTP TP-only path) with speculative decoding (EAGLE) removed. Default + # attention + fused shared experts + DSV4 FP4 sparse-attention indexer; + # mem-fraction 0.90. + PARALLEL_ARGS+=( --moe-runner-backend flashinfer_mxfp4 --disable-flashinfer-autotune + --enable-deepseek-v4-fp4-indexer ) - MEM_FRACTION_STATIC=0.95 - CHUNKED_PREFILL_SIZE=16384 + MEM_FRACTION_STATIC=0.90 + MODEL_ARGS=(--page-size 256) else PARALLEL_ARGS+=( --moe-runner-backend flashinfer_mxfp4 @@ -127,17 +180,13 @@ else ) fi -MODEL_ARGS=( - --attention-backend compressed - --page-size 256 - --disable-shared-experts-fusion -) - # AgentX concurrency counts live session trees, not individual requests. # Allow subagent fan-out to exceed CONC without clipping request bursts. MAX_RUNNING_REQUESTS=$((2 * CONC)) CUDA_GRAPH_MAX_BS=$CONC [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 +# Non-DP path keeps the concurrency-scaled decode graph; DP path set 544 above. +[ ${#CUDA_GRAPH_ARGS[@]} -eq 0 ] && CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS") export PYTHONNOUSERSITE=1 export TORCH_CUDA_ARCH_LIST=10.0 @@ -172,9 +221,9 @@ SGLANG_CMD=( --trust-remote-code "${PARALLEL_ARGS[@]}" --mem-fraction-static "$MEM_FRACTION_STATIC" - --swa-full-tokens-ratio 0.1 + --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + "${CUDA_GRAPH_ARGS[@]}" --allow-auto-truncate --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" --tool-call-parser deepseekv4 @@ -183,6 +232,7 @@ SGLANG_CMD=( --watchdog-timeout 1800 "${MODEL_ARGS[@]}" "${METRICS_ARGS[@]}" + "${EXTRA_ARGS[@]}" "${CACHE_ARGS[@]}" ) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6be8e006f..70f2cc585 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -14084,12 +14084,12 @@ dsv4-fp4-b200-sglang-agentic-hicache: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [8, 10, 16, 32, 40, 44] } + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [8, 16, 24] } # DEP without HiCache is already degraded at conc 52 (2,098 tok/s/GPU), # so resolve its pre-52 cliff instead of repeating the collapsed tail. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 24, 32, 38, 44, 48, 50, 52] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [16, 32, 38, 44, 50, 56, 64, 66, 68] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 38, 44, 48, 52, 54, 56, 58] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [16, 32, 38, 40, 44, 48, 52, 54, 56, 60, 64, 68] } # GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve # covers the middle/high-interactivity range omitted by the one-decode DEP @@ -14106,11 +14106,11 @@ dsv4-fp4-b300-sglang-agentic-hicache: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, kv-offloading: none, conc-list: [1, 4, 8, 16, 20, 24, 32] } - - { tp: 8, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 40, 48, 52, 56, 60, 64, 72] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16, 24, 32, 40, 64] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 128] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144, 196, 512] } + - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 8, 16] } + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 40] } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16, 24, 32, 40] } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 128] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 136, 144, 152, 196] } # DEP8 prefill uses an 8K batch because 16K OOMs in the FP4 MoE intermediate; # decode uses FULL_DECODE_ONLY after the controlled graph test restored decode diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c1a3c3aee..41d05ff2c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4699,3 +4699,15 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - dsv4-fp4-b200-sglang-agentic-hicache + - dsv4-fp4-b300-sglang-agentic-hicache + description: + - "B300: rework dsv4_fp4_b300_sglang.sh into three regimes selected at launch — DP-attention -> MegaMoE DeepGEMM; pure-TP (TP8/TP4) at conc<=16 -> low-latency; otherwise flashinfer_mxfp4 baseline" + - "B300 MegaMoE (DP-attention path): switch --moe-a2a-backend to megamoe with the DeepGEMM MegaMoE env stack (SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1, FIX_HASH_MEGA_MOE=1, USE_FAST_MASK_EP=1, FIX_MEGA_MOE_MEMORY=1, DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320, FIX_NEXTN_MEGA_MOE=1, SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0); add --enable-prefill-delayer, mem-fraction-static 0.85 (DEP8) / 0.93 (DEP4, whose heavier per-GPU weights need a higher floor to fit the KV pool), swa-full-tokens-ratio 0.075" + - "B300 MegaMoE rank-aware sizing: DEP8 uses chunked-prefill-size 65536 + cuda-graph-max-bs-decode 544; DEP4 uses chunked-prefill-size 32768 + cuda-graph-max-bs-decode 128 (effective per-DP chunk held at 8192)" + - "B300 low-latency (pure TP, conc<=16): --moe-runner-backend flashinfer_mxfp4 --disable-flashinfer-autotune --enable-deepseek-v4-fp4-indexer, mem-fraction-static 0.90, page-size 256 (SGLang DSV4 cookbook low-latency recipe; EAGLE speculative decoding intentionally omitted since it is unavailable here)" + - "B200: split mem-fraction-static (0.88 under DP-attention, 0.90 otherwise)" + - "nvidia-master.yaml: refine B200/B300 agentic-hicache conc-list search spaces" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2145