diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/rebuild-deepep.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/rebuild-deepep.sh new file mode 100644 index 0000000000..0be5dba59a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/rebuild-deepep.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -eux + +echo "=== Rebuilding DeepEP with kNumMaxTopK=16 for Qwen3.5 (topk=10) ===" + +DEEPEP_SRC="/sgl-workspace/DeepEP" + +if [ ! -d "$DEEPEP_SRC" ]; then + echo "ERROR: DeepEP source not found at $DEEPEP_SRC (mount via extra_mount)" + exit 1 +fi + +cd "$DEEPEP_SRC" + +# Resolve NVSHMEM by locating its header, not by directory name: a bare +# `find -name nvshmem -type d` can latch onto header-less source stubs +# (e.g. tilelang's bundled tvm src/runtime/contrib/nvshmem) and the build +# then dies with "fatal error: nvshmem.h: No such file or directory". +NVSHMEM_H=$(find /usr/local /opt /usr/lib /sgl-workspace -name "nvshmem.h" -path "*/include/*" \ + -not -path "*tilelang*" -not -path "*flashinfer*" -not -path "*DeepEP*" 2>/dev/null | head -1) +if [ -z "${NVSHMEM_H:-}" ]; then + echo "No NVSHMEM headers found in the image; installing nvidia-nvshmem-cu13..." + pip install --break-system-packages --quiet nvidia-nvshmem-cu13 + NVSHMEM_H=$(find /usr/local/lib/python3*/dist-packages/nvidia -name "nvshmem.h" -path "*/include/*" 2>/dev/null | head -1) +fi +if [ -z "${NVSHMEM_H:-}" ]; then + echo "ERROR: NVSHMEM headers not found in image or via pip" >&2 + exit 1 +fi +NVSHMEM_DIR=$(dirname "$(dirname "$NVSHMEM_H")") +echo "NVSHMEM_DIR=$NVSHMEM_DIR" + +# Fix missing nvshmem symlinks (container has .so.3 but not .so) +NVSHMEM_LIB="$NVSHMEM_DIR/lib" +if [ ! -f "$NVSHMEM_LIB/libnvshmem_host.so" ] && [ -f "$NVSHMEM_LIB/libnvshmem_host.so.3" ]; then + echo "Creating missing nvshmem symlinks..." + ln -sf libnvshmem_host.so.3 "$NVSHMEM_LIB/libnvshmem_host.so" +fi + +# Apply kNumMaxTopK=16 patch (Qwen3.5 uses topk=10, default kNumMaxTopK=8 is insufficient) +# Note: source has both kNumMaxTopK (uppercase) and kNumMaxTopk (lowercase) as separate variables +sed -i 's/kNumMaxTopK[[:space:]]*=[[:space:]]*[0-9][0-9]*/kNumMaxTopK = 16/g' csrc/kernels/internode_ll.cu +sed -i 's/kNumMaxTopk[[:space:]]*=[[:space:]]*[0-9][0-9]*/kNumMaxTopk = 16/g' csrc/kernels/internode_ll.cu + +# Verify the patch was applied +grep -q "kNumMaxTop. = 16" csrc/kernels/internode_ll.cu && echo "Patch verified: kNumMaxTopK/k=16" || { + echo "ERROR: kNumMaxTopK patch failed to apply!"; exit 1; +} + +# Build with full output so we can debug failures +# set -e will auto-exit on failure +TORCH_CUDA_ARCH_LIST="10.0" \ +NVSHMEM_DIR="$NVSHMEM_DIR" \ +pip install -e . --no-build-isolation 2>&1 + +echo "=== DeepEP rebuild complete ===" +python3 -c "import deep_ep; print('deep_ep imported successfully')" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-dep4-dep16.yaml new file mode 100644 index 0000000000..a30ac61660 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-dep4-dep16.yaml @@ -0,0 +1,155 @@ +name: "qwen3.5-1p1d-dep4-dep16" + +setup_script: rebuild-deepep.sh + +dynamo: + hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b + install: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 1 + +backend: + type: sglang + + prefill_environment: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + decode_environment: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + sglang_config: + prefill: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 4096 + mem-fraction-static: 0.80 + max-mamba-cache-size: 2048 + max-running-requests: 2048 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "512x1024x2048" \ No newline at end of file diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-tp4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-tp4-tp4.yaml new file mode 100644 index 0000000000..90eaa12aae --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-tp4-tp4.yaml @@ -0,0 +1,123 @@ +name: "qwen3.5-1p1d-tp4-tp4" + +sbatch_directives: + mem: "0" + +dynamo: + hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b + install: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + +backend: + type: sglang + + prefill_environment: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + + decode_environment: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_HEALTH_CHECK_TIMEOUT: "3600" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + + sglang_config: + prefill: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 4096 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "prefill" + + decode: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 4096 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "decode" + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 1 + random_range_ratio: 0.8 + concurrencies: "1x2x4x8x16x32x64" \ No newline at end of file diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/2p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/2p1d-dep4-dep16.yaml new file mode 100644 index 0000000000..35d2aba8e7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/2p1d-dep4-dep16.yaml @@ -0,0 +1,155 @@ +name: "qwen3.5-2p1d-dep4-dep16" + +setup_script: rebuild-deepep.sh + +dynamo: + hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b + install: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 2 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 4 + prefill_workers: 2 + decode_workers: 1 + +backend: + type: sglang + + prefill_environment: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + decode_environment: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + sglang_config: + prefill: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 8192 + mem-fraction-static: 0.75 + max-mamba-cache-size: 4096 + max-running-requests: 4096 + cuda-graph-max-bs: 256 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "4096" \ No newline at end of file diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml new file mode 100644 index 0000000000..0a938fb48c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml @@ -0,0 +1,121 @@ +name: "qwen3.5-1p1d-tp4-tp4" + +sbatch_directives: + mem: "0" + +dynamo: + hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b + install: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + +backend: + type: sglang + + prefill_environment: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + + decode_environment: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_HEALTH_CHECK_TIMEOUT: "3600" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + + sglang_config: + prefill: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 16384 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "prefill" + + decode: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 16384 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "decode" + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "1x2x4x8x16x32x64x128" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/4p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/4p1d-dep4-dep16.yaml new file mode 100644 index 0000000000..8b4b802669 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/4p1d-dep4-dep16.yaml @@ -0,0 +1,160 @@ +name: "qwen3.5-4p1d-dep4-dep16" + +setup_script: rebuild-deepep.sh + +sbatch_directives: + mem: "0" + +dynamo: + hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b + install: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 3 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 4 + decode_nodes: 4 + prefill_workers: 4 + decode_workers: 1 + +backend: + type: sglang + + prefill_environment: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + decode_environment: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + sglang_config: + prefill: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 16384 + mem-fraction-static: 0.80 + max-mamba-cache-size: 2048 + max-running-requests: 2048 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 20 + num_warmup_mult: 2 + random_range_ratio: 0.8 + concurrencies: "1024" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/8p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/8p1d-dep4-dep16.yaml new file mode 100644 index 0000000000..cf9ca607e2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/8p1d-dep4-dep16.yaml @@ -0,0 +1,161 @@ +name: "qwen3.5-8p1d-dep4-dep16" + +setup_script: rebuild-deepep.sh + +sbatch_directives: + mem: "0" + +infra: + etcd_nats_dedicated_node: true + +dynamo: + hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b + install: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 8 + decode_nodes: 4 + prefill_workers: 8 + decode_workers: 1 + +backend: + type: sglang + + prefill_environment: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + decode_environment: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + sglang_config: + prefill: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 16384 + mem-fraction-static: 0.80 + max-mamba-cache-size: 2048 + max-running-requests: 2048 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "2048x4096" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6be8e006f9..3d9e46399b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -14563,3 +14563,111 @@ glm5-fp4-gb300-dynamo-trt-mtp: ep: 16 dp-attn: true + +qwen3.5-fp8-gb300-dynamo-sglang: + image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: gb300 + precision: fp8 + framework: dynamo-sglang + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + # 1P1D STP: TP4 prefill + TP4 decode (pure tensor parallel). 2 nodes (1+1). + - spec-decoding: "none" + conc-list: [1, 2, 4, 8, 16, 32, 64] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-tp4-tp4.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + # 1P1D wide-EP: prefill DEP4 + decode DEP16. 5 nodes (1+4). + - spec-decoding: "none" + conc-list: [512, 1024, 2048] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-dep4-dep16.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 2P1D wide-EP: 2 prefill DEP4 + decode DEP16. 6 nodes (2+4). + - spec-decoding: "none" + conc-list: [4096] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/1k1k/2p1d-dep4-dep16.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - isl: 8192 + osl: 1024 + search-space: + # 1P1D STP: TP4 prefill + TP4 decode (pure tensor parallel). 2 nodes (1+1). + - spec-decoding: "none" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + # 4P1D wide-EP: 4 prefill DEP4 + decode DEP16. 8 nodes (4+4). + - spec-decoding: "none" + conc-list: [1024] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/8k1k/4p1d-dep4-dep16.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 8P1D wide-EP: 8 prefill DEP4 + decode DEP16. 12 nodes (8+4). + - spec-decoding: "none" + conc-list: [2048, 4096] + prefill: + num-worker: 8 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/8k1k/8p1d-dep4-dep16.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3425fd7c7c..b87c735fd1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4669,3 +4669,11 @@ - "Image: lmsysorg/sglang:v0.5.11-cu130" - "11 topologies across 1k/1k and 8k/1k (prefill TP4 + decode wide-EP: 8k/1k DEP16/24/32/40, 1k/1k DEP48/56; per-node TP4 low-latency); MTP flags: speculative-algorithm EAGLE, num-steps 2, eagle-topk 1, num-draft-tokens 3" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1907 + +- config-keys: + - qwen3.5-fp8-gb300-dynamo-sglang + description: + - "Add Qwen3.5-397B-A17B-FP8 GB300 disaggregated multinode SGLang benchmarks via Dynamo" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 5328ded02d..3e3fa61255 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -74,8 +74,13 @@ elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then # Qwen3.5 sglang recipes (qwen3.5-fp4). export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-NVFP4 export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp4" +elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then + # SRT_SLURM_MODEL_PREFIX must match the model.path alias used in our + # Qwen3.5 sglang recipes (qwen3.5-fp8). + export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-FP8 + export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" else - echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4, glm5-fp4, glm5-fp8, minimaxm2.5-fp4, minimaxm2.5-fp8, kimik2.5-fp4, qwen3.5-fp4" + echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4, glm5-fp4, glm5-fp8, minimaxm2.5-fp4, minimaxm2.5-fp8, kimik2.5-fp4, qwen3.5-fp4, qwen3.5-fp8" exit 1 fi @@ -184,10 +189,28 @@ elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.1" ]]; then cd "$SRT_REPO_DIR" git checkout main elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then - # Overlay our version-controlled Qwen3.5 recipes onto the submission branch. + # Overlay our version-controlled Qwen3.5 recipes onto the srt-slurm checkout. + # fp8 recipes pin dynamo by commit hash (source install), which needs the + # cargo/maturin bootstrap that only exists on srt-slurm:main — the + # sa-submission-q2-2026 sglang install path assumes maturin ships in the + # image, and the lmsysorg/sglang nightly-dev-cu13 image doesn't include it. + # Same branch the identical gb200-fp8 recipes run on. fp4 recipes pin + # dynamo by version (pip install) and stay on the submission branch they + # were validated against. git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout sa-submission-q2-2026 + if [[ $PRECISION == "fp8" ]]; then + git checkout main + # The stock configs/rebuild-deepep.sh (setup_script of the wide-EP + # fp8 recipes) resolves NVSHMEM by directory name, which can match a + # header-less stub in the cu13 nightly image (tilelang's bundled tvm + # contrib/nvshmem) and fail the DeepEP build with "nvshmem.h: No + # such file or directory". Overlay our header-based resolver. + cp "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/rebuild-deepep.sh" \ + configs/rebuild-deepep.sh + else + git checkout sa-submission-q2-2026 + fi mkdir -p recipes/sglang/qwen3.5 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" recipes/sglang/qwen3.5 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" ]]; then @@ -308,9 +331,12 @@ sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" # the GHA runner host (im-gb300-login-02, an x86 login node). It's required # whenever model.path resolves to the node-local /scratch NVMe that the login # node can't see: -# - the agentic path (DSv4-Pro checkpoint), and +# - the agentic path (DSv4-Pro checkpoint), # - glm5.1, whose GLM-5.1-NVFP4 weights are prestaged on the compute-node -# /scratch/models. +# /scratch/models, and +# - qwen3.5 fp8, whose weights are also on the compute-node /scratch/models +# and which runs on srt-slurm:main (the branch that has the preflight; +# qwen3.5 fp4 runs on sa-submission-q2-2026, which has none). # The engine still fails loudly at runtime if the path is genuinely missing on # the compute node. Other fixed-seq-len recipes resolve model.path to a # login-visible location, so keep the precheck enforced for them. @@ -318,7 +344,7 @@ SRTCTL_APPLY_ARGS=( -f "$CONFIG_FILE" --tags "gb300,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" ) -if [[ "$IS_AGENTIC" == "1" || "$MODEL_PREFIX" == "glm5.1" ]]; then +if [[ "$IS_AGENTIC" == "1" || "$MODEL_PREFIX" == "glm5.1" || ( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp8" ) ]]; then SRTCTL_APPLY_ARGS+=(--no-preflight) fi if [[ -n "$SRTCTL_SETUP_SCRIPT" ]]; then