diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh index f91419edb..f9df77df7 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh @@ -3,7 +3,12 @@ # MiniMax-M3 NVFP4 B300 single-node vLLM recipe. # Same shape as minimaxm3_fp8_b300.sh but uses the nvidia/MiniMax-M3-NVFP4 # checkpoint. MiniMax-M3 modelopt NVFP4 support (vllm-project/vllm PR #46380) is -# baked into the perf container image, so no runtime patch is needed. +# baked into the perf container image. +# +# At runtime the recipe swaps the image's FlashInfer for the first pinned +# nightly containing the upstream SM100 low-M MXFP8 split-K kernel +# (flashinfer-ai/flashinfer#3847), then backports the AutoTuner non-Tensor guard +# fix from flashinfer-ai/flashinfer#3918. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -19,6 +24,35 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME +# --- FlashInfer nightly + AutoTuner non-Tensor guard patch ------------------ +FLASHINFER_VERSION=0.6.15.dev20260710 +FLASHINFER_NIGHTLY_TAG=nightly-v0.6.15-20260710 +FLASHINFER_RELEASE_URL="https://github.com/flashinfer-ai/flashinfer/releases/download/${FLASHINFER_NIGHTLY_TAG}" + +python3 -m pip uninstall -y flashinfer-python flashinfer-cubin flashinfer-jit-cache + +python3 -m pip install \ + "${FLASHINFER_RELEASE_URL}/flashinfer_python-${FLASHINFER_VERSION}-py3-none-any.whl" \ + "${FLASHINFER_RELEASE_URL}/flashinfer_cubin-${FLASHINFER_VERSION}-py3-none-any.whl" \ + "${FLASHINFER_RELEASE_URL}/flashinfer_jit_cache-${FLASHINFER_VERSION}+cu130-cp39-abi3-manylinux_2_28_$(uname -m).whl" \ + || { echo "FlashInfer nightly install failed" >&2; exit 1; } + +# The pinned nightly predates flashinfer-ai/flashinfer#3918. Apply only its +# runtime fix; the upstream test change is intentionally not backported. +FLASHINFER_PATCH="$(dirname "$0")/patches/flashinfer-autotuner-non-tensor-guard.patch" +if ! command -v patch >/dev/null 2>&1; then + apt-get update -y && apt-get install -y --no-install-recommends patch \ + || { echo "Failed to install patch(1)" >&2; exit 1; } +fi +SITE_PACKAGES=$(dirname "$(python3 -c "import importlib.util; print(importlib.util.find_spec('flashinfer').submodule_search_locations[0])")") \ + || { echo "Could not locate the installed flashinfer package" >&2; exit 1; } +patch --dry-run -p1 -d "${SITE_PACKAGES}" < "${FLASHINFER_PATCH}" >/dev/null \ + || { echo "FlashInfer AutoTuner non-Tensor guard patch does not apply" >&2; exit 1; } +patch -p1 -d "${SITE_PACKAGES}" < "${FLASHINFER_PATCH}" \ + || { echo "FlashInfer AutoTuner non-Tensor guard patch failed" >&2; exit 1; } + +# ----------------------------------------------------------------------------- + if [[ -n "${MODEL_PATH:-}" ]]; then if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" @@ -39,6 +73,7 @@ SERVER_LOG=/workspace/server.log export VLLM_ENGINE_READY_TIMEOUT_S=3600 export VLLM_FLOAT32_MATMUL_PRECISION=high export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm +export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 if [ "${DP_ATTENTION}" = "true" ]; then PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" @@ -57,6 +92,7 @@ start_gpu_monitor set -x vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port $PORT \ $PARALLEL_ARGS \ +--attention_config.indexer_kv_dtype fp8 \ --gpu-memory-utilization 0.95 \ --max-model-len $MAX_MODEL_LEN \ --kv-cache-dtype fp8 \ diff --git a/benchmarks/single_node/fixed_seq_len/patches/flashinfer-autotuner-non-tensor-guard.patch b/benchmarks/single_node/fixed_seq_len/patches/flashinfer-autotuner-non-tensor-guard.patch new file mode 100644 index 000000000..711af8b84 --- /dev/null +++ b/benchmarks/single_node/fixed_seq_len/patches/flashinfer-autotuner-non-tensor-guard.patch @@ -0,0 +1,38 @@ +diff --git a/flashinfer/autotuner/autotuner.py b/flashinfer/autotuner/autotuner.py +index 54b90a0..76443ca 100644 +--- a/flashinfer/autotuner/autotuner.py ++++ b/flashinfer/autotuner/autotuner.py +@@ -2074,14 +2074,19 @@ class AutoTuner: + def _prepare_input_tensors_with_batches( + self, +- inputs: list[torch.Tensor], ++ inputs: list[Any], + tuning_config: TuningConfig, +- ) -> list[list[torch.Tensor]]: ++ ) -> list[list[Any]]: + """Create multiple input copies to flush the L2 cache between profiling iterations.""" + if not tuning_config.use_cold_l2_cache: + return [inputs] + +- one_buffer_bytes = sum(input.numel() * input.element_size() for input in inputs) ++ one_buffer_bytes = sum( ++ input.numel() * input.element_size() ++ if isinstance(input, torch.Tensor) ++ else 0 ++ for input in inputs ++ ) + if one_buffer_bytes <= 0: + logger.debug( + "[Autotuner] No tensor inputs or zero-sized tensors; falling back to single-batch profiling." + ) +@@ -2093,7 +2098,9 @@ class AutoTuner: + + inputs_list = [inputs] + for _ in range(num_buffers - 1): +- inputs_list.append(list(t.clone() for t in inputs)) ++ inputs_list.append( ++ [t.clone() if isinstance(t, torch.Tensor) else t for t in inputs] ++ ) + + logger.debug( + f"[Autotuner] use_cold_l2_cache={tuning_config.use_cold_l2_cache}, use {num_buffers} different tensors for profiling" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6be8e006f..3f3fc6c4c 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -13559,7 +13559,7 @@ minimaxm3-fp8-b300-vllm: # weights are pre-staged read-only at /scratch/models/MiniMax-M3-NVFP4 (added to # the STAGED_MODELS allow-list in launch_b300-nv.sh). minimaxm3-fp4-b300-vllm: - image: vllm/vllm-openai:nightly-93d8f834dd8acf33eb0e2a75b2711b628cb6e226 + image: vllm/vllm-openai:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: b300 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index be28208ab..e3c2f626b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4716,3 +4716,11 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - minimaxm3-fp4-b300-vllm + description: + - "Update the vLLM image to nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9" + - "Install FlashInfer 0.6.15.dev20260710 with upstream SM100 low-M MXFP8 split-K support" + - "Use the explicit FlashInfer TRT-LLM all-reduce backend override" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2124