[WIP] Update vllm B300 Minimax M3

benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh

@@ -3,7 +3,12 @@
 # MiniMax-M3 NVFP4 B300 single-node vLLM recipe.
 # Same shape as minimaxm3_fp8_b300.sh but uses the nvidia/MiniMax-M3-NVFP4
 # checkpoint. MiniMax-M3 modelopt NVFP4 support (vllm-project/vllm PR #46380) is
-# baked into the perf container image, so no runtime patch is needed.
+# baked into the perf container image.
+#
+# At runtime the recipe swaps the image's FlashInfer for the first pinned
+# nightly containing the upstream SM100 low-M MXFP8 split-K kernel
+# (flashinfer-ai/flashinfer#3847), then backports the AutoTuner non-Tensor guard
+# fix from flashinfer-ai/flashinfer#3918.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -19,6 +24,35 @@ check_env_vars \
     RANDOM_RANGE_RATIO \
     RESULT_FILENAME
 
+# --- FlashInfer nightly + AutoTuner non-Tensor guard patch ------------------
+FLASHINFER_VERSION=0.6.15.dev20260710
+FLASHINFER_NIGHTLY_TAG=nightly-v0.6.15-20260710
+FLASHINFER_RELEASE_URL="https://github.com/flashinfer-ai/flashinfer/releases/download/${FLASHINFER_NIGHTLY_TAG}"
+
+python3 -m pip uninstall -y flashinfer-python flashinfer-cubin flashinfer-jit-cache
+
+python3 -m pip install \
+    "${FLASHINFER_RELEASE_URL}/flashinfer_python-${FLASHINFER_VERSION}-py3-none-any.whl" \
+    "${FLASHINFER_RELEASE_URL}/flashinfer_cubin-${FLASHINFER_VERSION}-py3-none-any.whl" \
+    "${FLASHINFER_RELEASE_URL}/flashinfer_jit_cache-${FLASHINFER_VERSION}+cu130-cp39-abi3-manylinux_2_28_$(uname -m).whl" \
+    || { echo "FlashInfer nightly install failed" >&2; exit 1; }
+
+# The pinned nightly predates flashinfer-ai/flashinfer#3918. Apply only its
+# runtime fix; the upstream test change is intentionally not backported.
+FLASHINFER_PATCH="$(dirname "$0")/patches/flashinfer-autotuner-non-tensor-guard.patch"
+if ! command -v patch >/dev/null 2>&1; then
+    apt-get update -y && apt-get install -y --no-install-recommends patch \
+        || { echo "Failed to install patch(1)" >&2; exit 1; }
+fi
+SITE_PACKAGES=$(dirname "$(python3 -c "import importlib.util; print(importlib.util.find_spec('flashinfer').submodule_search_locations[0])")") \
+    || { echo "Could not locate the installed flashinfer package" >&2; exit 1; }
+patch --dry-run -p1 -d "${SITE_PACKAGES}" < "${FLASHINFER_PATCH}" >/dev/null \
+    || { echo "FlashInfer AutoTuner non-Tensor guard patch does not apply" >&2; exit 1; }
+patch -p1 -d "${SITE_PACKAGES}" < "${FLASHINFER_PATCH}" \
+    || { echo "FlashInfer AutoTuner non-Tensor guard patch failed" >&2; exit 1; }
+
+# -----------------------------------------------------------------------------
+
 if [[ -n "${MODEL_PATH:-}" ]]; then
     if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
         hf download "$MODEL" --local-dir "$MODEL_PATH"
@@ -38,7 +72,7 @@ SERVER_LOG=/workspace/server.log
 
 export VLLM_ENGINE_READY_TIMEOUT_S=3600
 export VLLM_FLOAT32_MATMUL_PRECISION=high
-export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm
+export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800
 
 if [ "${DP_ATTENTION}" = "true" ]; then
   PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel"
@@ -57,6 +91,7 @@ start_gpu_monitor
 set -x
 vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port $PORT \
 $PARALLEL_ARGS \
+--attention_config.indexer_kv_dtype fp8 \
 --gpu-memory-utilization 0.95 \
 --max-model-len $MAX_MODEL_LEN \
 --kv-cache-dtype fp8 \

benchmarks/single_node/fixed_seq_len/patches/flashinfer-autotuner-non-tensor-guard.patch

@@ -0,0 +1,38 @@
+diff --git a/flashinfer/autotuner/autotuner.py b/flashinfer/autotuner/autotuner.py
+index 54b90a0..76443ca 100644
+--- a/flashinfer/autotuner/autotuner.py
++++ b/flashinfer/autotuner/autotuner.py
+@@ -2074,14 +2074,19 @@ class AutoTuner:
+     def _prepare_input_tensors_with_batches(
+         self,
+-        inputs: list[torch.Tensor],
++        inputs: list[Any],
+         tuning_config: TuningConfig,
+-    ) -> list[list[torch.Tensor]]:
++    ) -> list[list[Any]]:
+         """Create multiple input copies to flush the L2 cache between profiling iterations."""
+         if not tuning_config.use_cold_l2_cache:
+             return [inputs]
+
+-        one_buffer_bytes = sum(input.numel() * input.element_size() for input in inputs)
++        one_buffer_bytes = sum(
++            input.numel() * input.element_size()
++            if isinstance(input, torch.Tensor)
++            else 0
++            for input in inputs
++        )
+         if one_buffer_bytes <= 0:
+             logger.debug(
+                 "[Autotuner] No tensor inputs or zero-sized tensors; falling back to single-batch profiling."
+             )
+@@ -2093,7 +2098,9 @@ class AutoTuner:
+
+         inputs_list = [inputs]
+         for _ in range(num_buffers - 1):
+-            inputs_list.append(list(t.clone() for t in inputs))
++            inputs_list.append(
++                [t.clone() if isinstance(t, torch.Tensor) else t for t in inputs]
++            )
+
+         logger.debug(
+             f"[Autotuner] use_cold_l2_cache={tuning_config.use_cold_l2_cache}, use {num_buffers} different tensors for profiling"

configs/nvidia-master.yaml

@@ -13559,7 +13559,7 @@ minimaxm3-fp8-b300-vllm:
 # weights are pre-staged read-only at /scratch/models/MiniMax-M3-NVFP4 (added to
 # the STAGED_MODELS allow-list in launch_b300-nv.sh).
 minimaxm3-fp4-b300-vllm:
-  image: vllm/vllm-openai:nightly-93d8f834dd8acf33eb0e2a75b2711b628cb6e226
+  image: vllm/vllm-openai:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9
   model: nvidia/MiniMax-M3-NVFP4
   model-prefix: minimaxm3
   runner: b300

perf-changelog.yaml

@@ -4723,3 +4723,11 @@
     - "Bump the SGLang ROCm image from 20260706 to 20260709 and retire the DeepSeek-V4 compress-state, DSA paged-MQA, AITER instruction-shape, and HiCache host-pool shims that are already included upstream."
     - "Remove the SWA re-prefill and unified-KV HiCache overlays after their upstream PRs merged; retain only patch code without a merged upstream equivalent."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2127
+
+- config-keys:
+    - minimaxm3-fp4-b300-vllm
+  description:
+    - "Update the vLLM image to nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9"
+    - "Install FlashInfer 0.6.15.dev20260710 with upstream SM100 low-M MXFP8 split-K support"
+    - "Remove the explicit FlashInfer TRT-LLM all-reduce backend override"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2124