diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index f89abadbe0..cb5bc921aa 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -415,7 +415,6 @@ steps: - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN - pytest -v -s v1/determinism/test_nvfp4_batch_invariant.py - - pytest -v -s v1/determinism/test_nvfp4_batch_invariant_scaled_mm.py - pytest -v -s v1/determinism/test_matmul_batch_invariant.py - pytest -v -s v1/determinism/test_cutlass_batch_invariance.py - pytest -v -s v1/determinism/test_online_batch_invariance.py diff --git a/csrc/libtorch_stable/quantization/fp4/nvfp4_blockwise_moe_kernel.cu b/csrc/libtorch_stable/quantization/fp4/nvfp4_blockwise_moe_kernel.cu index b044db6d32..ea33444063 100644 --- a/csrc/libtorch_stable/quantization/fp4/nvfp4_blockwise_moe_kernel.cu +++ b/csrc/libtorch_stable/quantization/fp4/nvfp4_blockwise_moe_kernel.cu @@ -274,6 +274,15 @@ void run_fp4_blockwise_scaled_group_mm_sm100( using Gemm1SM = cutlass::gemm::device::GemmUniversalAdapter; using Gemm = Gemm1SM; + static_assert( + cute::is_same_v< + typename Gemm::GemmKernel::TileScheduler, + cutlass::gemm::kernel::detail::PersistentTileSchedulerSm100Group< + ProblemShape, Gemm::GemmKernel::SchedulerPipelineStageCount>>, + "SM100 NVFP4 grouped GEMM must use PersistentTileSchedulerSm100Group " + "for batch invariance (VLLM_BATCH_INVARIANT=1). " + "If you want to change this, add a dedicated config/code path " + "for batch invariant mode, instead of relaxing this check."); using StrideA = typename Gemm::GemmKernel::InternalStrideA; using StrideB = typename Gemm::GemmKernel::InternalStrideB; using StrideC = typename Gemm::GemmKernel::InternalStrideC; @@ -473,6 +482,15 @@ void run_fp4_blockwise_scaled_group_mm_sm120( CollectiveEpilogue>; using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + static_assert( + cute::is_base_of_v< + cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperative, + typename Gemm::GemmKernel::CollectiveMainloop::DispatchPolicy:: + Schedule>, + "SM120 NVFP4 grouped GEMM must use a cooperative ptr-array mainloop " + "schedule for batch invariance (VLLM_BATCH_INVARIANT=1). " + "If you want to change this, add a dedicated config/code path " + "for batch invariant mode, instead of relaxing this check."); using StrideA = typename Gemm::GemmKernel::InternalStrideA; using StrideB = typename Gemm::GemmKernel::InternalStrideB; using StrideC = typename Gemm::GemmKernel::InternalStrideC; diff --git a/tests/v1/determinism/test_cutlass_batch_invariance.py b/tests/v1/determinism/test_cutlass_batch_invariance.py index 6df3f0a5f0..95403e50ff 100644 --- a/tests/v1/determinism/test_cutlass_batch_invariance.py +++ b/tests/v1/determinism/test_cutlass_batch_invariance.py @@ -1,22 +1,89 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CUTLASS batch-invariance tests. + +The NVFP4 CUTLASS tests in this file must run with ``VLLM_BATCH_INVARIANT=1`` +before the first relevant native kernel call. Do not run them in the same pytest +process after tests that intentionally exercise non-batch-invariant NVFP4 +scaled-mm kernels. +""" + +from typing import Any import pytest import torch import vllm.envs as envs +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from tests.kernels.moe.utils import make_dummy_moe_config, make_test_quant_config +from tests.kernels.quantization.nvfp4_utils import get_nvfp4_global_scale from tests.utils import TestFP8Layer, requires_fp8 +from vllm import _custom_ops as ops from vllm.model_executor.kernels.linear.scaled_mm.cutlass import ( CutlassFP8ScaledMMLinearKernel, ) +from vllm.model_executor.layers.fused_moe import fused_topk +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.all2all_utils import ( + maybe_make_prepare_finalize, +) +from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( + CutlassExpertsFp4, +) from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8DynamicTokenSym, kFp8StaticTensorSym, ) from vllm.platforms import current_platform +from vllm.utils.torch_utils import set_random_seed pytest.importorskip("torch.cuda") +_NVFP4_REQUIRES_SM100 = pytest.mark.skipif( + not current_platform.has_device_capability(100), + reason="Nvfp4 Requires compute capability of 10 or above.", +) + +_NVFP4_LINEAR_DTYPES = [torch.float16, torch.bfloat16] +_NVFP4_LINEAR_CONSISTENCY_SHAPES = [ + (256, 128, 4096), + (512, 256, 4096), + (256, 256, 2048), + (241, 160, 2048), + (401, 352, 1984), + (333, 320, 1008), + (287, 96, 4096), +] + +_NVFP4_MOE_DTYPES = (torch.bfloat16,) +_NVFP4_MOE_NUM_EXPERTS = (40, 64) +_NVFP4_MOE_TOPKS = (1, 4) + +_NVFP4_MOE_BATCH_INVARIANT_CASES = ( + { + "m": 12, + "n": 1024, + "k": 1024, + "subset_indices": ((0, 3, 7), (1, 4, 8, 10), (11, 5, 2, 9, 6)), + }, + { + "m": 129, + "n": 1024, + "k": 1024, + "subset_indices": ( + (0, 64, 128), + (1, 63, 64, 127), + (32, 33, 95, 96, 128), + ), + }, + { + "m": 73, + "n": 1472, + "k": 1536, + "subset_indices": ((0, 17, 35), (8, 36, 72), (1, 9, 33, 48, 64)), + }, +) + @pytest.fixture(autouse=True) def setup_cuda(): @@ -66,3 +133,230 @@ def test_cutlass_fp8_batch_invariant_fixed_config( torch.testing.assert_close(front_output, baseline, rtol=0, atol=0) torch.testing.assert_close(back_output, baseline, rtol=0, atol=0) + + +@_NVFP4_REQUIRES_SM100 +@pytest.mark.parametrize("dtype", _NVFP4_LINEAR_DTYPES) +@pytest.mark.parametrize("shape", _NVFP4_LINEAR_CONSISTENCY_SHAPES) +@torch.inference_mode() +def test_cutlass_nvfp4_scaled_mm_batch_invariant( + dtype: torch.dtype, + shape: tuple[int, int, int], +) -> None: + """Each row of a full-M GEMM must match its M=1 counterpart.""" + set_random_seed(12345) + m, n, packed_k = shape + k = packed_k * 2 # real K (FP4 elements) + + a_dtype = torch.randn((m, k), dtype=dtype, device="cuda") + b_dtype = torch.randn((n, k), dtype=dtype, device="cuda") + + a_global_scale = get_nvfp4_global_scale(a_dtype) + b_global_scale = get_nvfp4_global_scale(b_dtype) + alpha = 1.0 / (a_global_scale * b_global_scale) + + b_fp4, b_scale_interleaved = ops.scaled_fp4_quant(b_dtype, b_global_scale) + + a_fp4_full, a_sf_full = ops.scaled_fp4_quant(a_dtype, a_global_scale) + out_full = ops.cutlass_scaled_fp4_mm( + a_fp4_full, + b_fp4, + a_sf_full, + b_scale_interleaved, + alpha, + dtype, + ) + + for i in range(m): + a_row = a_dtype[i : i + 1] + a_fp4_row, a_sf_row = ops.scaled_fp4_quant(a_row, a_global_scale) + out_row = ops.cutlass_scaled_fp4_mm( + a_fp4_row, + b_fp4, + a_sf_row, + b_scale_interleaved, + alpha, + dtype, + ) + + assert torch.equal(out_full[i], out_row[0]), ( + f"VLLM_BATCH_INVARIANT: row {i} differs between M={m} and M=1: " + f"max_abs_diff={(out_full[i] - out_row[0]).abs().max().item()}" + ) + + +def _make_cutlass_fp4_moe_batch_invariant_case( + case_config: dict[str, Any], + activation: MoEActivation, + e: int, + topk: int, + dtype: torch.dtype, +) -> dict[str, Any]: + set_random_seed(7) + assert topk <= e + + hidden_states = ( + torch.randn((case_config["m"], case_config["k"]), device="cuda", dtype=dtype) + / 10 + ) + + w1_q, w2_q, quant_config = make_test_quant_config( + e, + case_config["n"], + case_config["k"], + in_dtype=dtype, + quant_dtype="nvfp4", + block_shape=None, + per_act_token_quant=False, + make_gate=activation.is_gated, + ) + + score = torch.randn((case_config["m"], e), device="cuda", dtype=dtype) + + moe_config = make_dummy_moe_config( + num_experts=e, + experts_per_token=topk, + hidden_dim=case_config["k"], + intermediate_size=case_config["n"], + in_dtype=dtype, + activation=activation, + ) + kernel = mk.FusedMoEKernel( + maybe_make_prepare_finalize( + moe=moe_config, + quant_config=quant_config, + allow_new_interface=True, + use_monolithic=False, + ), + CutlassExpertsFp4( + moe_config=moe_config, + quant_config=quant_config, + ), + ) + + return { + "config": case_config, + "hidden_states": hidden_states, + "score": score, + "kernel": kernel, + "w1_q": w1_q, + "w2_q": w2_q, + "activation": activation, + "e": e, + "topk": topk, + "dtype": dtype, + } + + +def _run_cutlass_fp4_moe( + case: dict[str, Any], + hidden_states: torch.Tensor, + score: torch.Tensor, +) -> torch.Tensor: + topk_weights, topk_ids, _ = fused_topk( + hidden_states, score, case["topk"], renormalize=False + ) + return case["kernel"].apply( + hidden_states=hidden_states, + w1=case["w1_q"], + w2=case["w2_q"], + topk_weights=topk_weights, + topk_ids=topk_ids, + global_num_experts=case["e"], + activation=case["activation"], + apply_router_weight_on_input=False, + expert_map=None, + ) + + +@_NVFP4_REQUIRES_SM100 +@pytest.mark.parametrize("case_config", _NVFP4_MOE_BATCH_INVARIANT_CASES) +@pytest.mark.parametrize("activation", [MoEActivation.SILU, MoEActivation.SWIGLUSTEP]) +@pytest.mark.parametrize("e", _NVFP4_MOE_NUM_EXPERTS) +@pytest.mark.parametrize("topk", _NVFP4_MOE_TOPKS) +@pytest.mark.parametrize("dtype", _NVFP4_MOE_DTYPES) +@torch.inference_mode() +def test_cutlass_nvfp4_moe_batch_invariant( + case_config: dict[str, Any], + activation: MoEActivation, + e: int, + topk: int, + dtype: torch.dtype, + default_vllm_config, + workspace_init, +) -> None: + case = _make_cutlass_fp4_moe_batch_invariant_case( + case_config, activation, e, topk, dtype + ) + case_id = f"{case['config']}, e={case['e']}, topk={case['topk']}" + # Establish the baseline output for the full mixed batch once, then + # compare every smaller replay against the corresponding slice here. + batch_output = _run_cutlass_fp4_moe(case, case["hidden_states"], case["score"]) + + assert CutlassExpertsFp4._supports_batch_invariance() + + # Re-run the whole batch with a different batch-row permutation to make + # sure the grouped GEMM result is invariant to how tokens are packed into + # expert work for scheduling. + indices = torch.arange( + case["hidden_states"].size(0), device=case["hidden_states"].device + ) + for perm_name, perm in ( + ("reversed", torch.flip(indices, dims=(0,))), + ("evens_then_odds", torch.cat((indices[::2], indices[1::2]))), + ): + permuted_output = _run_cutlass_fp4_moe( + case, + case["hidden_states"][perm], + case["score"][perm], + ) + torch.testing.assert_close( + batch_output[perm], + permuted_output, + atol=0, + rtol=0, + msg=( + f"{case_id}: permutation '{perm_name}' changed outputs " + "relative to the baseline batch order." + ), + ) + + # Re-run every batch-row as a batch-size-1 input and compare against the + # matching row from the full batch. + for idx in range(case["hidden_states"].size(0)): + single_idx = torch.tensor([idx], device=case["hidden_states"].device) + single_output = _run_cutlass_fp4_moe( + case, + case["hidden_states"][single_idx], + case["score"][single_idx], + ) + torch.testing.assert_close( + batch_output[single_idx], + single_output, + atol=0, + rtol=0, + msg=( + f"{case_id}: batch-row {idx} changed between full-batch " + "and batch-size-1 execution." + ), + ) + + # Re-run a few nontrivial sub-batches to catch interactions that only + # appear when multiple tokens are grouped together. + for subset_ids in case["config"]["subset_indices"]: + subset = torch.tensor(subset_ids, device=case["hidden_states"].device) + subset_output = _run_cutlass_fp4_moe( + case, + case["hidden_states"][subset], + case["score"][subset], + ) + torch.testing.assert_close( + batch_output[subset], + subset_output, + atol=0, + rtol=0, + msg=( + f"{case_id}: sub-batch {list(subset_ids)} changed " + "relative to full-batch execution." + ), + ) diff --git a/tests/v1/determinism/test_nvfp4_batch_invariant_scaled_mm.py b/tests/v1/determinism/test_nvfp4_batch_invariant_scaled_mm.py deleted file mode 100644 index 16bb26093a..0000000000 --- a/tests/v1/determinism/test_nvfp4_batch_invariant_scaled_mm.py +++ /dev/null @@ -1,101 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""NVFP4 CUTLASS GEMM tests that require ``VLLM_BATCH_INVARIANT=1``. - -Must run in a **fresh** pytest process: - - pytest tests/v1/determinism/test_nvfp4_batch_invariant_scaled_mm.py -v - -Do not share a session with ``tests/kernels/quantization/test_nvfp4_scaled_mm.py``: -the native code caches whether batch invariance is enabled on the first GEMM, and -if ``VLLM_BATCH_INVARIANT`` was not set at that moment, it stays disabled for the -rest of the process. -""" - -import os - -import pytest -import torch - -from tests.kernels.quantization.nvfp4_utils import get_nvfp4_global_scale -from vllm import _custom_ops as ops -from vllm.platforms import current_platform -from vllm.utils.torch_utils import set_random_seed - -if not current_platform.has_device_capability(100): - pytest.skip( - reason="Nvfp4 Requires compute capability of 10 or above.", - allow_module_level=True, - ) - -DTYPES = [torch.float16, torch.bfloat16] -SHAPES = [(128, 128, 64), (128, 128, 128), (256, 128, 64), (128, 256, 128)] -PAD_SHAPES = [(150, 128, 64), (128, 128, 96)] -SHAPES.extend(PAD_SHAPES) - - -CONSISTENCY_SHAPES = [ - (256, 128, 4096), - (512, 256, 4096), - (256, 256, 2048), - (241, 160, 2048), - (401, 352, 1984), - (333, 320, 1008), - (287, 96, 4096), -] - - -@pytest.mark.parametrize("dtype", DTYPES) -@pytest.mark.parametrize("shape", CONSISTENCY_SHAPES) -@torch.inference_mode() -def test_nvfp4_gemm_batch_invariance( - dtype: torch.dtype, - shape: tuple[int, int, int], -) -> None: - """Batch invariance: each row of a full-``M`` GEMM matches its ``M=1`` counterpart. - - For row ``i``, compares ``cutlass_scaled_fp4_mm`` run once over all ``M`` - rows against a separate call with ``A`` sliced to ``a_dtype[i : i+1]``. - Catches kernels whose reduction or scheduling depends on ``M`` or adjacent - rows. - """ - seed = int(os.getenv("VLLM_TEST_SEED", "12345")) - set_random_seed(seed) - m, n, packed_k = shape - k = packed_k * 2 # real K (FP4 elements) - - a_dtype = torch.randn((m, k), dtype=dtype, device="cuda") - b_dtype = torch.randn((n, k), dtype=dtype, device="cuda") - - a_global_scale = get_nvfp4_global_scale(a_dtype) - b_global_scale = get_nvfp4_global_scale(b_dtype) - alpha = 1.0 / (a_global_scale * b_global_scale) - - b_fp4, b_scale_interleaved = ops.scaled_fp4_quant(b_dtype, b_global_scale) - - a_fp4_full, a_sf_full = ops.scaled_fp4_quant(a_dtype, a_global_scale) - out_full = ops.cutlass_scaled_fp4_mm( - a_fp4_full, - b_fp4, - a_sf_full, - b_scale_interleaved, - alpha, - dtype, - ) - - for i in range(m): - a_row = a_dtype[i : i + 1] - a_fp4_row, a_sf_row = ops.scaled_fp4_quant(a_row, a_global_scale) - out_row = ops.cutlass_scaled_fp4_mm( - a_fp4_row, - b_fp4, - a_sf_row, - b_scale_interleaved, - alpha, - dtype, - ) - - assert torch.equal(out_full[i], out_row[0]), ( - f"VLLM_BATCH_INVARIANT: row {i} differs between M={m} and M=1: " - f"max_abs_diff={(out_full[i] - out_row[0]).abs().max().item()}" - ) diff --git a/vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py b/vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py index 68b3249163..ce787fc09e 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py @@ -734,6 +734,10 @@ class CutlassExpertsFp4(mk.FusedMoEExpertsModular): # needed for STANDARD activation format kernels in EP mode. return moe_parallel_config.ep_size == 1 + @staticmethod + def _supports_batch_invariance() -> bool: + return True + @staticmethod def activation_format() -> mk.FusedMoEActivationFormat: return mk.FusedMoEActivationFormat.Standard