diff --git a/tests/kernels/moe/test_batched_moe.py b/tests/kernels/moe/test_batched_moe.py index 9be58e6337..f0877f291e 100644 --- a/tests/kernels/moe/test_batched_moe.py +++ b/tests/kernels/moe/test_batched_moe.py @@ -119,6 +119,8 @@ class BatchedMMTensors: @pytest.mark.parametrize("dtype", DTYPES) @pytest.mark.parametrize("block_shape", [None, [128, 128]]) @pytest.mark.parametrize("per_act_token_quant", [False, True]) +# Tuned configs can pick a BLOCK_SIZE_K wider than the quantization block. +@pytest.mark.parametrize("fp8_block_k", [32, 256]) def test_batched_mm( num_experts: int, max_tokens_per_expert: int, @@ -127,6 +129,7 @@ def test_batched_mm( dtype: torch.dtype, block_shape: list[int] | None, per_act_token_quant: bool, + fp8_block_k: int, ): """Note: float8_e4m3fn is not supported on CUDA architecture < 89, and those tests will be skipped on unsupported hardware.""" @@ -149,6 +152,9 @@ def test_batched_mm( if per_act_token_quant and block_shape is not None: pytest.skip("Skip illegal quantization test.") + if fp8_block_k != 32 and not use_fp8_w8a8: + pytest.skip("BLOCK_SIZE_K is only varied for fp8.") + if dtype.itemsize == 1: act_dtype = torch.bfloat16 quant_dtype = dtype @@ -214,7 +220,7 @@ def test_batched_mm( config={ "BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 16, - "BLOCK_SIZE_K": 16 if dtype.itemsize > 1 else 32, + "BLOCK_SIZE_K": 16 if dtype.itemsize > 1 else fp8_block_k, }, per_act_token_quant=per_act_token_quant, block_shape=block_shape, diff --git a/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py b/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py index b8cd58635e..26297739b6 100644 --- a/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py @@ -450,6 +450,8 @@ def invoke_moe_batched_triton_kernel( BLOCK_M = config["BLOCK_SIZE_M"] BLOCK_N = config["BLOCK_SIZE_N"] BLOCK_K = config["BLOCK_SIZE_K"] + if block_shape is not None: + BLOCK_K = min(BLOCK_K, min(block_shape[0], block_shape[1])) grid = ( expert_num_tokens.size(0),