[Bugfix][MoE] Clamp BLOCK_SIZE_K to the FP8 quantization block in the batched Triton MoE (#60733)
This commit is contained in:
@@ -119,6 +119,8 @@ class BatchedMMTensors:
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("block_shape", [None, [128, 128]])
|
||||
@pytest.mark.parametrize("per_act_token_quant", [False, True])
|
||||
# Tuned configs can pick a BLOCK_SIZE_K wider than the quantization block.
|
||||
@pytest.mark.parametrize("fp8_block_k", [32, 256])
|
||||
def test_batched_mm(
|
||||
num_experts: int,
|
||||
max_tokens_per_expert: int,
|
||||
@@ -127,6 +129,7 @@ def test_batched_mm(
|
||||
dtype: torch.dtype,
|
||||
block_shape: list[int] | None,
|
||||
per_act_token_quant: bool,
|
||||
fp8_block_k: int,
|
||||
):
|
||||
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
|
||||
and those tests will be skipped on unsupported hardware."""
|
||||
@@ -149,6 +152,9 @@ def test_batched_mm(
|
||||
if per_act_token_quant and block_shape is not None:
|
||||
pytest.skip("Skip illegal quantization test.")
|
||||
|
||||
if fp8_block_k != 32 and not use_fp8_w8a8:
|
||||
pytest.skip("BLOCK_SIZE_K is only varied for fp8.")
|
||||
|
||||
if dtype.itemsize == 1:
|
||||
act_dtype = torch.bfloat16
|
||||
quant_dtype = dtype
|
||||
@@ -214,7 +220,7 @@ def test_batched_mm(
|
||||
config={
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 16,
|
||||
"BLOCK_SIZE_K": 16 if dtype.itemsize > 1 else 32,
|
||||
"BLOCK_SIZE_K": 16 if dtype.itemsize > 1 else fp8_block_k,
|
||||
},
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
|
||||
@@ -450,6 +450,8 @@ def invoke_moe_batched_triton_kernel(
|
||||
BLOCK_M = config["BLOCK_SIZE_M"]
|
||||
BLOCK_N = config["BLOCK_SIZE_N"]
|
||||
BLOCK_K = config["BLOCK_SIZE_K"]
|
||||
if block_shape is not None:
|
||||
BLOCK_K = min(BLOCK_K, min(block_shape[0], block_shape[1]))
|
||||
|
||||
grid = (
|
||||
expert_num_tokens.size(0),
|
||||
|
||||
Reference in New Issue
Block a user