[Kernel] Vectorized fp32 moe_sum reduction and support any topk (#46643)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -1243,15 +1243,27 @@ def test_batched_moe_align_block_size_opcheck():
|
||||
)
|
||||
|
||||
|
||||
# topk=8 covers topk > 4; k=511 covers the non-vectorized scalar path. The
|
||||
# layouts exercise contiguous input plus the two non-contiguous cases: a
|
||||
# transpose (strided hidden -> scalar gather) and a topk-slice (hidden still
|
||||
# contiguous -> vectorized).
|
||||
@pytest.mark.parametrize("m", [1, 33, 222])
|
||||
@pytest.mark.parametrize("topk", TOP_KS)
|
||||
@pytest.mark.parametrize("topk", [*TOP_KS, 8])
|
||||
@pytest.mark.parametrize("k", [128, 511, 1024])
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
def test_moe_sum(m: int, topk: int, k: int, dtype: torch.dtype):
|
||||
input = torch.randn((m, topk, k), device="cuda", dtype=dtype)
|
||||
@pytest.mark.parametrize("layout", ["contig", "transpose", "slice"])
|
||||
def test_moe_sum(m: int, topk: int, k: int, dtype: torch.dtype, layout: str):
|
||||
if layout == "transpose":
|
||||
input = torch.randn((m, k, topk), device="cuda", dtype=dtype).transpose(1, 2)
|
||||
elif layout == "slice":
|
||||
input = torch.randn((m, 2 * topk, k), device="cuda", dtype=dtype)[:, ::2, :]
|
||||
else:
|
||||
input = torch.randn((m, topk, k), device="cuda", dtype=dtype)
|
||||
assert input.is_contiguous() == (layout == "contig")
|
||||
actual = torch.empty((m, k), device="cuda", dtype=dtype)
|
||||
|
||||
expected = input.sum(dim=1)
|
||||
# Reduction accumulates in fp32.
|
||||
expected = input.float().sum(dim=1).to(dtype)
|
||||
torch.ops._moe_C.moe_sum(input, actual)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=0)
|
||||
|
||||
Reference in New Issue
Block a user