[MoE Refactor] Add sequence parallel tests to test_moe_layer.py (#41299)

Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
bnellnm
2026-05-12 21:52:19 -04:00
committed by GitHub
parent 07534b8782
commit 18f6bf5a21
4 changed files with 189 additions and 72 deletions
@@ -185,16 +185,21 @@ def flashinfer_alltoall_dispatch(
ep_size,
)
x_sf = MnnvlMoe.mnnvl_moe_alltoallv(
x_sf,
alltoall_info,
all2all_manager.workspace_tensor, # type: ignore[attr-defined]
ep_rank,
ep_size,
)
if x_sf is not None:
x_sf = MnnvlMoe.mnnvl_moe_alltoallv(
x_sf,
alltoall_info,
all2all_manager.workspace_tensor, # type: ignore[attr-defined]
ep_rank,
ep_size,
)
# Swizzle after the A2A if MoE kernel expects swizzled scales.
if quant_config.quant_dtype == "nvfp4" and quant_config.is_scale_swizzled:
if (
x_sf is not None
and quant_config.quant_dtype == "nvfp4"
and quant_config.is_scale_swizzled
):
if x_sf.element_size() == 1:
x_sf = x_sf.view(torch.uint8)
x_sf = nvfp4_block_scale_interleave(x_sf)