[MoE Refactor] Add sequence parallel tests to test_moe_layer.py (#41299)

Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
bnellnm
2026-05-12 21:52:19 -04:00
committed by GitHub
parent 07534b8782
commit 18f6bf5a21
4 changed files with 189 additions and 72 deletions
@@ -270,7 +270,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
input_tensor = input_.movedim(0, dim).contiguous()
if sizes is not None:
assert len(sizes) == world_size
assert len(sizes) == world_size, f"{len(sizes)} == {world_size}"
assert input_tensor.shape[0] == sum(sizes)
chunk_size = sizes[self.rank_in_group]
else: