[MoE Refactor] Add sequence parallel tests to test_moe_layer.py (#41299)
Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
@@ -270,7 +270,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
input_tensor = input_.movedim(0, dim).contiguous()
|
||||
|
||||
if sizes is not None:
|
||||
assert len(sizes) == world_size
|
||||
assert len(sizes) == world_size, f"{len(sizes)} == {world_size}"
|
||||
assert input_tensor.shape[0] == sum(sizes)
|
||||
chunk_size = sizes[self.rank_in_group]
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user