[MoE][Bugfix] Skip SP padded rows in grouped MoE routing (#56079)
Signed-off-by: Elvir Crncevic <elvircrn@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Tyler Michael Smith
parent
67e5b0acc9
commit
ac2f0ea82c
@@ -261,6 +261,7 @@ class UBatchWrapper:
|
||||
dp_metadata,
|
||||
batch_descriptor,
|
||||
cudagraph_runtime_mode,
|
||||
is_padding,
|
||||
) -> list[UbatchMetadata]:
|
||||
# Create one forward context per ubatch
|
||||
forward_contexts = []
|
||||
@@ -276,6 +277,11 @@ class UBatchWrapper:
|
||||
batch_descriptor=batch_descriptor,
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
slot_mapping=slot_mapping[i] if has_slot_mapping else None,
|
||||
is_padding=(
|
||||
is_padding[ubatch_slice.token_slice]
|
||||
if is_padding is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -373,6 +379,7 @@ class UBatchWrapper:
|
||||
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
slot_mapping = forward_context.slot_mapping
|
||||
is_padding = forward_context.is_padding
|
||||
num_tokens = sum(ubatch_slice.num_tokens for ubatch_slice in ubatch_slices)
|
||||
input_ids = kwargs["input_ids"]
|
||||
positions = kwargs["positions"]
|
||||
@@ -414,6 +421,7 @@ class UBatchWrapper:
|
||||
dp_metadata=ubatch_dp_metadata,
|
||||
batch_descriptor=batch_descriptor,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
is_padding=is_padding,
|
||||
)
|
||||
with self.sm_control:
|
||||
return self._capture_ubatches(ubatch_metadata, self.runnable)
|
||||
@@ -440,6 +448,7 @@ class UBatchWrapper:
|
||||
dp_metadata=ubatch_dp_metadata,
|
||||
batch_descriptor=batch_descriptor,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
is_padding=is_padding,
|
||||
)
|
||||
with self.sm_control:
|
||||
return self._run_ubatches(ubatch_metadata, self.runnable)
|
||||
|
||||
Reference in New Issue
Block a user