[MoE][Bugfix] Skip SP padded rows in grouped MoE routing (#56079)

Signed-off-by: Elvir Crncevic <elvircrn@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
Elvir Crnčević
2026-09-17 19:36:19 +00:00
committed by GitHub
co-authored by OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Tyler Michael Smith
parent 67e5b0acc9
commit ac2f0ea82c
12 changed files with 224 additions and 6 deletions
+9
View File
@@ -261,6 +261,7 @@ class UBatchWrapper:
dp_metadata,
batch_descriptor,
cudagraph_runtime_mode,
is_padding,
) -> list[UbatchMetadata]:
# Create one forward context per ubatch
forward_contexts = []
@@ -276,6 +277,11 @@ class UBatchWrapper:
batch_descriptor=batch_descriptor,
cudagraph_runtime_mode=cudagraph_runtime_mode,
slot_mapping=slot_mapping[i] if has_slot_mapping else None,
is_padding=(
is_padding[ubatch_slice.token_slice]
if is_padding is not None
else None
),
)
)
@@ -373,6 +379,7 @@ class UBatchWrapper:
attn_metadata = forward_context.attn_metadata
slot_mapping = forward_context.slot_mapping
is_padding = forward_context.is_padding
num_tokens = sum(ubatch_slice.num_tokens for ubatch_slice in ubatch_slices)
input_ids = kwargs["input_ids"]
positions = kwargs["positions"]
@@ -414,6 +421,7 @@ class UBatchWrapper:
dp_metadata=ubatch_dp_metadata,
batch_descriptor=batch_descriptor,
cudagraph_runtime_mode=CUDAGraphMode.NONE,
is_padding=is_padding,
)
with self.sm_control:
return self._capture_ubatches(ubatch_metadata, self.runnable)
@@ -440,6 +448,7 @@ class UBatchWrapper:
dp_metadata=ubatch_dp_metadata,
batch_descriptor=batch_descriptor,
cudagraph_runtime_mode=CUDAGraphMode.NONE,
is_padding=is_padding,
)
with self.sm_control:
return self._run_ubatches(ubatch_metadata, self.runnable)