[Bugfix][PCP] Align MoE routing padding with gathered tokens (#60946)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: wangyicong <wangyicong@bytedance.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Yicong Wang
2026-10-14 21:04:00 -04:00
committed by GitHub
co-authored by Lucas Wilkinson Claude Opus 5.5 OpenAI Codex
parent 8530aed2b7
commit bfdbfa1632
7 changed files with 180 additions and 26 deletions
+5 -2
View File
@@ -743,6 +743,9 @@ class ModelCudaGraphManager(CudaGraphManager):
# Capture with dummy rows marked as padding.
input_buffers.is_padding.fill_(True)
is_padding = input_buffers.is_padding[:num_tokens]
if pcp_manager is not None:
is_padding = pcp_manager.moe_is_padding(is_padding)
def forward_fn(cg_mode: CUDAGraphMode) -> None:
batch_descriptor = None
@@ -760,7 +763,7 @@ class ModelCudaGraphManager(CudaGraphManager):
num_tokens_across_dp=num_tokens_across_dp,
slot_mapping=slot_mappings,
batch_descriptor=batch_descriptor,
is_padding=input_buffers.is_padding[:num_tokens],
is_padding=is_padding,
):
if cg_mode == CUDAGraphMode.PIECEWISE:
# PIECEWISE graph (compiled PW or breakable, chosen inside
@@ -837,7 +840,7 @@ def prepare_inputs_to_capture(
num_reqs, num_tokens, input_buffers, max_query_len=max_query_len
)
if pcp_manager is not None:
input_batch = pcp_manager.prepare_inputs_to_capture(input_batch)
input_batch = pcp_manager.prepare_inputs_for_capture_or_profile(input_batch)
block_table_provider = pcp_manager or block_tables
input_block_tables = block_table_provider.get_dummy_block_tables(num_reqs)