[Bugfix][PCP] Align MoE routing padding with gathered tokens (#60946)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: wangyicong <wangyicong@bytedance.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
Claude Opus 5.5
OpenAI Codex
parent
8530aed2b7
commit
bfdbfa1632
@@ -743,6 +743,9 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
|
||||
# Capture with dummy rows marked as padding.
|
||||
input_buffers.is_padding.fill_(True)
|
||||
is_padding = input_buffers.is_padding[:num_tokens]
|
||||
if pcp_manager is not None:
|
||||
is_padding = pcp_manager.moe_is_padding(is_padding)
|
||||
|
||||
def forward_fn(cg_mode: CUDAGraphMode) -> None:
|
||||
batch_descriptor = None
|
||||
@@ -760,7 +763,7 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
slot_mapping=slot_mappings,
|
||||
batch_descriptor=batch_descriptor,
|
||||
is_padding=input_buffers.is_padding[:num_tokens],
|
||||
is_padding=is_padding,
|
||||
):
|
||||
if cg_mode == CUDAGraphMode.PIECEWISE:
|
||||
# PIECEWISE graph (compiled PW or breakable, chosen inside
|
||||
@@ -837,7 +840,7 @@ def prepare_inputs_to_capture(
|
||||
num_reqs, num_tokens, input_buffers, max_query_len=max_query_len
|
||||
)
|
||||
if pcp_manager is not None:
|
||||
input_batch = pcp_manager.prepare_inputs_to_capture(input_batch)
|
||||
input_batch = pcp_manager.prepare_inputs_for_capture_or_profile(input_batch)
|
||||
|
||||
block_table_provider = pcp_manager or block_tables
|
||||
input_block_tables = block_table_provider.get_dummy_block_tables(num_reqs)
|
||||
|
||||
Reference in New Issue
Block a user