[Optimization] Skip DP padding tokens in MoE (#46428)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Woosuk Kwon
2026-06-23 14:54:46 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 84f13374b3
commit 0a3e2dbc09
9 changed files with 154 additions and 1 deletions
+4
View File
@@ -471,6 +471,9 @@ class ModelCudaGraphManager(CudaGraphManager):
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
)
# Capture with dummy rows marked as padding.
input_buffers.is_padding.fill_(True)
def forward_fn(cg_mode: CUDAGraphMode) -> None:
batch_descriptor = None
if cg_mode == CUDAGraphMode.PIECEWISE:
@@ -488,6 +491,7 @@ class ModelCudaGraphManager(CudaGraphManager):
num_tokens_across_dp=num_tokens_across_dp,
slot_mapping=slot_mappings,
batch_descriptor=batch_descriptor,
is_padding=input_buffers.is_padding[:num_tokens],
):
if cg_mode == CUDAGraphMode.PIECEWISE:
# PIECEWISE graph (compiled PW or breakable, chosen inside