[Optimization] Skip DP padding tokens in MoE (#46428)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
84f13374b3
commit
0a3e2dbc09
@@ -471,6 +471,9 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
|
||||
)
|
||||
|
||||
# Capture with dummy rows marked as padding.
|
||||
input_buffers.is_padding.fill_(True)
|
||||
|
||||
def forward_fn(cg_mode: CUDAGraphMode) -> None:
|
||||
batch_descriptor = None
|
||||
if cg_mode == CUDAGraphMode.PIECEWISE:
|
||||
@@ -488,6 +491,7 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
slot_mapping=slot_mappings,
|
||||
batch_descriptor=batch_descriptor,
|
||||
is_padding=input_buffers.is_padding[:num_tokens],
|
||||
):
|
||||
if cg_mode == CUDAGraphMode.PIECEWISE:
|
||||
# PIECEWISE graph (compiled PW or breakable, chosen inside
|
||||
|
||||
Reference in New Issue
Block a user