[2/2][Model Runner V2] FULL CUDA graph capture for microbatched steps (DBO) (#51700)
Signed-off-by: specture724 <specture724@gmail.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -112,23 +112,45 @@ def sync_cudagraph_and_dp_padding(
|
||||
int(num_tokens_across_dp.min()),
|
||||
uniform_decode=uniform_decode_across_dp,
|
||||
):
|
||||
# Microbatching is all-or-nothing: the expert all-to-all is
|
||||
# collective, so every rank splits and pads to the same token count.
|
||||
# A rank too small to fill a microbatch does no work in it, like a
|
||||
# dummy run. Microbatched steps run eager; nothing is captured yet.
|
||||
# Expert all-to-all requires every rank to split and pad equally.
|
||||
# Empty microbatches run as dummy batches.
|
||||
ubatch_num_tokens = int(num_tokens_across_dp.max())
|
||||
return BatchExecutionDescriptor(
|
||||
cg_mode=CUDAGraphMode.NONE,
|
||||
num_tokens=ubatch_num_tokens,
|
||||
num_reqs=num_reqs,
|
||||
num_ubatches=get_num_ubatches(parallel_config),
|
||||
), DPSyncState(
|
||||
num_ubatches = get_num_ubatches(parallel_config)
|
||||
ubatch_desc = None
|
||||
if cudagraph_manager is not None:
|
||||
# Match a FULL microbatched CUDA graph, falling back to an eager
|
||||
# descriptor if no graph matches.
|
||||
ubatch_desc = cudagraph_manager.dispatch(
|
||||
num_reqs,
|
||||
ubatch_num_tokens,
|
||||
synced_uniform_token_count,
|
||||
num_active_loras=num_active_loras,
|
||||
num_ubatches=num_ubatches,
|
||||
)
|
||||
if 2 * int(num_tokens_across_dp.min()) < ubatch_desc.num_tokens:
|
||||
# If one rank has an empty second microbatch, run without
|
||||
# CUDA graphs.
|
||||
ubatch_desc = None
|
||||
if ubatch_desc is None:
|
||||
ubatch_desc = BatchExecutionDescriptor(
|
||||
cg_mode=CUDAGraphMode.NONE,
|
||||
num_tokens=ubatch_num_tokens,
|
||||
num_reqs=num_reqs,
|
||||
num_ubatches=num_ubatches,
|
||||
)
|
||||
# Refresh the token count to include CUDA graph padding.
|
||||
ubatch_num_tokens = ubatch_desc.num_tokens
|
||||
num_reqs = int(num_reqs_across_dp.max())
|
||||
if ubatch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
assert ubatch_desc.num_reqs is not None
|
||||
num_reqs = ubatch_desc.num_reqs
|
||||
return ubatch_desc, DPSyncState(
|
||||
num_tokens_across_dp=torch.full_like(
|
||||
num_tokens_across_dp, ubatch_num_tokens
|
||||
),
|
||||
uniform_token_count=synced_uniform_token_count,
|
||||
eager=True,
|
||||
num_reqs=int(num_reqs_across_dp.max()),
|
||||
eager=ubatch_desc.cg_mode == CUDAGraphMode.NONE,
|
||||
num_reqs=num_reqs,
|
||||
)
|
||||
|
||||
synced_cg_mode = CUDAGraphMode(int(cg_mode_across_dp.min().item()))
|
||||
|
||||
Reference in New Issue
Block a user