[2/2][Model Runner V2] FULL CUDA graph capture for microbatched steps (DBO) (#51700)

Signed-off-by: specture724 <specture724@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Anjie Hou
2026-09-13 22:42:33 -07:00
committed by GitHub
co-authored by Codex
parent 67111973ee
commit 73d2a8cf86
6 changed files with 407 additions and 80 deletions
+34 -12
View File
@@ -112,23 +112,45 @@ def sync_cudagraph_and_dp_padding(
int(num_tokens_across_dp.min()),
uniform_decode=uniform_decode_across_dp,
):
# Microbatching is all-or-nothing: the expert all-to-all is
# collective, so every rank splits and pads to the same token count.
# A rank too small to fill a microbatch does no work in it, like a
# dummy run. Microbatched steps run eager; nothing is captured yet.
# Expert all-to-all requires every rank to split and pad equally.
# Empty microbatches run as dummy batches.
ubatch_num_tokens = int(num_tokens_across_dp.max())
return BatchExecutionDescriptor(
cg_mode=CUDAGraphMode.NONE,
num_tokens=ubatch_num_tokens,
num_reqs=num_reqs,
num_ubatches=get_num_ubatches(parallel_config),
), DPSyncState(
num_ubatches = get_num_ubatches(parallel_config)
ubatch_desc = None
if cudagraph_manager is not None:
# Match a FULL microbatched CUDA graph, falling back to an eager
# descriptor if no graph matches.
ubatch_desc = cudagraph_manager.dispatch(
num_reqs,
ubatch_num_tokens,
synced_uniform_token_count,
num_active_loras=num_active_loras,
num_ubatches=num_ubatches,
)
if 2 * int(num_tokens_across_dp.min()) < ubatch_desc.num_tokens:
# If one rank has an empty second microbatch, run without
# CUDA graphs.
ubatch_desc = None
if ubatch_desc is None:
ubatch_desc = BatchExecutionDescriptor(
cg_mode=CUDAGraphMode.NONE,
num_tokens=ubatch_num_tokens,
num_reqs=num_reqs,
num_ubatches=num_ubatches,
)
# Refresh the token count to include CUDA graph padding.
ubatch_num_tokens = ubatch_desc.num_tokens
num_reqs = int(num_reqs_across_dp.max())
if ubatch_desc.cg_mode == CUDAGraphMode.FULL:
assert ubatch_desc.num_reqs is not None
num_reqs = ubatch_desc.num_reqs
return ubatch_desc, DPSyncState(
num_tokens_across_dp=torch.full_like(
num_tokens_across_dp, ubatch_num_tokens
),
uniform_token_count=synced_uniform_token_count,
eager=True,
num_reqs=int(num_reqs_across_dp.max()),
eager=ubatch_desc.cg_mode == CUDAGraphMode.NONE,
num_reqs=num_reqs,
)
synced_cg_mode = CUDAGraphMode(int(cg_mode_across_dp.min().item()))