[MRV2] Miscellaneous code structure updates (#60754)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-10-09 17:58:09 -07:00
committed by GitHub
parent 9f1c783886
commit 7d1d8660d6
18 changed files with 140 additions and 109 deletions
+19 -19
View File
@@ -152,7 +152,7 @@ def sync_cudagraph_and_dp_padding(
num_reqs=num_reqs,
)
synced_cg_mode = CUDAGraphMode(int(cg_mode_across_dp.min().item()))
synced_cg_mode = CUDAGraphMode(int(cg_mode_across_dp.min()))
# If any rank wants to run eager, all ranks run eager
if synced_cg_mode == CUDAGraphMode.NONE:
@@ -175,13 +175,13 @@ def sync_cudagraph_and_dp_padding(
"cudagraph_manager should only be None during profile run, "
"where synced_cg_mode must be NONE across all DP ranks"
)
synced_num_tokens = int(num_tokens_across_dp.max().item())
synced_num_tokens = int(num_tokens_across_dp.max())
# Varlen decode graphs are selected by the query-length bound, so ranks must agree
# on it or they pad to different token counts below.
synced_max_query_len: int | None = None
if bool(torch.all(max_query_lens_across_dp != -1).item()):
synced_max_query_len = int(max_query_lens_across_dp.max().item())
if bool(torch.all(max_query_lens_across_dp != -1)):
synced_max_query_len = int(max_query_lens_across_dp.max())
# Dispatch for the final synced values, use num_reqs instead of synced_num_reqs
# so we don't perform request padding for PIECEWISE graphs.
@@ -223,12 +223,12 @@ def dispatch_cg_and_sync_dp(
parallel_config: ParallelConfig | None = None,
allow_ubatching: bool = False,
uniform_decode: bool = False,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
) -> tuple[BatchExecutionDescriptor, DPSyncState | None]:
"""Pick a cudagraph descriptor for this batch, agreeing it across DP ranks.
Runs a collective when dp_size > 1 so every rank dispatches to the same
shape. Pass `dp_sync` from a dispatch already made over this same batch (a
shape. Pass `dp_sync_state` from a dispatch already made over this same batch (a
drafter's prefill runs the target's batch shape) to reuse that agreement
instead, with no collective.
@@ -238,7 +238,7 @@ def dispatch_cg_and_sync_dp(
num_reqs: Requests in this rank's batch.
num_tokens: Tokens in this rank's batch, already padded by the caller.
uniform_token_count: Per-request token count if this rank's batch is a
uniform decode, else None. `dp_sync.uniform_token_count` takes its
uniform decode, else None. `dp_sync_state.uniform_token_count` takes its
place when a sync is reused, since that one is agreed across ranks.
dp_size: Data-parallel world size. 1 skips all cross-rank work.
dp_rank: This rank's index in the DP group.
@@ -254,7 +254,7 @@ def dispatch_cg_and_sync_dp(
microbatches. Agreed across ranks before it takes effect.
uniform_decode: Whether this rank's batch is a uniform decode, used
to pick the microbatching split.
dp_sync: Agreement from a prior dispatch over this same batch, to reuse.
dp_sync_state: Agreement from a prior dispatch over this same batch, to reuse.
Must come from a batch with this same padded `num_tokens` and the
same `uniform_token_count`; `num_reqs` may differ, as neither
depends on it. Passing a sync from a different batch is a caller
@@ -265,7 +265,7 @@ def dispatch_cg_and_sync_dp(
dispatch to reuse. It is None when `dp_size` is 1 or no rank has work.
"""
reuse_eager = dp_sync is not None and dp_sync.eager
reuse_eager = dp_sync_state is not None and dp_sync_state.eager
if need_eager or reuse_eager:
batch_desc = BatchExecutionDescriptor(
@@ -282,7 +282,7 @@ def dispatch_cg_and_sync_dp(
batch_desc = cudagraph_manager.dispatch(
num_reqs,
num_tokens,
dp_sync.uniform_token_count if dp_sync is not None else uniform_token_count,
dp_sync_state.uniform_token_count if dp_sync_state else uniform_token_count,
num_active_loras=num_active_loras,
max_query_len=max_query_len,
)
@@ -290,24 +290,24 @@ def dispatch_cg_and_sync_dp(
if dp_size == 1:
return batch_desc, None
if dp_sync is not None:
assert dp_sync.num_tokens_across_dp[dp_rank] == num_tokens, (
if dp_sync_state is not None:
assert dp_sync_state.num_tokens_across_dp[dp_rank] == num_tokens, (
"reusing a DP sync taken over a different batch"
)
assert (
dp_sync.uniform_token_count is None
or uniform_token_count == dp_sync.uniform_token_count
dp_sync_state.uniform_token_count is None
or uniform_token_count == dp_sync_state.uniform_token_count
), "reusing a DP sync taken over a different batch"
if not dp_sync.eager and batch_desc.num_tokens != num_tokens:
if not dp_sync_state.eager and batch_desc.num_tokens != num_tokens:
# Capture sizes can differ between managers, so this one may
# pad further. Every rank pads alike, so report what will run.
dp_sync = replace(
dp_sync,
dp_sync_state = replace(
dp_sync_state,
num_tokens_across_dp=torch.full_like(
dp_sync.num_tokens_across_dp, batch_desc.num_tokens
dp_sync_state.num_tokens_across_dp, batch_desc.num_tokens
),
)
return batch_desc, dp_sync
return batch_desc, dp_sync_state
return sync_cudagraph_and_dp_padding(
cudagraph_manager,