[MRV2] Miscellaneous code structure updates (#60754)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -152,7 +152,7 @@ def sync_cudagraph_and_dp_padding(
|
||||
num_reqs=num_reqs,
|
||||
)
|
||||
|
||||
synced_cg_mode = CUDAGraphMode(int(cg_mode_across_dp.min().item()))
|
||||
synced_cg_mode = CUDAGraphMode(int(cg_mode_across_dp.min()))
|
||||
|
||||
# If any rank wants to run eager, all ranks run eager
|
||||
if synced_cg_mode == CUDAGraphMode.NONE:
|
||||
@@ -175,13 +175,13 @@ def sync_cudagraph_and_dp_padding(
|
||||
"cudagraph_manager should only be None during profile run, "
|
||||
"where synced_cg_mode must be NONE across all DP ranks"
|
||||
)
|
||||
synced_num_tokens = int(num_tokens_across_dp.max().item())
|
||||
synced_num_tokens = int(num_tokens_across_dp.max())
|
||||
|
||||
# Varlen decode graphs are selected by the query-length bound, so ranks must agree
|
||||
# on it or they pad to different token counts below.
|
||||
synced_max_query_len: int | None = None
|
||||
if bool(torch.all(max_query_lens_across_dp != -1).item()):
|
||||
synced_max_query_len = int(max_query_lens_across_dp.max().item())
|
||||
if bool(torch.all(max_query_lens_across_dp != -1)):
|
||||
synced_max_query_len = int(max_query_lens_across_dp.max())
|
||||
|
||||
# Dispatch for the final synced values, use num_reqs instead of synced_num_reqs
|
||||
# so we don't perform request padding for PIECEWISE graphs.
|
||||
@@ -223,12 +223,12 @@ def dispatch_cg_and_sync_dp(
|
||||
parallel_config: ParallelConfig | None = None,
|
||||
allow_ubatching: bool = False,
|
||||
uniform_decode: bool = False,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
dp_sync_state: DPSyncState | None = None,
|
||||
) -> tuple[BatchExecutionDescriptor, DPSyncState | None]:
|
||||
"""Pick a cudagraph descriptor for this batch, agreeing it across DP ranks.
|
||||
|
||||
Runs a collective when dp_size > 1 so every rank dispatches to the same
|
||||
shape. Pass `dp_sync` from a dispatch already made over this same batch (a
|
||||
shape. Pass `dp_sync_state` from a dispatch already made over this same batch (a
|
||||
drafter's prefill runs the target's batch shape) to reuse that agreement
|
||||
instead, with no collective.
|
||||
|
||||
@@ -238,7 +238,7 @@ def dispatch_cg_and_sync_dp(
|
||||
num_reqs: Requests in this rank's batch.
|
||||
num_tokens: Tokens in this rank's batch, already padded by the caller.
|
||||
uniform_token_count: Per-request token count if this rank's batch is a
|
||||
uniform decode, else None. `dp_sync.uniform_token_count` takes its
|
||||
uniform decode, else None. `dp_sync_state.uniform_token_count` takes its
|
||||
place when a sync is reused, since that one is agreed across ranks.
|
||||
dp_size: Data-parallel world size. 1 skips all cross-rank work.
|
||||
dp_rank: This rank's index in the DP group.
|
||||
@@ -254,7 +254,7 @@ def dispatch_cg_and_sync_dp(
|
||||
microbatches. Agreed across ranks before it takes effect.
|
||||
uniform_decode: Whether this rank's batch is a uniform decode, used
|
||||
to pick the microbatching split.
|
||||
dp_sync: Agreement from a prior dispatch over this same batch, to reuse.
|
||||
dp_sync_state: Agreement from a prior dispatch over this same batch, to reuse.
|
||||
Must come from a batch with this same padded `num_tokens` and the
|
||||
same `uniform_token_count`; `num_reqs` may differ, as neither
|
||||
depends on it. Passing a sync from a different batch is a caller
|
||||
@@ -265,7 +265,7 @@ def dispatch_cg_and_sync_dp(
|
||||
dispatch to reuse. It is None when `dp_size` is 1 or no rank has work.
|
||||
|
||||
"""
|
||||
reuse_eager = dp_sync is not None and dp_sync.eager
|
||||
reuse_eager = dp_sync_state is not None and dp_sync_state.eager
|
||||
|
||||
if need_eager or reuse_eager:
|
||||
batch_desc = BatchExecutionDescriptor(
|
||||
@@ -282,7 +282,7 @@ def dispatch_cg_and_sync_dp(
|
||||
batch_desc = cudagraph_manager.dispatch(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
dp_sync.uniform_token_count if dp_sync is not None else uniform_token_count,
|
||||
dp_sync_state.uniform_token_count if dp_sync_state else uniform_token_count,
|
||||
num_active_loras=num_active_loras,
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
@@ -290,24 +290,24 @@ def dispatch_cg_and_sync_dp(
|
||||
if dp_size == 1:
|
||||
return batch_desc, None
|
||||
|
||||
if dp_sync is not None:
|
||||
assert dp_sync.num_tokens_across_dp[dp_rank] == num_tokens, (
|
||||
if dp_sync_state is not None:
|
||||
assert dp_sync_state.num_tokens_across_dp[dp_rank] == num_tokens, (
|
||||
"reusing a DP sync taken over a different batch"
|
||||
)
|
||||
assert (
|
||||
dp_sync.uniform_token_count is None
|
||||
or uniform_token_count == dp_sync.uniform_token_count
|
||||
dp_sync_state.uniform_token_count is None
|
||||
or uniform_token_count == dp_sync_state.uniform_token_count
|
||||
), "reusing a DP sync taken over a different batch"
|
||||
if not dp_sync.eager and batch_desc.num_tokens != num_tokens:
|
||||
if not dp_sync_state.eager and batch_desc.num_tokens != num_tokens:
|
||||
# Capture sizes can differ between managers, so this one may
|
||||
# pad further. Every rank pads alike, so report what will run.
|
||||
dp_sync = replace(
|
||||
dp_sync,
|
||||
dp_sync_state = replace(
|
||||
dp_sync_state,
|
||||
num_tokens_across_dp=torch.full_like(
|
||||
dp_sync.num_tokens_across_dp, batch_desc.num_tokens
|
||||
dp_sync_state.num_tokens_across_dp, batch_desc.num_tokens
|
||||
),
|
||||
)
|
||||
return batch_desc, dp_sync
|
||||
return batch_desc, dp_sync_state
|
||||
|
||||
return sync_cudagraph_and_dp_padding(
|
||||
cudagraph_manager,
|
||||
|
||||
Reference in New Issue
Block a user