[MRV2] Miscellaneous code structure updates (#60754)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-10-09 17:58:09 -07:00
committed by GitHub
parent 9f1c783886
commit 7d1d8660d6
18 changed files with 140 additions and 109 deletions
@@ -355,7 +355,7 @@ class DFlashSpeculator(DraftModelSpeculator):
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -446,8 +446,10 @@ class DFlashSpeculator(DraftModelSpeculator):
)
batch_sync, num_batch_tokens = (
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, self.num_query_per_req)
if dp_sync is not None
self._build_uniform_batch_dp_sync(
dp_sync_state, num_reqs, self.num_query_per_req
)
if dp_sync_state is not None
else (None, num_query_tokens)
)
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
@@ -459,7 +461,7 @@ class DFlashSpeculator(DraftModelSpeculator):
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
dp_sync=batch_sync,
dp_sync_state=batch_sync,
)
num_tokens_padded = batch_desc.num_tokens
num_tokens_across_dp = (