[MRV2] Miscellaneous code structure updates (#60754)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-10-09 17:58:09 -07:00
committed by GitHub
parent 9f1c783886
commit 7d1d8660d6
18 changed files with 140 additions and 109 deletions
@@ -355,7 +355,7 @@ class DFlashSpeculator(DraftModelSpeculator):
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -446,8 +446,10 @@ class DFlashSpeculator(DraftModelSpeculator):
)
batch_sync, num_batch_tokens = (
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, self.num_query_per_req)
if dp_sync is not None
self._build_uniform_batch_dp_sync(
dp_sync_state, num_reqs, self.num_query_per_req
)
if dp_sync_state is not None
else (None, num_query_tokens)
)
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
@@ -459,7 +461,7 @@ class DFlashSpeculator(DraftModelSpeculator):
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
dp_sync=batch_sync,
dp_sync_state=batch_sync,
)
num_tokens_padded = batch_desc.num_tokens
num_tokens_across_dp = (
@@ -98,7 +98,7 @@ class ExtractHiddenStatesSpeculator(DraftModelSpeculator):
next_prefill_tokens: torch.Tensor,
temperature: torch.Tensor,
seeds: torch.Tensor,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -145,9 +145,7 @@ class ExtractHiddenStatesSpeculator(DraftModelSpeculator):
draft_attn_metadata,
self.vllm_config,
num_tokens=num_tokens,
num_tokens_across_dp=(
dp_sync.num_tokens_across_dp if dp_sync is not None else None
),
num_tokens_across_dp=dp_sync_state and dp_sync_state.num_tokens_across_dp,
cudagraph_runtime_mode=CUDAGraphMode.NONE,
slot_mapping=draft_slot_mappings,
is_padding=input_batch.is_padding[:num_tokens],
@@ -150,7 +150,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -201,7 +201,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
dp_sync=dp_sync,
dp_sync_state=dp_sync_state,
)
num_tokens_across_dp = (
batch_sync.num_tokens_across_dp if batch_sync is not None else None
@@ -226,7 +226,7 @@ class NgramGPUSpeculator(BaseSpeculator):
next_prefill_tokens: torch.Tensor,
temperature: torch.Tensor,
seeds: torch.Tensor,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
+5 -5
View File
@@ -92,7 +92,7 @@ class BaseSpeculator(ABC):
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -525,18 +525,18 @@ class DraftModelSpeculator(BaseSpeculator):
def _build_uniform_batch_dp_sync(
self,
target_dp_sync: DPSyncState,
target_dp_sync_state: DPSyncState,
num_reqs: int,
num_query_per_req: int = 1,
) -> tuple[DPSyncState, int]:
num_batch_tokens = target_dp_sync.num_reqs * num_query_per_req
num_batch_tokens = target_dp_sync_state.num_reqs * num_query_per_req
assert num_reqs * num_query_per_req <= num_batch_tokens, (
"reusing a DP sync that does not cover this batch's requests"
)
return replace(
target_dp_sync,
target_dp_sync_state,
num_tokens_across_dp=torch.full_like(
target_dp_sync.num_tokens_across_dp, num_batch_tokens
target_dp_sync_state.num_tokens_across_dp, num_batch_tokens
),
uniform_token_count=num_query_per_req,
eager=False,
@@ -350,7 +350,7 @@ class StandaloneARSpeculator(DraftModelSpeculator):
next_prefill_tokens: torch.Tensor,
temperature: torch.Tensor,
seeds: torch.Tensor,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -362,7 +362,7 @@ class StandaloneARSpeculator(DraftModelSpeculator):
num_reqs = input_batch.num_reqs
skip_attn = dummy_run and skip_attn_for_dummy_run
num_tokens_across_dp = (
dp_sync.num_tokens_across_dp if dp_sync is not None else None
dp_sync_state.num_tokens_across_dp if dp_sync_state is not None else None
)
# Copy per-request temperature/seeds/idx_mapping into pre-allocated
@@ -219,7 +219,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -299,7 +299,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
dp_sync=dp_sync,
dp_sync_state=dp_sync_state,
)
num_tokens_across_dp = (
prefill_batch_sync.num_tokens_across_dp
@@ -352,8 +352,10 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
)
decode_batch_sync, num_batch_tokens = (
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, num_query_per_req=1)
if dp_sync is not None
self._build_uniform_batch_dp_sync(
dp_sync_state, num_reqs, num_query_per_req=1
)
if dp_sync_state is not None
else (None, num_reqs)
)
# Each request produces exactly 1 token per draft generation step,
@@ -366,7 +368,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
dp_sync=decode_batch_sync,
dp_sync_state=decode_batch_sync,
)
if self.decode_cudagraph_manager is not None:
decode_batch_desc = self.decode_cudagraph_manager.specialize_spec_tokens(
+19
View File
@@ -13,6 +13,25 @@ from vllm.v1.worker.gpu.input_batch import InputBatch
logger = init_logger(__name__)
def get_drafter_hidden_states(
model: torch.nn.Module, hidden_states: torch.Tensor
) -> torch.Tensor:
"""Hidden states fed to the drafter.
Targets such as DeepSeek V4 expose the pre-hc_head residual through
get_mtp_target_hidden_states(). The buffer is sized at
max_num_batched_tokens and only allocated for drafters that consume
target hidden states, so None means "use the regular hidden states".
"""
get_target_hidden_states = getattr(model, "get_mtp_target_hidden_states", None)
if get_target_hidden_states is None:
return hidden_states
target_hidden_states = get_target_hidden_states()
if target_hidden_states is None:
return hidden_states
return target_hidden_states[: hidden_states.shape[0]]
def get_pp_safe_draft_load_config(load_config: LoadConfig) -> LoadConfig:
"""Avoid collectives that include PP ranks without a draft model."""
if get_pp_group().world_size > 1 and load_config.load_format == "fastsafetensors":