[MRV2] Miscellaneous code structure updates (#60754)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-10-09 17:58:09 -07:00
committed by GitHub
parent 9f1c783886
commit 7d1d8660d6
18 changed files with 140 additions and 109 deletions
+1 -1
View File
@@ -147,7 +147,7 @@ def _propose(
next_prefill_tokens=torch.zeros(B, dtype=torch.int32, device=DEVICE),
temperature=torch.zeros(B, dtype=torch.float32, device=DEVICE),
seeds=torch.zeros(B, dtype=torch.int64, device=DEVICE),
dp_sync=None,
dp_sync_state=None,
)
return drafts.cpu().tolist()
+6 -6
View File
@@ -63,13 +63,11 @@ def test_load_start_phase(
request_indices = torch.tensor([3, 1])
request_ids = ["first", "second"]
attn_metadata = {"layer": object()}
connector.pre_forward(
output,
request_state_indices=request_indices,
request_ids=request_ids,
attn_metadata=attn_metadata,
input_batch = SimpleNamespace(
idx_mapping=request_indices, req_ids=request_ids, num_tokens=5
)
attn_metadata = {"layer": object()}
connector.pre_forward(output, input_batch, attn_metadata=attn_metadata)
assert events == (
["handle", "bind", "start"] if has_sync_kv_loads else ["handle", "bind"]
)
@@ -80,6 +78,8 @@ def test_load_start_phase(
kwargs = connector.kv_connector.start_load_kv.call_args.kwargs
assert kwargs["request_state_indices"] is request_indices
assert kwargs["request_ids"] is request_ids
assert kwargs["num_tokens"] == 5
assert kwargs["scheduler_output"] is output
assert kwargs["attn_metadata"] is attn_metadata
# A subsequent step without a forward must not reuse the prior batch.
+5 -5
View File
@@ -21,6 +21,7 @@ from vllm.v1.kv_cache_interface import (
from vllm.v1.worker.gpu.async_utils import async_copy_to_np
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.model_runner import ExecuteModelState, GPUModelRunner
from vllm.v1.worker.gpu.spec_decode.utils import get_drafter_hidden_states
def test_non_last_pp_rank_uses_global_batch_for_sample_feedback():
@@ -43,7 +44,7 @@ def test_non_last_pp_rank_uses_global_batch_for_sample_feedback():
slot_mappings_by_layer=None,
hidden_states=None,
aux_hidden_states=None,
dp_sync=None,
dp_sync_state=None,
finished_req_ids=set(),
ec_connector_output=None,
cudagraph_stats=None,
@@ -342,16 +343,15 @@ def test_capture_model_profile_only_skips_lock(monkeypatch):
@pytest.mark.parametrize("target_buffer", ["absent", "none", "tensor"])
def test_get_drafter_hidden_states_tolerates_missing_target_buffer(target_buffer):
"""Targets allocate the MTP hidden buffer only for hidden-state drafters."""
runner = GPUModelRunner.__new__(GPUModelRunner)
hidden_states = torch.zeros(4, 8)
buffer = torch.arange(16 * 8, dtype=torch.float32).view(16, 8)
if target_buffer == "absent":
runner.model = SimpleNamespace()
model = SimpleNamespace()
else:
returned = buffer if target_buffer == "tensor" else None
runner.model = SimpleNamespace(get_mtp_target_hidden_states=lambda: returned)
model = SimpleNamespace(get_mtp_target_hidden_states=lambda: returned)
out = runner._get_drafter_hidden_states(hidden_states)
out = get_drafter_hidden_states(model, hidden_states)
if target_buffer == "tensor":
assert torch.equal(out, buffer[:4])
@@ -185,7 +185,7 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
slot_mappings_by_layer=None,
hidden_states=None,
aux_hidden_states=None,
dp_sync=None,
dp_sync_state=None,
finished_req_ids=set(),
ec_connector_output=None,
routed_experts=None,
+5 -5
View File
@@ -350,10 +350,10 @@ def test_decode_threshold_only_applies_when_every_rank_is_a_uniform_decode():
def test_microbatching_pads_all_ranks_to_the_largest():
"""Ranks must run the same token count so microbatch sizes line up."""
desc, dp_sync = _sync_dp([200, 256])
desc, dp_sync_state = _sync_dp([200, 256])
assert desc.num_tokens == 256
assert dp_sync is not None
assert dp_sync.num_tokens_across_dp.tolist() == [256, 256]
assert dp_sync_state is not None
assert dp_sync_state.num_tokens_across_dp.tolist() == [256, 256]
def test_microbatching_survives_a_rank_that_cannot_fill_it():
@@ -959,7 +959,7 @@ def test_microbatched_graph_needs_every_rank_to_reach_the_split():
assert desc.num_ubatches == 2
# Rank 1 pads the group up to a 128-token graph, but rank 0 only reaches 33.
desc, dp_sync = _sync_dp([33, 128], uniform, cudagraph_manager=manager)
desc, dp_sync_state = _sync_dp([33, 128], uniform, cudagraph_manager=manager)
assert desc.cg_mode == CUDAGraphMode.NONE
assert desc.num_ubatches == 2
assert dp_sync is not None and dp_sync.eager
assert dp_sync_state is not None and dp_sync_state.eager
+2 -2
View File
@@ -1091,12 +1091,12 @@ class VllmBackend:
# TODO(patchy): the V1 torch.compile ngram-gpu kernel causes vllm
# torch compile cache errors. The V2 implementation is pure Triton and
# does not need the cache disabled.
is_ngram_gpu_enabled = (
is_v1_ngram_gpu_enabled = (
vllm_config.speculative_config is not None
and vllm_config.speculative_config.use_ngram_gpu()
and not vllm_config.use_v2_model_runner
)
disable_cache = disable_cache or is_ngram_gpu_enabled
disable_cache = disable_cache or is_v1_ngram_gpu_enabled
if disable_cache:
logger.info("%svLLM's torch.compile cache is disabled.", self._log_prefix())
+19 -19
View File
@@ -152,7 +152,7 @@ def sync_cudagraph_and_dp_padding(
num_reqs=num_reqs,
)
synced_cg_mode = CUDAGraphMode(int(cg_mode_across_dp.min().item()))
synced_cg_mode = CUDAGraphMode(int(cg_mode_across_dp.min()))
# If any rank wants to run eager, all ranks run eager
if synced_cg_mode == CUDAGraphMode.NONE:
@@ -175,13 +175,13 @@ def sync_cudagraph_and_dp_padding(
"cudagraph_manager should only be None during profile run, "
"where synced_cg_mode must be NONE across all DP ranks"
)
synced_num_tokens = int(num_tokens_across_dp.max().item())
synced_num_tokens = int(num_tokens_across_dp.max())
# Varlen decode graphs are selected by the query-length bound, so ranks must agree
# on it or they pad to different token counts below.
synced_max_query_len: int | None = None
if bool(torch.all(max_query_lens_across_dp != -1).item()):
synced_max_query_len = int(max_query_lens_across_dp.max().item())
if bool(torch.all(max_query_lens_across_dp != -1)):
synced_max_query_len = int(max_query_lens_across_dp.max())
# Dispatch for the final synced values, use num_reqs instead of synced_num_reqs
# so we don't perform request padding for PIECEWISE graphs.
@@ -223,12 +223,12 @@ def dispatch_cg_and_sync_dp(
parallel_config: ParallelConfig | None = None,
allow_ubatching: bool = False,
uniform_decode: bool = False,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
) -> tuple[BatchExecutionDescriptor, DPSyncState | None]:
"""Pick a cudagraph descriptor for this batch, agreeing it across DP ranks.
Runs a collective when dp_size > 1 so every rank dispatches to the same
shape. Pass `dp_sync` from a dispatch already made over this same batch (a
shape. Pass `dp_sync_state` from a dispatch already made over this same batch (a
drafter's prefill runs the target's batch shape) to reuse that agreement
instead, with no collective.
@@ -238,7 +238,7 @@ def dispatch_cg_and_sync_dp(
num_reqs: Requests in this rank's batch.
num_tokens: Tokens in this rank's batch, already padded by the caller.
uniform_token_count: Per-request token count if this rank's batch is a
uniform decode, else None. `dp_sync.uniform_token_count` takes its
uniform decode, else None. `dp_sync_state.uniform_token_count` takes its
place when a sync is reused, since that one is agreed across ranks.
dp_size: Data-parallel world size. 1 skips all cross-rank work.
dp_rank: This rank's index in the DP group.
@@ -254,7 +254,7 @@ def dispatch_cg_and_sync_dp(
microbatches. Agreed across ranks before it takes effect.
uniform_decode: Whether this rank's batch is a uniform decode, used
to pick the microbatching split.
dp_sync: Agreement from a prior dispatch over this same batch, to reuse.
dp_sync_state: Agreement from a prior dispatch over this same batch, to reuse.
Must come from a batch with this same padded `num_tokens` and the
same `uniform_token_count`; `num_reqs` may differ, as neither
depends on it. Passing a sync from a different batch is a caller
@@ -265,7 +265,7 @@ def dispatch_cg_and_sync_dp(
dispatch to reuse. It is None when `dp_size` is 1 or no rank has work.
"""
reuse_eager = dp_sync is not None and dp_sync.eager
reuse_eager = dp_sync_state is not None and dp_sync_state.eager
if need_eager or reuse_eager:
batch_desc = BatchExecutionDescriptor(
@@ -282,7 +282,7 @@ def dispatch_cg_and_sync_dp(
batch_desc = cudagraph_manager.dispatch(
num_reqs,
num_tokens,
dp_sync.uniform_token_count if dp_sync is not None else uniform_token_count,
dp_sync_state.uniform_token_count if dp_sync_state else uniform_token_count,
num_active_loras=num_active_loras,
max_query_len=max_query_len,
)
@@ -290,24 +290,24 @@ def dispatch_cg_and_sync_dp(
if dp_size == 1:
return batch_desc, None
if dp_sync is not None:
assert dp_sync.num_tokens_across_dp[dp_rank] == num_tokens, (
if dp_sync_state is not None:
assert dp_sync_state.num_tokens_across_dp[dp_rank] == num_tokens, (
"reusing a DP sync taken over a different batch"
)
assert (
dp_sync.uniform_token_count is None
or uniform_token_count == dp_sync.uniform_token_count
dp_sync_state.uniform_token_count is None
or uniform_token_count == dp_sync_state.uniform_token_count
), "reusing a DP sync taken over a different batch"
if not dp_sync.eager and batch_desc.num_tokens != num_tokens:
if not dp_sync_state.eager and batch_desc.num_tokens != num_tokens:
# Capture sizes can differ between managers, so this one may
# pad further. Every rank pads alike, so report what will run.
dp_sync = replace(
dp_sync,
dp_sync_state = replace(
dp_sync_state,
num_tokens_across_dp=torch.full_like(
dp_sync.num_tokens_across_dp, batch_desc.num_tokens
dp_sync_state.num_tokens_across_dp, batch_desc.num_tokens
),
)
return batch_desc, dp_sync
return batch_desc, dp_sync_state
return sync_cudagraph_and_dp_padding(
cudagraph_manager,
+36 -3
View File
@@ -24,12 +24,18 @@ from vllm.v1.outputs import (
if TYPE_CHECKING:
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.worker.gpu.input_batch import InputBatch
class KVConnector:
"""KVConnector interface used by GPUModelRunner."""
def pre_forward(self, scheduler_output: "SchedulerOutput", **kwargs: Any) -> None:
def pre_forward(
self,
scheduler_output: "SchedulerOutput",
input_batch: "InputBatch | None" = None,
attn_metadata: dict[str, Any] | None = None,
) -> None:
pass
def finish_forward(self) -> None:
@@ -61,7 +67,12 @@ class ActiveKVConnector(KVConnector):
self._pending_load_kwargs: dict[str, Any] | None = None
self._disabled = False
def pre_forward(self, scheduler_output: "SchedulerOutput", **kwargs: Any) -> None:
def pre_forward(
self,
scheduler_output: "SchedulerOutput",
input_batch: "InputBatch | None" = None,
attn_metadata: dict[str, Any] | None = None,
) -> None:
if self._disabled:
return
@@ -69,7 +80,9 @@ class ActiveKVConnector(KVConnector):
assert kv_connector_metadata is not None
self.kv_connector.handle_preemptions(kv_connector_metadata)
self.kv_connector.bind_connector_metadata(kv_connector_metadata)
self._pending_load_kwargs = kwargs
self._pending_load_kwargs = self._build_load_kwargs(
scheduler_output, input_batch, attn_metadata
)
if scheduler_output.has_sync_kv_loads:
# Sync loads need to run before this step's forward.
@@ -77,6 +90,26 @@ class ActiveKVConnector(KVConnector):
# Otherwise start the async load after forward to keep submission
# off the critical path.
@staticmethod
def _build_load_kwargs(
scheduler_output: "SchedulerOutput",
input_batch: "InputBatch | None",
attn_metadata: dict[str, Any] | None,
) -> dict[str, Any]:
"""Keyword args for the connector's start_load_kv()."""
if input_batch is None:
# No forward this step.
return {}
load_kwargs: dict[str, Any] = dict(
scheduler_output=scheduler_output,
request_state_indices=input_batch.idx_mapping,
request_ids=input_batch.req_ids,
num_tokens=input_batch.num_tokens,
)
if attn_metadata is not None:
load_kwargs["attn_metadata"] = attn_metadata
return load_kwargs
def _start_load_kv(self) -> None:
load_kwargs = self._pending_load_kwargs
assert load_kwargs is not None
+20 -43
View File
@@ -162,7 +162,10 @@ from vllm.v1.worker.gpu.spec_decode.rejection_sampler import (
get_max_chunk_logits,
)
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
from vllm.v1.worker.gpu.spec_decode.utils import (
DraftTokensHandler,
get_drafter_hidden_states,
)
from vllm.v1.worker.gpu.states import RequestState
from vllm.v1.worker.gpu.structured_outputs import (
StructuredOutputsWorker,
@@ -578,9 +581,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def get_encoder_timing_stats(self) -> dict[str, dict[str, float | int]]:
encoder_runner = getattr(self.model_state, "encoder_runner", None)
if encoder_runner is None:
return {}
return encoder_runner.get_encoder_timing_stats()
return encoder_runner.get_encoder_timing_stats() if encoder_runner else {}
def get_kv_cache_spec(self):
kv_cache_spec = get_kv_cache_spec(self.vllm_config)
@@ -905,7 +906,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
slot_mappings_by_layer = self.execute_model_state.slot_mappings_by_layer
hidden_states = self.execute_model_state.hidden_states
aux_hidden_states = self.execute_model_state.aux_hidden_states
dp_sync = self.execute_model_state.dp_sync
dp_sync_state = self.execute_model_state.dp_sync_state
self.execute_model_state = None
self.step_timing.forward_end()
@@ -921,7 +922,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
mm_inputs = [], all_false
spec_hidden_states = self._get_drafter_hidden_states(hidden_states)
spec_hidden_states = get_drafter_hidden_states(self.model, hidden_states)
if isinstance(self.sampler, GPUWatermarkSampler):
self.speculator.prepare_watermarking(
self.sampler, input_batch.idx_mapping
@@ -943,7 +944,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
next_prefill_tokens=self.req_states.next_prefill_tokens,
temperature=self.sampler.sampling_states.temperature.gpu,
seeds=self.sampler.sampling_states.seeds.gpu,
dp_sync=dp_sync,
dp_sync_state=dp_sync_state,
dummy_run=True,
skip_attn_for_dummy_run=skip_attn,
mm_inputs=mm_inputs,
@@ -1771,7 +1772,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# cross-attention cache with dynamic encoder outputs.
skip_compiled = True
batch_desc, dp_sync = dispatch_cg_and_sync_dp(
batch_desc, dp_sync_state = dispatch_cg_and_sync_dp(
self.cudagraph_manager,
num_reqs,
num_toks,
@@ -1983,13 +1984,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
self.step_timing.forward_start()
connector_kwargs = dict(
scheduler_output=scheduler_output,
request_state_indices=input_batch.idx_mapping,
request_ids=input_batch.req_ids,
num_tokens=input_batch.num_tokens,
)
# Run model.
if batch_desc.cg_mode == CUDAGraphMode.FULL:
# Use explicit cudagraph replay for FULL mode.
@@ -1997,7 +1991,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# because they are already copied to the CUDA graph input buffers.
assert self.cudagraph_manager is not None
self.kv_connector.pre_forward(
**connector_kwargs, attn_metadata=attn_metadata
scheduler_output, input_batch, attn_metadata=attn_metadata
)
model_output = self.cudagraph_manager.run_fullgraph(batch_desc)
else:
@@ -2013,16 +2007,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.vllm_config,
num_tokens=input_batch.num_tokens_after_padding,
cudagraph_runtime_mode=batch_desc.cg_mode,
num_tokens_across_dp=(
dp_sync.num_tokens_across_dp if dp_sync is not None else None
),
num_tokens_across_dp=dp_sync_state
and dp_sync_state.num_tokens_across_dp,
batch_descriptor=batch_descriptor,
ubatch_slices=ubatch_slices,
slot_mapping=slot_mappings_by_layer,
skip_compiled=skip_compiled,
is_padding=input_batch.is_padding,
):
self.kv_connector.pre_forward(**connector_kwargs)
self.kv_connector.pre_forward(scheduler_output, input_batch)
if ubatch_state is not None:
assert self.ubatch_runner is not None
model_output = self.ubatch_runner.run(
@@ -2064,7 +2057,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
slot_mappings_by_layer=slot_mappings_by_layer,
hidden_states=hidden_states,
aux_hidden_states=aux_hidden_states,
dp_sync=dp_sync,
dp_sync_state=dp_sync_state,
finished_req_ids=finished_req_ids,
ec_connector_output=ec_connector_output,
cudagraph_stats=cudagraph_stats,
@@ -2080,24 +2073,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
return None
def _get_drafter_hidden_states(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Hidden states fed to the drafter.
Targets such as DeepSeek V4 expose the pre-hc_head residual through
get_mtp_target_hidden_states(). The buffer is sized at
max_num_batched_tokens and only allocated for drafters that consume
target hidden states, so None means "use the regular hidden states".
"""
get_target_hidden_states = getattr(
self.model, "get_mtp_target_hidden_states", None
)
if get_target_hidden_states is None:
return hidden_states
target_hidden_states = get_target_hidden_states()
if target_hidden_states is None:
return hidden_states
return target_hidden_states[: hidden_states.shape[0]]
@torch.inference_mode()
@step_eplb_after()
def sample_tokens(
@@ -2112,7 +2087,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
slot_mappings_by_layer = self.execute_model_state.slot_mappings_by_layer
hidden_states = self.execute_model_state.hidden_states
aux_hidden_states = self.execute_model_state.aux_hidden_states
dp_sync = self.execute_model_state.dp_sync
dp_sync_state = self.execute_model_state.dp_sync_state
finished_req_ids = self.execute_model_state.finished_req_ids
ec_connector_output = self.execute_model_state.ec_connector_output
cudagraph_stats = self.execute_model_state.cudagraph_stats
@@ -2243,7 +2218,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.speculator.observe_verification(
input_batch.idx_mapping, num_sampled, num_rejected
)
spec_hidden_states = self._get_drafter_hidden_states(draft_hidden_states)
spec_hidden_states = get_drafter_hidden_states(
self.model, draft_hidden_states
)
if isinstance(self.sampler, GPUWatermarkSampler):
self.speculator.prepare_watermarking(
self.sampler, input_batch.idx_mapping
@@ -2262,7 +2239,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.sampler.sampling_states.temperature.gpu,
self.sampler.sampling_states.seeds.gpu,
num_speculative_tokens=num_spec_tokens,
dp_sync=dp_sync,
dp_sync_state=dp_sync_state,
mm_inputs=mm_inputs,
)
if num_spec_tokens < self.num_speculative_steps:
@@ -2419,7 +2396,7 @@ class ExecuteModelState(NamedTuple):
slot_mappings_by_layer: dict[str, torch.Tensor] | None
hidden_states: torch.Tensor | None
aux_hidden_states: list[torch.Tensor] | None
dp_sync: DPSyncState | None
dp_sync_state: DPSyncState | None
finished_req_ids: set[str]
ec_connector_output: ECConnectorOutput | None
cudagraph_stats: CUDAGraphStat | None
@@ -135,7 +135,7 @@ class EncoderDecoderModelState(ModelState):
if for_capture:
max_seq_len = self.max_model_len
else:
max_seq_len = int(seq_lens_cpu_upper_bound[:num_reqs].max().item())
max_seq_len = int(seq_lens_cpu_upper_bound[:num_reqs].max())
attn_metadata = build_attn_metadata(
attn_groups=attn_groups,
num_reqs=num_reqs,
@@ -355,7 +355,7 @@ class DFlashSpeculator(DraftModelSpeculator):
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -446,8 +446,10 @@ class DFlashSpeculator(DraftModelSpeculator):
)
batch_sync, num_batch_tokens = (
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, self.num_query_per_req)
if dp_sync is not None
self._build_uniform_batch_dp_sync(
dp_sync_state, num_reqs, self.num_query_per_req
)
if dp_sync_state is not None
else (None, num_query_tokens)
)
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
@@ -459,7 +461,7 @@ class DFlashSpeculator(DraftModelSpeculator):
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
dp_sync=batch_sync,
dp_sync_state=batch_sync,
)
num_tokens_padded = batch_desc.num_tokens
num_tokens_across_dp = (
@@ -98,7 +98,7 @@ class ExtractHiddenStatesSpeculator(DraftModelSpeculator):
next_prefill_tokens: torch.Tensor,
temperature: torch.Tensor,
seeds: torch.Tensor,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -145,9 +145,7 @@ class ExtractHiddenStatesSpeculator(DraftModelSpeculator):
draft_attn_metadata,
self.vllm_config,
num_tokens=num_tokens,
num_tokens_across_dp=(
dp_sync.num_tokens_across_dp if dp_sync is not None else None
),
num_tokens_across_dp=dp_sync_state and dp_sync_state.num_tokens_across_dp,
cudagraph_runtime_mode=CUDAGraphMode.NONE,
slot_mapping=draft_slot_mappings,
is_padding=input_batch.is_padding[:num_tokens],
@@ -150,7 +150,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -201,7 +201,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
dp_sync=dp_sync,
dp_sync_state=dp_sync_state,
)
num_tokens_across_dp = (
batch_sync.num_tokens_across_dp if batch_sync is not None else None
@@ -226,7 +226,7 @@ class NgramGPUSpeculator(BaseSpeculator):
next_prefill_tokens: torch.Tensor,
temperature: torch.Tensor,
seeds: torch.Tensor,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
+5 -5
View File
@@ -92,7 +92,7 @@ class BaseSpeculator(ABC):
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -525,18 +525,18 @@ class DraftModelSpeculator(BaseSpeculator):
def _build_uniform_batch_dp_sync(
self,
target_dp_sync: DPSyncState,
target_dp_sync_state: DPSyncState,
num_reqs: int,
num_query_per_req: int = 1,
) -> tuple[DPSyncState, int]:
num_batch_tokens = target_dp_sync.num_reqs * num_query_per_req
num_batch_tokens = target_dp_sync_state.num_reqs * num_query_per_req
assert num_reqs * num_query_per_req <= num_batch_tokens, (
"reusing a DP sync that does not cover this batch's requests"
)
return replace(
target_dp_sync,
target_dp_sync_state,
num_tokens_across_dp=torch.full_like(
target_dp_sync.num_tokens_across_dp, num_batch_tokens
target_dp_sync_state.num_tokens_across_dp, num_batch_tokens
),
uniform_token_count=num_query_per_req,
eager=False,
@@ -350,7 +350,7 @@ class StandaloneARSpeculator(DraftModelSpeculator):
next_prefill_tokens: torch.Tensor,
temperature: torch.Tensor,
seeds: torch.Tensor,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -362,7 +362,7 @@ class StandaloneARSpeculator(DraftModelSpeculator):
num_reqs = input_batch.num_reqs
skip_attn = dummy_run and skip_attn_for_dummy_run
num_tokens_across_dp = (
dp_sync.num_tokens_across_dp if dp_sync is not None else None
dp_sync_state.num_tokens_across_dp if dp_sync_state is not None else None
)
# Copy per-request temperature/seeds/idx_mapping into pre-allocated
@@ -219,7 +219,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
dp_sync: DPSyncState | None = None,
dp_sync_state: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -299,7 +299,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
dp_sync=dp_sync,
dp_sync_state=dp_sync_state,
)
num_tokens_across_dp = (
prefill_batch_sync.num_tokens_across_dp
@@ -352,8 +352,10 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
)
decode_batch_sync, num_batch_tokens = (
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, num_query_per_req=1)
if dp_sync is not None
self._build_uniform_batch_dp_sync(
dp_sync_state, num_reqs, num_query_per_req=1
)
if dp_sync_state is not None
else (None, num_reqs)
)
# Each request produces exactly 1 token per draft generation step,
@@ -366,7 +368,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
dp_sync=decode_batch_sync,
dp_sync_state=decode_batch_sync,
)
if self.decode_cudagraph_manager is not None:
decode_batch_desc = self.decode_cudagraph_manager.specialize_spec_tokens(
+19
View File
@@ -13,6 +13,25 @@ from vllm.v1.worker.gpu.input_batch import InputBatch
logger = init_logger(__name__)
def get_drafter_hidden_states(
model: torch.nn.Module, hidden_states: torch.Tensor
) -> torch.Tensor:
"""Hidden states fed to the drafter.
Targets such as DeepSeek V4 expose the pre-hc_head residual through
get_mtp_target_hidden_states(). The buffer is sized at
max_num_batched_tokens and only allocated for drafters that consume
target hidden states, so None means "use the regular hidden states".
"""
get_target_hidden_states = getattr(model, "get_mtp_target_hidden_states", None)
if get_target_hidden_states is None:
return hidden_states
target_hidden_states = get_target_hidden_states()
if target_hidden_states is None:
return hidden_states
return target_hidden_states[: hidden_states.shape[0]]
def get_pp_safe_draft_load_config(load_config: LoadConfig) -> LoadConfig:
"""Avoid collectives that include PP ranks without a draft model."""
if get_pp_group().world_size > 1 and load_config.load_format == "fastsafetensors":