[MRV2] Miscellaneous code structure updates (#60754)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -147,7 +147,7 @@ def _propose(
|
||||
next_prefill_tokens=torch.zeros(B, dtype=torch.int32, device=DEVICE),
|
||||
temperature=torch.zeros(B, dtype=torch.float32, device=DEVICE),
|
||||
seeds=torch.zeros(B, dtype=torch.int64, device=DEVICE),
|
||||
dp_sync=None,
|
||||
dp_sync_state=None,
|
||||
)
|
||||
return drafts.cpu().tolist()
|
||||
|
||||
|
||||
@@ -63,13 +63,11 @@ def test_load_start_phase(
|
||||
|
||||
request_indices = torch.tensor([3, 1])
|
||||
request_ids = ["first", "second"]
|
||||
attn_metadata = {"layer": object()}
|
||||
connector.pre_forward(
|
||||
output,
|
||||
request_state_indices=request_indices,
|
||||
request_ids=request_ids,
|
||||
attn_metadata=attn_metadata,
|
||||
input_batch = SimpleNamespace(
|
||||
idx_mapping=request_indices, req_ids=request_ids, num_tokens=5
|
||||
)
|
||||
attn_metadata = {"layer": object()}
|
||||
connector.pre_forward(output, input_batch, attn_metadata=attn_metadata)
|
||||
assert events == (
|
||||
["handle", "bind", "start"] if has_sync_kv_loads else ["handle", "bind"]
|
||||
)
|
||||
@@ -80,6 +78,8 @@ def test_load_start_phase(
|
||||
kwargs = connector.kv_connector.start_load_kv.call_args.kwargs
|
||||
assert kwargs["request_state_indices"] is request_indices
|
||||
assert kwargs["request_ids"] is request_ids
|
||||
assert kwargs["num_tokens"] == 5
|
||||
assert kwargs["scheduler_output"] is output
|
||||
assert kwargs["attn_metadata"] is attn_metadata
|
||||
|
||||
# A subsequent step without a forward must not reuse the prior batch.
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
from vllm.v1.worker.gpu.async_utils import async_copy_to_np
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.model_runner import ExecuteModelState, GPUModelRunner
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import get_drafter_hidden_states
|
||||
|
||||
|
||||
def test_non_last_pp_rank_uses_global_batch_for_sample_feedback():
|
||||
@@ -43,7 +44,7 @@ def test_non_last_pp_rank_uses_global_batch_for_sample_feedback():
|
||||
slot_mappings_by_layer=None,
|
||||
hidden_states=None,
|
||||
aux_hidden_states=None,
|
||||
dp_sync=None,
|
||||
dp_sync_state=None,
|
||||
finished_req_ids=set(),
|
||||
ec_connector_output=None,
|
||||
cudagraph_stats=None,
|
||||
@@ -342,16 +343,15 @@ def test_capture_model_profile_only_skips_lock(monkeypatch):
|
||||
@pytest.mark.parametrize("target_buffer", ["absent", "none", "tensor"])
|
||||
def test_get_drafter_hidden_states_tolerates_missing_target_buffer(target_buffer):
|
||||
"""Targets allocate the MTP hidden buffer only for hidden-state drafters."""
|
||||
runner = GPUModelRunner.__new__(GPUModelRunner)
|
||||
hidden_states = torch.zeros(4, 8)
|
||||
buffer = torch.arange(16 * 8, dtype=torch.float32).view(16, 8)
|
||||
if target_buffer == "absent":
|
||||
runner.model = SimpleNamespace()
|
||||
model = SimpleNamespace()
|
||||
else:
|
||||
returned = buffer if target_buffer == "tensor" else None
|
||||
runner.model = SimpleNamespace(get_mtp_target_hidden_states=lambda: returned)
|
||||
model = SimpleNamespace(get_mtp_target_hidden_states=lambda: returned)
|
||||
|
||||
out = runner._get_drafter_hidden_states(hidden_states)
|
||||
out = get_drafter_hidden_states(model, hidden_states)
|
||||
|
||||
if target_buffer == "tensor":
|
||||
assert torch.equal(out, buffer[:4])
|
||||
|
||||
@@ -185,7 +185,7 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
||||
slot_mappings_by_layer=None,
|
||||
hidden_states=None,
|
||||
aux_hidden_states=None,
|
||||
dp_sync=None,
|
||||
dp_sync_state=None,
|
||||
finished_req_ids=set(),
|
||||
ec_connector_output=None,
|
||||
routed_experts=None,
|
||||
|
||||
@@ -350,10 +350,10 @@ def test_decode_threshold_only_applies_when_every_rank_is_a_uniform_decode():
|
||||
|
||||
def test_microbatching_pads_all_ranks_to_the_largest():
|
||||
"""Ranks must run the same token count so microbatch sizes line up."""
|
||||
desc, dp_sync = _sync_dp([200, 256])
|
||||
desc, dp_sync_state = _sync_dp([200, 256])
|
||||
assert desc.num_tokens == 256
|
||||
assert dp_sync is not None
|
||||
assert dp_sync.num_tokens_across_dp.tolist() == [256, 256]
|
||||
assert dp_sync_state is not None
|
||||
assert dp_sync_state.num_tokens_across_dp.tolist() == [256, 256]
|
||||
|
||||
|
||||
def test_microbatching_survives_a_rank_that_cannot_fill_it():
|
||||
@@ -959,7 +959,7 @@ def test_microbatched_graph_needs_every_rank_to_reach_the_split():
|
||||
assert desc.num_ubatches == 2
|
||||
|
||||
# Rank 1 pads the group up to a 128-token graph, but rank 0 only reaches 33.
|
||||
desc, dp_sync = _sync_dp([33, 128], uniform, cudagraph_manager=manager)
|
||||
desc, dp_sync_state = _sync_dp([33, 128], uniform, cudagraph_manager=manager)
|
||||
assert desc.cg_mode == CUDAGraphMode.NONE
|
||||
assert desc.num_ubatches == 2
|
||||
assert dp_sync is not None and dp_sync.eager
|
||||
assert dp_sync_state is not None and dp_sync_state.eager
|
||||
|
||||
@@ -1091,12 +1091,12 @@ class VllmBackend:
|
||||
# TODO(patchy): the V1 torch.compile ngram-gpu kernel causes vllm
|
||||
# torch compile cache errors. The V2 implementation is pure Triton and
|
||||
# does not need the cache disabled.
|
||||
is_ngram_gpu_enabled = (
|
||||
is_v1_ngram_gpu_enabled = (
|
||||
vllm_config.speculative_config is not None
|
||||
and vllm_config.speculative_config.use_ngram_gpu()
|
||||
and not vllm_config.use_v2_model_runner
|
||||
)
|
||||
disable_cache = disable_cache or is_ngram_gpu_enabled
|
||||
disable_cache = disable_cache or is_v1_ngram_gpu_enabled
|
||||
|
||||
if disable_cache:
|
||||
logger.info("%svLLM's torch.compile cache is disabled.", self._log_prefix())
|
||||
|
||||
@@ -152,7 +152,7 @@ def sync_cudagraph_and_dp_padding(
|
||||
num_reqs=num_reqs,
|
||||
)
|
||||
|
||||
synced_cg_mode = CUDAGraphMode(int(cg_mode_across_dp.min().item()))
|
||||
synced_cg_mode = CUDAGraphMode(int(cg_mode_across_dp.min()))
|
||||
|
||||
# If any rank wants to run eager, all ranks run eager
|
||||
if synced_cg_mode == CUDAGraphMode.NONE:
|
||||
@@ -175,13 +175,13 @@ def sync_cudagraph_and_dp_padding(
|
||||
"cudagraph_manager should only be None during profile run, "
|
||||
"where synced_cg_mode must be NONE across all DP ranks"
|
||||
)
|
||||
synced_num_tokens = int(num_tokens_across_dp.max().item())
|
||||
synced_num_tokens = int(num_tokens_across_dp.max())
|
||||
|
||||
# Varlen decode graphs are selected by the query-length bound, so ranks must agree
|
||||
# on it or they pad to different token counts below.
|
||||
synced_max_query_len: int | None = None
|
||||
if bool(torch.all(max_query_lens_across_dp != -1).item()):
|
||||
synced_max_query_len = int(max_query_lens_across_dp.max().item())
|
||||
if bool(torch.all(max_query_lens_across_dp != -1)):
|
||||
synced_max_query_len = int(max_query_lens_across_dp.max())
|
||||
|
||||
# Dispatch for the final synced values, use num_reqs instead of synced_num_reqs
|
||||
# so we don't perform request padding for PIECEWISE graphs.
|
||||
@@ -223,12 +223,12 @@ def dispatch_cg_and_sync_dp(
|
||||
parallel_config: ParallelConfig | None = None,
|
||||
allow_ubatching: bool = False,
|
||||
uniform_decode: bool = False,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
dp_sync_state: DPSyncState | None = None,
|
||||
) -> tuple[BatchExecutionDescriptor, DPSyncState | None]:
|
||||
"""Pick a cudagraph descriptor for this batch, agreeing it across DP ranks.
|
||||
|
||||
Runs a collective when dp_size > 1 so every rank dispatches to the same
|
||||
shape. Pass `dp_sync` from a dispatch already made over this same batch (a
|
||||
shape. Pass `dp_sync_state` from a dispatch already made over this same batch (a
|
||||
drafter's prefill runs the target's batch shape) to reuse that agreement
|
||||
instead, with no collective.
|
||||
|
||||
@@ -238,7 +238,7 @@ def dispatch_cg_and_sync_dp(
|
||||
num_reqs: Requests in this rank's batch.
|
||||
num_tokens: Tokens in this rank's batch, already padded by the caller.
|
||||
uniform_token_count: Per-request token count if this rank's batch is a
|
||||
uniform decode, else None. `dp_sync.uniform_token_count` takes its
|
||||
uniform decode, else None. `dp_sync_state.uniform_token_count` takes its
|
||||
place when a sync is reused, since that one is agreed across ranks.
|
||||
dp_size: Data-parallel world size. 1 skips all cross-rank work.
|
||||
dp_rank: This rank's index in the DP group.
|
||||
@@ -254,7 +254,7 @@ def dispatch_cg_and_sync_dp(
|
||||
microbatches. Agreed across ranks before it takes effect.
|
||||
uniform_decode: Whether this rank's batch is a uniform decode, used
|
||||
to pick the microbatching split.
|
||||
dp_sync: Agreement from a prior dispatch over this same batch, to reuse.
|
||||
dp_sync_state: Agreement from a prior dispatch over this same batch, to reuse.
|
||||
Must come from a batch with this same padded `num_tokens` and the
|
||||
same `uniform_token_count`; `num_reqs` may differ, as neither
|
||||
depends on it. Passing a sync from a different batch is a caller
|
||||
@@ -265,7 +265,7 @@ def dispatch_cg_and_sync_dp(
|
||||
dispatch to reuse. It is None when `dp_size` is 1 or no rank has work.
|
||||
|
||||
"""
|
||||
reuse_eager = dp_sync is not None and dp_sync.eager
|
||||
reuse_eager = dp_sync_state is not None and dp_sync_state.eager
|
||||
|
||||
if need_eager or reuse_eager:
|
||||
batch_desc = BatchExecutionDescriptor(
|
||||
@@ -282,7 +282,7 @@ def dispatch_cg_and_sync_dp(
|
||||
batch_desc = cudagraph_manager.dispatch(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
dp_sync.uniform_token_count if dp_sync is not None else uniform_token_count,
|
||||
dp_sync_state.uniform_token_count if dp_sync_state else uniform_token_count,
|
||||
num_active_loras=num_active_loras,
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
@@ -290,24 +290,24 @@ def dispatch_cg_and_sync_dp(
|
||||
if dp_size == 1:
|
||||
return batch_desc, None
|
||||
|
||||
if dp_sync is not None:
|
||||
assert dp_sync.num_tokens_across_dp[dp_rank] == num_tokens, (
|
||||
if dp_sync_state is not None:
|
||||
assert dp_sync_state.num_tokens_across_dp[dp_rank] == num_tokens, (
|
||||
"reusing a DP sync taken over a different batch"
|
||||
)
|
||||
assert (
|
||||
dp_sync.uniform_token_count is None
|
||||
or uniform_token_count == dp_sync.uniform_token_count
|
||||
dp_sync_state.uniform_token_count is None
|
||||
or uniform_token_count == dp_sync_state.uniform_token_count
|
||||
), "reusing a DP sync taken over a different batch"
|
||||
if not dp_sync.eager and batch_desc.num_tokens != num_tokens:
|
||||
if not dp_sync_state.eager and batch_desc.num_tokens != num_tokens:
|
||||
# Capture sizes can differ between managers, so this one may
|
||||
# pad further. Every rank pads alike, so report what will run.
|
||||
dp_sync = replace(
|
||||
dp_sync,
|
||||
dp_sync_state = replace(
|
||||
dp_sync_state,
|
||||
num_tokens_across_dp=torch.full_like(
|
||||
dp_sync.num_tokens_across_dp, batch_desc.num_tokens
|
||||
dp_sync_state.num_tokens_across_dp, batch_desc.num_tokens
|
||||
),
|
||||
)
|
||||
return batch_desc, dp_sync
|
||||
return batch_desc, dp_sync_state
|
||||
|
||||
return sync_cudagraph_and_dp_padding(
|
||||
cudagraph_manager,
|
||||
|
||||
@@ -24,12 +24,18 @@ from vllm.v1.outputs import (
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
|
||||
|
||||
class KVConnector:
|
||||
"""KVConnector interface used by GPUModelRunner."""
|
||||
|
||||
def pre_forward(self, scheduler_output: "SchedulerOutput", **kwargs: Any) -> None:
|
||||
def pre_forward(
|
||||
self,
|
||||
scheduler_output: "SchedulerOutput",
|
||||
input_batch: "InputBatch | None" = None,
|
||||
attn_metadata: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
def finish_forward(self) -> None:
|
||||
@@ -61,7 +67,12 @@ class ActiveKVConnector(KVConnector):
|
||||
self._pending_load_kwargs: dict[str, Any] | None = None
|
||||
self._disabled = False
|
||||
|
||||
def pre_forward(self, scheduler_output: "SchedulerOutput", **kwargs: Any) -> None:
|
||||
def pre_forward(
|
||||
self,
|
||||
scheduler_output: "SchedulerOutput",
|
||||
input_batch: "InputBatch | None" = None,
|
||||
attn_metadata: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
if self._disabled:
|
||||
return
|
||||
|
||||
@@ -69,7 +80,9 @@ class ActiveKVConnector(KVConnector):
|
||||
assert kv_connector_metadata is not None
|
||||
self.kv_connector.handle_preemptions(kv_connector_metadata)
|
||||
self.kv_connector.bind_connector_metadata(kv_connector_metadata)
|
||||
self._pending_load_kwargs = kwargs
|
||||
self._pending_load_kwargs = self._build_load_kwargs(
|
||||
scheduler_output, input_batch, attn_metadata
|
||||
)
|
||||
|
||||
if scheduler_output.has_sync_kv_loads:
|
||||
# Sync loads need to run before this step's forward.
|
||||
@@ -77,6 +90,26 @@ class ActiveKVConnector(KVConnector):
|
||||
# Otherwise start the async load after forward to keep submission
|
||||
# off the critical path.
|
||||
|
||||
@staticmethod
|
||||
def _build_load_kwargs(
|
||||
scheduler_output: "SchedulerOutput",
|
||||
input_batch: "InputBatch | None",
|
||||
attn_metadata: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Keyword args for the connector's start_load_kv()."""
|
||||
if input_batch is None:
|
||||
# No forward this step.
|
||||
return {}
|
||||
load_kwargs: dict[str, Any] = dict(
|
||||
scheduler_output=scheduler_output,
|
||||
request_state_indices=input_batch.idx_mapping,
|
||||
request_ids=input_batch.req_ids,
|
||||
num_tokens=input_batch.num_tokens,
|
||||
)
|
||||
if attn_metadata is not None:
|
||||
load_kwargs["attn_metadata"] = attn_metadata
|
||||
return load_kwargs
|
||||
|
||||
def _start_load_kv(self) -> None:
|
||||
load_kwargs = self._pending_load_kwargs
|
||||
assert load_kwargs is not None
|
||||
|
||||
@@ -162,7 +162,10 @@ from vllm.v1.worker.gpu.spec_decode.rejection_sampler import (
|
||||
get_max_chunk_logits,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import (
|
||||
DraftTokensHandler,
|
||||
get_drafter_hidden_states,
|
||||
)
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
from vllm.v1.worker.gpu.structured_outputs import (
|
||||
StructuredOutputsWorker,
|
||||
@@ -578,9 +581,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
def get_encoder_timing_stats(self) -> dict[str, dict[str, float | int]]:
|
||||
encoder_runner = getattr(self.model_state, "encoder_runner", None)
|
||||
if encoder_runner is None:
|
||||
return {}
|
||||
return encoder_runner.get_encoder_timing_stats()
|
||||
return encoder_runner.get_encoder_timing_stats() if encoder_runner else {}
|
||||
|
||||
def get_kv_cache_spec(self):
|
||||
kv_cache_spec = get_kv_cache_spec(self.vllm_config)
|
||||
@@ -905,7 +906,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
slot_mappings_by_layer = self.execute_model_state.slot_mappings_by_layer
|
||||
hidden_states = self.execute_model_state.hidden_states
|
||||
aux_hidden_states = self.execute_model_state.aux_hidden_states
|
||||
dp_sync = self.execute_model_state.dp_sync
|
||||
dp_sync_state = self.execute_model_state.dp_sync_state
|
||||
self.execute_model_state = None
|
||||
|
||||
self.step_timing.forward_end()
|
||||
@@ -921,7 +922,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
mm_inputs = [], all_false
|
||||
|
||||
spec_hidden_states = self._get_drafter_hidden_states(hidden_states)
|
||||
spec_hidden_states = get_drafter_hidden_states(self.model, hidden_states)
|
||||
if isinstance(self.sampler, GPUWatermarkSampler):
|
||||
self.speculator.prepare_watermarking(
|
||||
self.sampler, input_batch.idx_mapping
|
||||
@@ -943,7 +944,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
next_prefill_tokens=self.req_states.next_prefill_tokens,
|
||||
temperature=self.sampler.sampling_states.temperature.gpu,
|
||||
seeds=self.sampler.sampling_states.seeds.gpu,
|
||||
dp_sync=dp_sync,
|
||||
dp_sync_state=dp_sync_state,
|
||||
dummy_run=True,
|
||||
skip_attn_for_dummy_run=skip_attn,
|
||||
mm_inputs=mm_inputs,
|
||||
@@ -1771,7 +1772,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# cross-attention cache with dynamic encoder outputs.
|
||||
skip_compiled = True
|
||||
|
||||
batch_desc, dp_sync = dispatch_cg_and_sync_dp(
|
||||
batch_desc, dp_sync_state = dispatch_cg_and_sync_dp(
|
||||
self.cudagraph_manager,
|
||||
num_reqs,
|
||||
num_toks,
|
||||
@@ -1983,13 +1984,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
self.step_timing.forward_start()
|
||||
|
||||
connector_kwargs = dict(
|
||||
scheduler_output=scheduler_output,
|
||||
request_state_indices=input_batch.idx_mapping,
|
||||
request_ids=input_batch.req_ids,
|
||||
num_tokens=input_batch.num_tokens,
|
||||
)
|
||||
|
||||
# Run model.
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
# Use explicit cudagraph replay for FULL mode.
|
||||
@@ -1997,7 +1991,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# because they are already copied to the CUDA graph input buffers.
|
||||
assert self.cudagraph_manager is not None
|
||||
self.kv_connector.pre_forward(
|
||||
**connector_kwargs, attn_metadata=attn_metadata
|
||||
scheduler_output, input_batch, attn_metadata=attn_metadata
|
||||
)
|
||||
model_output = self.cudagraph_manager.run_fullgraph(batch_desc)
|
||||
else:
|
||||
@@ -2013,16 +2007,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.vllm_config,
|
||||
num_tokens=input_batch.num_tokens_after_padding,
|
||||
cudagraph_runtime_mode=batch_desc.cg_mode,
|
||||
num_tokens_across_dp=(
|
||||
dp_sync.num_tokens_across_dp if dp_sync is not None else None
|
||||
),
|
||||
num_tokens_across_dp=dp_sync_state
|
||||
and dp_sync_state.num_tokens_across_dp,
|
||||
batch_descriptor=batch_descriptor,
|
||||
ubatch_slices=ubatch_slices,
|
||||
slot_mapping=slot_mappings_by_layer,
|
||||
skip_compiled=skip_compiled,
|
||||
is_padding=input_batch.is_padding,
|
||||
):
|
||||
self.kv_connector.pre_forward(**connector_kwargs)
|
||||
self.kv_connector.pre_forward(scheduler_output, input_batch)
|
||||
if ubatch_state is not None:
|
||||
assert self.ubatch_runner is not None
|
||||
model_output = self.ubatch_runner.run(
|
||||
@@ -2064,7 +2057,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
slot_mappings_by_layer=slot_mappings_by_layer,
|
||||
hidden_states=hidden_states,
|
||||
aux_hidden_states=aux_hidden_states,
|
||||
dp_sync=dp_sync,
|
||||
dp_sync_state=dp_sync_state,
|
||||
finished_req_ids=finished_req_ids,
|
||||
ec_connector_output=ec_connector_output,
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
@@ -2080,24 +2073,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
return None
|
||||
|
||||
def _get_drafter_hidden_states(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Hidden states fed to the drafter.
|
||||
|
||||
Targets such as DeepSeek V4 expose the pre-hc_head residual through
|
||||
get_mtp_target_hidden_states(). The buffer is sized at
|
||||
max_num_batched_tokens and only allocated for drafters that consume
|
||||
target hidden states, so None means "use the regular hidden states".
|
||||
"""
|
||||
get_target_hidden_states = getattr(
|
||||
self.model, "get_mtp_target_hidden_states", None
|
||||
)
|
||||
if get_target_hidden_states is None:
|
||||
return hidden_states
|
||||
target_hidden_states = get_target_hidden_states()
|
||||
if target_hidden_states is None:
|
||||
return hidden_states
|
||||
return target_hidden_states[: hidden_states.shape[0]]
|
||||
|
||||
@torch.inference_mode()
|
||||
@step_eplb_after()
|
||||
def sample_tokens(
|
||||
@@ -2112,7 +2087,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
slot_mappings_by_layer = self.execute_model_state.slot_mappings_by_layer
|
||||
hidden_states = self.execute_model_state.hidden_states
|
||||
aux_hidden_states = self.execute_model_state.aux_hidden_states
|
||||
dp_sync = self.execute_model_state.dp_sync
|
||||
dp_sync_state = self.execute_model_state.dp_sync_state
|
||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||
ec_connector_output = self.execute_model_state.ec_connector_output
|
||||
cudagraph_stats = self.execute_model_state.cudagraph_stats
|
||||
@@ -2243,7 +2218,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.speculator.observe_verification(
|
||||
input_batch.idx_mapping, num_sampled, num_rejected
|
||||
)
|
||||
spec_hidden_states = self._get_drafter_hidden_states(draft_hidden_states)
|
||||
spec_hidden_states = get_drafter_hidden_states(
|
||||
self.model, draft_hidden_states
|
||||
)
|
||||
if isinstance(self.sampler, GPUWatermarkSampler):
|
||||
self.speculator.prepare_watermarking(
|
||||
self.sampler, input_batch.idx_mapping
|
||||
@@ -2262,7 +2239,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.sampler.sampling_states.temperature.gpu,
|
||||
self.sampler.sampling_states.seeds.gpu,
|
||||
num_speculative_tokens=num_spec_tokens,
|
||||
dp_sync=dp_sync,
|
||||
dp_sync_state=dp_sync_state,
|
||||
mm_inputs=mm_inputs,
|
||||
)
|
||||
if num_spec_tokens < self.num_speculative_steps:
|
||||
@@ -2419,7 +2396,7 @@ class ExecuteModelState(NamedTuple):
|
||||
slot_mappings_by_layer: dict[str, torch.Tensor] | None
|
||||
hidden_states: torch.Tensor | None
|
||||
aux_hidden_states: list[torch.Tensor] | None
|
||||
dp_sync: DPSyncState | None
|
||||
dp_sync_state: DPSyncState | None
|
||||
finished_req_ids: set[str]
|
||||
ec_connector_output: ECConnectorOutput | None
|
||||
cudagraph_stats: CUDAGraphStat | None
|
||||
|
||||
@@ -135,7 +135,7 @@ class EncoderDecoderModelState(ModelState):
|
||||
if for_capture:
|
||||
max_seq_len = self.max_model_len
|
||||
else:
|
||||
max_seq_len = int(seq_lens_cpu_upper_bound[:num_reqs].max().item())
|
||||
max_seq_len = int(seq_lens_cpu_upper_bound[:num_reqs].max())
|
||||
attn_metadata = build_attn_metadata(
|
||||
attn_groups=attn_groups,
|
||||
num_reqs=num_reqs,
|
||||
|
||||
@@ -355,7 +355,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
temperature: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
seeds: torch.Tensor,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
dp_sync_state: DPSyncState | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
@@ -446,8 +446,10 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
)
|
||||
|
||||
batch_sync, num_batch_tokens = (
|
||||
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, self.num_query_per_req)
|
||||
if dp_sync is not None
|
||||
self._build_uniform_batch_dp_sync(
|
||||
dp_sync_state, num_reqs, self.num_query_per_req
|
||||
)
|
||||
if dp_sync_state is not None
|
||||
else (None, num_query_tokens)
|
||||
)
|
||||
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
|
||||
@@ -459,7 +461,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
dp_size=self.dp_size,
|
||||
dp_rank=self.dp_rank,
|
||||
need_eager=is_profile,
|
||||
dp_sync=batch_sync,
|
||||
dp_sync_state=batch_sync,
|
||||
)
|
||||
num_tokens_padded = batch_desc.num_tokens
|
||||
num_tokens_across_dp = (
|
||||
|
||||
@@ -98,7 +98,7 @@ class ExtractHiddenStatesSpeculator(DraftModelSpeculator):
|
||||
next_prefill_tokens: torch.Tensor,
|
||||
temperature: torch.Tensor,
|
||||
seeds: torch.Tensor,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
dp_sync_state: DPSyncState | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
@@ -145,9 +145,7 @@ class ExtractHiddenStatesSpeculator(DraftModelSpeculator):
|
||||
draft_attn_metadata,
|
||||
self.vllm_config,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_across_dp=(
|
||||
dp_sync.num_tokens_across_dp if dp_sync is not None else None
|
||||
),
|
||||
num_tokens_across_dp=dp_sync_state and dp_sync_state.num_tokens_across_dp,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
slot_mapping=draft_slot_mappings,
|
||||
is_padding=input_batch.is_padding[:num_tokens],
|
||||
|
||||
@@ -150,7 +150,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
|
||||
temperature: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
seeds: torch.Tensor,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
dp_sync_state: DPSyncState | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
@@ -201,7 +201,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
|
||||
dp_size=self.dp_size,
|
||||
dp_rank=self.dp_rank,
|
||||
need_eager=is_profile,
|
||||
dp_sync=dp_sync,
|
||||
dp_sync_state=dp_sync_state,
|
||||
)
|
||||
num_tokens_across_dp = (
|
||||
batch_sync.num_tokens_across_dp if batch_sync is not None else None
|
||||
|
||||
@@ -226,7 +226,7 @@ class NgramGPUSpeculator(BaseSpeculator):
|
||||
next_prefill_tokens: torch.Tensor,
|
||||
temperature: torch.Tensor,
|
||||
seeds: torch.Tensor,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
dp_sync_state: DPSyncState | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
|
||||
@@ -92,7 +92,7 @@ class BaseSpeculator(ABC):
|
||||
temperature: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
seeds: torch.Tensor,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
dp_sync_state: DPSyncState | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
@@ -525,18 +525,18 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
|
||||
def _build_uniform_batch_dp_sync(
|
||||
self,
|
||||
target_dp_sync: DPSyncState,
|
||||
target_dp_sync_state: DPSyncState,
|
||||
num_reqs: int,
|
||||
num_query_per_req: int = 1,
|
||||
) -> tuple[DPSyncState, int]:
|
||||
num_batch_tokens = target_dp_sync.num_reqs * num_query_per_req
|
||||
num_batch_tokens = target_dp_sync_state.num_reqs * num_query_per_req
|
||||
assert num_reqs * num_query_per_req <= num_batch_tokens, (
|
||||
"reusing a DP sync that does not cover this batch's requests"
|
||||
)
|
||||
return replace(
|
||||
target_dp_sync,
|
||||
target_dp_sync_state,
|
||||
num_tokens_across_dp=torch.full_like(
|
||||
target_dp_sync.num_tokens_across_dp, num_batch_tokens
|
||||
target_dp_sync_state.num_tokens_across_dp, num_batch_tokens
|
||||
),
|
||||
uniform_token_count=num_query_per_req,
|
||||
eager=False,
|
||||
|
||||
@@ -350,7 +350,7 @@ class StandaloneARSpeculator(DraftModelSpeculator):
|
||||
next_prefill_tokens: torch.Tensor,
|
||||
temperature: torch.Tensor,
|
||||
seeds: torch.Tensor,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
dp_sync_state: DPSyncState | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
@@ -362,7 +362,7 @@ class StandaloneARSpeculator(DraftModelSpeculator):
|
||||
num_reqs = input_batch.num_reqs
|
||||
skip_attn = dummy_run and skip_attn_for_dummy_run
|
||||
num_tokens_across_dp = (
|
||||
dp_sync.num_tokens_across_dp if dp_sync is not None else None
|
||||
dp_sync_state.num_tokens_across_dp if dp_sync_state is not None else None
|
||||
)
|
||||
|
||||
# Copy per-request temperature/seeds/idx_mapping into pre-allocated
|
||||
|
||||
@@ -219,7 +219,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
|
||||
temperature: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
seeds: torch.Tensor,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
dp_sync_state: DPSyncState | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
@@ -299,7 +299,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
|
||||
dp_size=self.dp_size,
|
||||
dp_rank=self.dp_rank,
|
||||
need_eager=is_profile,
|
||||
dp_sync=dp_sync,
|
||||
dp_sync_state=dp_sync_state,
|
||||
)
|
||||
num_tokens_across_dp = (
|
||||
prefill_batch_sync.num_tokens_across_dp
|
||||
@@ -352,8 +352,10 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
|
||||
)
|
||||
|
||||
decode_batch_sync, num_batch_tokens = (
|
||||
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, num_query_per_req=1)
|
||||
if dp_sync is not None
|
||||
self._build_uniform_batch_dp_sync(
|
||||
dp_sync_state, num_reqs, num_query_per_req=1
|
||||
)
|
||||
if dp_sync_state is not None
|
||||
else (None, num_reqs)
|
||||
)
|
||||
# Each request produces exactly 1 token per draft generation step,
|
||||
@@ -366,7 +368,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
|
||||
dp_size=self.dp_size,
|
||||
dp_rank=self.dp_rank,
|
||||
need_eager=is_profile,
|
||||
dp_sync=decode_batch_sync,
|
||||
dp_sync_state=decode_batch_sync,
|
||||
)
|
||||
if self.decode_cudagraph_manager is not None:
|
||||
decode_batch_desc = self.decode_cudagraph_manager.specialize_spec_tokens(
|
||||
|
||||
@@ -13,6 +13,25 @@ from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def get_drafter_hidden_states(
|
||||
model: torch.nn.Module, hidden_states: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Hidden states fed to the drafter.
|
||||
|
||||
Targets such as DeepSeek V4 expose the pre-hc_head residual through
|
||||
get_mtp_target_hidden_states(). The buffer is sized at
|
||||
max_num_batched_tokens and only allocated for drafters that consume
|
||||
target hidden states, so None means "use the regular hidden states".
|
||||
"""
|
||||
get_target_hidden_states = getattr(model, "get_mtp_target_hidden_states", None)
|
||||
if get_target_hidden_states is None:
|
||||
return hidden_states
|
||||
target_hidden_states = get_target_hidden_states()
|
||||
if target_hidden_states is None:
|
||||
return hidden_states
|
||||
return target_hidden_states[: hidden_states.shape[0]]
|
||||
|
||||
|
||||
def get_pp_safe_draft_load_config(load_config: LoadConfig) -> LoadConfig:
|
||||
"""Avoid collectives that include PP ranks without a draft model."""
|
||||
if get_pp_group().world_size > 1 and load_config.load_format == "fastsafetensors":
|
||||
|
||||
Reference in New Issue
Block a user