[MRV2] Miscellaneous code structure updates (#60754)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-10-09 17:58:09 -07:00
committed by GitHub
parent 9f1c783886
commit 7d1d8660d6
18 changed files with 140 additions and 109 deletions
+20 -43
View File
@@ -162,7 +162,10 @@ from vllm.v1.worker.gpu.spec_decode.rejection_sampler import (
get_max_chunk_logits,
)
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
from vllm.v1.worker.gpu.spec_decode.utils import (
DraftTokensHandler,
get_drafter_hidden_states,
)
from vllm.v1.worker.gpu.states import RequestState
from vllm.v1.worker.gpu.structured_outputs import (
StructuredOutputsWorker,
@@ -578,9 +581,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def get_encoder_timing_stats(self) -> dict[str, dict[str, float | int]]:
encoder_runner = getattr(self.model_state, "encoder_runner", None)
if encoder_runner is None:
return {}
return encoder_runner.get_encoder_timing_stats()
return encoder_runner.get_encoder_timing_stats() if encoder_runner else {}
def get_kv_cache_spec(self):
kv_cache_spec = get_kv_cache_spec(self.vllm_config)
@@ -905,7 +906,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
slot_mappings_by_layer = self.execute_model_state.slot_mappings_by_layer
hidden_states = self.execute_model_state.hidden_states
aux_hidden_states = self.execute_model_state.aux_hidden_states
dp_sync = self.execute_model_state.dp_sync
dp_sync_state = self.execute_model_state.dp_sync_state
self.execute_model_state = None
self.step_timing.forward_end()
@@ -921,7 +922,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
mm_inputs = [], all_false
spec_hidden_states = self._get_drafter_hidden_states(hidden_states)
spec_hidden_states = get_drafter_hidden_states(self.model, hidden_states)
if isinstance(self.sampler, GPUWatermarkSampler):
self.speculator.prepare_watermarking(
self.sampler, input_batch.idx_mapping
@@ -943,7 +944,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
next_prefill_tokens=self.req_states.next_prefill_tokens,
temperature=self.sampler.sampling_states.temperature.gpu,
seeds=self.sampler.sampling_states.seeds.gpu,
dp_sync=dp_sync,
dp_sync_state=dp_sync_state,
dummy_run=True,
skip_attn_for_dummy_run=skip_attn,
mm_inputs=mm_inputs,
@@ -1771,7 +1772,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# cross-attention cache with dynamic encoder outputs.
skip_compiled = True
batch_desc, dp_sync = dispatch_cg_and_sync_dp(
batch_desc, dp_sync_state = dispatch_cg_and_sync_dp(
self.cudagraph_manager,
num_reqs,
num_toks,
@@ -1983,13 +1984,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
self.step_timing.forward_start()
connector_kwargs = dict(
scheduler_output=scheduler_output,
request_state_indices=input_batch.idx_mapping,
request_ids=input_batch.req_ids,
num_tokens=input_batch.num_tokens,
)
# Run model.
if batch_desc.cg_mode == CUDAGraphMode.FULL:
# Use explicit cudagraph replay for FULL mode.
@@ -1997,7 +1991,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# because they are already copied to the CUDA graph input buffers.
assert self.cudagraph_manager is not None
self.kv_connector.pre_forward(
**connector_kwargs, attn_metadata=attn_metadata
scheduler_output, input_batch, attn_metadata=attn_metadata
)
model_output = self.cudagraph_manager.run_fullgraph(batch_desc)
else:
@@ -2013,16 +2007,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.vllm_config,
num_tokens=input_batch.num_tokens_after_padding,
cudagraph_runtime_mode=batch_desc.cg_mode,
num_tokens_across_dp=(
dp_sync.num_tokens_across_dp if dp_sync is not None else None
),
num_tokens_across_dp=dp_sync_state
and dp_sync_state.num_tokens_across_dp,
batch_descriptor=batch_descriptor,
ubatch_slices=ubatch_slices,
slot_mapping=slot_mappings_by_layer,
skip_compiled=skip_compiled,
is_padding=input_batch.is_padding,
):
self.kv_connector.pre_forward(**connector_kwargs)
self.kv_connector.pre_forward(scheduler_output, input_batch)
if ubatch_state is not None:
assert self.ubatch_runner is not None
model_output = self.ubatch_runner.run(
@@ -2064,7 +2057,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
slot_mappings_by_layer=slot_mappings_by_layer,
hidden_states=hidden_states,
aux_hidden_states=aux_hidden_states,
dp_sync=dp_sync,
dp_sync_state=dp_sync_state,
finished_req_ids=finished_req_ids,
ec_connector_output=ec_connector_output,
cudagraph_stats=cudagraph_stats,
@@ -2080,24 +2073,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
return None
def _get_drafter_hidden_states(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Hidden states fed to the drafter.
Targets such as DeepSeek V4 expose the pre-hc_head residual through
get_mtp_target_hidden_states(). The buffer is sized at
max_num_batched_tokens and only allocated for drafters that consume
target hidden states, so None means "use the regular hidden states".
"""
get_target_hidden_states = getattr(
self.model, "get_mtp_target_hidden_states", None
)
if get_target_hidden_states is None:
return hidden_states
target_hidden_states = get_target_hidden_states()
if target_hidden_states is None:
return hidden_states
return target_hidden_states[: hidden_states.shape[0]]
@torch.inference_mode()
@step_eplb_after()
def sample_tokens(
@@ -2112,7 +2087,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
slot_mappings_by_layer = self.execute_model_state.slot_mappings_by_layer
hidden_states = self.execute_model_state.hidden_states
aux_hidden_states = self.execute_model_state.aux_hidden_states
dp_sync = self.execute_model_state.dp_sync
dp_sync_state = self.execute_model_state.dp_sync_state
finished_req_ids = self.execute_model_state.finished_req_ids
ec_connector_output = self.execute_model_state.ec_connector_output
cudagraph_stats = self.execute_model_state.cudagraph_stats
@@ -2243,7 +2218,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.speculator.observe_verification(
input_batch.idx_mapping, num_sampled, num_rejected
)
spec_hidden_states = self._get_drafter_hidden_states(draft_hidden_states)
spec_hidden_states = get_drafter_hidden_states(
self.model, draft_hidden_states
)
if isinstance(self.sampler, GPUWatermarkSampler):
self.speculator.prepare_watermarking(
self.sampler, input_batch.idx_mapping
@@ -2262,7 +2239,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.sampler.sampling_states.temperature.gpu,
self.sampler.sampling_states.seeds.gpu,
num_speculative_tokens=num_spec_tokens,
dp_sync=dp_sync,
dp_sync_state=dp_sync_state,
mm_inputs=mm_inputs,
)
if num_spec_tokens < self.num_speculative_steps:
@@ -2419,7 +2396,7 @@ class ExecuteModelState(NamedTuple):
slot_mappings_by_layer: dict[str, torch.Tensor] | None
hidden_states: torch.Tensor | None
aux_hidden_states: list[torch.Tensor] | None
dp_sync: DPSyncState | None
dp_sync_state: DPSyncState | None
finished_req_ids: set[str]
ec_connector_output: ECConnectorOutput | None
cudagraph_stats: CUDAGraphStat | None