[MRV2] Miscellaneous code structure updates (#60754)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -162,7 +162,10 @@ from vllm.v1.worker.gpu.spec_decode.rejection_sampler import (
|
||||
get_max_chunk_logits,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import (
|
||||
DraftTokensHandler,
|
||||
get_drafter_hidden_states,
|
||||
)
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
from vllm.v1.worker.gpu.structured_outputs import (
|
||||
StructuredOutputsWorker,
|
||||
@@ -578,9 +581,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
def get_encoder_timing_stats(self) -> dict[str, dict[str, float | int]]:
|
||||
encoder_runner = getattr(self.model_state, "encoder_runner", None)
|
||||
if encoder_runner is None:
|
||||
return {}
|
||||
return encoder_runner.get_encoder_timing_stats()
|
||||
return encoder_runner.get_encoder_timing_stats() if encoder_runner else {}
|
||||
|
||||
def get_kv_cache_spec(self):
|
||||
kv_cache_spec = get_kv_cache_spec(self.vllm_config)
|
||||
@@ -905,7 +906,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
slot_mappings_by_layer = self.execute_model_state.slot_mappings_by_layer
|
||||
hidden_states = self.execute_model_state.hidden_states
|
||||
aux_hidden_states = self.execute_model_state.aux_hidden_states
|
||||
dp_sync = self.execute_model_state.dp_sync
|
||||
dp_sync_state = self.execute_model_state.dp_sync_state
|
||||
self.execute_model_state = None
|
||||
|
||||
self.step_timing.forward_end()
|
||||
@@ -921,7 +922,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
mm_inputs = [], all_false
|
||||
|
||||
spec_hidden_states = self._get_drafter_hidden_states(hidden_states)
|
||||
spec_hidden_states = get_drafter_hidden_states(self.model, hidden_states)
|
||||
if isinstance(self.sampler, GPUWatermarkSampler):
|
||||
self.speculator.prepare_watermarking(
|
||||
self.sampler, input_batch.idx_mapping
|
||||
@@ -943,7 +944,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
next_prefill_tokens=self.req_states.next_prefill_tokens,
|
||||
temperature=self.sampler.sampling_states.temperature.gpu,
|
||||
seeds=self.sampler.sampling_states.seeds.gpu,
|
||||
dp_sync=dp_sync,
|
||||
dp_sync_state=dp_sync_state,
|
||||
dummy_run=True,
|
||||
skip_attn_for_dummy_run=skip_attn,
|
||||
mm_inputs=mm_inputs,
|
||||
@@ -1771,7 +1772,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# cross-attention cache with dynamic encoder outputs.
|
||||
skip_compiled = True
|
||||
|
||||
batch_desc, dp_sync = dispatch_cg_and_sync_dp(
|
||||
batch_desc, dp_sync_state = dispatch_cg_and_sync_dp(
|
||||
self.cudagraph_manager,
|
||||
num_reqs,
|
||||
num_toks,
|
||||
@@ -1983,13 +1984,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
self.step_timing.forward_start()
|
||||
|
||||
connector_kwargs = dict(
|
||||
scheduler_output=scheduler_output,
|
||||
request_state_indices=input_batch.idx_mapping,
|
||||
request_ids=input_batch.req_ids,
|
||||
num_tokens=input_batch.num_tokens,
|
||||
)
|
||||
|
||||
# Run model.
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
# Use explicit cudagraph replay for FULL mode.
|
||||
@@ -1997,7 +1991,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# because they are already copied to the CUDA graph input buffers.
|
||||
assert self.cudagraph_manager is not None
|
||||
self.kv_connector.pre_forward(
|
||||
**connector_kwargs, attn_metadata=attn_metadata
|
||||
scheduler_output, input_batch, attn_metadata=attn_metadata
|
||||
)
|
||||
model_output = self.cudagraph_manager.run_fullgraph(batch_desc)
|
||||
else:
|
||||
@@ -2013,16 +2007,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.vllm_config,
|
||||
num_tokens=input_batch.num_tokens_after_padding,
|
||||
cudagraph_runtime_mode=batch_desc.cg_mode,
|
||||
num_tokens_across_dp=(
|
||||
dp_sync.num_tokens_across_dp if dp_sync is not None else None
|
||||
),
|
||||
num_tokens_across_dp=dp_sync_state
|
||||
and dp_sync_state.num_tokens_across_dp,
|
||||
batch_descriptor=batch_descriptor,
|
||||
ubatch_slices=ubatch_slices,
|
||||
slot_mapping=slot_mappings_by_layer,
|
||||
skip_compiled=skip_compiled,
|
||||
is_padding=input_batch.is_padding,
|
||||
):
|
||||
self.kv_connector.pre_forward(**connector_kwargs)
|
||||
self.kv_connector.pre_forward(scheduler_output, input_batch)
|
||||
if ubatch_state is not None:
|
||||
assert self.ubatch_runner is not None
|
||||
model_output = self.ubatch_runner.run(
|
||||
@@ -2064,7 +2057,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
slot_mappings_by_layer=slot_mappings_by_layer,
|
||||
hidden_states=hidden_states,
|
||||
aux_hidden_states=aux_hidden_states,
|
||||
dp_sync=dp_sync,
|
||||
dp_sync_state=dp_sync_state,
|
||||
finished_req_ids=finished_req_ids,
|
||||
ec_connector_output=ec_connector_output,
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
@@ -2080,24 +2073,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
return None
|
||||
|
||||
def _get_drafter_hidden_states(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Hidden states fed to the drafter.
|
||||
|
||||
Targets such as DeepSeek V4 expose the pre-hc_head residual through
|
||||
get_mtp_target_hidden_states(). The buffer is sized at
|
||||
max_num_batched_tokens and only allocated for drafters that consume
|
||||
target hidden states, so None means "use the regular hidden states".
|
||||
"""
|
||||
get_target_hidden_states = getattr(
|
||||
self.model, "get_mtp_target_hidden_states", None
|
||||
)
|
||||
if get_target_hidden_states is None:
|
||||
return hidden_states
|
||||
target_hidden_states = get_target_hidden_states()
|
||||
if target_hidden_states is None:
|
||||
return hidden_states
|
||||
return target_hidden_states[: hidden_states.shape[0]]
|
||||
|
||||
@torch.inference_mode()
|
||||
@step_eplb_after()
|
||||
def sample_tokens(
|
||||
@@ -2112,7 +2087,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
slot_mappings_by_layer = self.execute_model_state.slot_mappings_by_layer
|
||||
hidden_states = self.execute_model_state.hidden_states
|
||||
aux_hidden_states = self.execute_model_state.aux_hidden_states
|
||||
dp_sync = self.execute_model_state.dp_sync
|
||||
dp_sync_state = self.execute_model_state.dp_sync_state
|
||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||
ec_connector_output = self.execute_model_state.ec_connector_output
|
||||
cudagraph_stats = self.execute_model_state.cudagraph_stats
|
||||
@@ -2243,7 +2218,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.speculator.observe_verification(
|
||||
input_batch.idx_mapping, num_sampled, num_rejected
|
||||
)
|
||||
spec_hidden_states = self._get_drafter_hidden_states(draft_hidden_states)
|
||||
spec_hidden_states = get_drafter_hidden_states(
|
||||
self.model, draft_hidden_states
|
||||
)
|
||||
if isinstance(self.sampler, GPUWatermarkSampler):
|
||||
self.speculator.prepare_watermarking(
|
||||
self.sampler, input_batch.idx_mapping
|
||||
@@ -2262,7 +2239,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.sampler.sampling_states.temperature.gpu,
|
||||
self.sampler.sampling_states.seeds.gpu,
|
||||
num_speculative_tokens=num_spec_tokens,
|
||||
dp_sync=dp_sync,
|
||||
dp_sync_state=dp_sync_state,
|
||||
mm_inputs=mm_inputs,
|
||||
)
|
||||
if num_spec_tokens < self.num_speculative_steps:
|
||||
@@ -2419,7 +2396,7 @@ class ExecuteModelState(NamedTuple):
|
||||
slot_mappings_by_layer: dict[str, torch.Tensor] | None
|
||||
hidden_states: torch.Tensor | None
|
||||
aux_hidden_states: list[torch.Tensor] | None
|
||||
dp_sync: DPSyncState | None
|
||||
dp_sync_state: DPSyncState | None
|
||||
finished_req_ids: set[str]
|
||||
ec_connector_output: ECConnectorOutput | None
|
||||
cudagraph_stats: CUDAGraphStat | None
|
||||
|
||||
Reference in New Issue
Block a user