[MRV2] Miscellaneous code structure updates (#60754)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -13,6 +13,25 @@ from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def get_drafter_hidden_states(
|
||||
model: torch.nn.Module, hidden_states: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Hidden states fed to the drafter.
|
||||
|
||||
Targets such as DeepSeek V4 expose the pre-hc_head residual through
|
||||
get_mtp_target_hidden_states(). The buffer is sized at
|
||||
max_num_batched_tokens and only allocated for drafters that consume
|
||||
target hidden states, so None means "use the regular hidden states".
|
||||
"""
|
||||
get_target_hidden_states = getattr(model, "get_mtp_target_hidden_states", None)
|
||||
if get_target_hidden_states is None:
|
||||
return hidden_states
|
||||
target_hidden_states = get_target_hidden_states()
|
||||
if target_hidden_states is None:
|
||||
return hidden_states
|
||||
return target_hidden_states[: hidden_states.shape[0]]
|
||||
|
||||
|
||||
def get_pp_safe_draft_load_config(load_config: LoadConfig) -> LoadConfig:
|
||||
"""Avoid collectives that include PP ranks without a draft model."""
|
||||
if get_pp_group().world_size > 1 and load_config.load_format == "fastsafetensors":
|
||||
|
||||
Reference in New Issue
Block a user