[Model Runner V2][Spec Decode] Handle tuple hidden states from MTP draft models (#46786)
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
|
||||
AutoRegressiveSpeculator,
|
||||
)
|
||||
|
||||
|
||||
class _TestSpeculator(AutoRegressiveSpeculator):
|
||||
def load_draft_model(self, target_model, target_attn_layer_names):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class _DraftModel(torch.nn.Module):
|
||||
def __init__(self, output: torch.Tensor | tuple[torch.Tensor, torch.Tensor]):
|
||||
super().__init__()
|
||||
self.output = output
|
||||
|
||||
def forward(self, **kwargs):
|
||||
return self.output
|
||||
|
||||
|
||||
def _make_speculator(
|
||||
monkeypatch,
|
||||
output: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
) -> _TestSpeculator:
|
||||
monkeypatch.setattr(
|
||||
spec_module,
|
||||
"set_forward_context",
|
||||
lambda *args, **kwargs: nullcontext(),
|
||||
)
|
||||
|
||||
speculator = object.__new__(_TestSpeculator)
|
||||
speculator.supports_mm_inputs = False
|
||||
speculator.vllm_config = None
|
||||
speculator.input_buffers = SimpleNamespace(
|
||||
input_ids=torch.arange(4),
|
||||
positions=torch.arange(4),
|
||||
)
|
||||
speculator.hidden_states = torch.zeros(4, 3)
|
||||
speculator.model = _DraftModel(output)
|
||||
return speculator
|
||||
|
||||
|
||||
def test_run_model_unpacks_tuple_return_for_mtp(monkeypatch):
|
||||
logits_hidden = torch.full((4, 3), 1.0)
|
||||
feedback_hidden = torch.full((4, 3), 2.0)
|
||||
speculator = _make_speculator(monkeypatch, (logits_hidden, feedback_hidden))
|
||||
|
||||
actual_logits_hidden, actual_feedback_hidden = speculator._run_model(
|
||||
4,
|
||||
attn_metadata=None,
|
||||
slot_mappings=None,
|
||||
num_tokens_across_dp=None,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
)
|
||||
|
||||
assert actual_logits_hidden is logits_hidden
|
||||
assert actual_feedback_hidden is feedback_hidden
|
||||
|
||||
|
||||
def test_run_model_reuses_tensor_return_for_mtp(monkeypatch):
|
||||
hidden = torch.full((4, 3), 1.0)
|
||||
speculator = _make_speculator(monkeypatch, hidden)
|
||||
|
||||
actual_logits_hidden, actual_feedback_hidden = speculator._run_model(
|
||||
4,
|
||||
attn_metadata=None,
|
||||
slot_mappings=None,
|
||||
num_tokens_across_dp=None,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
)
|
||||
|
||||
assert actual_logits_hidden is hidden
|
||||
assert actual_feedback_hidden is hidden
|
||||
@@ -60,16 +60,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
"""
|
||||
return True
|
||||
|
||||
@property
|
||||
def model_returns_tuple(self) -> bool:
|
||||
"""
|
||||
Whether the draft model's forward() returns a tuple.
|
||||
|
||||
True: returns (last_hidden_states, hidden_states) — Eagle, Gemma4 MTP.
|
||||
False: returns a single tensor used for both — standard MTP (DeepSeek).
|
||||
"""
|
||||
return True
|
||||
|
||||
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||
# Initialize cudagraph manager for draft prefill (draft position 0).
|
||||
self.prefill_cudagraph_manager = PrefillSpeculatorCudaGraphManager(
|
||||
@@ -328,7 +318,9 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
else:
|
||||
# Eager (NONE): call the raw model directly.
|
||||
ret_hidden_states = self.model(**model_inputs)
|
||||
if self.model_returns_tuple:
|
||||
# Some MTP models declare a single-tensor contract but return
|
||||
# (logits_hidden, feedback_hidden) for final-norm correctness.
|
||||
if isinstance(ret_hidden_states, tuple):
|
||||
last_hidden_states, hidden_states = ret_hidden_states
|
||||
else:
|
||||
last_hidden_states = ret_hidden_states
|
||||
|
||||
@@ -30,13 +30,6 @@ class Gemma4Speculator(AutoRegressiveSpeculator):
|
||||
# No new KV slots are written, so positions and seq_lens stay fixed.
|
||||
return False
|
||||
|
||||
@property
|
||||
def model_returns_tuple(self) -> bool:
|
||||
# forward() returns (draft_hidden_states, backbone_hidden_states).
|
||||
# The proposer uses draft_hidden_states for compute_logits and
|
||||
# backbone_hidden_states for the hidden-state feedback buffer.
|
||||
return True
|
||||
|
||||
def load_draft_model(
|
||||
self,
|
||||
target_model: nn.Module,
|
||||
|
||||
@@ -10,10 +10,6 @@ from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
|
||||
|
||||
|
||||
class MTPSpeculator(AutoRegressiveSpeculator):
|
||||
@property
|
||||
def model_returns_tuple(self) -> bool:
|
||||
return False
|
||||
|
||||
def load_draft_model(
|
||||
self,
|
||||
target_model: nn.Module,
|
||||
|
||||
Reference in New Issue
Block a user