[Model Runner V2][Spec Decode] Handle tuple hidden states from MTP draft models (#46786)

This commit is contained in:
Chauncey
2026-06-27 18:38:07 -07:00
committed by GitHub
parent 798185d438
commit 11a12305c0
4 changed files with 85 additions and 22 deletions
@@ -0,0 +1,82 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import nullcontext
from types import SimpleNamespace
import torch
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
class _TestSpeculator(AutoRegressiveSpeculator):
def load_draft_model(self, target_model, target_attn_layer_names):
raise NotImplementedError
class _DraftModel(torch.nn.Module):
def __init__(self, output: torch.Tensor | tuple[torch.Tensor, torch.Tensor]):
super().__init__()
self.output = output
def forward(self, **kwargs):
return self.output
def _make_speculator(
monkeypatch,
output: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
) -> _TestSpeculator:
monkeypatch.setattr(
spec_module,
"set_forward_context",
lambda *args, **kwargs: nullcontext(),
)
speculator = object.__new__(_TestSpeculator)
speculator.supports_mm_inputs = False
speculator.vllm_config = None
speculator.input_buffers = SimpleNamespace(
input_ids=torch.arange(4),
positions=torch.arange(4),
)
speculator.hidden_states = torch.zeros(4, 3)
speculator.model = _DraftModel(output)
return speculator
def test_run_model_unpacks_tuple_return_for_mtp(monkeypatch):
logits_hidden = torch.full((4, 3), 1.0)
feedback_hidden = torch.full((4, 3), 2.0)
speculator = _make_speculator(monkeypatch, (logits_hidden, feedback_hidden))
actual_logits_hidden, actual_feedback_hidden = speculator._run_model(
4,
attn_metadata=None,
slot_mappings=None,
num_tokens_across_dp=None,
cudagraph_runtime_mode=CUDAGraphMode.NONE,
)
assert actual_logits_hidden is logits_hidden
assert actual_feedback_hidden is feedback_hidden
def test_run_model_reuses_tensor_return_for_mtp(monkeypatch):
hidden = torch.full((4, 3), 1.0)
speculator = _make_speculator(monkeypatch, hidden)
actual_logits_hidden, actual_feedback_hidden = speculator._run_model(
4,
attn_metadata=None,
slot_mappings=None,
num_tokens_across_dp=None,
cudagraph_runtime_mode=CUDAGraphMode.NONE,
)
assert actual_logits_hidden is hidden
assert actual_feedback_hidden is hidden
@@ -60,16 +60,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
"""
return True
@property
def model_returns_tuple(self) -> bool:
"""
Whether the draft model's forward() returns a tuple.
True: returns (last_hidden_states, hidden_states) — Eagle, Gemma4 MTP.
False: returns a single tensor used for both — standard MTP (DeepSeek).
"""
return True
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
# Initialize cudagraph manager for draft prefill (draft position 0).
self.prefill_cudagraph_manager = PrefillSpeculatorCudaGraphManager(
@@ -328,7 +318,9 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
else:
# Eager (NONE): call the raw model directly.
ret_hidden_states = self.model(**model_inputs)
if self.model_returns_tuple:
# Some MTP models declare a single-tensor contract but return
# (logits_hidden, feedback_hidden) for final-norm correctness.
if isinstance(ret_hidden_states, tuple):
last_hidden_states, hidden_states = ret_hidden_states
else:
last_hidden_states = ret_hidden_states
@@ -30,13 +30,6 @@ class Gemma4Speculator(AutoRegressiveSpeculator):
# No new KV slots are written, so positions and seq_lens stay fixed.
return False
@property
def model_returns_tuple(self) -> bool:
# forward() returns (draft_hidden_states, backbone_hidden_states).
# The proposer uses draft_hidden_states for compute_logits and
# backbone_hidden_states for the hidden-state feedback buffer.
return True
def load_draft_model(
self,
target_model: nn.Module,
@@ -10,10 +10,6 @@ from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
class MTPSpeculator(AutoRegressiveSpeculator):
@property
def model_returns_tuple(self) -> bool:
return False
def load_draft_model(
self,
target_model: nn.Module,