[Model Runner V2][Spec Decode] Handle tuple hidden states from MTP draft models (#46786)

This commit is contained in:
Chauncey
2026-06-27 18:38:07 -07:00
committed by GitHub
parent 798185d438
commit 11a12305c0
4 changed files with 85 additions and 22 deletions
@@ -0,0 +1,82 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import nullcontext
from types import SimpleNamespace
import torch
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
class _TestSpeculator(AutoRegressiveSpeculator):
def load_draft_model(self, target_model, target_attn_layer_names):
raise NotImplementedError
class _DraftModel(torch.nn.Module):
def __init__(self, output: torch.Tensor | tuple[torch.Tensor, torch.Tensor]):
super().__init__()
self.output = output
def forward(self, **kwargs):
return self.output
def _make_speculator(
monkeypatch,
output: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
) -> _TestSpeculator:
monkeypatch.setattr(
spec_module,
"set_forward_context",
lambda *args, **kwargs: nullcontext(),
)
speculator = object.__new__(_TestSpeculator)
speculator.supports_mm_inputs = False
speculator.vllm_config = None
speculator.input_buffers = SimpleNamespace(
input_ids=torch.arange(4),
positions=torch.arange(4),
)
speculator.hidden_states = torch.zeros(4, 3)
speculator.model = _DraftModel(output)
return speculator
def test_run_model_unpacks_tuple_return_for_mtp(monkeypatch):
logits_hidden = torch.full((4, 3), 1.0)
feedback_hidden = torch.full((4, 3), 2.0)
speculator = _make_speculator(monkeypatch, (logits_hidden, feedback_hidden))
actual_logits_hidden, actual_feedback_hidden = speculator._run_model(
4,
attn_metadata=None,
slot_mappings=None,
num_tokens_across_dp=None,
cudagraph_runtime_mode=CUDAGraphMode.NONE,
)
assert actual_logits_hidden is logits_hidden
assert actual_feedback_hidden is feedback_hidden
def test_run_model_reuses_tensor_return_for_mtp(monkeypatch):
hidden = torch.full((4, 3), 1.0)
speculator = _make_speculator(monkeypatch, hidden)
actual_logits_hidden, actual_feedback_hidden = speculator._run_model(
4,
attn_metadata=None,
slot_mappings=None,
num_tokens_across_dp=None,
cudagraph_runtime_mode=CUDAGraphMode.NONE,
)
assert actual_logits_hidden is hidden
assert actual_feedback_hidden is hidden