[Model Runner V2][Spec Decode] Handle tuple hidden states from MTP draft models (#46786)
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
|
||||
AutoRegressiveSpeculator,
|
||||
)
|
||||
|
||||
|
||||
class _TestSpeculator(AutoRegressiveSpeculator):
|
||||
def load_draft_model(self, target_model, target_attn_layer_names):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class _DraftModel(torch.nn.Module):
|
||||
def __init__(self, output: torch.Tensor | tuple[torch.Tensor, torch.Tensor]):
|
||||
super().__init__()
|
||||
self.output = output
|
||||
|
||||
def forward(self, **kwargs):
|
||||
return self.output
|
||||
|
||||
|
||||
def _make_speculator(
|
||||
monkeypatch,
|
||||
output: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
) -> _TestSpeculator:
|
||||
monkeypatch.setattr(
|
||||
spec_module,
|
||||
"set_forward_context",
|
||||
lambda *args, **kwargs: nullcontext(),
|
||||
)
|
||||
|
||||
speculator = object.__new__(_TestSpeculator)
|
||||
speculator.supports_mm_inputs = False
|
||||
speculator.vllm_config = None
|
||||
speculator.input_buffers = SimpleNamespace(
|
||||
input_ids=torch.arange(4),
|
||||
positions=torch.arange(4),
|
||||
)
|
||||
speculator.hidden_states = torch.zeros(4, 3)
|
||||
speculator.model = _DraftModel(output)
|
||||
return speculator
|
||||
|
||||
|
||||
def test_run_model_unpacks_tuple_return_for_mtp(monkeypatch):
|
||||
logits_hidden = torch.full((4, 3), 1.0)
|
||||
feedback_hidden = torch.full((4, 3), 2.0)
|
||||
speculator = _make_speculator(monkeypatch, (logits_hidden, feedback_hidden))
|
||||
|
||||
actual_logits_hidden, actual_feedback_hidden = speculator._run_model(
|
||||
4,
|
||||
attn_metadata=None,
|
||||
slot_mappings=None,
|
||||
num_tokens_across_dp=None,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
)
|
||||
|
||||
assert actual_logits_hidden is logits_hidden
|
||||
assert actual_feedback_hidden is feedback_hidden
|
||||
|
||||
|
||||
def test_run_model_reuses_tensor_return_for_mtp(monkeypatch):
|
||||
hidden = torch.full((4, 3), 1.0)
|
||||
speculator = _make_speculator(monkeypatch, hidden)
|
||||
|
||||
actual_logits_hidden, actual_feedback_hidden = speculator._run_model(
|
||||
4,
|
||||
attn_metadata=None,
|
||||
slot_mappings=None,
|
||||
num_tokens_across_dp=None,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
)
|
||||
|
||||
assert actual_logits_hidden is hidden
|
||||
assert actual_feedback_hidden is hidden
|
||||
Reference in New Issue
Block a user