diff --git a/tests/v1/worker/test_gpu_autoregressive_speculator.py b/tests/v1/worker/test_gpu_autoregressive_speculator.py new file mode 100644 index 0000000000..940fb375a9 --- /dev/null +++ b/tests/v1/worker/test_gpu_autoregressive_speculator.py @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from contextlib import nullcontext +from types import SimpleNamespace + +import torch + +from vllm.config.compilation import CUDAGraphMode +from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module +from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( + AutoRegressiveSpeculator, +) + + +class _TestSpeculator(AutoRegressiveSpeculator): + def load_draft_model(self, target_model, target_attn_layer_names): + raise NotImplementedError + + +class _DraftModel(torch.nn.Module): + def __init__(self, output: torch.Tensor | tuple[torch.Tensor, torch.Tensor]): + super().__init__() + self.output = output + + def forward(self, **kwargs): + return self.output + + +def _make_speculator( + monkeypatch, + output: torch.Tensor | tuple[torch.Tensor, torch.Tensor], +) -> _TestSpeculator: + monkeypatch.setattr( + spec_module, + "set_forward_context", + lambda *args, **kwargs: nullcontext(), + ) + + speculator = object.__new__(_TestSpeculator) + speculator.supports_mm_inputs = False + speculator.vllm_config = None + speculator.input_buffers = SimpleNamespace( + input_ids=torch.arange(4), + positions=torch.arange(4), + ) + speculator.hidden_states = torch.zeros(4, 3) + speculator.model = _DraftModel(output) + return speculator + + +def test_run_model_unpacks_tuple_return_for_mtp(monkeypatch): + logits_hidden = torch.full((4, 3), 1.0) + feedback_hidden = torch.full((4, 3), 2.0) + speculator = _make_speculator(monkeypatch, (logits_hidden, feedback_hidden)) + + actual_logits_hidden, actual_feedback_hidden = speculator._run_model( + 4, + attn_metadata=None, + slot_mappings=None, + num_tokens_across_dp=None, + cudagraph_runtime_mode=CUDAGraphMode.NONE, + ) + + assert actual_logits_hidden is logits_hidden + assert actual_feedback_hidden is feedback_hidden + + +def test_run_model_reuses_tensor_return_for_mtp(monkeypatch): + hidden = torch.full((4, 3), 1.0) + speculator = _make_speculator(monkeypatch, hidden) + + actual_logits_hidden, actual_feedback_hidden = speculator._run_model( + 4, + attn_metadata=None, + slot_mappings=None, + num_tokens_across_dp=None, + cudagraph_runtime_mode=CUDAGraphMode.NONE, + ) + + assert actual_logits_hidden is hidden + assert actual_feedback_hidden is hidden diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index 775c06f7b8..422d3ac690 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -60,16 +60,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): """ return True - @property - def model_returns_tuple(self) -> bool: - """ - Whether the draft model's forward() returns a tuple. - - True: returns (last_hidden_states, hidden_states) — Eagle, Gemma4 MTP. - False: returns a single tensor used for both — standard MTP (DeepSeek). - """ - return True - def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None: # Initialize cudagraph manager for draft prefill (draft position 0). self.prefill_cudagraph_manager = PrefillSpeculatorCudaGraphManager( @@ -328,7 +318,9 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): else: # Eager (NONE): call the raw model directly. ret_hidden_states = self.model(**model_inputs) - if self.model_returns_tuple: + # Some MTP models declare a single-tensor contract but return + # (logits_hidden, feedback_hidden) for final-norm correctness. + if isinstance(ret_hidden_states, tuple): last_hidden_states, hidden_states = ret_hidden_states else: last_hidden_states = ret_hidden_states diff --git a/vllm/v1/worker/gpu/spec_decode/gemma4/speculator.py b/vllm/v1/worker/gpu/spec_decode/gemma4/speculator.py index fcbea5d101..dfa2c68010 100644 --- a/vllm/v1/worker/gpu/spec_decode/gemma4/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/gemma4/speculator.py @@ -30,13 +30,6 @@ class Gemma4Speculator(AutoRegressiveSpeculator): # No new KV slots are written, so positions and seq_lens stay fixed. return False - @property - def model_returns_tuple(self) -> bool: - # forward() returns (draft_hidden_states, backbone_hidden_states). - # The proposer uses draft_hidden_states for compute_logits and - # backbone_hidden_states for the hidden-state feedback buffer. - return True - def load_draft_model( self, target_model: nn.Module, diff --git a/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py b/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py index e6abb0be83..4b9354f23e 100644 --- a/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py @@ -10,10 +10,6 @@ from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model class MTPSpeculator(AutoRegressiveSpeculator): - @property - def model_returns_tuple(self) -> bool: - return False - def load_draft_model( self, target_model: nn.Module,