[Feature][Model Runner V2] Support extract_hidden_states speculation (#49811)

Signed-off-by: Zupeng Wang <71580390+zupengwang@users.noreply.github.com>
Signed-off-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: OpenAI <support@openai.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Zupeng Wang
2026-08-20 11:13:43 -04:00
committed by GitHub
co-authored by OpenAI Misha Goin OpenAI Codex
parent cb09dd7488
commit 01af92e175
6 changed files with 314 additions and 2 deletions
+15
View File
@@ -5,6 +5,7 @@ import logging
import os
from dataclasses import MISSING, Field, asdict, dataclass, field
from types import SimpleNamespace
from typing import cast
from unittest.mock import patch
import pydantic
@@ -263,6 +264,20 @@ def test_dsa_models_select_matching_mtp(model_type, expected_architecture):
assert hf_config.architectures == [expected_architecture]
def test_v2_model_runner_supports_extract_hidden_states():
config = VllmConfig()
config.speculative_config = cast(
SpeculativeConfig,
SimpleNamespace(
method="extract_hidden_states",
parallel_drafting=False,
enable_adaptive_verification=False,
),
)
assert config._get_v2_model_runner_unsupported_features() == []
@pytest.mark.parametrize(
("use_v2_model_runner", "expected_capture_sizes"),
[
@@ -0,0 +1,132 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import nullcontext
from types import SimpleNamespace
from typing import Any, cast
import pytest
import torch
from vllm.v1.worker.gpu.spec_decode import extract_hidden_states as spec_module
from vllm.v1.worker.gpu.spec_decode import init_speculator
from vllm.v1.worker.gpu.spec_decode.extract_hidden_states import (
ExtractHiddenStatesSpeculator,
)
class _RecordingModel(torch.nn.Module):
def forward(self, *, hidden_states: torch.Tensor) -> None:
self.hidden_states = hidden_states.clone()
def test_init_requires_greedy_draft_sampling():
vllm_config = cast(
Any,
SimpleNamespace(
speculative_config=SimpleNamespace(draft_sample_method="probabilistic")
),
)
with pytest.raises(ValueError, match="only supports draft_sample_method='greedy'"):
ExtractHiddenStatesSpeculator(vllm_config, torch.device("cpu"))
def test_init_speculator_dispatches_extract_hidden_states(monkeypatch):
vllm_config = cast(
Any,
SimpleNamespace(
speculative_config=SimpleNamespace(method="extract_hidden_states")
),
)
device = torch.device("cpu")
def fake_speculator(config, target_device):
return config, target_device
monkeypatch.setattr(spec_module, "ExtractHiddenStatesSpeculator", fake_speculator)
assert init_speculator(vllm_config, device) == (vllm_config, device)
def test_propose_caches_hidden_states_and_returns_sampled_tokens(monkeypatch):
contexts = []
def fake_set_forward_context(*args, **kwargs):
contexts.append((args, kwargs))
return nullcontext()
monkeypatch.setattr(spec_module, "set_forward_context", fake_set_forward_context)
layer_name = "cache_only_layers.2"
speculator = object.__new__(ExtractHiddenStatesSpeculator)
speculator.vllm_config = cast(Any, SimpleNamespace())
speculator.num_hidden_states = 2
speculator.hidden_states = torch.zeros(4, 2, 3)
speculator.draft_attn_layer_names = {layer_name}
speculator.model = _RecordingModel()
input_batch = cast(
Any,
SimpleNamespace(
idx_mapping=torch.tensor([2, 0], dtype=torch.int32),
is_padding=torch.zeros(4, dtype=torch.bool),
),
)
aux_hidden_states = [
torch.full((4, 3), 1.0),
torch.full((4, 3), 2.0),
]
attn_metadata = {layer_name: object(), "target_layer": object()}
slot_mappings = {
layer_name: torch.arange(4),
"target_layer": torch.arange(4),
}
last_sampled = torch.tensor([[10], [11], [12]], dtype=torch.int64)
draft_tokens = speculator.propose(
input_batch=input_batch,
attn_metadata=attn_metadata,
slot_mappings=slot_mappings,
last_hidden_states=torch.empty(0),
aux_hidden_states=aux_hidden_states,
num_sampled=torch.empty(0),
num_rejected=torch.empty(0),
last_sampled=last_sampled,
next_prefill_tokens=torch.empty(0),
temperature=torch.empty(0),
seeds=torch.empty(0),
)
expected_hidden_states = torch.stack(aux_hidden_states, dim=1)
assert torch.equal(speculator.model.hidden_states, expected_hidden_states)
assert torch.equal(draft_tokens, torch.tensor([[12], [10]]))
assert len(contexts) == 1
args, kwargs = contexts[0]
assert args[0] == {layer_name: attn_metadata[layer_name]}
assert kwargs["num_tokens"] == 4
assert set(kwargs["slot_mapping"]) == {layer_name}
assert torch.equal(kwargs["slot_mapping"][layer_name], slot_mappings[layer_name])
def test_propose_requires_aux_hidden_states():
speculator = object.__new__(ExtractHiddenStatesSpeculator)
speculator.num_hidden_states = 2
input_batch = cast(
Any, SimpleNamespace(idx_mapping=torch.tensor([0], dtype=torch.int32))
)
with pytest.raises(ValueError, match="aux_hidden_states are required"):
speculator.propose(
input_batch=input_batch,
attn_metadata={},
slot_mappings={},
last_hidden_states=torch.empty(0),
aux_hidden_states=None,
num_sampled=torch.empty(0),
num_rejected=torch.empty(0),
last_sampled=torch.tensor([[10]]),
next_prefill_tokens=torch.empty(0),
temperature=torch.empty(0),
seeds=torch.empty(0),
)
+1
View File
@@ -2429,6 +2429,7 @@ class VllmConfig:
"mtp",
"dflash",
"dspark",
"extract_hidden_states",
):
unsupported.append(f"speculative method '{speculative_config.method}'")
+6 -1
View File
@@ -247,7 +247,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.is_last_pp_rank:
self.speculator = init_speculator(self.vllm_config, self.device)
if self.speculative_config.method in ("eagle3", "dflash", "dspark"):
if self.speculative_config.method in (
"eagle3",
"dflash",
"dspark",
"extract_hidden_states",
):
# Drafting may require auxiliary hidden states from target model outputs
self.use_aux_hidden_state_outputs = True
if self.use_pp:
+7 -1
View File
@@ -8,7 +8,13 @@ from vllm.config import VllmConfig
def init_speculator(vllm_config: VllmConfig, device: torch.device):
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
if speculative_config.method == "dflash":
if speculative_config.method == "extract_hidden_states":
from vllm.v1.worker.gpu.spec_decode.extract_hidden_states import (
ExtractHiddenStatesSpeculator,
)
return ExtractHiddenStatesSpeculator(vllm_config, device)
elif speculative_config.method == "dflash":
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
DFlashSpeculator,
)
@@ -0,0 +1,153 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import torch
import torch.nn as nn
from vllm.compilation.backends import set_model_tag
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.forward_context import set_forward_context
from vllm.model_executor.model_loader import get_model
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
class ExtractHiddenStatesSpeculator(DraftModelSpeculator):
"""Cache target hidden states while returning always-accepted draft tokens."""
def __init__(self, vllm_config: VllmConfig, device: torch.device):
assert vllm_config.speculative_config is not None
if vllm_config.speculative_config.draft_sample_method != "greedy":
raise ValueError(
"extract_hidden_states only supports draft_sample_method='greedy'"
)
super().__init__(vllm_config, device)
if self.num_speculative_steps != 1:
raise ValueError(
"extract_hidden_states requires num_speculative_tokens to be 1"
)
if self.speculative_config.disable_padded_drafter_batch:
raise ValueError(
"disable_padded_drafter_batch is not supported with "
"extract_hidden_states method"
)
self.supports_mm_inputs = False
layer_ids = getattr(
self.draft_model_config.hf_config,
"eagle_aux_hidden_state_layer_ids",
None,
)
if not layer_ids:
raise ValueError(
"eagle_aux_hidden_state_layer_ids must be set in the draft "
"model config for extract_hidden_states method"
)
self.num_hidden_states = len(layer_ids)
assert isinstance(self.dtype, torch.dtype)
self.hidden_states = torch.zeros(
self.max_num_tokens,
self.num_hidden_states,
self.vllm_config.model_config.get_hidden_size(),
dtype=self.dtype,
device=device,
)
def load_draft_model(
self,
target_model: nn.Module,
target_attn_layer_names: set[str],
) -> nn.Module:
del target_model, target_attn_layer_names
with set_model_tag("extract_hidden_states"):
return get_model(
vllm_config=self.vllm_config,
model_config=self.draft_model_config,
)
def load_model(self, target_model: nn.Module) -> None:
super().load_model(target_model)
if len(self.draft_attn_layer_names) != 1:
raise ValueError(
"ExtractHiddenStatesModel should have exactly one attention "
f"layer, found {len(self.draft_attn_layer_names)}"
)
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
del cudagraph_mode
def capture(self) -> None:
return None
@torch.inference_mode()
def propose(
self,
input_batch: InputBatch,
attn_metadata: dict[str, Any],
slot_mappings: dict[str, torch.Tensor],
last_hidden_states: torch.Tensor,
aux_hidden_states: list[torch.Tensor] | None,
num_sampled: torch.Tensor,
num_rejected: torch.Tensor,
last_sampled: torch.Tensor,
next_prefill_tokens: torch.Tensor,
temperature: torch.Tensor,
seeds: torch.Tensor,
num_tokens_across_dp: torch.Tensor | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
is_profile: bool = False,
) -> torch.Tensor:
del (
last_hidden_states,
num_sampled,
num_rejected,
next_prefill_tokens,
temperature,
seeds,
dummy_run,
mm_inputs,
is_profile,
)
draft_tokens = last_sampled[input_batch.idx_mapping, :1]
if skip_attn_for_dummy_run:
return draft_tokens
if aux_hidden_states is None:
raise ValueError(
"aux_hidden_states are required when using extract_hidden_states"
)
if len(aux_hidden_states) != self.num_hidden_states:
raise ValueError(
f"Expected {self.num_hidden_states} auxiliary hidden states, "
f"got {len(aux_hidden_states)}"
)
stacked_hidden_states = torch.stack(aux_hidden_states, dim=1)
num_tokens = stacked_hidden_states.shape[0]
self.hidden_states[:num_tokens].copy_(stacked_hidden_states)
draft_attn_metadata = {
name: attn_metadata[name] for name in self.draft_attn_layer_names
}
draft_slot_mappings = {
name: slot_mappings[name][:num_tokens]
for name in self.draft_attn_layer_names
}
with set_forward_context(
draft_attn_metadata,
self.vllm_config,
num_tokens=num_tokens,
num_tokens_across_dp=num_tokens_across_dp,
cudagraph_runtime_mode=CUDAGraphMode.NONE,
slot_mapping=draft_slot_mappings,
is_padding=input_batch.is_padding[:num_tokens],
):
self.model(hidden_states=self.hidden_states[:num_tokens])
return draft_tokens