[Feature][Model Runner V2] Support extract_hidden_states speculation (#49811)
Signed-off-by: Zupeng Wang <71580390+zupengwang@users.noreply.github.com> Signed-off-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: OpenAI <support@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI
Misha Goin
OpenAI Codex
parent
cb09dd7488
commit
01af92e175
@@ -5,6 +5,7 @@ import logging
|
||||
import os
|
||||
from dataclasses import MISSING, Field, asdict, dataclass, field
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
from unittest.mock import patch
|
||||
|
||||
import pydantic
|
||||
@@ -263,6 +264,20 @@ def test_dsa_models_select_matching_mtp(model_type, expected_architecture):
|
||||
assert hf_config.architectures == [expected_architecture]
|
||||
|
||||
|
||||
def test_v2_model_runner_supports_extract_hidden_states():
|
||||
config = VllmConfig()
|
||||
config.speculative_config = cast(
|
||||
SpeculativeConfig,
|
||||
SimpleNamespace(
|
||||
method="extract_hidden_states",
|
||||
parallel_drafting=False,
|
||||
enable_adaptive_verification=False,
|
||||
),
|
||||
)
|
||||
|
||||
assert config._get_v2_model_runner_unsupported_features() == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("use_v2_model_runner", "expected_capture_sizes"),
|
||||
[
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.worker.gpu.spec_decode import extract_hidden_states as spec_module
|
||||
from vllm.v1.worker.gpu.spec_decode import init_speculator
|
||||
from vllm.v1.worker.gpu.spec_decode.extract_hidden_states import (
|
||||
ExtractHiddenStatesSpeculator,
|
||||
)
|
||||
|
||||
|
||||
class _RecordingModel(torch.nn.Module):
|
||||
def forward(self, *, hidden_states: torch.Tensor) -> None:
|
||||
self.hidden_states = hidden_states.clone()
|
||||
|
||||
|
||||
def test_init_requires_greedy_draft_sampling():
|
||||
vllm_config = cast(
|
||||
Any,
|
||||
SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(draft_sample_method="probabilistic")
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="only supports draft_sample_method='greedy'"):
|
||||
ExtractHiddenStatesSpeculator(vllm_config, torch.device("cpu"))
|
||||
|
||||
|
||||
def test_init_speculator_dispatches_extract_hidden_states(monkeypatch):
|
||||
vllm_config = cast(
|
||||
Any,
|
||||
SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(method="extract_hidden_states")
|
||||
),
|
||||
)
|
||||
device = torch.device("cpu")
|
||||
|
||||
def fake_speculator(config, target_device):
|
||||
return config, target_device
|
||||
|
||||
monkeypatch.setattr(spec_module, "ExtractHiddenStatesSpeculator", fake_speculator)
|
||||
|
||||
assert init_speculator(vllm_config, device) == (vllm_config, device)
|
||||
|
||||
|
||||
def test_propose_caches_hidden_states_and_returns_sampled_tokens(monkeypatch):
|
||||
contexts = []
|
||||
|
||||
def fake_set_forward_context(*args, **kwargs):
|
||||
contexts.append((args, kwargs))
|
||||
return nullcontext()
|
||||
|
||||
monkeypatch.setattr(spec_module, "set_forward_context", fake_set_forward_context)
|
||||
|
||||
layer_name = "cache_only_layers.2"
|
||||
speculator = object.__new__(ExtractHiddenStatesSpeculator)
|
||||
speculator.vllm_config = cast(Any, SimpleNamespace())
|
||||
speculator.num_hidden_states = 2
|
||||
speculator.hidden_states = torch.zeros(4, 2, 3)
|
||||
speculator.draft_attn_layer_names = {layer_name}
|
||||
speculator.model = _RecordingModel()
|
||||
|
||||
input_batch = cast(
|
||||
Any,
|
||||
SimpleNamespace(
|
||||
idx_mapping=torch.tensor([2, 0], dtype=torch.int32),
|
||||
is_padding=torch.zeros(4, dtype=torch.bool),
|
||||
),
|
||||
)
|
||||
aux_hidden_states = [
|
||||
torch.full((4, 3), 1.0),
|
||||
torch.full((4, 3), 2.0),
|
||||
]
|
||||
attn_metadata = {layer_name: object(), "target_layer": object()}
|
||||
slot_mappings = {
|
||||
layer_name: torch.arange(4),
|
||||
"target_layer": torch.arange(4),
|
||||
}
|
||||
last_sampled = torch.tensor([[10], [11], [12]], dtype=torch.int64)
|
||||
|
||||
draft_tokens = speculator.propose(
|
||||
input_batch=input_batch,
|
||||
attn_metadata=attn_metadata,
|
||||
slot_mappings=slot_mappings,
|
||||
last_hidden_states=torch.empty(0),
|
||||
aux_hidden_states=aux_hidden_states,
|
||||
num_sampled=torch.empty(0),
|
||||
num_rejected=torch.empty(0),
|
||||
last_sampled=last_sampled,
|
||||
next_prefill_tokens=torch.empty(0),
|
||||
temperature=torch.empty(0),
|
||||
seeds=torch.empty(0),
|
||||
)
|
||||
|
||||
expected_hidden_states = torch.stack(aux_hidden_states, dim=1)
|
||||
assert torch.equal(speculator.model.hidden_states, expected_hidden_states)
|
||||
assert torch.equal(draft_tokens, torch.tensor([[12], [10]]))
|
||||
|
||||
assert len(contexts) == 1
|
||||
args, kwargs = contexts[0]
|
||||
assert args[0] == {layer_name: attn_metadata[layer_name]}
|
||||
assert kwargs["num_tokens"] == 4
|
||||
assert set(kwargs["slot_mapping"]) == {layer_name}
|
||||
assert torch.equal(kwargs["slot_mapping"][layer_name], slot_mappings[layer_name])
|
||||
|
||||
|
||||
def test_propose_requires_aux_hidden_states():
|
||||
speculator = object.__new__(ExtractHiddenStatesSpeculator)
|
||||
speculator.num_hidden_states = 2
|
||||
input_batch = cast(
|
||||
Any, SimpleNamespace(idx_mapping=torch.tensor([0], dtype=torch.int32))
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="aux_hidden_states are required"):
|
||||
speculator.propose(
|
||||
input_batch=input_batch,
|
||||
attn_metadata={},
|
||||
slot_mappings={},
|
||||
last_hidden_states=torch.empty(0),
|
||||
aux_hidden_states=None,
|
||||
num_sampled=torch.empty(0),
|
||||
num_rejected=torch.empty(0),
|
||||
last_sampled=torch.tensor([[10]]),
|
||||
next_prefill_tokens=torch.empty(0),
|
||||
temperature=torch.empty(0),
|
||||
seeds=torch.empty(0),
|
||||
)
|
||||
Reference in New Issue
Block a user