[Feature][Model Runner V2] Support extract_hidden_states speculation (#49811)
Signed-off-by: Zupeng Wang <71580390+zupengwang@users.noreply.github.com> Signed-off-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: OpenAI <support@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI
Misha Goin
OpenAI Codex
parent
cb09dd7488
commit
01af92e175
@@ -5,6 +5,7 @@ import logging
|
||||
import os
|
||||
from dataclasses import MISSING, Field, asdict, dataclass, field
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
from unittest.mock import patch
|
||||
|
||||
import pydantic
|
||||
@@ -263,6 +264,20 @@ def test_dsa_models_select_matching_mtp(model_type, expected_architecture):
|
||||
assert hf_config.architectures == [expected_architecture]
|
||||
|
||||
|
||||
def test_v2_model_runner_supports_extract_hidden_states():
|
||||
config = VllmConfig()
|
||||
config.speculative_config = cast(
|
||||
SpeculativeConfig,
|
||||
SimpleNamespace(
|
||||
method="extract_hidden_states",
|
||||
parallel_drafting=False,
|
||||
enable_adaptive_verification=False,
|
||||
),
|
||||
)
|
||||
|
||||
assert config._get_v2_model_runner_unsupported_features() == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("use_v2_model_runner", "expected_capture_sizes"),
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user