[Spec Decode] Support hybrid attention models in extract_hidden_states (#39949)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
5794c65f8c
commit
2f821faeae
@@ -83,7 +83,7 @@ def register_predictable_model():
|
||||
|
||||
|
||||
def test_extract_hidden_states_with_predictable_dummy_model(
|
||||
predictable_llama_config_path, tmp_path
|
||||
predictable_llama_config_path, tmp_path, monkeypatch
|
||||
):
|
||||
"""Comprehensive test using a predictable dummy model with synthetic weights.
|
||||
|
||||
@@ -94,6 +94,12 @@ def test_extract_hidden_states_with_predictable_dummy_model(
|
||||
3. Layer ordering is preserved correctly (non-sequential layer IDs)
|
||||
4. Multiple prompts of different lengths produce consistent layer values
|
||||
"""
|
||||
# Force fork so the engine worker inherits the autouse fixture's
|
||||
# ModelRegistry.register_model("PredictableLlamaForCausalLM", ...).
|
||||
# Spawn (the CI default) starts a fresh Python process that wouldn't
|
||||
# see the registration.
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "fork")
|
||||
|
||||
# Test with non-sequential layer ordering to verify correct association
|
||||
layer_ids = [5, 2, 10]
|
||||
num_layers = len(layer_ids)
|
||||
@@ -153,3 +159,55 @@ def test_extract_hidden_states_with_predictable_dummy_model(
|
||||
f"but got mean={layer_hidden.mean():.3f}, "
|
||||
f"min={layer_hidden.min():.3f}, max={layer_hidden.max():.3f}"
|
||||
)
|
||||
|
||||
|
||||
def test_extract_hidden_states_qwen35_hybrid_smoke(tmp_path):
|
||||
"""Smoke test for Qwen3.5 hybrid (mamba + full-attention) models.
|
||||
Uses load_format="dummy" to just check shape/plumbing.
|
||||
"""
|
||||
layer_ids = [5, 11, 17]
|
||||
hidden_size = 1024 # Qwen/Qwen3.5-0.8B hidden_size
|
||||
|
||||
llm = LLM(
|
||||
model="Qwen/Qwen3.5-0.8B",
|
||||
speculative_config={
|
||||
"method": "extract_hidden_states",
|
||||
"num_speculative_tokens": 1,
|
||||
"draft_model_config": {
|
||||
"hf_config": {"eagle_aux_hidden_state_layer_ids": layer_ids}
|
||||
},
|
||||
},
|
||||
kv_transfer_config={
|
||||
"kv_connector": "ExampleHiddenStatesConnector",
|
||||
"kv_role": "kv_producer",
|
||||
"kv_connector_extra_config": {"shared_storage_path": str(tmp_path)},
|
||||
},
|
||||
max_model_len=256,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=0.4,
|
||||
load_format="dummy",
|
||||
)
|
||||
|
||||
prompts = ["Hello world", "Test prompt with several tokens"]
|
||||
sampling_params = SamplingParams(max_tokens=1, temperature=0.0)
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
del llm
|
||||
gc.collect()
|
||||
|
||||
assert len(outputs) == len(prompts)
|
||||
for output in outputs:
|
||||
assert output.kv_transfer_params is not None
|
||||
hidden_states_path = output.kv_transfer_params.get("hidden_states_path")
|
||||
assert hidden_states_path is not None
|
||||
assert os.path.exists(hidden_states_path)
|
||||
|
||||
with safe_open(hidden_states_path, "pt") as f:
|
||||
token_ids = f.get_tensor("token_ids")
|
||||
hidden_states = f.get_tensor("hidden_states")
|
||||
|
||||
assert torch.equal(token_ids, torch.tensor(output.prompt_token_ids))
|
||||
assert hidden_states.shape == (
|
||||
len(output.prompt_token_ids),
|
||||
len(layer_ids),
|
||||
hidden_size,
|
||||
)
|
||||
|
||||
@@ -11,7 +11,6 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import KVTransferConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorRole
|
||||
|
||||
# ruff: noqa: E501
|
||||
@@ -44,11 +43,9 @@ class DecodeBenchTestRunner:
|
||||
|
||||
# Create vllm config with DecodeBenchConnector
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=block_size, max_num_batched_tokens=1000
|
||||
)
|
||||
vllm_config.kv_transfer_config = KVTransferConfig(
|
||||
block_size=block_size,
|
||||
max_num_batched_tokens=1000,
|
||||
kv_connector="DecodeBenchConnector",
|
||||
kv_role="kv_both",
|
||||
)
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
|
||||
@@ -35,10 +35,11 @@ def _make_empty_scheduler_output():
|
||||
|
||||
|
||||
def test_kv_connector_mixin_clears_metadata():
|
||||
vllm_config = create_vllm_config()
|
||||
vllm_config.kv_transfer_config.kv_connector = "TestExampleConnector"
|
||||
vllm_config.kv_transfer_config.kv_role = "kv_both"
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config["name"] = "unit"
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="TestExampleConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"name": "unit"},
|
||||
)
|
||||
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=0, kv_cache_tensors=[], kv_cache_groups=[]
|
||||
|
||||
@@ -130,17 +130,14 @@ KVConnectorFactory.register_connector(
|
||||
@pytest.fixture
|
||||
def mc() -> MultiConnector:
|
||||
"""MultiConnector using two mocked connectors"""
|
||||
vllm_config = create_vllm_config()
|
||||
|
||||
mock_connector_config = {
|
||||
"kv_connector": "MockConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_module_path": "tests.v1.kv_connector.unit.test_multi_connector",
|
||||
}
|
||||
|
||||
vllm_config.kv_transfer_config = KVTransferConfig(
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MultiConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={
|
||||
"connectors": [mock_connector_config, mock_connector_config],
|
||||
},
|
||||
@@ -405,39 +402,35 @@ def test_multi_connector_handle_preemptions_integration():
|
||||
|
||||
try:
|
||||
# Configure MultiConnector with two TestExampleConnectors
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="MultiConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={
|
||||
"connectors": [
|
||||
{
|
||||
"kv_connector": "TestExampleConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {
|
||||
"shared_storage_path": str(storage_path / "s1"),
|
||||
"name": "preempt1",
|
||||
},
|
||||
"kv_connector_module_path": "tests.v1.kv_connector.unit.utils",
|
||||
connectors_extra_config = {
|
||||
"connectors": [
|
||||
{
|
||||
"kv_connector": "TestExampleConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {
|
||||
"shared_storage_path": str(storage_path / "s1"),
|
||||
"name": "preempt1",
|
||||
},
|
||||
{
|
||||
"kv_connector": "TestExampleConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {
|
||||
"shared_storage_path": str(storage_path / "s2"),
|
||||
"name": "preempt2",
|
||||
},
|
||||
"kv_connector_module_path": "tests.v1.kv_connector.unit.utils",
|
||||
"kv_connector_module_path": "tests.v1.kv_connector.unit.utils",
|
||||
},
|
||||
{
|
||||
"kv_connector": "TestExampleConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {
|
||||
"shared_storage_path": str(storage_path / "s2"),
|
||||
"name": "preempt2",
|
||||
},
|
||||
]
|
||||
},
|
||||
)
|
||||
"kv_connector_module_path": "tests.v1.kv_connector.unit.utils",
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=16,
|
||||
max_num_batched_tokens=100,
|
||||
kv_connector_extra_config=kv_transfer_config.kv_connector_extra_config,
|
||||
kv_connector="MultiConnector",
|
||||
kv_connector_extra_config=connectors_extra_config,
|
||||
)
|
||||
vllm_config.kv_transfer_config = kv_transfer_config
|
||||
|
||||
# Create scheduler - this initializes the MultiConnector with SCHEDULER role
|
||||
scheduler = create_scheduler(vllm_config, num_blocks=10)
|
||||
@@ -973,7 +966,6 @@ def test_multi_connector_worker_metadata(mc):
|
||||
|
||||
def _make_multi_connector(connector_names: list[str]) -> MultiConnector:
|
||||
"""Build a MultiConnector wrapping the given registered connectors."""
|
||||
vllm_config = create_vllm_config()
|
||||
connectors = [
|
||||
{
|
||||
"kv_connector": name,
|
||||
@@ -982,9 +974,8 @@ def _make_multi_connector(connector_names: list[str]) -> MultiConnector:
|
||||
}
|
||||
for name in connector_names
|
||||
]
|
||||
vllm_config.kv_transfer_config = KVTransferConfig(
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MultiConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"connectors": connectors},
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
|
||||
@@ -102,6 +102,7 @@ def create_vllm_config(
|
||||
attention_backend: str | None = None,
|
||||
kv_load_failure_policy: Literal["recompute", "fail"] = "fail",
|
||||
kv_connector: str = "NixlConnector",
|
||||
kv_connector_module_path: str | None = None,
|
||||
kv_role: str = "kv_both",
|
||||
disable_hybrid_kv_cache_manager: bool | None = None,
|
||||
) -> VllmConfig:
|
||||
@@ -130,6 +131,7 @@ def create_vllm_config(
|
||||
)
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector=kv_connector,
|
||||
kv_connector_module_path=kv_connector_module_path,
|
||||
kv_role=kv_role,
|
||||
enable_permute_local_kv=enable_permute_local_kv,
|
||||
kv_connector_extra_config=kv_connector_extra_config or {},
|
||||
|
||||
Reference in New Issue
Block a user