Hidden states extraction improvements (#43805)

Signed-off-by: Fynn Schmitt-Ulms <fschmitt@redhat.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Fynn Schmitt-Ulms
2026-06-11 09:44:45 -04:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ab3a1fd2e6
commit ebc6ef971a
7 changed files with 454 additions and 233 deletions
@@ -1,44 +1,40 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
import os
import tempfile
import pytest
import torch
from safetensors import safe_open
from tests.utils import create_new_process_for_each_test, multi_gpu_test
from vllm import LLM, ModelRegistry, SamplingParams
from vllm.distributed.kv_transfer.kv_connector.v1 import (
example_hidden_states_connector,
)
def get_and_check_output(output, expected_shape):
assert output.kv_transfer_params is not None
hidden_states_path = output.kv_transfer_params.get("hidden_states_path")
assert hidden_states_path is not None
assert os.path.exists(hidden_states_path)
# Load and verify the saved tensors
with safe_open(hidden_states_path, "pt") as f:
# Check that token_ids and hidden_states are present
tensor_names = f.keys()
assert "token_ids" in tensor_names
assert "hidden_states" in tensor_names
obj = example_hidden_states_connector.load_hidden_states(hidden_states_path)
token_ids = obj["token_ids"]
hidden_states = obj["hidden_states"]
token_ids = f.get_tensor("token_ids")
hidden_states = f.get_tensor("hidden_states")
prompt_token_ids = output.prompt_token_ids
assert torch.equal(token_ids, torch.tensor(prompt_token_ids))
prompt_token_ids = output.prompt_token_ids
assert torch.equal(token_ids, torch.tensor(prompt_token_ids))
assert hidden_states.shape == expected_shape
assert hidden_states.shape == expected_shape
# Verify hidden_states are not all zeros (i.e., they were actually computed)
assert not torch.allclose(hidden_states, torch.zeros_like(hidden_states))
# Verify hidden_states are not all zeros (i.e., they were actually computed)
assert not torch.allclose(hidden_states, torch.zeros_like(hidden_states))
return token_ids, hidden_states
@pytest.fixture(scope="module")
@pytest.fixture
def predictable_llama_config_path(tmp_path_factory):
"""Create a minimal LlamaConfig for PredictableLlamaForCausalLM."""
from transformers import LlamaConfig, LlamaTokenizerFast
@@ -53,7 +49,7 @@ def predictable_llama_config_path(tmp_path_factory):
num_hidden_layers=24, # Enough layers to test various layer_ids
num_attention_heads=4,
num_key_value_heads=4,
max_position_embeddings=128,
max_position_embeddings=1024,
architectures=["PredictableLlamaForCausalLM"],
)
@@ -85,24 +81,25 @@ def register_predictable_model():
def test_extract_hidden_states_with_predictable_dummy_model(
predictable_llama_config_path, tmp_path, monkeypatch
):
"""Comprehensive test using a predictable dummy model with synthetic weights.
"""Test hidden-state extraction with a predictable dummy model.
The PredictableLlamaForCausalLM outputs deterministic hidden states where
each layer produces values equal to (layer_index). This test verifies:
1. Hidden states are correctly extracted from requested layers
2. Values match the expected predictable pattern
3. Layer ordering is preserved correctly (non-sequential layer IDs)
4. Multiple prompts of different lengths produce consistent layer values
Tests 3 scenarios:
1. **Basic extraction**: non-sequential layer ordering, multiple prompts
of varying length — verifies correct layer association and
deterministic values.
2. **Chunked prefill**: max_num_batched_tokens=128 with ~500-token
prompts so each is split across multiple scheduler iterations —
verifies hidden states are reassembled correctly.
3. **Per-request options**: custom hidden_states_path and
include_output_tokens — verifies per-request kv_transfer_params
plumbing.
"""
# Force fork so the engine worker inherits the autouse fixture's
# ModelRegistry.register_model("PredictableLlamaForCausalLM", ...).
# Spawn (the CI default) starts a fresh Python process that wouldn't
# see the registration.
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "fork")
# Test with non-sequential layer ordering to verify correct association
layer_ids = [5, 2, 10]
num_layers = len(layer_ids)
max_num_batched_tokens = 128
llm = LLM(
model=predictable_llama_config_path,
@@ -116,16 +113,21 @@ def test_extract_hidden_states_with_predictable_dummy_model(
kv_transfer_config={
"kv_connector": "ExampleHiddenStatesConnector",
"kv_role": "kv_producer",
"kv_connector_extra_config": {"shared_storage_path": tmp_path},
"kv_connector_extra_config": {
"shared_storage_path": tmp_path,
"allow_custom_save_path": True,
},
},
max_model_len=128,
max_model_len=1024,
max_num_batched_tokens=max_num_batched_tokens,
enforce_eager=True,
enable_chunked_prefill=False,
trust_remote_code=True,
load_format="dummy", # Don't try to load real weights
load_format="dummy",
)
# Test with multiple prompts of different lengths
hidden_size = llm.llm_engine.model_config.get_hidden_size()
# --- Scenario 1: basic extraction with non-sequential layers ----------
prompts = [
"Short",
"Medium length",
@@ -133,15 +135,10 @@ def test_extract_hidden_states_with_predictable_dummy_model(
"Much longer prompt with many tokens", # repeated prompt
]
sampling_params = SamplingParams(max_tokens=1, temperature=0.0)
hidden_size = llm.llm_engine.model_config.get_hidden_size()
outputs = llm.generate(prompts, sampling_params)
del llm
gc.collect()
assert len(outputs) == len(prompts)
for output in outputs:
# hidden_states shape is [prompt_len, num_hidden_layers, hidden_size]
expected_shape = (
len(output.prompt_token_ids),
num_layers,
@@ -156,12 +153,100 @@ def test_extract_hidden_states_with_predictable_dummy_model(
torch.full_like(layer_hidden, layer_id),
atol=1e-5,
), (
f"Layer {layer_id} at position {idx} should output {float(layer_id)}, "
f"but got mean={layer_hidden.mean():.3f}, "
f"min={layer_hidden.min():.3f}, max={layer_hidden.max():.3f}"
f"Layer {layer_id} at position {idx} should output "
f"{float(layer_id)}, but got mean="
f"{layer_hidden.mean():.3f}, min="
f"{layer_hidden.min():.3f}, max={layer_hidden.max():.3f}"
)
# --- Scenario 2: chunked prefill with long prompts --------------------
long_prompt = " ".join(["word"] * 500)
chunked_prompts = [
long_prompt,
long_prompt + " extra tokens here",
"Short",
]
outputs = llm.generate(chunked_prompts, sampling_params)
assert len(outputs) == len(chunked_prompts)
for output in outputs:
prompt_len = len(output.prompt_token_ids)
expected_shape = (prompt_len, num_layers, hidden_size)
_token_ids, hidden_states = get_and_check_output(output, expected_shape)
for idx, layer_id in enumerate(layer_ids):
layer_hidden = hidden_states[:, idx, :]
assert torch.allclose(
layer_hidden,
torch.full_like(layer_hidden, layer_id),
atol=1e-5,
), (
f"Layer {layer_id} at position {idx} should output "
f"{float(layer_id)}, but got mean="
f"{layer_hidden.mean():.3f}, min="
f"{layer_hidden.min():.3f}, max="
f"{layer_hidden.max():.3f}. "
f"prompt_len={prompt_len}, "
f"max_num_batched_tokens={max_num_batched_tokens}"
)
# --- Scenario 3: per-request options ----------------------------------
max_tokens = 5
custom_path = os.path.join(tmp_path, "subdir", "custom.safetensors")
sampling_params_list = [
SamplingParams(max_tokens=max_tokens, temperature=0.0),
SamplingParams(
max_tokens=max_tokens,
temperature=0.0,
extra_args={
"kv_transfer_params": {
"hidden_states_path": custom_path,
"include_output_tokens": True,
}
},
),
]
per_req_prompts = ["Short", "Medium length"]
outputs = llm.generate(per_req_prompts, sampling_params_list)
# First output: prompt-only hidden states, default path
out0 = outputs[0]
path0 = out0.kv_transfer_params["hidden_states_path"]
assert path0 != custom_path
obj0 = example_hidden_states_connector.load_hidden_states(path0)
assert torch.equal(obj0["token_ids"], torch.tensor(out0.prompt_token_ids))
assert obj0["hidden_states"].shape == (
len(out0.prompt_token_ids),
num_layers,
hidden_size,
)
example_hidden_states_connector.cleanup_hidden_states(path0)
# Second output: prompt + output tokens, custom path
out1 = outputs[1]
assert out1.kv_transfer_params["hidden_states_path"] == custom_path
obj1 = example_hidden_states_connector.load_hidden_states(custom_path)
token_ids = obj1["token_ids"]
hidden_states = obj1["hidden_states"]
# The final output token was never an input to the model, so its hidden
# state is not in the cache — hence the -1.
total_tokens = len(out1.prompt_token_ids) + len(out1.outputs[0].token_ids) - 1
assert token_ids.shape[0] == total_tokens
assert hidden_states.shape == (total_tokens, num_layers, hidden_size)
# Verify predictable layer values hold for all tokens (prompt + output)
for idx, layer_id in enumerate(layer_ids):
layer_hidden = hidden_states[:, idx, :]
assert torch.allclose(
layer_hidden,
torch.full_like(layer_hidden, layer_id),
atol=1e-5,
)
example_hidden_states_connector.cleanup_hidden_states(custom_path)
@create_new_process_for_each_test()
def test_extract_hidden_states_qwen35_hybrid_smoke(tmp_path):
"""Smoke test for Qwen3.5 hybrid (mamba + full-attention) models.
Uses load_format="dummy" to just check shape/plumbing.
@@ -185,7 +270,6 @@ def test_extract_hidden_states_qwen35_hybrid_smoke(tmp_path):
},
max_model_len=256,
enforce_eager=True,
enable_chunked_prefill=False,
gpu_memory_utilization=0.4,
load_format="dummy",
)
@@ -193,19 +277,68 @@ def test_extract_hidden_states_qwen35_hybrid_smoke(tmp_path):
prompts = ["Hello world", "Test prompt with several tokens"]
sampling_params = SamplingParams(max_tokens=1, temperature=0.0)
outputs = llm.generate(prompts, sampling_params)
del llm
gc.collect()
assert len(outputs) == len(prompts)
for output in outputs:
assert output.kv_transfer_params is not None
hidden_states_path = output.kv_transfer_params.get("hidden_states_path")
assert hidden_states_path is not None
assert os.path.exists(hidden_states_path)
with safe_open(hidden_states_path, "pt") as f:
token_ids = f.get_tensor("token_ids")
hidden_states = f.get_tensor("hidden_states")
obj = example_hidden_states_connector.load_hidden_states(hidden_states_path)
token_ids = obj["token_ids"]
hidden_states = obj["hidden_states"]
assert torch.equal(token_ids, torch.tensor(output.prompt_token_ids))
assert hidden_states.shape == (
len(output.prompt_token_ids),
len(layer_ids),
hidden_size,
)
@pytest.mark.timeout(60)
@multi_gpu_test(num_gpus=2)
@create_new_process_for_each_test()
def test_extract_hidden_states_tp2():
"""Test that hidden states extraction works with tensor_parallel_size=2."""
tmp_dir = tempfile.mkdtemp()
layer_ids = [5, 11, 17]
hidden_size = 1024 # Qwen/Qwen3-0.6B hidden_size
llm = LLM(
model="Qwen/Qwen3-0.6B",
tensor_parallel_size=2,
speculative_config={
"method": "extract_hidden_states",
"num_speculative_tokens": 1,
"draft_model_config": {
"hf_config": {"eagle_aux_hidden_state_layer_ids": layer_ids}
},
},
kv_transfer_config={
"kv_connector": "ExampleHiddenStatesConnector",
"kv_role": "kv_producer",
"kv_connector_extra_config": {"shared_storage_path": tmp_dir},
},
max_model_len=256,
enforce_eager=True,
gpu_memory_utilization=0.4,
load_format="dummy",
)
prompts = ["Hello world", "Test prompt with several tokens"]
sampling_params = SamplingParams(max_tokens=1, temperature=0.0)
outputs = llm.generate(prompts, sampling_params)
assert len(outputs) == len(prompts)
for output in outputs:
assert output.kv_transfer_params is not None
hidden_states_path = output.kv_transfer_params.get("hidden_states_path")
assert hidden_states_path is not None
obj = example_hidden_states_connector.load_hidden_states(hidden_states_path)
token_ids = obj["token_ids"]
hidden_states = obj["hidden_states"]
assert torch.equal(token_ids, torch.tensor(output.prompt_token_ids))
assert hidden_states.shape == (
@@ -69,7 +69,6 @@ def _create_proposer(
scheduler_config=SchedulerConfig(
max_model_len=model_config.max_model_len,
is_encoder_decoder=model_config.is_encoder_decoder,
enable_chunked_prefill=False,
),
attention_config=AttentionConfig(),
)
@@ -120,7 +119,6 @@ def test_proposer_initialization_missing_layer_ids():
scheduler_config=SchedulerConfig(
max_model_len=model_config.max_model_len,
is_encoder_decoder=model_config.is_encoder_decoder,
enable_chunked_prefill=False,
),
attention_config=AttentionConfig(),
)