Hidden states extraction improvements (#43805)

Signed-off-by: Fynn Schmitt-Ulms <fschmitt@redhat.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Fynn Schmitt-Ulms
2026-06-11 09:44:45 -04:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ab3a1fd2e6
commit ebc6ef971a
7 changed files with 454 additions and 233 deletions
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import tempfile
from vllm import LLM, SamplingParams
@@ -18,7 +19,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1 import (
with tempfile.TemporaryDirectory() as tmpdirname:
llm = LLM(
model="Qwen/Qwen3-8B", # Your target model
enable_chunked_prefill=False, # required
speculative_config={
"method": "extract_hidden_states",
"num_speculative_tokens": 1,
@@ -38,13 +38,30 @@ with tempfile.TemporaryDirectory() as tmpdirname:
kv_role="kv_producer",
kv_connector_extra_config={
"shared_storage_path": tmpdirname,
"allow_custom_save_path": True,
},
),
)
prompts = ["Generate a sentence with hidden states", "Write a python function"]
sampling_params = SamplingParams(max_tokens=1)
outputs = llm.generate(prompts, sampling_params)
# One request uses defaults, the other uses a custom save path and
# includes output token hidden states via per-request kv_transfer_params.
sampling_params_list = [
SamplingParams(max_tokens=1),
SamplingParams(
max_tokens=10,
extra_args={
"kv_transfer_params": {
"hidden_states_path": os.path.join(
tmpdirname, "custom_output.safetensors"
),
"include_output_tokens": True,
}
},
),
]
outputs = llm.generate(prompts, sampling_params_list)
for output in outputs:
print("\nPrompt:", output.prompt)
@@ -52,16 +69,16 @@ with tempfile.TemporaryDirectory() as tmpdirname:
hidden_states_path = output.kv_transfer_params.get("hidden_states_path")
assert hidden_states_path is not None
print("Prompt hidden states path:", hidden_states_path)
print("Hidden states path:", hidden_states_path)
obj = example_hidden_states_connector.load_hidden_states(hidden_states_path)
token_ids = obj["token_ids"]
hidden_states = obj["hidden_states"]
print("Extracted token ids:", token_ids) # Matches prompt token ids
print("Extracted token ids:", token_ids)
print(
"Extracted hidden states shape:", hidden_states.shape
) # [prompt_len, num_extracted_layers, hidden_size]
) # [num_tokens, num_extracted_layers, hidden_size]
print("Extracted hidden states:", hidden_states)
example_hidden_states_connector.cleanup_hidden_states(hidden_states_path)