Hidden states extraction improvements (#43805)
Signed-off-by: Fynn Schmitt-Ulms <fschmitt@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ab3a1fd2e6
commit
ebc6ef971a
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
@@ -18,7 +19,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1 import (
|
||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
llm = LLM(
|
||||
model="Qwen/Qwen3-8B", # Your target model
|
||||
enable_chunked_prefill=False, # required
|
||||
speculative_config={
|
||||
"method": "extract_hidden_states",
|
||||
"num_speculative_tokens": 1,
|
||||
@@ -38,13 +38,30 @@ with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
kv_role="kv_producer",
|
||||
kv_connector_extra_config={
|
||||
"shared_storage_path": tmpdirname,
|
||||
"allow_custom_save_path": True,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
prompts = ["Generate a sentence with hidden states", "Write a python function"]
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
|
||||
# One request uses defaults, the other uses a custom save path and
|
||||
# includes output token hidden states via per-request kv_transfer_params.
|
||||
sampling_params_list = [
|
||||
SamplingParams(max_tokens=1),
|
||||
SamplingParams(
|
||||
max_tokens=10,
|
||||
extra_args={
|
||||
"kv_transfer_params": {
|
||||
"hidden_states_path": os.path.join(
|
||||
tmpdirname, "custom_output.safetensors"
|
||||
),
|
||||
"include_output_tokens": True,
|
||||
}
|
||||
},
|
||||
),
|
||||
]
|
||||
outputs = llm.generate(prompts, sampling_params_list)
|
||||
|
||||
for output in outputs:
|
||||
print("\nPrompt:", output.prompt)
|
||||
@@ -52,16 +69,16 @@ with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
|
||||
hidden_states_path = output.kv_transfer_params.get("hidden_states_path")
|
||||
assert hidden_states_path is not None
|
||||
print("Prompt hidden states path:", hidden_states_path)
|
||||
print("Hidden states path:", hidden_states_path)
|
||||
|
||||
obj = example_hidden_states_connector.load_hidden_states(hidden_states_path)
|
||||
token_ids = obj["token_ids"]
|
||||
hidden_states = obj["hidden_states"]
|
||||
|
||||
print("Extracted token ids:", token_ids) # Matches prompt token ids
|
||||
print("Extracted token ids:", token_ids)
|
||||
print(
|
||||
"Extracted hidden states shape:", hidden_states.shape
|
||||
) # [prompt_len, num_extracted_layers, hidden_size]
|
||||
) # [num_tokens, num_extracted_layers, hidden_size]
|
||||
print("Extracted hidden states:", hidden_states)
|
||||
|
||||
example_hidden_states_connector.cleanup_hidden_states(hidden_states_path)
|
||||
|
||||
Reference in New Issue
Block a user