[Perf] Optimize hidden state extraction logic (#37374)

Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
Benjamin Chislett
2026-05-22 18:23:08 -04:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent f743254143
commit 4e2eba28be
9 changed files with 772 additions and 65 deletions
@@ -1,11 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import fcntl
import os
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass, field
from functools import partial
from importlib.metadata import version
from typing import TYPE_CHECKING, Any
import safetensors
import torch
from packaging.version import Version
from safetensors.torch import load_file, save_file
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
@@ -14,6 +19,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorRole,
SupportsHMA,
)
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
@@ -36,6 +42,39 @@ def extract_from_kv_cache(
return kv_cache[slot_mapping // block_size, slot_mapping % block_size][:num_tokens]
def load_hidden_states(path: str) -> dict[str, torch.Tensor]:
"""Load hidden states written by ExampleHiddenStatesConnector.
Blocks (without polling) until the async write is complete by
acquiring a shared flock on the companion lock file. The kernel
puts the caller to sleep until the writer releases its exclusive lock.
Args:
path: The file path returned in kv_transfer_params["hidden_states_path"].
Returns:
Dict with "hidden_states" and "token_ids" tensors.
"""
lock_path = path + ".lock"
with open(lock_path) as lf:
fcntl.flock(lf, fcntl.LOCK_SH) # sleeps until writer releases LOCK_EX
data = load_file(path, device="cpu")
return data
def cleanup_hidden_states(path: str, keep_hidden_states: bool = False) -> None:
"""Clean up hidden states file and lock file after loading.
If keep_hidden_states is True, only removes the lock file
and keeps the hidden states file.
"""
lock_path = path + ".lock"
if os.path.exists(lock_path):
os.remove(lock_path)
if not keep_hidden_states and os.path.exists(path):
os.remove(path)
@dataclass
class ReqMeta:
# Request ID
@@ -112,6 +151,13 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
logger.info(self._kv_transfer_config)
logger.info("Shared storage path is %s", self._storage_path)
if Version(version("safetensors")) < Version("0.8.0"):
logger.warning(
"safetensors < 0.8.0 holds the GIL during save_file, which "
"serializes the writer thread pool and hurts throughput. "
"Upgrade to safetensors >= 0.8.0 for better performance."
)
assert self._vllm_config.speculative_config is not None, (
"ExampleHiddenStatesConnector only works when using "
"'extract_hidden_states' speculative method"
@@ -125,17 +171,97 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
self._active_requests: dict[str, NewRequestData] = {}
self._req_blocks: dict[str, list[int]] = {}
# Async write infrastructure (worker-side).
# Dedicated CUDA stream for DtoH copies so they don't block
# the default stream (model forward). Thread pool for disk writes.
self._copy_stream: torch.cuda.Stream | None = None # lazy init
self._executor = ThreadPoolExecutor(
max_workers=self._kv_transfer_config.get_from_extra_config(
"num_writer_threads", 8
),
thread_name_prefix="vllm-hs-save",
)
# Whether to use a filesystem lock when writing files to shared storage.
# This is necessary for online transfer clients to avoid incomplete reads,
# but can be disabled for offline tasks that run tasks in batches to completion
self.use_lock = self._kv_transfer_config.get_from_extra_config(
"use_synchronization_lock", True
)
# (tensors_dict, copy_done_event, filename, req_id) queued by
# save_kv_layer, submitted to thread pool by wait_for_save.
self._pending_copies: list[
tuple[dict[str, torch.Tensor], torch.cuda.Event, str, str]
] = []
# req_id → in-flight disk-write Future for that req_id.
self._req_futures: dict[str, Future] = {}
# req_id → CUDA event marking completion of the DtoH copy. Once
# this event is complete the request is considered "done sending"
# by get_finished; clients block on the per-file flock to wait for
# the disk write itself.
self._req_copy_events: dict[str, torch.cuda.Event] = {}
# req_ids reported as finished-generating by the scheduler,
# accumulated across get_finished calls.
self._accumulated_finished_req_ids: set[str] = set()
def _get_copy_stream(self) -> torch.cuda.Stream:
"""Lazily create the copy stream (CUDA must be initialized)."""
if self._copy_stream is None:
self._copy_stream = torch.cuda.Stream()
return self._copy_stream
# ==============================
# Worker-side methods
# ==============================
def start_load_kv(self, *args, **kwargs: Any) -> None:
pass # Empty implementation of abstract method
pass # Store-only connector — nothing to load
def wait_for_layer_load(self, layer_name: str) -> None:
pass # Empty implementation of abstract method
pass # Store-only connector — nothing to load
def wait_for_save(self):
pass # Empty implementation of abstract method
"""Submit pending async copies to the thread pool for disk write.
For each pending write we acquire an exclusive flock on a
companion ``.lock`` file **before** submitting to the thread pool.
The thread worker releases the lock after the data file is fully
written. Clients call :func:`load_hidden_states` which takes a
shared flock — the kernel sleeps the client until the writer is
done. Because ``wait_for_save`` runs before the worker returns
output to the scheduler, the lock file is guaranteed to exist
(and be held) by the time the client receives the path.
The lock can be disabled via the "use_synchronization_lock" extra config.
"""
for tensors, event, filename, req_id in self._pending_copies:
prior = self._req_futures.get(req_id)
assert prior is None, "Found another KV transfer request with same req_id!"
lock_fd = None
if self.use_lock:
# Create/open the lock file and acquire an exclusive lock.
# The lock is held by this fd; the thread worker will close
# the fd after writing, which releases the lock.
lock_path = filename + ".lock"
lock_fd = os.open(
lock_path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o644
)
fcntl.flock(lock_fd, fcntl.LOCK_EX)
future = self._executor.submit(
self._write_tensors, tensors, event, filename, lock_fd
)
self._req_copy_events[req_id] = event
self._req_futures[req_id] = future
future.add_done_callback(partial(self._on_write_done, req_id))
self._pending_copies.clear()
def _on_write_done(self, req_id: str, future: Future) -> None:
"""Surface any exception from the disk-write thread and drop the
completed future from the in-flight tracking dict."""
self._req_futures.pop(req_id, None)
exc = future.exception()
if exc is not None:
logger.error("Hidden-states write failed for req_id=%s: %r", req_id, exc)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
from vllm.model_executor.models.extract_hidden_states import (
@@ -151,6 +277,26 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
f"Expected 1 CacheOnlyAttentionLayer, got {len(self.cache_layers)}"
)
@staticmethod
def _write_tensors(
tensors: dict[str, torch.Tensor],
event: torch.cuda.Event,
filename: str,
lock_fd: int | None,
) -> None:
"""Thread worker: wait for async DtoH copy, write to disk, release lock.
``lock_fd`` is an open file descriptor on the companion ``.lock``
file with ``LOCK_EX`` already held. Closing it releases the lock,
which unblocks any client sleeping on ``LOCK_SH``.
"""
try:
event.synchronize()
save_file(tensors, filename)
finally:
if lock_fd is not None:
os.close(lock_fd) # releases LOCK_EX
def save_kv_layer(
self,
layer_name: str,
@@ -161,6 +307,10 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
"""Start saving the KV cache of the layer from vLLM's paged buffer
to the connector.
Launches an async DtoH copy on a dedicated CUDA stream. The
actual disk write is deferred to wait_for_save() which submits
it to a thread pool.
Args:
layer_name (str): the name of the layer.
kv_layer (torch.Tensor): the paged KV buffer of the current
@@ -184,21 +334,46 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
os.makedirs(self._storage_path, exist_ok=True)
slot_mapping = attn_metadata.slot_mapping
copy_stream = self._get_copy_stream()
# Ensure the copy stream sees all prior writes on the default stream.
ready_event = torch.cuda.Event()
ready_event.record()
copy_stream.wait_event(ready_event)
slot_mapping = get_forward_context().slot_mapping[layer_name] # type: ignore
offset = 0
for request in connector_metadata.requests:
num_tokens = request.token_ids.shape[0]
req_slot_mapping = slot_mapping[offset : offset + num_tokens]
offset += num_tokens
with torch.cuda.stream(copy_stream):
req_slot_mapping_gpu = slot_mapping[offset : offset + num_tokens]
assert req_slot_mapping_gpu.device == kv_layer.device
offset += num_tokens
hidden_states = extract_from_kv_cache(
kv_layer, req_slot_mapping, num_tokens
hidden_states_gpu = extract_from_kv_cache(
kv_layer, req_slot_mapping_gpu, num_tokens
)
# Async DtoH copy into pinned host memory.
pinned_hs = torch.empty_like(
hidden_states_gpu, device="cpu", pin_memory=True
)
pinned_hs.copy_(hidden_states_gpu, non_blocking=True)
# Record completion of this copy on the copy stream.
copy_done = torch.cuda.Event()
copy_done.record(copy_stream)
# token_ids is already on CPU (created in ReqMeta.make_meta).
assert not request.token_ids.is_cuda, (
"Expected token_ids on CPU, got CUDA tensor"
)
tensors = {
"hidden_states": hidden_states.detach().cpu(),
"token_ids": request.token_ids.detach().cpu(),
"hidden_states": pinned_hs,
"token_ids": request.token_ids.clone(),
}
safetensors.torch.save_file(tensors, request.filename)
self._pending_copies.append(
(tensors, copy_done, request.filename, request.req_id)
)
# ==============================
# Scheduler-side methods
@@ -258,31 +433,6 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
self._active_requests[new_req.req_id] = new_req
self._req_blocks[new_req.req_id] = list(new_req.block_ids[0])
cached_reqs = scheduler_output.scheduled_cached_reqs
for i, req_id in enumerate(cached_reqs.req_ids):
if req_id not in self._active_requests:
continue
new_block_ids = cached_reqs.new_block_ids[i]
cached_req = self._active_requests[req_id]
req_block_ids = self._req_blocks[req_id]
if new_block_ids is None:
continue
block_ids = new_block_ids[0]
req_block_ids.extend(block_ids)
filename = os.path.join(self._storage_path, f"{req_id}.safetensors")
meta.add_request(
req_id=req_id,
filename=filename,
token_ids=cached_req.prompt_token_ids or [],
new_req=False,
)
return meta
def request_finished(
@@ -309,7 +459,31 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
_ = self._active_requests.pop(req_id, None)
_ = self._req_blocks.pop(req_id, None)
return False, {"hidden_states_path": req_filename}
return True, {"hidden_states_path": req_filename}
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""Poll DtoH-copy completion for requests that finished generating.
The scheduler passes finished_req_ids to tell the worker which
requests are done generating. We accumulate these across calls
and return a request as "finished sending" once its DtoH copy
event is complete (or if it never had a pending copy). The
subsequent disk write may still be in flight; clients block on
the per-file flock to wait for it.
"""
self._accumulated_finished_req_ids.update(finished_req_ids)
done_sending: set[str] = set()
for req_id in list(self._accumulated_finished_req_ids):
event = self._req_copy_events.get(req_id)
if event is None or event.query():
self._req_copy_events.pop(req_id, None)
done_sending.add(req_id)
self._accumulated_finished_req_ids.discard(req_id)
return done_sending or None, None
def request_finished_all_groups(
self,