[Perf] Optimize hidden state extraction logic (#37374)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
f743254143
commit
4e2eba28be
@@ -1,11 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import fcntl
|
||||
import os
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from functools import partial
|
||||
from importlib.metadata import version
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import safetensors
|
||||
import torch
|
||||
from packaging.version import Version
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
@@ -14,6 +19,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorRole,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
|
||||
@@ -36,6 +42,39 @@ def extract_from_kv_cache(
|
||||
return kv_cache[slot_mapping // block_size, slot_mapping % block_size][:num_tokens]
|
||||
|
||||
|
||||
def load_hidden_states(path: str) -> dict[str, torch.Tensor]:
|
||||
"""Load hidden states written by ExampleHiddenStatesConnector.
|
||||
|
||||
Blocks (without polling) until the async write is complete by
|
||||
acquiring a shared flock on the companion lock file. The kernel
|
||||
puts the caller to sleep until the writer releases its exclusive lock.
|
||||
|
||||
Args:
|
||||
path: The file path returned in kv_transfer_params["hidden_states_path"].
|
||||
|
||||
Returns:
|
||||
Dict with "hidden_states" and "token_ids" tensors.
|
||||
"""
|
||||
lock_path = path + ".lock"
|
||||
with open(lock_path) as lf:
|
||||
fcntl.flock(lf, fcntl.LOCK_SH) # sleeps until writer releases LOCK_EX
|
||||
data = load_file(path, device="cpu")
|
||||
return data
|
||||
|
||||
|
||||
def cleanup_hidden_states(path: str, keep_hidden_states: bool = False) -> None:
|
||||
"""Clean up hidden states file and lock file after loading.
|
||||
|
||||
If keep_hidden_states is True, only removes the lock file
|
||||
and keeps the hidden states file.
|
||||
"""
|
||||
lock_path = path + ".lock"
|
||||
if os.path.exists(lock_path):
|
||||
os.remove(lock_path)
|
||||
if not keep_hidden_states and os.path.exists(path):
|
||||
os.remove(path)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReqMeta:
|
||||
# Request ID
|
||||
@@ -112,6 +151,13 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
logger.info(self._kv_transfer_config)
|
||||
logger.info("Shared storage path is %s", self._storage_path)
|
||||
|
||||
if Version(version("safetensors")) < Version("0.8.0"):
|
||||
logger.warning(
|
||||
"safetensors < 0.8.0 holds the GIL during save_file, which "
|
||||
"serializes the writer thread pool and hurts throughput. "
|
||||
"Upgrade to safetensors >= 0.8.0 for better performance."
|
||||
)
|
||||
|
||||
assert self._vllm_config.speculative_config is not None, (
|
||||
"ExampleHiddenStatesConnector only works when using "
|
||||
"'extract_hidden_states' speculative method"
|
||||
@@ -125,17 +171,97 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
self._active_requests: dict[str, NewRequestData] = {}
|
||||
self._req_blocks: dict[str, list[int]] = {}
|
||||
|
||||
# Async write infrastructure (worker-side).
|
||||
# Dedicated CUDA stream for DtoH copies so they don't block
|
||||
# the default stream (model forward). Thread pool for disk writes.
|
||||
self._copy_stream: torch.cuda.Stream | None = None # lazy init
|
||||
self._executor = ThreadPoolExecutor(
|
||||
max_workers=self._kv_transfer_config.get_from_extra_config(
|
||||
"num_writer_threads", 8
|
||||
),
|
||||
thread_name_prefix="vllm-hs-save",
|
||||
)
|
||||
# Whether to use a filesystem lock when writing files to shared storage.
|
||||
# This is necessary for online transfer clients to avoid incomplete reads,
|
||||
# but can be disabled for offline tasks that run tasks in batches to completion
|
||||
self.use_lock = self._kv_transfer_config.get_from_extra_config(
|
||||
"use_synchronization_lock", True
|
||||
)
|
||||
# (tensors_dict, copy_done_event, filename, req_id) queued by
|
||||
# save_kv_layer, submitted to thread pool by wait_for_save.
|
||||
self._pending_copies: list[
|
||||
tuple[dict[str, torch.Tensor], torch.cuda.Event, str, str]
|
||||
] = []
|
||||
# req_id → in-flight disk-write Future for that req_id.
|
||||
self._req_futures: dict[str, Future] = {}
|
||||
# req_id → CUDA event marking completion of the DtoH copy. Once
|
||||
# this event is complete the request is considered "done sending"
|
||||
# by get_finished; clients block on the per-file flock to wait for
|
||||
# the disk write itself.
|
||||
self._req_copy_events: dict[str, torch.cuda.Event] = {}
|
||||
# req_ids reported as finished-generating by the scheduler,
|
||||
# accumulated across get_finished calls.
|
||||
self._accumulated_finished_req_ids: set[str] = set()
|
||||
|
||||
def _get_copy_stream(self) -> torch.cuda.Stream:
|
||||
"""Lazily create the copy stream (CUDA must be initialized)."""
|
||||
if self._copy_stream is None:
|
||||
self._copy_stream = torch.cuda.Stream()
|
||||
return self._copy_stream
|
||||
|
||||
# ==============================
|
||||
# Worker-side methods
|
||||
# ==============================
|
||||
def start_load_kv(self, *args, **kwargs: Any) -> None:
|
||||
pass # Empty implementation of abstract method
|
||||
pass # Store-only connector — nothing to load
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
pass # Empty implementation of abstract method
|
||||
pass # Store-only connector — nothing to load
|
||||
|
||||
def wait_for_save(self):
|
||||
pass # Empty implementation of abstract method
|
||||
"""Submit pending async copies to the thread pool for disk write.
|
||||
|
||||
For each pending write we acquire an exclusive flock on a
|
||||
companion ``.lock`` file **before** submitting to the thread pool.
|
||||
The thread worker releases the lock after the data file is fully
|
||||
written. Clients call :func:`load_hidden_states` which takes a
|
||||
shared flock — the kernel sleeps the client until the writer is
|
||||
done. Because ``wait_for_save`` runs before the worker returns
|
||||
output to the scheduler, the lock file is guaranteed to exist
|
||||
(and be held) by the time the client receives the path.
|
||||
|
||||
The lock can be disabled via the "use_synchronization_lock" extra config.
|
||||
"""
|
||||
for tensors, event, filename, req_id in self._pending_copies:
|
||||
prior = self._req_futures.get(req_id)
|
||||
assert prior is None, "Found another KV transfer request with same req_id!"
|
||||
|
||||
lock_fd = None
|
||||
if self.use_lock:
|
||||
# Create/open the lock file and acquire an exclusive lock.
|
||||
# The lock is held by this fd; the thread worker will close
|
||||
# the fd after writing, which releases the lock.
|
||||
lock_path = filename + ".lock"
|
||||
lock_fd = os.open(
|
||||
lock_path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o644
|
||||
)
|
||||
fcntl.flock(lock_fd, fcntl.LOCK_EX)
|
||||
|
||||
future = self._executor.submit(
|
||||
self._write_tensors, tensors, event, filename, lock_fd
|
||||
)
|
||||
self._req_copy_events[req_id] = event
|
||||
self._req_futures[req_id] = future
|
||||
future.add_done_callback(partial(self._on_write_done, req_id))
|
||||
self._pending_copies.clear()
|
||||
|
||||
def _on_write_done(self, req_id: str, future: Future) -> None:
|
||||
"""Surface any exception from the disk-write thread and drop the
|
||||
completed future from the in-flight tracking dict."""
|
||||
self._req_futures.pop(req_id, None)
|
||||
exc = future.exception()
|
||||
if exc is not None:
|
||||
logger.error("Hidden-states write failed for req_id=%s: %r", req_id, exc)
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
from vllm.model_executor.models.extract_hidden_states import (
|
||||
@@ -151,6 +277,26 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
f"Expected 1 CacheOnlyAttentionLayer, got {len(self.cache_layers)}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _write_tensors(
|
||||
tensors: dict[str, torch.Tensor],
|
||||
event: torch.cuda.Event,
|
||||
filename: str,
|
||||
lock_fd: int | None,
|
||||
) -> None:
|
||||
"""Thread worker: wait for async DtoH copy, write to disk, release lock.
|
||||
|
||||
``lock_fd`` is an open file descriptor on the companion ``.lock``
|
||||
file with ``LOCK_EX`` already held. Closing it releases the lock,
|
||||
which unblocks any client sleeping on ``LOCK_SH``.
|
||||
"""
|
||||
try:
|
||||
event.synchronize()
|
||||
save_file(tensors, filename)
|
||||
finally:
|
||||
if lock_fd is not None:
|
||||
os.close(lock_fd) # releases LOCK_EX
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
@@ -161,6 +307,10 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""Start saving the KV cache of the layer from vLLM's paged buffer
|
||||
to the connector.
|
||||
|
||||
Launches an async DtoH copy on a dedicated CUDA stream. The
|
||||
actual disk write is deferred to wait_for_save() which submits
|
||||
it to a thread pool.
|
||||
|
||||
Args:
|
||||
layer_name (str): the name of the layer.
|
||||
kv_layer (torch.Tensor): the paged KV buffer of the current
|
||||
@@ -184,21 +334,46 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
os.makedirs(self._storage_path, exist_ok=True)
|
||||
|
||||
slot_mapping = attn_metadata.slot_mapping
|
||||
copy_stream = self._get_copy_stream()
|
||||
|
||||
# Ensure the copy stream sees all prior writes on the default stream.
|
||||
ready_event = torch.cuda.Event()
|
||||
ready_event.record()
|
||||
copy_stream.wait_event(ready_event)
|
||||
|
||||
slot_mapping = get_forward_context().slot_mapping[layer_name] # type: ignore
|
||||
offset = 0
|
||||
for request in connector_metadata.requests:
|
||||
num_tokens = request.token_ids.shape[0]
|
||||
req_slot_mapping = slot_mapping[offset : offset + num_tokens]
|
||||
offset += num_tokens
|
||||
with torch.cuda.stream(copy_stream):
|
||||
req_slot_mapping_gpu = slot_mapping[offset : offset + num_tokens]
|
||||
assert req_slot_mapping_gpu.device == kv_layer.device
|
||||
offset += num_tokens
|
||||
|
||||
hidden_states = extract_from_kv_cache(
|
||||
kv_layer, req_slot_mapping, num_tokens
|
||||
hidden_states_gpu = extract_from_kv_cache(
|
||||
kv_layer, req_slot_mapping_gpu, num_tokens
|
||||
)
|
||||
# Async DtoH copy into pinned host memory.
|
||||
pinned_hs = torch.empty_like(
|
||||
hidden_states_gpu, device="cpu", pin_memory=True
|
||||
)
|
||||
pinned_hs.copy_(hidden_states_gpu, non_blocking=True)
|
||||
|
||||
# Record completion of this copy on the copy stream.
|
||||
copy_done = torch.cuda.Event()
|
||||
copy_done.record(copy_stream)
|
||||
|
||||
# token_ids is already on CPU (created in ReqMeta.make_meta).
|
||||
assert not request.token_ids.is_cuda, (
|
||||
"Expected token_ids on CPU, got CUDA tensor"
|
||||
)
|
||||
tensors = {
|
||||
"hidden_states": hidden_states.detach().cpu(),
|
||||
"token_ids": request.token_ids.detach().cpu(),
|
||||
"hidden_states": pinned_hs,
|
||||
"token_ids": request.token_ids.clone(),
|
||||
}
|
||||
safetensors.torch.save_file(tensors, request.filename)
|
||||
self._pending_copies.append(
|
||||
(tensors, copy_done, request.filename, request.req_id)
|
||||
)
|
||||
|
||||
# ==============================
|
||||
# Scheduler-side methods
|
||||
@@ -258,31 +433,6 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
self._active_requests[new_req.req_id] = new_req
|
||||
self._req_blocks[new_req.req_id] = list(new_req.block_ids[0])
|
||||
|
||||
cached_reqs = scheduler_output.scheduled_cached_reqs
|
||||
for i, req_id in enumerate(cached_reqs.req_ids):
|
||||
if req_id not in self._active_requests:
|
||||
continue
|
||||
|
||||
new_block_ids = cached_reqs.new_block_ids[i]
|
||||
|
||||
cached_req = self._active_requests[req_id]
|
||||
req_block_ids = self._req_blocks[req_id]
|
||||
|
||||
if new_block_ids is None:
|
||||
continue
|
||||
|
||||
block_ids = new_block_ids[0]
|
||||
|
||||
req_block_ids.extend(block_ids)
|
||||
filename = os.path.join(self._storage_path, f"{req_id}.safetensors")
|
||||
|
||||
meta.add_request(
|
||||
req_id=req_id,
|
||||
filename=filename,
|
||||
token_ids=cached_req.prompt_token_ids or [],
|
||||
new_req=False,
|
||||
)
|
||||
|
||||
return meta
|
||||
|
||||
def request_finished(
|
||||
@@ -309,7 +459,31 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
_ = self._active_requests.pop(req_id, None)
|
||||
_ = self._req_blocks.pop(req_id, None)
|
||||
|
||||
return False, {"hidden_states_path": req_filename}
|
||||
return True, {"hidden_states_path": req_filename}
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""Poll DtoH-copy completion for requests that finished generating.
|
||||
|
||||
The scheduler passes finished_req_ids to tell the worker which
|
||||
requests are done generating. We accumulate these across calls
|
||||
and return a request as "finished sending" once its DtoH copy
|
||||
event is complete (or if it never had a pending copy). The
|
||||
subsequent disk write may still be in flight; clients block on
|
||||
the per-file flock to wait for it.
|
||||
"""
|
||||
self._accumulated_finished_req_ids.update(finished_req_ids)
|
||||
|
||||
done_sending: set[str] = set()
|
||||
for req_id in list(self._accumulated_finished_req_ids):
|
||||
event = self._req_copy_events.get(req_id)
|
||||
if event is None or event.query():
|
||||
self._req_copy_events.pop(req_id, None)
|
||||
done_sending.add(req_id)
|
||||
self._accumulated_finished_req_ids.discard(req_id)
|
||||
|
||||
return done_sending or None, None
|
||||
|
||||
def request_finished_all_groups(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user