[Nixl][PD] Lease renewal TTL KV blocks on P (#41383)
Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
@@ -510,6 +510,14 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
pass
|
||||
|
||||
def on_new_request(self, request: "Request") -> None:
|
||||
"""Called by the scheduler when a new request is added.
|
||||
|
||||
Connectors can override this to inspect the request and perform
|
||||
bookkeeping. The default implementation is a no-op.
|
||||
"""
|
||||
return
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
"""
|
||||
Update KVConnector state from worker-side connectors output.
|
||||
|
||||
@@ -389,6 +389,10 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# Call with empty blocks for other connectors.
|
||||
c.update_state_after_alloc(request, empty_blocks, 0)
|
||||
|
||||
def on_new_request(self, request: "Request") -> None:
|
||||
for c in self._connectors:
|
||||
c.on_new_request(request)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> MultiKVConnectorMetadata:
|
||||
|
||||
@@ -44,6 +44,7 @@ from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
|
||||
from vllm.v1.attention.backends.utils import get_kv_cache_layout
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
@@ -156,6 +157,14 @@ class NixlConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def on_new_request(self, request: "Request") -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.on_new_request(request)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_connector_output(connector_output)
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
|
||||
@@ -6,7 +6,7 @@ from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds, EngineId
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorHandshakeMetadata,
|
||||
KVConnectorMetadata,
|
||||
@@ -33,8 +33,9 @@ GET_META_MSG = b"get_meta_msg"
|
||||
# 1: Initial version with compatibility checking
|
||||
# 2: Add remote_request_id to kv_transfer_params
|
||||
# 3: Add physical_blocks_per_logical_kv_block to NixlAgentMetadata
|
||||
# 4: Add KV block lease renewal through heartbeats
|
||||
#
|
||||
NIXL_CONNECTOR_VERSION: int = 3
|
||||
NIXL_CONNECTOR_VERSION: int = 4
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -133,6 +134,16 @@ def compute_nixl_compatibility_hash(
|
||||
return compat_hash
|
||||
|
||||
|
||||
@dataclass
|
||||
class HeartbeatInfo:
|
||||
"""Heartbeat data for a single remote engine, sent from D worker to P."""
|
||||
|
||||
req_ids: set[ReqId]
|
||||
host: str
|
||||
port: int
|
||||
tp_size: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class RemoteMeta:
|
||||
block_ids: BlockIds
|
||||
@@ -158,6 +169,8 @@ class NixlConnectorMetadata(KVConnectorMetadata):
|
||||
self.reqs_to_send: dict[ReqId, float] = {}
|
||||
self.reqs_in_batch: set[ReqId] = set()
|
||||
self.reqs_not_processed: set[ReqId] = set()
|
||||
# Heartbeat data grouped by remote engine, sent by D worker to P.
|
||||
self.heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {}
|
||||
|
||||
def _add_new_req(
|
||||
self,
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
GET_META_MSG,
|
||||
HeartbeatInfo,
|
||||
NixlConnectorMetadata,
|
||||
NixlHandshakePayload,
|
||||
ReqId,
|
||||
@@ -41,6 +42,7 @@ if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -65,6 +67,13 @@ class NixlConnectorScheduler:
|
||||
+ vllm_config.parallel_config.data_parallel_index
|
||||
)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
self._kv_lease_duration: int = (
|
||||
vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"kv_lease_duration", 30
|
||||
)
|
||||
)
|
||||
# NOTE (NickLucche): For now we use a hardcoded value for a simpler interface.
|
||||
self._heartbeat_interval = self._kv_lease_duration // 6
|
||||
if current_platform.device_type == "cpu":
|
||||
self.use_host_buffer = False
|
||||
else:
|
||||
@@ -104,6 +113,13 @@ class NixlConnectorScheduler:
|
||||
# remote prefill or aborted.
|
||||
self._reqs_not_processed: set[ReqId] = set()
|
||||
|
||||
# Heartbeat tracking: requests needing periodic lease-renewal heartbeats to
|
||||
# remote P-side, stored as ready-to-send HeartbeatInfo grouped by remote engine
|
||||
self._heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {}
|
||||
# Reverse lookup: local req_id -> (engine_id, remote_req_id) for O(1) removal
|
||||
self._heartbeat_req_engine: dict[ReqId, tuple[EngineId, ReqId]] = {}
|
||||
self._last_heartbeat_time: float = 0.0
|
||||
|
||||
# Gather Sliding Window sizes for each kv cache group (if any) in number of
|
||||
# blocks per KV cache group. This is used to clip the local attention window.
|
||||
sw_sizes_tokens: list[tuple[int, int]] = [
|
||||
@@ -135,12 +151,19 @@ class NixlConnectorScheduler:
|
||||
"bidirectional_kv_xfer", False
|
||||
)
|
||||
)
|
||||
self.decoder_kv_blocks_ttl = (
|
||||
vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"decoder_kv_blocks_ttl", 480
|
||||
)
|
||||
)
|
||||
|
||||
if self.is_bidirectional_kv_xfer_enabled and self.kv_recompute_threshold > 0:
|
||||
logger.info(
|
||||
"Bidirectional KV transfer is enabled and the kv "
|
||||
"recompute threshold is set to %d tokens",
|
||||
"recompute threshold is set to %d tokens."
|
||||
"KV blocks on D are released after a TTL of %d seconds.",
|
||||
self.kv_recompute_threshold,
|
||||
self.decoder_kv_blocks_ttl,
|
||||
)
|
||||
|
||||
def shutdown(self):
|
||||
@@ -149,6 +172,50 @@ class NixlConnectorScheduler:
|
||||
self._nixl_handshake_listener_t.join()
|
||||
self._nixl_handshake_listener_t = None
|
||||
|
||||
def on_new_request(self, request: "Request") -> None:
|
||||
"""Track a request that may need heartbeats."""
|
||||
params = request.kv_transfer_params
|
||||
# NOTE (NickLucche) This excludes request meant for P, ie heartbeats are
|
||||
# effectively disabled for Bidirectional KV transfer.
|
||||
if params is None or not params.get("do_remote_prefill"):
|
||||
return
|
||||
# Only track if all required remote fields are present.
|
||||
remote_engine_id = params.get("remote_engine_id")
|
||||
remote_request_id = params.get("remote_request_id")
|
||||
host = params.get("remote_host")
|
||||
port = params.get("remote_port")
|
||||
tp_size = params.get("tp_size")
|
||||
if (
|
||||
remote_engine_id is None
|
||||
or remote_request_id is None
|
||||
or host is None
|
||||
or port is None
|
||||
or tp_size is None
|
||||
):
|
||||
return
|
||||
if remote_engine_id not in self._heartbeat_by_engine:
|
||||
self._heartbeat_by_engine[remote_engine_id] = HeartbeatInfo(
|
||||
req_ids=set(),
|
||||
host=host,
|
||||
port=port,
|
||||
tp_size=tp_size,
|
||||
)
|
||||
self._heartbeat_by_engine[remote_engine_id].req_ids.add(remote_request_id)
|
||||
self._heartbeat_req_engine[request.request_id] = (
|
||||
remote_engine_id,
|
||||
remote_request_id,
|
||||
)
|
||||
|
||||
def _stop_heartbeat(self, req_id: ReqId) -> None:
|
||||
"""Remove *req_id* from heartbeat tracking (if tracked)."""
|
||||
if key := self._heartbeat_req_engine.pop(req_id, None):
|
||||
engine_id, remote_id = key
|
||||
if info := self._heartbeat_by_engine.get(engine_id):
|
||||
info.req_ids.discard(remote_id)
|
||||
if not info.req_ids:
|
||||
# Clean up empty engines so we don't leak a key when remote dies.
|
||||
del self._heartbeat_by_engine[engine_id]
|
||||
|
||||
def get_sw_clipped_blocks(self, block_ids: BlockIds) -> BlockIds:
|
||||
"""
|
||||
Clip the number of blocks to the sliding window size for each kv cache group
|
||||
@@ -490,6 +557,13 @@ class NixlConnectorScheduler:
|
||||
meta.reqs_in_batch = self._reqs_in_batch
|
||||
meta.reqs_not_processed = self._reqs_not_processed
|
||||
|
||||
# Package heartbeats, throttled by heartbeat_interval.
|
||||
if self._heartbeat_by_engine:
|
||||
now = time.perf_counter()
|
||||
if now - self._last_heartbeat_time >= self._heartbeat_interval:
|
||||
self._last_heartbeat_time = now
|
||||
meta.heartbeat_by_engine = self._heartbeat_by_engine
|
||||
|
||||
# Clear the list once workers start the transfers
|
||||
self._reqs_need_recv.clear()
|
||||
self._reqs_in_batch = set()
|
||||
@@ -498,6 +572,11 @@ class NixlConnectorScheduler:
|
||||
|
||||
return meta
|
||||
|
||||
def update_connector_output(self, connector_output: "KVConnectorOutput") -> None:
|
||||
"""Stop heartbeating for requests whose KV transfer completed."""
|
||||
for req_id in connector_output.finished_recving or ():
|
||||
self._stop_heartbeat(req_id)
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
@@ -523,6 +602,10 @@ class NixlConnectorScheduler:
|
||||
is_p_node = bool(params.get("do_remote_decode"))
|
||||
is_d_node = not is_p_node
|
||||
|
||||
# Stop heartbeating for aborted requests that never reached finished_recving:
|
||||
# normal path cleans up in update_connector_output.
|
||||
self._stop_heartbeat(request.request_id)
|
||||
|
||||
if params.get("do_remote_prefill"):
|
||||
# If do_remote_prefill is still True when the request is finished,
|
||||
# update_state_after_alloc must not have been called (the request
|
||||
@@ -556,14 +639,19 @@ class NixlConnectorScheduler:
|
||||
remote_num_tokens = 0
|
||||
if delay_free_blocks:
|
||||
# Prefill request on remote. It will be read from D upon completion
|
||||
request_kv_blocks_ttl = self._kv_lease_duration
|
||||
if is_d_node:
|
||||
# For blocks pinned on D, use a simpler timeout for now instead of a
|
||||
# lease mechanism as turn2 request is client-driven.
|
||||
request_kv_blocks_ttl = self.decoder_kv_blocks_ttl
|
||||
logger.debug(
|
||||
"NIXLConnector request_finished(%s) waiting for %d seconds "
|
||||
"for remote decode to fetch blocks",
|
||||
"before releasing blocks",
|
||||
request.request_id,
|
||||
envs.VLLM_NIXL_ABORT_REQUEST_TIMEOUT,
|
||||
request_kv_blocks_ttl,
|
||||
)
|
||||
self._reqs_need_send[request.request_id] = (
|
||||
time.perf_counter() + envs.VLLM_NIXL_ABORT_REQUEST_TIMEOUT
|
||||
time.perf_counter() + request_kv_blocks_ttl
|
||||
)
|
||||
# NOTE HMA will "mark" empty/null blocks in groups with 0s (eg SWA ones),
|
||||
# trimming down after allocating for the whole sequence length. Empty
|
||||
|
||||
@@ -18,7 +18,6 @@ import numpy as np
|
||||
import torch
|
||||
import zmq
|
||||
|
||||
from vllm import envs
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
BlockIds,
|
||||
EngineId,
|
||||
@@ -217,6 +216,12 @@ class NixlConnectorWorker:
|
||||
self.nixl_backends = vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"backends", ["UCX"]
|
||||
)
|
||||
kv_lease_duration: int = vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"kv_lease_duration", 30
|
||||
)
|
||||
# NOTE (NickLucche): For now we use a hardcoded value for a simpler interface.
|
||||
self._lease_extension = kv_lease_duration * 2 // 3
|
||||
|
||||
self._is_hma_required = (
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
and any(
|
||||
@@ -688,23 +693,38 @@ class NixlConnectorWorker:
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
def _background_nixl_handshake(
|
||||
self, req_id: str, remote_engine_id: EngineId, meta: ReqMeta
|
||||
):
|
||||
# Do NIXL handshake in background and add to _ready_requests when done.
|
||||
fut = self._handshake_futures.get(remote_engine_id)
|
||||
if fut is None:
|
||||
assert meta.remote is not None
|
||||
def _ensure_handshake(
|
||||
self,
|
||||
engine_id: EngineId,
|
||||
host: str,
|
||||
port: int,
|
||||
tp_size: int,
|
||||
) -> Future[dict[int, str]] | None:
|
||||
"""
|
||||
Ensure a handshake is in-flight (or already done) for *engine_id*.
|
||||
|
||||
Returns the ``Future`` if a handshake is pending (or was just
|
||||
started), or ``None`` if the handshake already completed
|
||||
successfully. Callers can attach per-request callbacks to the
|
||||
returned future.
|
||||
Failures to handshake are logged and the request is marked as failed.
|
||||
"""
|
||||
with self._handshake_lock:
|
||||
if engine_id in self._remote_agents:
|
||||
return None
|
||||
fut = self._handshake_futures.get(engine_id)
|
||||
if fut is not None:
|
||||
return fut
|
||||
fut = self._handshake_initiation_executor.submit(
|
||||
self._nixl_handshake,
|
||||
meta.remote.host,
|
||||
meta.remote.port,
|
||||
meta.tp_size,
|
||||
remote_engine_id,
|
||||
host,
|
||||
port,
|
||||
tp_size,
|
||||
engine_id,
|
||||
)
|
||||
self._handshake_futures[remote_engine_id] = fut
|
||||
self._handshake_futures[engine_id] = fut
|
||||
|
||||
def done_callback(f: Future[dict[int, str]], eid=remote_engine_id):
|
||||
def done_callback(f: Future[dict[int, str]], eid=engine_id):
|
||||
with self._handshake_lock:
|
||||
del self._handshake_futures[eid]
|
||||
try:
|
||||
@@ -718,15 +738,30 @@ class NixlConnectorWorker:
|
||||
)
|
||||
|
||||
fut.add_done_callback(done_callback)
|
||||
return fut
|
||||
|
||||
# check handshake success before proceeding with request
|
||||
def _background_nixl_handshake(
|
||||
self, req_id: str, remote_engine_id: EngineId, meta: ReqMeta
|
||||
):
|
||||
# Do NIXL handshake in background and add to _ready_requests when done.
|
||||
assert meta.remote is not None
|
||||
fut = self._ensure_handshake(
|
||||
remote_engine_id,
|
||||
meta.remote.host,
|
||||
meta.remote.port,
|
||||
meta.tp_size,
|
||||
)
|
||||
if fut is None:
|
||||
# Already handshaked — only happens if caller does not pre-check.
|
||||
self._ready_requests.put((req_id, meta))
|
||||
return
|
||||
|
||||
# Check handshake success before proceeding with request.
|
||||
def request_ready(f: Future[Any], entry=(req_id, meta)):
|
||||
try:
|
||||
# check if handshake succeeded
|
||||
f.result()
|
||||
self._ready_requests.put(entry)
|
||||
except Exception as e:
|
||||
# handshake failed - mark blocks as invalid
|
||||
self._log_failure(
|
||||
failure_type="handshake_failed",
|
||||
req_id=req_id,
|
||||
@@ -1721,10 +1756,9 @@ class NixlConnectorWorker:
|
||||
self.xfer_stats.record_kv_expired_req()
|
||||
logger.warning(
|
||||
"Releasing expired KV blocks for request %s which were "
|
||||
"retrieved by %d decode worker(s) within %d seconds.",
|
||||
"retrieved by %d remote worker(s) before lease expired.",
|
||||
req_id,
|
||||
count,
|
||||
envs.VLLM_NIXL_ABORT_REQUEST_TIMEOUT,
|
||||
)
|
||||
self._reqs_to_process.remove(req_id)
|
||||
del self._reqs_to_send[req_id]
|
||||
@@ -1737,12 +1771,22 @@ class NixlConnectorWorker:
|
||||
Get req_ids which got a remote xfer message. When multiple consumers
|
||||
are reading from the same producer (heterogeneous TP scenario), wait
|
||||
for all consumers to be done pulling.
|
||||
|
||||
Also handles heartbeat notifications ("HB:req1,req2,...") by
|
||||
extending the lease on the referenced requests.
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
notified_req_ids: set[str] = set()
|
||||
for notifs in self.nixl_wrapper.get_new_notifs().values():
|
||||
for notif in notifs:
|
||||
req_id, tp_size = notif.decode("utf-8").rsplit(":", 1)
|
||||
msg = notif.decode("utf-8")
|
||||
|
||||
# Handle heartbeat messages from D-side.
|
||||
if msg.startswith("HB:"):
|
||||
self._handle_heartbeat(msg[3:])
|
||||
continue
|
||||
|
||||
req_id, tp_size = msg.rsplit(":", 1)
|
||||
if (
|
||||
req_id not in self._reqs_to_send
|
||||
and req_id not in self._reqs_to_process
|
||||
@@ -1777,6 +1821,27 @@ class NixlConnectorWorker:
|
||||
self._reqs_to_send.pop(req_id, None)
|
||||
return notified_req_ids
|
||||
|
||||
def _handle_heartbeat(self, payload: str) -> None:
|
||||
"""Extend leases for requests referenced in a heartbeat.
|
||||
|
||||
Args:
|
||||
payload: comma-separated P-side request IDs, e.g.
|
||||
"req_abc,req_def".
|
||||
"""
|
||||
new_expiry = time.perf_counter() + self._lease_extension
|
||||
for req_id in payload.split(","):
|
||||
if req_id in self._reqs_to_send:
|
||||
old = self._reqs_to_send[req_id]
|
||||
self._reqs_to_send[req_id] = max(old, new_expiry)
|
||||
logger.debug(
|
||||
"Heartbeat extended lease for request %s "
|
||||
"by %ds (old_expiry=%.1f, new_expiry=%.1f)",
|
||||
req_id,
|
||||
self._lease_extension,
|
||||
old,
|
||||
new_expiry,
|
||||
)
|
||||
|
||||
def _pop_done_transfers(self, transfers: dict[str, list[int]]) -> set[str]:
|
||||
"""
|
||||
Pop completed xfers by checking for DONE state.
|
||||
@@ -1897,6 +1962,37 @@ class NixlConnectorWorker:
|
||||
if req_id in self._reqs_to_process:
|
||||
self._reqs_to_send[req_id] = expiration_time
|
||||
|
||||
# Send heartbeats to P-side engines to keep KV blocks alive while
|
||||
# requests sit in the D scheduler WAITING queue.
|
||||
self._send_heartbeats(metadata)
|
||||
|
||||
def _send_heartbeats(self, metadata: NixlConnectorMetadata) -> None:
|
||||
"""
|
||||
Send heartbeat notifications to remote engines, extending lease on KV blocks.
|
||||
"""
|
||||
for engine_id, hb_info in metadata.heartbeat_by_engine.items():
|
||||
# Proactive handshake (this request may still be in waiting queue) so
|
||||
# the **next** heartbeat for this remote can go through.
|
||||
if (
|
||||
self._ensure_handshake(
|
||||
engine_id, hb_info.host, hb_info.port, hb_info.tp_size
|
||||
)
|
||||
is not None
|
||||
):
|
||||
continue # handshake is still pending
|
||||
|
||||
# Build the heartbeat message: "HB:req1,req2,..."
|
||||
hb_msg = ("HB:" + ",".join(hb_info.req_ids)).encode()
|
||||
for agent_name in self._remote_agents[engine_id].values():
|
||||
try:
|
||||
self.nixl_wrapper.send_notif(agent_name, notif_msg=hb_msg)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Failed to send heartbeat to engine %s",
|
||||
engine_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _read_blocks_for_req(self, req_id: str, meta: ReqMeta):
|
||||
assert meta.remote is not None and self.transfer_topo is not None
|
||||
engine_id = meta.remote.engine_id
|
||||
|
||||
Reference in New Issue
Block a user