[Nixl][PD] Lease renewal TTL KV blocks on P (#41383)

Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
Nicolò Lucchesi
2026-05-11 09:27:30 +00:00
committed by GitHub
parent 9efdddca28
commit 770e9bd6b3
13 changed files with 462 additions and 48 deletions
@@ -510,6 +510,14 @@ class KVConnectorBase_V1(ABC):
"""
pass
def on_new_request(self, request: "Request") -> None:
"""Called by the scheduler when a new request is added.
Connectors can override this to inspect the request and perform
bookkeeping. The default implementation is a no-op.
"""
return
def update_connector_output(self, connector_output: KVConnectorOutput):
"""
Update KVConnector state from worker-side connectors output.
@@ -389,6 +389,10 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
# Call with empty blocks for other connectors.
c.update_state_after_alloc(request, empty_blocks, 0)
def on_new_request(self, request: "Request") -> None:
for c in self._connectors:
c.on_new_request(request)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> MultiKVConnectorMetadata:
@@ -44,6 +44,7 @@ from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
from vllm.v1.attention.backends.utils import get_kv_cache_layout
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import MambaSpec
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
@@ -156,6 +157,14 @@ class NixlConnector(KVConnectorBase_V1, SupportsHMA):
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def on_new_request(self, request: "Request") -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.on_new_request(request)
def update_connector_output(self, connector_output: KVConnectorOutput):
assert self.connector_scheduler is not None
self.connector_scheduler.update_connector_output(connector_output)
def request_finished(
self,
request: "Request",
@@ -6,7 +6,7 @@ from dataclasses import dataclass
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds, EngineId
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorHandshakeMetadata,
KVConnectorMetadata,
@@ -33,8 +33,9 @@ GET_META_MSG = b"get_meta_msg"
# 1: Initial version with compatibility checking
# 2: Add remote_request_id to kv_transfer_params
# 3: Add physical_blocks_per_logical_kv_block to NixlAgentMetadata
# 4: Add KV block lease renewal through heartbeats
#
NIXL_CONNECTOR_VERSION: int = 3
NIXL_CONNECTOR_VERSION: int = 4
@dataclass
@@ -133,6 +134,16 @@ def compute_nixl_compatibility_hash(
return compat_hash
@dataclass
class HeartbeatInfo:
"""Heartbeat data for a single remote engine, sent from D worker to P."""
req_ids: set[ReqId]
host: str
port: int
tp_size: int
@dataclass
class RemoteMeta:
block_ids: BlockIds
@@ -158,6 +169,8 @@ class NixlConnectorMetadata(KVConnectorMetadata):
self.reqs_to_send: dict[ReqId, float] = {}
self.reqs_in_batch: set[ReqId] = set()
self.reqs_not_processed: set[ReqId] = set()
# Heartbeat data grouped by remote engine, sent by D worker to P.
self.heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {}
def _add_new_req(
self,
@@ -21,6 +21,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
GET_META_MSG,
HeartbeatInfo,
NixlConnectorMetadata,
NixlHandshakePayload,
ReqId,
@@ -41,6 +42,7 @@ if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
@@ -65,6 +67,13 @@ class NixlConnectorScheduler:
+ vllm_config.parallel_config.data_parallel_index
)
assert vllm_config.kv_transfer_config is not None
self._kv_lease_duration: int = (
vllm_config.kv_transfer_config.get_from_extra_config(
"kv_lease_duration", 30
)
)
# NOTE (NickLucche): For now we use a hardcoded value for a simpler interface.
self._heartbeat_interval = self._kv_lease_duration // 6
if current_platform.device_type == "cpu":
self.use_host_buffer = False
else:
@@ -104,6 +113,13 @@ class NixlConnectorScheduler:
# remote prefill or aborted.
self._reqs_not_processed: set[ReqId] = set()
# Heartbeat tracking: requests needing periodic lease-renewal heartbeats to
# remote P-side, stored as ready-to-send HeartbeatInfo grouped by remote engine
self._heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {}
# Reverse lookup: local req_id -> (engine_id, remote_req_id) for O(1) removal
self._heartbeat_req_engine: dict[ReqId, tuple[EngineId, ReqId]] = {}
self._last_heartbeat_time: float = 0.0
# Gather Sliding Window sizes for each kv cache group (if any) in number of
# blocks per KV cache group. This is used to clip the local attention window.
sw_sizes_tokens: list[tuple[int, int]] = [
@@ -135,12 +151,19 @@ class NixlConnectorScheduler:
"bidirectional_kv_xfer", False
)
)
self.decoder_kv_blocks_ttl = (
vllm_config.kv_transfer_config.get_from_extra_config(
"decoder_kv_blocks_ttl", 480
)
)
if self.is_bidirectional_kv_xfer_enabled and self.kv_recompute_threshold > 0:
logger.info(
"Bidirectional KV transfer is enabled and the kv "
"recompute threshold is set to %d tokens",
"recompute threshold is set to %d tokens."
"KV blocks on D are released after a TTL of %d seconds.",
self.kv_recompute_threshold,
self.decoder_kv_blocks_ttl,
)
def shutdown(self):
@@ -149,6 +172,50 @@ class NixlConnectorScheduler:
self._nixl_handshake_listener_t.join()
self._nixl_handshake_listener_t = None
def on_new_request(self, request: "Request") -> None:
"""Track a request that may need heartbeats."""
params = request.kv_transfer_params
# NOTE (NickLucche) This excludes request meant for P, ie heartbeats are
# effectively disabled for Bidirectional KV transfer.
if params is None or not params.get("do_remote_prefill"):
return
# Only track if all required remote fields are present.
remote_engine_id = params.get("remote_engine_id")
remote_request_id = params.get("remote_request_id")
host = params.get("remote_host")
port = params.get("remote_port")
tp_size = params.get("tp_size")
if (
remote_engine_id is None
or remote_request_id is None
or host is None
or port is None
or tp_size is None
):
return
if remote_engine_id not in self._heartbeat_by_engine:
self._heartbeat_by_engine[remote_engine_id] = HeartbeatInfo(
req_ids=set(),
host=host,
port=port,
tp_size=tp_size,
)
self._heartbeat_by_engine[remote_engine_id].req_ids.add(remote_request_id)
self._heartbeat_req_engine[request.request_id] = (
remote_engine_id,
remote_request_id,
)
def _stop_heartbeat(self, req_id: ReqId) -> None:
"""Remove *req_id* from heartbeat tracking (if tracked)."""
if key := self._heartbeat_req_engine.pop(req_id, None):
engine_id, remote_id = key
if info := self._heartbeat_by_engine.get(engine_id):
info.req_ids.discard(remote_id)
if not info.req_ids:
# Clean up empty engines so we don't leak a key when remote dies.
del self._heartbeat_by_engine[engine_id]
def get_sw_clipped_blocks(self, block_ids: BlockIds) -> BlockIds:
"""
Clip the number of blocks to the sliding window size for each kv cache group
@@ -490,6 +557,13 @@ class NixlConnectorScheduler:
meta.reqs_in_batch = self._reqs_in_batch
meta.reqs_not_processed = self._reqs_not_processed
# Package heartbeats, throttled by heartbeat_interval.
if self._heartbeat_by_engine:
now = time.perf_counter()
if now - self._last_heartbeat_time >= self._heartbeat_interval:
self._last_heartbeat_time = now
meta.heartbeat_by_engine = self._heartbeat_by_engine
# Clear the list once workers start the transfers
self._reqs_need_recv.clear()
self._reqs_in_batch = set()
@@ -498,6 +572,11 @@ class NixlConnectorScheduler:
return meta
def update_connector_output(self, connector_output: "KVConnectorOutput") -> None:
"""Stop heartbeating for requests whose KV transfer completed."""
for req_id in connector_output.finished_recving or ():
self._stop_heartbeat(req_id)
def request_finished(
self,
request: "Request",
@@ -523,6 +602,10 @@ class NixlConnectorScheduler:
is_p_node = bool(params.get("do_remote_decode"))
is_d_node = not is_p_node
# Stop heartbeating for aborted requests that never reached finished_recving:
# normal path cleans up in update_connector_output.
self._stop_heartbeat(request.request_id)
if params.get("do_remote_prefill"):
# If do_remote_prefill is still True when the request is finished,
# update_state_after_alloc must not have been called (the request
@@ -556,14 +639,19 @@ class NixlConnectorScheduler:
remote_num_tokens = 0
if delay_free_blocks:
# Prefill request on remote. It will be read from D upon completion
request_kv_blocks_ttl = self._kv_lease_duration
if is_d_node:
# For blocks pinned on D, use a simpler timeout for now instead of a
# lease mechanism as turn2 request is client-driven.
request_kv_blocks_ttl = self.decoder_kv_blocks_ttl
logger.debug(
"NIXLConnector request_finished(%s) waiting for %d seconds "
"for remote decode to fetch blocks",
"before releasing blocks",
request.request_id,
envs.VLLM_NIXL_ABORT_REQUEST_TIMEOUT,
request_kv_blocks_ttl,
)
self._reqs_need_send[request.request_id] = (
time.perf_counter() + envs.VLLM_NIXL_ABORT_REQUEST_TIMEOUT
time.perf_counter() + request_kv_blocks_ttl
)
# NOTE HMA will "mark" empty/null blocks in groups with 0s (eg SWA ones),
# trimming down after allocating for the whole sequence length. Empty
@@ -18,7 +18,6 @@ import numpy as np
import torch
import zmq
from vllm import envs
from vllm.distributed.kv_transfer.kv_connector.utils import (
BlockIds,
EngineId,
@@ -217,6 +216,12 @@ class NixlConnectorWorker:
self.nixl_backends = vllm_config.kv_transfer_config.get_from_extra_config(
"backends", ["UCX"]
)
kv_lease_duration: int = vllm_config.kv_transfer_config.get_from_extra_config(
"kv_lease_duration", 30
)
# NOTE (NickLucche): For now we use a hardcoded value for a simpler interface.
self._lease_extension = kv_lease_duration * 2 // 3
self._is_hma_required = (
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
and any(
@@ -688,23 +693,38 @@ class NixlConnectorWorker:
stacklevel=2,
)
def _background_nixl_handshake(
self, req_id: str, remote_engine_id: EngineId, meta: ReqMeta
):
# Do NIXL handshake in background and add to _ready_requests when done.
fut = self._handshake_futures.get(remote_engine_id)
if fut is None:
assert meta.remote is not None
def _ensure_handshake(
self,
engine_id: EngineId,
host: str,
port: int,
tp_size: int,
) -> Future[dict[int, str]] | None:
"""
Ensure a handshake is in-flight (or already done) for *engine_id*.
Returns the ``Future`` if a handshake is pending (or was just
started), or ``None`` if the handshake already completed
successfully. Callers can attach per-request callbacks to the
returned future.
Failures to handshake are logged and the request is marked as failed.
"""
with self._handshake_lock:
if engine_id in self._remote_agents:
return None
fut = self._handshake_futures.get(engine_id)
if fut is not None:
return fut
fut = self._handshake_initiation_executor.submit(
self._nixl_handshake,
meta.remote.host,
meta.remote.port,
meta.tp_size,
remote_engine_id,
host,
port,
tp_size,
engine_id,
)
self._handshake_futures[remote_engine_id] = fut
self._handshake_futures[engine_id] = fut
def done_callback(f: Future[dict[int, str]], eid=remote_engine_id):
def done_callback(f: Future[dict[int, str]], eid=engine_id):
with self._handshake_lock:
del self._handshake_futures[eid]
try:
@@ -718,15 +738,30 @@ class NixlConnectorWorker:
)
fut.add_done_callback(done_callback)
return fut
# check handshake success before proceeding with request
def _background_nixl_handshake(
self, req_id: str, remote_engine_id: EngineId, meta: ReqMeta
):
# Do NIXL handshake in background and add to _ready_requests when done.
assert meta.remote is not None
fut = self._ensure_handshake(
remote_engine_id,
meta.remote.host,
meta.remote.port,
meta.tp_size,
)
if fut is None:
# Already handshaked — only happens if caller does not pre-check.
self._ready_requests.put((req_id, meta))
return
# Check handshake success before proceeding with request.
def request_ready(f: Future[Any], entry=(req_id, meta)):
try:
# check if handshake succeeded
f.result()
self._ready_requests.put(entry)
except Exception as e:
# handshake failed - mark blocks as invalid
self._log_failure(
failure_type="handshake_failed",
req_id=req_id,
@@ -1721,10 +1756,9 @@ class NixlConnectorWorker:
self.xfer_stats.record_kv_expired_req()
logger.warning(
"Releasing expired KV blocks for request %s which were "
"retrieved by %d decode worker(s) within %d seconds.",
"retrieved by %d remote worker(s) before lease expired.",
req_id,
count,
envs.VLLM_NIXL_ABORT_REQUEST_TIMEOUT,
)
self._reqs_to_process.remove(req_id)
del self._reqs_to_send[req_id]
@@ -1737,12 +1771,22 @@ class NixlConnectorWorker:
Get req_ids which got a remote xfer message. When multiple consumers
are reading from the same producer (heterogeneous TP scenario), wait
for all consumers to be done pulling.
Also handles heartbeat notifications ("HB:req1,req2,...") by
extending the lease on the referenced requests.
"""
assert self.transfer_topo is not None
notified_req_ids: set[str] = set()
for notifs in self.nixl_wrapper.get_new_notifs().values():
for notif in notifs:
req_id, tp_size = notif.decode("utf-8").rsplit(":", 1)
msg = notif.decode("utf-8")
# Handle heartbeat messages from D-side.
if msg.startswith("HB:"):
self._handle_heartbeat(msg[3:])
continue
req_id, tp_size = msg.rsplit(":", 1)
if (
req_id not in self._reqs_to_send
and req_id not in self._reqs_to_process
@@ -1777,6 +1821,27 @@ class NixlConnectorWorker:
self._reqs_to_send.pop(req_id, None)
return notified_req_ids
def _handle_heartbeat(self, payload: str) -> None:
"""Extend leases for requests referenced in a heartbeat.
Args:
payload: comma-separated P-side request IDs, e.g.
"req_abc,req_def".
"""
new_expiry = time.perf_counter() + self._lease_extension
for req_id in payload.split(","):
if req_id in self._reqs_to_send:
old = self._reqs_to_send[req_id]
self._reqs_to_send[req_id] = max(old, new_expiry)
logger.debug(
"Heartbeat extended lease for request %s "
"by %ds (old_expiry=%.1f, new_expiry=%.1f)",
req_id,
self._lease_extension,
old,
new_expiry,
)
def _pop_done_transfers(self, transfers: dict[str, list[int]]) -> set[str]:
"""
Pop completed xfers by checking for DONE state.
@@ -1897,6 +1962,37 @@ class NixlConnectorWorker:
if req_id in self._reqs_to_process:
self._reqs_to_send[req_id] = expiration_time
# Send heartbeats to P-side engines to keep KV blocks alive while
# requests sit in the D scheduler WAITING queue.
self._send_heartbeats(metadata)
def _send_heartbeats(self, metadata: NixlConnectorMetadata) -> None:
"""
Send heartbeat notifications to remote engines, extending lease on KV blocks.
"""
for engine_id, hb_info in metadata.heartbeat_by_engine.items():
# Proactive handshake (this request may still be in waiting queue) so
# the **next** heartbeat for this remote can go through.
if (
self._ensure_handshake(
engine_id, hb_info.host, hb_info.port, hb_info.tp_size
)
is not None
):
continue # handshake is still pending
# Build the heartbeat message: "HB:req1,req2,..."
hb_msg = ("HB:" + ",".join(hb_info.req_ids)).encode()
for agent_name in self._remote_agents[engine_id].values():
try:
self.nixl_wrapper.send_notif(agent_name, notif_msg=hb_msg)
except Exception:
logger.debug(
"Failed to send heartbeat to engine %s",
engine_id,
exc_info=True,
)
def _read_blocks_for_req(self, req_id: str, meta: ReqMeta):
assert meta.remote is not None and self.transfer_topo is not None
engine_id = meta.remote.engine_id