[NIXL] Avoid reading expired blocks in bidirectional turn-2 read (#47021)

Signed-off-by: Tomer Gilad <tgilad@nvidia.com>
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
tomerg-nvidia
2026-07-14 13:03:41 +00:00
committed by GitHub
co-authored by NickLucche
parent b6754f536e
commit 7a74a9662b
10 changed files with 264 additions and 22 deletions
@@ -322,8 +322,13 @@ class NixlBaseConnectorScheduler:
)
if msg != GET_META_MSG:
logger.warning("Connection listener got unexpected message %s", msg)
# Echo our perf_counter so P can estimate the clock offset.
# perf_counter is only comparable within a process, so this
# listener must run in the same process that stamps the block
# expiry deadline (`_reqs_need_send`).
ts = msgspec.msgpack.encode(time.perf_counter())
sock.send_multipart(
(identity, b"", encoded_data[(target_pp_rank, target_tp_rank)])
(identity, b"", encoded_data[(target_pp_rank, target_tp_rank)], ts)
)
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
@@ -266,6 +266,12 @@ class NixlBaseConnectorWorker:
# NOTE (NickLucche): For now we use a hardcoded value for a simpler interface.
self._lease_extension = kv_lease_duration * 2 // 3
self._bidirectional_kv_xfer_enabled: bool = (
vllm_config.kv_transfer_config.get_from_extra_config(
"bidirectional_kv_xfer", False
)
)
self._is_hma_required = (
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
and any(
@@ -340,6 +346,8 @@ class NixlBaseConnectorWorker:
self._remote_agents: dict[EngineId, dict[tuple[int, int], str]] = defaultdict(
dict
)
# Map of engine_id -> clock offset.
self._engine_clock_offset: dict[EngineId, float] = {}
# Metadata.
self.engine_id: EngineId = engine_id
@@ -470,7 +478,9 @@ class NixlBaseConnectorWorker:
thread_name_prefix="vllm-nixl-handshake-initiator",
)
self._ready_requests = queue.Queue[tuple[ReqId, ReqMeta]]()
self._handshake_futures: dict[EngineId, Future[dict[tuple[int, int], str]]] = {}
self._handshake_futures: dict[
EngineId, Future[tuple[dict[tuple[int, int], str], float]]
] = {}
# Protects _handshake_futures and _remote_agents.
self._handshake_lock = threading.RLock()
@@ -559,7 +569,7 @@ class NixlBaseConnectorWorker:
expected_engine_id: str,
remote_pp_size: int = 1,
notif_agents_only: bool = False,
) -> dict[tuple[int, int], str]:
) -> tuple[dict[tuple[int, int], str], float]:
"""Do a NIXL handshake with a remote instance."""
# the first time we connect to a remote agent.
@@ -583,6 +593,11 @@ class NixlBaseConnectorWorker:
p_remote_ranks = self.transfer_topo.handshake_target_ranks(remote_tp_size)
remote_rank_to_agent_name: dict[tuple[int, int], str] = {}
path = make_zmq_path("tcp", host, port)
# Clock offset to the peer, estimated from the handshake round-trip.
# Keep the lowest-RTT sample: hop cost is ~uniform across ranks, so a
# higher RTT is just noise that skews the midpoint estimate.
best_rtt = float("inf")
best_offset: float | None = None
with zmq_ctx(zmq.REQ, path) as sock:
for remote_pp_rank, remote_rank in itertools.product(
@@ -595,15 +610,24 @@ class NixlBaseConnectorWorker:
remote_rank,
)
start_time = time.perf_counter()
# Send query for the request.
msg = msgspec.msgpack.encode(
(GET_META_MSG, remote_pp_rank, remote_rank)
)
# Set receive timeout to 5 seconds to avoid hanging on dead server
sock.setsockopt(zmq.RCVTIMEO, 5000) # milliseconds
start_time = time.perf_counter()
sock.send(msg)
handshake_bytes = sock.recv()
reply_parts = sock.recv_multipart()
recv_time = time.perf_counter()
assert len(reply_parts) == 2
handshake_bytes = reply_parts[0]
remote_perf = msgspec.msgpack.decode(reply_parts[1])
rtt = recv_time - start_time
if rtt < best_rtt:
best_rtt = rtt
best_offset = remote_perf - (start_time + recv_time) / 2
# Decode handshake payload to get compatibility hash
handshake_decoder = msgspec.msgpack.Decoder(NixlHandshakePayload)
@@ -681,7 +705,9 @@ class NixlBaseConnectorWorker:
)
remote_ranks = (remote_pp_rank, remote_rank)
remote_rank_to_agent_name[remote_ranks] = remote_agent_name
return remote_rank_to_agent_name
assert best_offset is not None
return remote_rank_to_agent_name, best_offset
def _add_notif_only_remote_agent(
self, metadata: NixlAgentMetadata, remote_tp_size: int
@@ -818,7 +844,7 @@ class NixlBaseConnectorWorker:
tp_size: int,
pp_size: int = 1,
notif_agents_only: bool = False,
) -> Future[dict[tuple[int, int], str]] | None:
) -> Future[tuple[dict[tuple[int, int], str], float]] | None:
"""
Ensure a handshake is in-flight (or already done) for *engine_id*.
@@ -846,11 +872,16 @@ class NixlBaseConnectorWorker:
)
self._handshake_futures[engine_id] = fut
def done_callback(f: Future[dict[tuple[int, int], str]], eid=engine_id):
def done_callback(
f: Future[tuple[dict[tuple[int, int], str], float]],
eid=engine_id,
):
with self._handshake_lock:
del self._handshake_futures[eid]
try:
self._remote_agents[eid] = f.result()
remote_agents, clock_offset = f.result()
self._remote_agents[eid] = remote_agents
self._engine_clock_offset[eid] = clock_offset
self._engine_last_active[eid] = time.perf_counter()
except Exception as e:
self._log_failure(
@@ -2421,6 +2452,8 @@ class NixlBaseConnectorWorker:
if self.transfer_topo is not None:
self.transfer_topo.unregister_remote_engine(engine_id)
# Drop the cached clock offset; it is re-measured on the next handshake.
self._engine_clock_offset.pop(engine_id, None)
# Push P-side engines are tracked in _remote_agents but not in
# _engine_last_active (they don't participate in stale eviction), so
# tolerate a missing entry.
@@ -39,8 +39,10 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
# 2: Add remote_request_id to kv_transfer_params
# 3: Add physical_blocks_per_logical_kv_block to NixlAgentMetadata
# 4: Add KV block lease renewal through heartbeats
# 5: Add remote_blocks_expiry_time to kv_transfer_params + handshake
# clock-sync timestamp
#
NIXL_CONNECTOR_VERSION: int = 4
NIXL_CONNECTOR_VERSION: int = 5
@dataclass
@@ -157,6 +159,7 @@ class RemoteMeta:
port: int
engine_id: str
request_id: str
blocks_expiry_time: float | None = None
@dataclass
@@ -225,5 +228,6 @@ class NixlConnectorMetadata(KVConnectorMetadata):
request_id=kv_transfer_params["remote_request_id"],
host=kv_transfer_params["remote_host"],
port=kv_transfer_params["remote_port"],
blocks_expiry_time=kv_transfer_params.get("remote_blocks_expiry_time"),
)
self.reqs_to_recv[request_id] = req
@@ -238,6 +238,7 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
# remove the conditional below
delay_free_blocks = any(len(group) > 0 for group in block_ids)
remote_num_tokens = 0
blocks_expiry_time = None
if delay_free_blocks:
# Prefill request on remote. It will be read from D upon completion
request_kv_blocks_ttl = self._kv_lease_duration
@@ -254,6 +255,9 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
self._reqs_need_send[request.request_id] = (
time.perf_counter() + request_kv_blocks_ttl
)
if is_d_node:
# D blocks expiry time exported for the turn-2 readback.
blocks_expiry_time = self._reqs_need_send[request.request_id]
# NOTE HMA will "mark" empty/null blocks in groups with 0s (eg SWA ones),
# trimming down after allocating for the whole sequence length. Empty
# blocks are always at the start of the list.
@@ -272,4 +276,5 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
remote_port=self.side_channel_port,
tp_size=self.vllm_config.parallel_config.tensor_parallel_size,
remote_num_tokens=remote_num_tokens,
remote_blocks_expiry_time=blocks_expiry_time,
)
@@ -25,6 +25,10 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
# Slack (seconds) subtracted from D's exported block-expiry deadline on the turn-2
# readback, absorbing clock-offset error and read latency.
_KV_BLOCKS_EXPIRY_SAFETY_MARGIN = 5.0
class NixlPullConnectorWorker(NixlBaseConnectorWorker):
"""Pull-specific (READ) worker logic."""
@@ -98,12 +102,34 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
# requests sit in the D scheduler WAITING queue.
self._send_heartbeats(metadata)
def _is_turn2_read_expired(self, meta: ReqMeta) -> bool:
"""Whether D's cached blocks for this turn-2 readback have (nearly) expired."""
assert meta.remote is not None
blocks_expiry_time = meta.remote.blocks_expiry_time
# Deadline may be absent (router may not forward it) -> read as usual.
if blocks_expiry_time is None or not meta.local_physical_block_ids:
return False
clock_offset = self._engine_clock_offset[meta.remote.engine_id]
deadline = blocks_expiry_time - clock_offset
return time.perf_counter() + _KV_BLOCKS_EXPIRY_SAFETY_MARGIN >= deadline
def _read_blocks_for_req(self, req_id: str, meta: ReqMeta):
assert meta.remote is not None and self.transfer_topo is not None
engine_id = meta.remote.engine_id
# Update last activity from this remote. Mind that cleanup is done on main
# thread (this one), so we don't race on this structure.
self._engine_last_active[engine_id] = time.perf_counter()
if self._bidirectional_kv_xfer_enabled and self._is_turn2_read_expired(meta):
logger.warning(
"Declining expired remote read for %s from engine %s.",
req_id,
engine_id,
)
self.xfer_stats.record_kv_expired_req()
self._handle_failed_transfer(req_id, None)
return
plan = self.tp_mappings[engine_id]
remote_info = self.transfer_topo.get_engine_info(engine_id)
tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size)
@@ -294,7 +294,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
return
def _on_handshake(
f: Future[dict[tuple[int, int], str]],
f: Future[tuple[dict[tuple[int, int], str], float]],
rid: str = req_id,
rd: dict[str, Any] = reg_data,
) -> None:
@@ -456,7 +456,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
if decode_engine_id in self._remote_agents:
return True
try:
remote_agents = self._nixl_handshake(
remote_agents, _ = self._nixl_handshake(
decode_host,
decode_port,
decode_tp_size,