[KV Connector][NIXL][Bugfix] Fix NIXL handshake failures not honoring kv_load_failure_policy (#40364)

When NIXL handshake fails (e.g., due to compatibility hash mismatch
between prefill and decode instances), requests fail with "engine dead"
error instead of gracefully falling back to local recomputation as configured
by kv_load_failure_policy='recompute'.

Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
Nicolò Lucchesi
2026-05-11 09:37:21 +00:00
committed by GitHub
parent 770e9bd6b3
commit 5672d100ed
2 changed files with 162 additions and 25 deletions
@@ -388,10 +388,12 @@ class NixlConnectorWorker:
# Set of requests that have been part of a batch, regardless of status.
self._reqs_to_process: set[ReqId] = set()
# invalid blocks from failed NIXL operations
self._invalid_block_ids: set[int] = set()
# Invalid blocks from failed NIXL operations (thread-safe queue of block ids)
self._invalid_block_ids: queue.Queue[set[int]] = queue.Queue()
# requests that skipped transfer (handshake or transfer failures)
self._failed_recv_reqs: set[ReqId] = set()
# Uses Queue for thread-safe cross-thread coordination with the
# background handshake thread, matching the _ready_requests pattern.
self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue()
# Handshake metadata of this worker for NIXL transfers.
self.xfer_handshake_metadata: NixlHandshakePayload | None = None
@@ -768,11 +770,7 @@ class NixlConnectorWorker:
error=e,
meta=meta,
)
if (
req_meta := self._recving_metadata.get(req_id)
) and not self._is_hma_required:
self._invalid_block_ids.update(req_meta.local_block_ids[0])
self._failed_recv_reqs.add(req_id)
self._handle_failed_transfer(req_id, None)
fut.add_done_callback(request_ready)
@@ -1696,17 +1694,26 @@ class NixlConnectorWorker:
done_sending = self._get_new_notifs()
done_recving = self._pop_done_transfers(self._recving_transfers)
# add requests that skipped transfer to done_recving
done_recving.update(self._failed_recv_reqs)
self._failed_recv_reqs.clear()
# Drain queue of requests where handshake or transfer setup failed.
failed_recv_reqs = set[ReqId]()
while not self._failed_recv_reqs.empty():
try:
failed_recv_reqs.add(self._failed_recv_reqs.get_nowait())
except queue.Empty:
break
# Add failed requests to done_recving for scheduler tracking
# (blocks are already marked invalid, scheduler will handle recompute)
done_recving.update(failed_recv_reqs)
if len(done_sending) > 0 or len(done_recving) > 0:
logger.debug(
"Rank %s, get_finished: %s requests done sending "
"and %s requests done recving",
"and %s requests done recving (%s failed)",
self.tp_rank,
len(done_sending),
len(done_recving),
len(failed_recv_reqs),
)
block_ids_for_blocksize_post_process = defaultdict(list)
@@ -1715,6 +1722,15 @@ class NixlConnectorWorker:
# clean up metadata for completed requests
meta = self._recving_metadata.pop(req_id, None)
assert meta is not None, f"{req_id} not found in recving_metadata list"
# Skip KV sync and post-processing for failed requests
if req_id in failed_recv_reqs:
logger.warning(
"Skipping KV post-processing for failed request %s",
req_id,
)
continue
assert meta.remote is not None
if self.use_host_buffer:
self.sync_recved_kv_to_device(req_id, meta)
@@ -1889,7 +1905,7 @@ class NixlConnectorWorker:
transfers[req_id] = in_progress
return done_req_ids
def _handle_failed_transfer(self, req_id: str, handle: int):
def _handle_failed_transfer(self, req_id: str, handle: int | None):
"""
Handle a failed transfer by marking all (logical) blocks as invalid and
recording the failure.
@@ -1901,8 +1917,10 @@ class NixlConnectorWorker:
# Use .get() here as the metadata cleanup is handled by get_finished()
# TODO (NickLucche) handle failed transfer for HMA.
if (meta := self._recving_metadata.get(req_id)) and not self._is_hma_required:
self._invalid_block_ids.update(meta.local_block_ids[0])
self.nixl_wrapper.release_xfer_handle(handle)
self._invalid_block_ids.put(set(meta.local_block_ids[0]))
self._failed_recv_reqs.put(req_id)
if handle is not None:
self.nixl_wrapper.release_xfer_handle(handle)
self.xfer_stats.record_failed_transfer()
def start_load_kv(self, metadata: NixlConnectorMetadata):
@@ -2221,14 +2239,7 @@ class NixlConnectorWorker:
dst_engine_id=dst_engine_id,
remote_rank=remote_rank,
)
if (
meta := self._recving_metadata.get(request_id)
) and not self._is_hma_required:
self._invalid_block_ids.update(meta.local_block_ids[0])
self.xfer_stats.record_failed_transfer()
if handle is not None:
self.nixl_wrapper.release_xfer_handle(handle)
self._failed_recv_reqs.add(request_id)
self._handle_failed_transfer(request_id, handle)
def get_mapped_blocks(
self, block_ids: np.ndarray, block_size_ratio: int
@@ -2366,8 +2377,13 @@ class NixlConnectorWorker:
This is called by the scheduler to identify blocks that need
to be retried after a NIXL transfer failure.
"""
result = self._invalid_block_ids
self._invalid_block_ids = set()
# Drain the queue (thread-safe, no lock needed).
result: set[int] = set()
while not self._invalid_block_ids.empty():
try:
result.update(self._invalid_block_ids.get_nowait())
except queue.Empty:
break
return result
def __del__(self):