[KV Connector][NIXL][Bugfix] Fix NIXL handshake failures not honoring kv_load_failure_policy (#40364)
When NIXL handshake fails (e.g., due to compatibility hash mismatch between prefill and decode instances), requests fail with "engine dead" error instead of gracefully falling back to local recomputation as configured by kv_load_failure_policy='recompute'. Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
@@ -2267,6 +2267,127 @@ def test_transfer_setup_failure_returns_finished(default_vllm_config, dist_init)
|
||||
assert request_id in done_recving
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
||||
FailingNixlWrapper,
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"failure_mode",
|
||||
[
|
||||
"handshake",
|
||||
"transfer_setup",
|
||||
"transfer_failed",
|
||||
"transfer_exception",
|
||||
],
|
||||
)
|
||||
def test_failed_request_skips_kv_postprocessing(
|
||||
default_vllm_config, dist_init, failure_mode
|
||||
):
|
||||
"""Test that failed requests skip KV sync and post-processing in
|
||||
get_finished().
|
||||
|
||||
This is the core safety behavior: when a KV transfer fails at any stage,
|
||||
the request must still appear in done_recving (so the scheduler can apply
|
||||
kv_load_failure_policy), but sync_recved_kv_to_device and post-processing
|
||||
must NOT be called since no valid KV data was received.
|
||||
|
||||
Covers all failure paths that involve an actual (attempted) KV transfer:
|
||||
- handshake: add_remote_agent raises during async handshake
|
||||
- transfer_setup: make_prepped_xfer raises before handle is in transfers
|
||||
- transfer_failed: check_xfer_state returns bad state ("ERR") in
|
||||
_pop_done_transfers — this is the path that previously had the bug
|
||||
where post-processing was NOT skipped
|
||||
- transfer_exception: check_xfer_state raises in _pop_done_transfers
|
||||
|
||||
Note: notification_failed (send_notif raises on the full-cache-hit path)
|
||||
is intentionally excluded. That path is a best-effort D→P courtesy
|
||||
notification; the blocks are already in D's cache, so no KV transfer
|
||||
was attempted and done_recving is correctly empty.
|
||||
"""
|
||||
# Map each failure mode to the FailingNixlWrapper attribute to set.
|
||||
_WRAPPER_CONFIG: dict[str, str] = {
|
||||
"handshake": "fail_handshake",
|
||||
"transfer_setup": "fail_transfer_setup",
|
||||
"transfer_failed": "fail_transfer_state",
|
||||
"transfer_exception": "fail_transfer_exception",
|
||||
}
|
||||
|
||||
# Use enable_permute_local_kv=True so that
|
||||
# post_process_device_kv_on_receive would be called on the success path,
|
||||
# making the assertion meaningful (not trivially true).
|
||||
vllm_config = create_vllm_config(enable_permute_local_kv=True)
|
||||
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
connector.engine_id,
|
||||
hand_shake_latency=0.1 if failure_mode == "handshake" else 0,
|
||||
)
|
||||
worker = connector.connector_worker
|
||||
setattr(worker.nixl_wrapper, _WRAPPER_CONFIG[failure_mode], True)
|
||||
|
||||
request_id = f"test_{failure_mode}_skip_postprocess"
|
||||
metadata = NixlConnectorMetadata()
|
||||
metadata.add_new_req_to_recv(
|
||||
request_id=request_id,
|
||||
local_block_ids=([1, 2, 3],),
|
||||
kv_transfer_params={
|
||||
"remote_block_ids": ([4, 5, 6],),
|
||||
"remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
|
||||
"remote_request_id": f"prefill-{request_id}",
|
||||
"remote_host": "localhost",
|
||||
"remote_port": 1234,
|
||||
"remote_tp_size": 1,
|
||||
},
|
||||
)
|
||||
connector.bind_connector_metadata(metadata)
|
||||
|
||||
dummy_ctx = ForwardContext(
|
||||
no_compile_layers={},
|
||||
attn_metadata={},
|
||||
slot_mapping={},
|
||||
)
|
||||
connector.start_load_kv(dummy_ctx)
|
||||
|
||||
if failure_mode == "handshake":
|
||||
# Wait for async handshake to fail.
|
||||
time.sleep(0.3)
|
||||
else:
|
||||
# All other modes: let the handshake complete, then process the
|
||||
# ready_requests queue. For transfer_failed / transfer_exception the
|
||||
# handle ends up in _recving_transfers; the failure surfaces in
|
||||
# get_finished() via _pop_done_transfers below.
|
||||
connector.bind_connector_metadata(NixlConnectorMetadata())
|
||||
time.sleep(0.1)
|
||||
connector.start_load_kv(dummy_ctx)
|
||||
|
||||
# Spy on sync_recved_kv_to_device and post_process_device_kv_on_receive
|
||||
# to verify they are NOT called for the failed request.
|
||||
with (
|
||||
patch.object(worker, "sync_recved_kv_to_device") as mock_sync,
|
||||
patch.object(worker, "post_process_device_kv_on_receive") as mock_postprocess,
|
||||
):
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
|
||||
# The failed request must appear in done_recving so the scheduler
|
||||
# can handle it (e.g., trigger recompute via kv_load_failure_policy).
|
||||
assert request_id in done_recving
|
||||
|
||||
# Critical: KV sync and post-processing must NOT have been called
|
||||
# since no valid KV data was received for the failed request.
|
||||
mock_sync.assert_not_called()
|
||||
mock_postprocess.assert_not_called()
|
||||
|
||||
# Metadata for the request should have been cleaned up.
|
||||
assert request_id not in worker._recving_metadata
|
||||
|
||||
# Blocks should have been marked as invalid.
|
||||
invalid_blocks = connector.get_block_ids_with_load_errors()
|
||||
assert invalid_blocks == {1, 2, 3}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mismatch_type,config_overrides,version_override,should_fail,enforce_handshake_compat",
|
||||
[
|
||||
|
||||
@@ -388,10 +388,12 @@ class NixlConnectorWorker:
|
||||
# Set of requests that have been part of a batch, regardless of status.
|
||||
self._reqs_to_process: set[ReqId] = set()
|
||||
|
||||
# invalid blocks from failed NIXL operations
|
||||
self._invalid_block_ids: set[int] = set()
|
||||
# Invalid blocks from failed NIXL operations (thread-safe queue of block ids)
|
||||
self._invalid_block_ids: queue.Queue[set[int]] = queue.Queue()
|
||||
# requests that skipped transfer (handshake or transfer failures)
|
||||
self._failed_recv_reqs: set[ReqId] = set()
|
||||
# Uses Queue for thread-safe cross-thread coordination with the
|
||||
# background handshake thread, matching the _ready_requests pattern.
|
||||
self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue()
|
||||
|
||||
# Handshake metadata of this worker for NIXL transfers.
|
||||
self.xfer_handshake_metadata: NixlHandshakePayload | None = None
|
||||
@@ -768,11 +770,7 @@ class NixlConnectorWorker:
|
||||
error=e,
|
||||
meta=meta,
|
||||
)
|
||||
if (
|
||||
req_meta := self._recving_metadata.get(req_id)
|
||||
) and not self._is_hma_required:
|
||||
self._invalid_block_ids.update(req_meta.local_block_ids[0])
|
||||
self._failed_recv_reqs.add(req_id)
|
||||
self._handle_failed_transfer(req_id, None)
|
||||
|
||||
fut.add_done_callback(request_ready)
|
||||
|
||||
@@ -1696,17 +1694,26 @@ class NixlConnectorWorker:
|
||||
done_sending = self._get_new_notifs()
|
||||
done_recving = self._pop_done_transfers(self._recving_transfers)
|
||||
|
||||
# add requests that skipped transfer to done_recving
|
||||
done_recving.update(self._failed_recv_reqs)
|
||||
self._failed_recv_reqs.clear()
|
||||
# Drain queue of requests where handshake or transfer setup failed.
|
||||
failed_recv_reqs = set[ReqId]()
|
||||
while not self._failed_recv_reqs.empty():
|
||||
try:
|
||||
failed_recv_reqs.add(self._failed_recv_reqs.get_nowait())
|
||||
except queue.Empty:
|
||||
break
|
||||
|
||||
# Add failed requests to done_recving for scheduler tracking
|
||||
# (blocks are already marked invalid, scheduler will handle recompute)
|
||||
done_recving.update(failed_recv_reqs)
|
||||
|
||||
if len(done_sending) > 0 or len(done_recving) > 0:
|
||||
logger.debug(
|
||||
"Rank %s, get_finished: %s requests done sending "
|
||||
"and %s requests done recving",
|
||||
"and %s requests done recving (%s failed)",
|
||||
self.tp_rank,
|
||||
len(done_sending),
|
||||
len(done_recving),
|
||||
len(failed_recv_reqs),
|
||||
)
|
||||
|
||||
block_ids_for_blocksize_post_process = defaultdict(list)
|
||||
@@ -1715,6 +1722,15 @@ class NixlConnectorWorker:
|
||||
# clean up metadata for completed requests
|
||||
meta = self._recving_metadata.pop(req_id, None)
|
||||
assert meta is not None, f"{req_id} not found in recving_metadata list"
|
||||
|
||||
# Skip KV sync and post-processing for failed requests
|
||||
if req_id in failed_recv_reqs:
|
||||
logger.warning(
|
||||
"Skipping KV post-processing for failed request %s",
|
||||
req_id,
|
||||
)
|
||||
continue
|
||||
|
||||
assert meta.remote is not None
|
||||
if self.use_host_buffer:
|
||||
self.sync_recved_kv_to_device(req_id, meta)
|
||||
@@ -1889,7 +1905,7 @@ class NixlConnectorWorker:
|
||||
transfers[req_id] = in_progress
|
||||
return done_req_ids
|
||||
|
||||
def _handle_failed_transfer(self, req_id: str, handle: int):
|
||||
def _handle_failed_transfer(self, req_id: str, handle: int | None):
|
||||
"""
|
||||
Handle a failed transfer by marking all (logical) blocks as invalid and
|
||||
recording the failure.
|
||||
@@ -1901,8 +1917,10 @@ class NixlConnectorWorker:
|
||||
# Use .get() here as the metadata cleanup is handled by get_finished()
|
||||
# TODO (NickLucche) handle failed transfer for HMA.
|
||||
if (meta := self._recving_metadata.get(req_id)) and not self._is_hma_required:
|
||||
self._invalid_block_ids.update(meta.local_block_ids[0])
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
self._invalid_block_ids.put(set(meta.local_block_ids[0]))
|
||||
self._failed_recv_reqs.put(req_id)
|
||||
if handle is not None:
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
|
||||
def start_load_kv(self, metadata: NixlConnectorMetadata):
|
||||
@@ -2221,14 +2239,7 @@ class NixlConnectorWorker:
|
||||
dst_engine_id=dst_engine_id,
|
||||
remote_rank=remote_rank,
|
||||
)
|
||||
if (
|
||||
meta := self._recving_metadata.get(request_id)
|
||||
) and not self._is_hma_required:
|
||||
self._invalid_block_ids.update(meta.local_block_ids[0])
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
if handle is not None:
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
self._failed_recv_reqs.add(request_id)
|
||||
self._handle_failed_transfer(request_id, handle)
|
||||
|
||||
def get_mapped_blocks(
|
||||
self, block_ids: np.ndarray, block_size_ratio: int
|
||||
@@ -2366,8 +2377,13 @@ class NixlConnectorWorker:
|
||||
This is called by the scheduler to identify blocks that need
|
||||
to be retried after a NIXL transfer failure.
|
||||
"""
|
||||
result = self._invalid_block_ids
|
||||
self._invalid_block_ids = set()
|
||||
# Drain the queue (thread-safe, no lock needed).
|
||||
result: set[int] = set()
|
||||
while not self._invalid_block_ids.empty():
|
||||
try:
|
||||
result.update(self._invalid_block_ids.get_nowait())
|
||||
except queue.Empty:
|
||||
break
|
||||
return result
|
||||
|
||||
def __del__(self):
|
||||
|
||||
Reference in New Issue
Block a user