diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 35370efa03..3f92b183dc 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -2267,6 +2267,127 @@ def test_transfer_setup_failure_returns_finished(default_vllm_config, dist_init) assert request_id in done_recving +@patch( + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + FailingNixlWrapper, +) +@pytest.mark.parametrize( + "failure_mode", + [ + "handshake", + "transfer_setup", + "transfer_failed", + "transfer_exception", + ], +) +def test_failed_request_skips_kv_postprocessing( + default_vllm_config, dist_init, failure_mode +): + """Test that failed requests skip KV sync and post-processing in + get_finished(). + + This is the core safety behavior: when a KV transfer fails at any stage, + the request must still appear in done_recving (so the scheduler can apply + kv_load_failure_policy), but sync_recved_kv_to_device and post-processing + must NOT be called since no valid KV data was received. + + Covers all failure paths that involve an actual (attempted) KV transfer: + - handshake: add_remote_agent raises during async handshake + - transfer_setup: make_prepped_xfer raises before handle is in transfers + - transfer_failed: check_xfer_state returns bad state ("ERR") in + _pop_done_transfers — this is the path that previously had the bug + where post-processing was NOT skipped + - transfer_exception: check_xfer_state raises in _pop_done_transfers + + Note: notification_failed (send_notif raises on the full-cache-hit path) + is intentionally excluded. That path is a best-effort D→P courtesy + notification; the blocks are already in D's cache, so no KV transfer + was attempted and done_recving is correctly empty. + """ + # Map each failure mode to the FailingNixlWrapper attribute to set. + _WRAPPER_CONFIG: dict[str, str] = { + "handshake": "fail_handshake", + "transfer_setup": "fail_transfer_setup", + "transfer_failed": "fail_transfer_state", + "transfer_exception": "fail_transfer_exception", + } + + # Use enable_permute_local_kv=True so that + # post_process_device_kv_on_receive would be called on the success path, + # making the assertion meaningful (not trivially true). + vllm_config = create_vllm_config(enable_permute_local_kv=True) + + connector = NixlConnector( + vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + ) + connector.connector_worker = FakeNixlConnectorWorker( + vllm_config, + connector.engine_id, + hand_shake_latency=0.1 if failure_mode == "handshake" else 0, + ) + worker = connector.connector_worker + setattr(worker.nixl_wrapper, _WRAPPER_CONFIG[failure_mode], True) + + request_id = f"test_{failure_mode}_skip_postprocess" + metadata = NixlConnectorMetadata() + metadata.add_new_req_to_recv( + request_id=request_id, + local_block_ids=([1, 2, 3],), + kv_transfer_params={ + "remote_block_ids": ([4, 5, 6],), + "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, + "remote_request_id": f"prefill-{request_id}", + "remote_host": "localhost", + "remote_port": 1234, + "remote_tp_size": 1, + }, + ) + connector.bind_connector_metadata(metadata) + + dummy_ctx = ForwardContext( + no_compile_layers={}, + attn_metadata={}, + slot_mapping={}, + ) + connector.start_load_kv(dummy_ctx) + + if failure_mode == "handshake": + # Wait for async handshake to fail. + time.sleep(0.3) + else: + # All other modes: let the handshake complete, then process the + # ready_requests queue. For transfer_failed / transfer_exception the + # handle ends up in _recving_transfers; the failure surfaces in + # get_finished() via _pop_done_transfers below. + connector.bind_connector_metadata(NixlConnectorMetadata()) + time.sleep(0.1) + connector.start_load_kv(dummy_ctx) + + # Spy on sync_recved_kv_to_device and post_process_device_kv_on_receive + # to verify they are NOT called for the failed request. + with ( + patch.object(worker, "sync_recved_kv_to_device") as mock_sync, + patch.object(worker, "post_process_device_kv_on_receive") as mock_postprocess, + ): + _, done_recving = connector.get_finished(finished_req_ids=set()) + + # The failed request must appear in done_recving so the scheduler + # can handle it (e.g., trigger recompute via kv_load_failure_policy). + assert request_id in done_recving + + # Critical: KV sync and post-processing must NOT have been called + # since no valid KV data was received for the failed request. + mock_sync.assert_not_called() + mock_postprocess.assert_not_called() + + # Metadata for the request should have been cleaned up. + assert request_id not in worker._recving_metadata + + # Blocks should have been marked as invalid. + invalid_blocks = connector.get_block_ids_with_load_errors() + assert invalid_blocks == {1, 2, 3} + + @pytest.mark.parametrize( "mismatch_type,config_overrides,version_override,should_fail,enforce_handshake_compat", [ diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py index a46367a9ab..f7d46af20e 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py @@ -388,10 +388,12 @@ class NixlConnectorWorker: # Set of requests that have been part of a batch, regardless of status. self._reqs_to_process: set[ReqId] = set() - # invalid blocks from failed NIXL operations - self._invalid_block_ids: set[int] = set() + # Invalid blocks from failed NIXL operations (thread-safe queue of block ids) + self._invalid_block_ids: queue.Queue[set[int]] = queue.Queue() # requests that skipped transfer (handshake or transfer failures) - self._failed_recv_reqs: set[ReqId] = set() + # Uses Queue for thread-safe cross-thread coordination with the + # background handshake thread, matching the _ready_requests pattern. + self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue() # Handshake metadata of this worker for NIXL transfers. self.xfer_handshake_metadata: NixlHandshakePayload | None = None @@ -768,11 +770,7 @@ class NixlConnectorWorker: error=e, meta=meta, ) - if ( - req_meta := self._recving_metadata.get(req_id) - ) and not self._is_hma_required: - self._invalid_block_ids.update(req_meta.local_block_ids[0]) - self._failed_recv_reqs.add(req_id) + self._handle_failed_transfer(req_id, None) fut.add_done_callback(request_ready) @@ -1696,17 +1694,26 @@ class NixlConnectorWorker: done_sending = self._get_new_notifs() done_recving = self._pop_done_transfers(self._recving_transfers) - # add requests that skipped transfer to done_recving - done_recving.update(self._failed_recv_reqs) - self._failed_recv_reqs.clear() + # Drain queue of requests where handshake or transfer setup failed. + failed_recv_reqs = set[ReqId]() + while not self._failed_recv_reqs.empty(): + try: + failed_recv_reqs.add(self._failed_recv_reqs.get_nowait()) + except queue.Empty: + break + + # Add failed requests to done_recving for scheduler tracking + # (blocks are already marked invalid, scheduler will handle recompute) + done_recving.update(failed_recv_reqs) if len(done_sending) > 0 or len(done_recving) > 0: logger.debug( "Rank %s, get_finished: %s requests done sending " - "and %s requests done recving", + "and %s requests done recving (%s failed)", self.tp_rank, len(done_sending), len(done_recving), + len(failed_recv_reqs), ) block_ids_for_blocksize_post_process = defaultdict(list) @@ -1715,6 +1722,15 @@ class NixlConnectorWorker: # clean up metadata for completed requests meta = self._recving_metadata.pop(req_id, None) assert meta is not None, f"{req_id} not found in recving_metadata list" + + # Skip KV sync and post-processing for failed requests + if req_id in failed_recv_reqs: + logger.warning( + "Skipping KV post-processing for failed request %s", + req_id, + ) + continue + assert meta.remote is not None if self.use_host_buffer: self.sync_recved_kv_to_device(req_id, meta) @@ -1889,7 +1905,7 @@ class NixlConnectorWorker: transfers[req_id] = in_progress return done_req_ids - def _handle_failed_transfer(self, req_id: str, handle: int): + def _handle_failed_transfer(self, req_id: str, handle: int | None): """ Handle a failed transfer by marking all (logical) blocks as invalid and recording the failure. @@ -1901,8 +1917,10 @@ class NixlConnectorWorker: # Use .get() here as the metadata cleanup is handled by get_finished() # TODO (NickLucche) handle failed transfer for HMA. if (meta := self._recving_metadata.get(req_id)) and not self._is_hma_required: - self._invalid_block_ids.update(meta.local_block_ids[0]) - self.nixl_wrapper.release_xfer_handle(handle) + self._invalid_block_ids.put(set(meta.local_block_ids[0])) + self._failed_recv_reqs.put(req_id) + if handle is not None: + self.nixl_wrapper.release_xfer_handle(handle) self.xfer_stats.record_failed_transfer() def start_load_kv(self, metadata: NixlConnectorMetadata): @@ -2221,14 +2239,7 @@ class NixlConnectorWorker: dst_engine_id=dst_engine_id, remote_rank=remote_rank, ) - if ( - meta := self._recving_metadata.get(request_id) - ) and not self._is_hma_required: - self._invalid_block_ids.update(meta.local_block_ids[0]) - self.xfer_stats.record_failed_transfer() - if handle is not None: - self.nixl_wrapper.release_xfer_handle(handle) - self._failed_recv_reqs.add(request_id) + self._handle_failed_transfer(request_id, handle) def get_mapped_blocks( self, block_ids: np.ndarray, block_size_ratio: int @@ -2366,8 +2377,13 @@ class NixlConnectorWorker: This is called by the scheduler to identify blocks that need to be retried after a NIXL transfer failure. """ - result = self._invalid_block_ids - self._invalid_block_ids = set() + # Drain the queue (thread-safe, no lock needed). + result: set[int] = set() + while not self._invalid_block_ids.empty(): + try: + result.update(self._invalid_block_ids.get_nowait()) + except queue.Empty: + break return result def __del__(self):