diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 9b54f626f2..298886d160 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -341,6 +341,10 @@ def test_abort_immediately_remote_prefill_enqueues_empty_recv(): assert req_meta.remote.request_id == f"prefill-{42}" # do_remote_prefill is consumed by request_finished to prevent re-issuing. assert request.kv_transfer_params["do_remote_prefill"] is False + # The scheduler is not waiting on this recv -- the request is already gone + # from self.requests, so reporting it would trip `assert req_id in + # self.requests` in _update_from_kv_xfer_finished. + assert req_meta.awaiting_kvs is False @patch( @@ -2695,6 +2699,74 @@ class FailingNixlWrapper(FakeNixlWrapper): return super().check_xfer_state(handle) +@patch( + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", + FailingNixlWrapper, +) +@pytest.mark.parametrize("awaiting_kvs", [True, False]) +@pytest.mark.parametrize("notification_fails", [False, True]) +def test_empty_recv_is_reported_only_when_awaited( + default_vllm_config, dist_init, awaiting_kvs, notification_fails +): + """An empty recv is reported iff the scheduler parked the request on it. + + _read_blocks returns early when the local block list is empty -- D already + holds the KV and only the producer needs telling. Two kinds of caller reach + that path, and they need opposite handling: + + Parked (awaiting_kvs=True, a full prefix cache hit). Without a report the + request is named in neither _recving_transfers nor _failed_recv_reqs and + never reaches finished_recving. The scheduler has no other way to release a + WAITING_FOR_REMOTE_KVS request, and there is no timeout, so it sits in + skipped_waiting holding its blocks for the life of the process. + + Notify-only (awaiting_kvs=False): request_finished seeding an empty recv to + free P's blocks for a request aborted before it was scheduled, or a readback + on a request that stays RUNNING. Reporting either one crashes the engine + core -- _update_from_kv_xfer_finished asserts the id is still in + self.requests and that the request is parked or finished. + + A failed notification changes neither: the KV is local either way, and the + producer frees its own blocks on a timeout. + """ + vllm_config = create_vllm_config() + connector = NixlConnector( + vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + ) + connector.connector_worker = FakeNixlConnectorWorker( + vllm_config, + connector.engine_id, + hand_shake_latency=0.0, + kv_cache_config=connector._kv_cache_config, + ) + connector.connector_worker.nixl_wrapper.fail_send_notif = notification_fails + + request_id = f"test_empty_recv_awaited_{awaiting_kvs}" + metadata = NixlConnectorMetadata() + metadata.add_new_req_to_recv( + request_id=request_id, + local_block_ids=(), # empty: the whole prompt is already cached locally + kv_transfer_params={ + "remote_block_ids": [[20, 21, 22]], + "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, + "remote_request_id": f"prefill-{request_id}", + "remote_host": "localhost", + "remote_port": 1234, + "remote_tp_size": 1, + }, + awaiting_kvs=awaiting_kvs, + ) + connector.bind_connector_metadata(metadata) + dummy_ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={}) + connector.start_load_kv(dummy_ctx) + connector.bind_connector_metadata(NixlConnectorMetadata()) + time.sleep(0.1) + connector.start_load_kv(dummy_ctx) + + _, done_recving = connector.get_finished(finished_req_ids=set()) + assert (request_id in done_recving) is awaiting_kvs + + @patch( "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FailingNixlWrapper, diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index 259e9536c1..60eb2d6614 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -149,8 +149,49 @@ def test_update_state_after_alloc_tracks_cached_blocks_per_group(): scheduler.update_state_after_alloc(request, blocks, num_external_tokens=2) - _, _, local_num_computed_blocks = scheduler._reqs_need_recv[request.request_id] + _, _, local_num_computed_blocks, awaiting_kvs = scheduler._reqs_need_recv[ + request.request_id + ] assert local_num_computed_blocks == (2, 1) + # There are external tokens to pull, so the scheduler parks the request and + # is waiting to see it in finished_recving. + assert awaiting_kvs is True + + +@pytest.mark.cpu_test +def test_full_local_hit_is_not_awaited_by_the_scheduler(): + """A recv with num_external_tokens == 0 must not be flagged as awaited. + + The request keeps running -- there is nothing to pull, only a notification + owed to the producer -- so the worker must not report it in + finished_recving. Doing so trips `assert RequestStatus.is_finished` in + _update_from_kv_xfer_finished, which only tolerates a finished recv for a + request that is parked or already done.""" + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import ( + NixlPullConnectorScheduler, + ) + from vllm.v1.core.kv_cache_manager import KVCacheBlocks + from vllm.v1.core.kv_cache_utils import KVCacheBlock + + scheduler = object.__new__(NixlPullConnectorScheduler) + scheduler._reqs_in_batch = set() + scheduler._reqs_need_save = {} + scheduler._reqs_need_recv = {} + scheduler.use_host_buffer = False + scheduler.is_bidirectional_kv_xfer_enabled = False + scheduler._is_hma_required = False + scheduler.kv_cache_config = MagicMock( + select_transfer_block_ids=lambda block_ids: block_ids + ) + + blocks = KVCacheBlocks(blocks=([KVCacheBlock(block_id=0, _block_hash=object())],)) + request = create_request(do_remote_prefill=True) + + scheduler.update_state_after_alloc(request, blocks, num_external_tokens=0) + + _, local_block_ids, _, awaiting_kvs = scheduler._reqs_need_recv[request.request_id] + assert not local_block_ids # full local hit: nothing to pull + assert awaiting_kvs is False @pytest.mark.cpu_test diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py index 5c01d99696..9cfd100730 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py @@ -113,7 +113,7 @@ class NixlBaseConnectorScheduler: # New requests are added by update_state_after_alloc in # the scheduler. Used to make metadata passed to Worker. self._reqs_need_recv: dict[ - ReqId, tuple[Request, BlockIds, tuple[int, ...]] + ReqId, tuple[Request, BlockIds, tuple[int, ...], bool] ] = {} self._reqs_need_save: dict[ReqId, Request] = {} # Reqs to send and their expiration time @@ -459,13 +459,19 @@ class NixlBaseConnectorScheduler: meta = NixlConnectorMetadata() # Loop through scheduled reqs and convert to ReqMeta. - for req_id, (req, block_ids, cached) in self._reqs_need_recv.items(): + for req_id, ( + req, + block_ids, + cached, + awaiting_kvs, + ) in self._reqs_need_recv.items(): assert req.kv_transfer_params is not None meta.add_new_req_to_recv( request_id=req_id, local_block_ids=block_ids, kv_transfer_params=req.kv_transfer_params, local_num_computed_blocks=cached, + awaiting_kvs=awaiting_kvs, ) if self.use_host_buffer: diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py index f61fdfa835..be32a6969d 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py @@ -246,6 +246,9 @@ class ReqMeta: remote_block_size: int | None = None # Remote producer pipeline-parallel size (push mode, D side). pp_size: int = 1 + # True only when the scheduler parked the request in WAITING_FOR_REMOTE_KVS + # and expects it in finished_recving; notify-only recvs must not be reported. + awaiting_kvs: bool = False class NixlConnectorMetadata(KVConnectorMetadata): @@ -275,6 +278,7 @@ class NixlConnectorMetadata(KVConnectorMetadata): local_block_ids: BlockIds, kv_transfer_params: dict[str, Any], local_num_computed_blocks: tuple[int, ...] = (), + awaiting_kvs: bool = False, ) -> ReqMeta: return ReqMeta( local_block_ids=local_block_ids, @@ -285,6 +289,7 @@ class NixlConnectorMetadata(KVConnectorMetadata): remote_block_size=kv_transfer_params.get("remote_block_size"), pp_size=kv_transfer_params.get("pp_size", 1), local_num_computed_blocks=local_num_computed_blocks, + awaiting_kvs=awaiting_kvs, ) def add_new_req_to_save( @@ -303,11 +308,13 @@ class NixlConnectorMetadata(KVConnectorMetadata): local_block_ids: BlockIds, kv_transfer_params: dict[str, Any], local_num_computed_blocks: tuple[int, ...] = (), + awaiting_kvs: bool = False, ): req = self._add_new_req( local_block_ids, kv_transfer_params, local_num_computed_blocks, + awaiting_kvs, ) req.remote = RemoteMeta( block_ids=kv_transfer_params["remote_block_ids"], diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_scheduler.py index 446126c328..9b9012a532 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_scheduler.py @@ -172,6 +172,9 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler): request, local_block_ids, local_num_computed_blocks, + # Parked in WAITING_FOR_REMOTE_KVS only when there is + # something to pull; a full local hit stays RUNNING. + num_external_tokens > 0, ) else: @@ -224,7 +227,7 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler): # To avoid stranding the prefill blocks in the prefill instance, # we must add empty block_ids to _reqs_need_recv so that our # worker side will notify and free blocks in the prefill instance. - self._reqs_need_recv[request.request_id] = (request, [], ()) + self._reqs_need_recv[request.request_id] = (request, [], (), False) params["do_remote_prefill"] = False return False, None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py index 59205b943a..f787639cfe 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -303,6 +303,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): local_dram_handle=local_dram_handle, remote_xfer_side_handle=remote_xfer_side_handle, expected_consumers=plan.local_consumers, + awaiting_kvs=meta.awaiting_kvs, ): return @@ -327,6 +328,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): local_dram_handle: int | None, remote_xfer_side_handle: int, expected_consumers: int, + awaiting_kvs: bool, ) -> bool: """ Post a READ point-to-point xfer request from a single local worker to @@ -387,6 +389,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): remote_agent_name=agent_name, ) self.xfer_stats.record_failed_notification() + # Report even on notification failure: the KV is already local, and + # an unreported parked request would hold its blocks forever. + # Notify-only recvs must stay unreported (scheduler asserts). + if awaiting_kvs: + self._recving_transfers.setdefault(request_id, []) return True if read_spec.block_ids_by_region: diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py index c11a63bb34..a007e0c5a3 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py @@ -198,7 +198,7 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler): # ReqMeta without a KeyError — the actual remote block IDs are # learned by P over the NIXL handshake at WRITE time. params["remote_block_ids"] = () - self._reqs_need_recv[request.request_id] = (request, local_block_ids, ()) + self._reqs_need_recv[request.request_id] = (request, local_block_ids, (), False) # Mark as processed so a re-entry (e.g. preemption + reschedule) # doesn't re-stage the registration. @@ -240,7 +240,7 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler): # recv so the worker emits a notif that lets P free them. # Seed remote_block_ids so add_new_req_to_recv won't KeyError. params["remote_block_ids"] = () - self._reqs_need_recv[request.request_id] = (request, [], ()) + self._reqs_need_recv[request.request_id] = (request, [], (), False) params["do_remote_prefill"] = False return False, None