diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py index 292c3eea16..441e8e3676 100644 --- a/tests/v1/kv_connector/unit/test_multi_connector.py +++ b/tests/v1/kv_connector/unit/test_multi_connector.py @@ -272,7 +272,7 @@ def test_multi_example_connector_consistency(): "set_xfer_handshake_metadata_pp_aware", "on_new_request", "get_num_new_matched_tokens 0", - "update_state_after_alloc num_blocks=[0] 0", + "update_state_after_alloc num_blocks=[7] 0", "build_connector_meta", ] # First three events are from initialization (register_kv_caches, @@ -292,7 +292,7 @@ def test_multi_example_connector_consistency(): "set_xfer_handshake_metadata_pp_aware", "on_new_request", "get_num_new_matched_tokens 0", - "update_state_after_alloc num_blocks=[0] 0", + "update_state_after_alloc num_blocks=[7] 0", "build_connector_meta", ] assert events["storage2-WORKER"][:8] == [ @@ -315,9 +315,9 @@ def test_multi_example_connector_consistency(): events = get_connector_events() # get_num_new_matched_tokens will return new tokens from the first - # connector so update_state_after_alloc will be with allocated blocks - # on that one but with zero blocks for others (first nonzero match is - # chosen). + # connector (first nonzero match is chosen), so update_state_after_alloc + # will report those external tokens on that one. Other connectors still + # receive the request's real blocks but with 0 external tokens. storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"]) storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"]) assert storage1_scheduler_events[:4] == [ @@ -329,7 +329,7 @@ def test_multi_example_connector_consistency(): assert storage2_scheduler_events[:4] == [ "on_new_request", "get_num_new_matched_tokens 0", - "update_state_after_alloc num_blocks=[0] 0", + "update_state_after_alloc num_blocks=[7] 0", "build_connector_meta", ] @@ -345,15 +345,15 @@ def test_multi_example_connector_consistency(): events = get_connector_events() # get_num_new_matched_tokens will be called for both connectors but will - # return 0 from the first connector, but the second connector should have - # a hit, so update_state_after_alloc will only be called with allocated - # blocks for the second connector. + # return 0 from the first connector, while the second connector has a hit. + # Both connectors receive the request's real blocks, but only the chosen + # (second) connector reports external tokens. storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"]) storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"]) assert storage1_scheduler_events[:4] == [ "on_new_request", "get_num_new_matched_tokens 0", - "update_state_after_alloc num_blocks=[0] 0", + "update_state_after_alloc num_blocks=[7] 0", "build_connector_meta", ] assert storage2_scheduler_events[:4] == [ diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/base.py b/vllm/distributed/kv_transfer/kv_connector/v1/base.py index 954fedafe8..c0eb247297 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/base.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/base.py @@ -498,11 +498,16 @@ class KVConnectorBase_V1(ABC): asynchronously loaded into, and second when any additional blocks are allocated, after the load/transfer is complete. + Decide whether to load based on ``num_external_tokens``, not on + whether ``blocks`` is empty: ``blocks`` may be non-empty even when + ``num_external_tokens == 0`` (e.g. a non-chosen sub-connector of + MultiConnector still receives the request's real blocks). + Args: request (Request): the request object. blocks (KVCacheBlocks): the blocks allocated for the request. - num_external_tokens (int): the number of tokens that will be - loaded from the external KV cache. + num_external_tokens (int): the number of tokens to load from the + external KV cache. 0 means nothing should be loaded. """ pass diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index de21a1398e..3d0d216d99 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -550,17 +550,19 @@ class MoRIIOConnectorScheduler: # remote_engine_id is returned by the prefill's request_finished. # host/ports come from the request_id (parsed in add_new_req). if "remote_engine_id" in params: - # If remote_blocks and num_external_tokens = 0, we have - # a full prefix cache hit on the D worker. We need to call - # send_notify in _read_blocks to free the memory on the P. - - # Get unhashed blocks to pull from remote. - local_block_ids = blocks.get_block_ids()[0] - assert len(local_block_ids) <= len(remote_block_ids) - if len(local_block_ids) == len(remote_block_ids): - pass + if num_external_tokens > 0: + # Get unhashed blocks to pull from remote. + local_block_ids = blocks.get_block_ids()[0] + assert len(local_block_ids) <= len(remote_block_ids) + if len(local_block_ids) != len(remote_block_ids): + local_block_ids = remote_block_ids[ + -len(local_block_ids) : + ] else: - local_block_ids = remote_block_ids[-len(local_block_ids) :] + # If remote_blocks and num_external_tokens = 0, we have + # a full prefix cache hit on the D worker. We need to call + # send_notify in _read_blocks to free the memory on the P. + local_block_ids = [] self._reqs_need_recv[request.request_id] = ( request, @@ -593,7 +595,11 @@ class MoRIIOConnectorScheduler: ) remote_notify_port = int(remote_notify_port) - block_ids = blocks.get_block_ids()[0] + # num_external_tokens == 0: nothing to push, so don't tell the + # producer to write into these blocks. + block_notify_list = ( + blocks.get_block_ids()[0] if num_external_tokens > 0 else [] + ) for tp_index in range(self.tp_size): target_port = remote_notify_port + get_port_offset( @@ -603,7 +609,7 @@ class MoRIIOConnectorScheduler: self.send_notify_block( req_id=request.request_id, transfer_id=request.kv_transfer_params["transfer_id"], - block_notify_list=block_ids, + block_notify_list=block_notify_list, host=remote_host, port=target_port, ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py index 89ec412e5c..f3e89f6623 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py @@ -403,14 +403,13 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA): self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int ): chosen_connector = self._requests_to_connector.get(request.request_id, -1) - empty_blocks = blocks.new_empty() for i, c in enumerate(self._connectors): if i == chosen_connector: # Forward call to the chosen connector (if any). c.update_state_after_alloc(request, blocks, num_external_tokens) else: - # Call with empty blocks for other connectors. - c.update_state_after_alloc(request, empty_blocks, 0) + # Other connectors still receive the request's real blocks + c.update_state_after_alloc(request, blocks, 0) def on_new_request(self, request: "Request") -> None: for c in self._connectors: