[KVConnector] MultiConnector: give every sub-connector the request's real blocks in update_state_after_alloc (#46865)

Signed-off-by: deng451e <838677410@qq.com>
This commit is contained in:
weishu
2026-07-09 11:10:39 -07:00
committed by GitHub
parent e08a915146
commit 2285cfca46
4 changed files with 37 additions and 27 deletions
@@ -272,7 +272,7 @@ def test_multi_example_connector_consistency():
"set_xfer_handshake_metadata_pp_aware",
"on_new_request",
"get_num_new_matched_tokens 0",
"update_state_after_alloc num_blocks=[0] 0",
"update_state_after_alloc num_blocks=[7] 0",
"build_connector_meta",
]
# First three events are from initialization (register_kv_caches,
@@ -292,7 +292,7 @@ def test_multi_example_connector_consistency():
"set_xfer_handshake_metadata_pp_aware",
"on_new_request",
"get_num_new_matched_tokens 0",
"update_state_after_alloc num_blocks=[0] 0",
"update_state_after_alloc num_blocks=[7] 0",
"build_connector_meta",
]
assert events["storage2-WORKER"][:8] == [
@@ -315,9 +315,9 @@ def test_multi_example_connector_consistency():
events = get_connector_events()
# get_num_new_matched_tokens will return new tokens from the first
# connector so update_state_after_alloc will be with allocated blocks
# on that one but with zero blocks for others (first nonzero match is
# chosen).
# connector (first nonzero match is chosen), so update_state_after_alloc
# will report those external tokens on that one. Other connectors still
# receive the request's real blocks but with 0 external tokens.
storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"])
storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"])
assert storage1_scheduler_events[:4] == [
@@ -329,7 +329,7 @@ def test_multi_example_connector_consistency():
assert storage2_scheduler_events[:4] == [
"on_new_request",
"get_num_new_matched_tokens 0",
"update_state_after_alloc num_blocks=[0] 0",
"update_state_after_alloc num_blocks=[7] 0",
"build_connector_meta",
]
@@ -345,15 +345,15 @@ def test_multi_example_connector_consistency():
events = get_connector_events()
# get_num_new_matched_tokens will be called for both connectors but will
# return 0 from the first connector, but the second connector should have
# a hit, so update_state_after_alloc will only be called with allocated
# blocks for the second connector.
# return 0 from the first connector, while the second connector has a hit.
# Both connectors receive the request's real blocks, but only the chosen
# (second) connector reports external tokens.
storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"])
storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"])
assert storage1_scheduler_events[:4] == [
"on_new_request",
"get_num_new_matched_tokens 0",
"update_state_after_alloc num_blocks=[0] 0",
"update_state_after_alloc num_blocks=[7] 0",
"build_connector_meta",
]
assert storage2_scheduler_events[:4] == [
@@ -498,11 +498,16 @@ class KVConnectorBase_V1(ABC):
asynchronously loaded into, and second when any additional blocks
are allocated, after the load/transfer is complete.
Decide whether to load based on ``num_external_tokens``, not on
whether ``blocks`` is empty: ``blocks`` may be non-empty even when
``num_external_tokens == 0`` (e.g. a non-chosen sub-connector of
MultiConnector still receives the request's real blocks).
Args:
request (Request): the request object.
blocks (KVCacheBlocks): the blocks allocated for the request.
num_external_tokens (int): the number of tokens that will be
loaded from the external KV cache.
num_external_tokens (int): the number of tokens to load from the
external KV cache. 0 means nothing should be loaded.
"""
pass
@@ -550,17 +550,19 @@ class MoRIIOConnectorScheduler:
# remote_engine_id is returned by the prefill's request_finished.
# host/ports come from the request_id (parsed in add_new_req).
if "remote_engine_id" in params:
# If remote_blocks and num_external_tokens = 0, we have
# a full prefix cache hit on the D worker. We need to call
# send_notify in _read_blocks to free the memory on the P.
# Get unhashed blocks to pull from remote.
local_block_ids = blocks.get_block_ids()[0]
assert len(local_block_ids) <= len(remote_block_ids)
if len(local_block_ids) == len(remote_block_ids):
pass
if num_external_tokens > 0:
# Get unhashed blocks to pull from remote.
local_block_ids = blocks.get_block_ids()[0]
assert len(local_block_ids) <= len(remote_block_ids)
if len(local_block_ids) != len(remote_block_ids):
local_block_ids = remote_block_ids[
-len(local_block_ids) :
]
else:
local_block_ids = remote_block_ids[-len(local_block_ids) :]
# If remote_blocks and num_external_tokens = 0, we have
# a full prefix cache hit on the D worker. We need to call
# send_notify in _read_blocks to free the memory on the P.
local_block_ids = []
self._reqs_need_recv[request.request_id] = (
request,
@@ -593,7 +595,11 @@ class MoRIIOConnectorScheduler:
)
remote_notify_port = int(remote_notify_port)
block_ids = blocks.get_block_ids()[0]
# num_external_tokens == 0: nothing to push, so don't tell the
# producer to write into these blocks.
block_notify_list = (
blocks.get_block_ids()[0] if num_external_tokens > 0 else []
)
for tp_index in range(self.tp_size):
target_port = remote_notify_port + get_port_offset(
@@ -603,7 +609,7 @@ class MoRIIOConnectorScheduler:
self.send_notify_block(
req_id=request.request_id,
transfer_id=request.kv_transfer_params["transfer_id"],
block_notify_list=block_ids,
block_notify_list=block_notify_list,
host=remote_host,
port=target_port,
)
@@ -403,14 +403,13 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
chosen_connector = self._requests_to_connector.get(request.request_id, -1)
empty_blocks = blocks.new_empty()
for i, c in enumerate(self._connectors):
if i == chosen_connector:
# Forward call to the chosen connector (if any).
c.update_state_after_alloc(request, blocks, num_external_tokens)
else:
# Call with empty blocks for other connectors.
c.update_state_after_alloc(request, empty_blocks, 0)
# Other connectors still receive the request's real blocks
c.update_state_after_alloc(request, blocks, 0)
def on_new_request(self, request: "Request") -> None:
for c in self._connectors: