[KVConnector] MultiConnector: give every sub-connector the request's real blocks in update_state_after_alloc (#46865)
Signed-off-by: deng451e <838677410@qq.com>
This commit is contained in:
@@ -272,7 +272,7 @@ def test_multi_example_connector_consistency():
|
||||
"set_xfer_handshake_metadata_pp_aware",
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[0] 0",
|
||||
"update_state_after_alloc num_blocks=[7] 0",
|
||||
"build_connector_meta",
|
||||
]
|
||||
# First three events are from initialization (register_kv_caches,
|
||||
@@ -292,7 +292,7 @@ def test_multi_example_connector_consistency():
|
||||
"set_xfer_handshake_metadata_pp_aware",
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[0] 0",
|
||||
"update_state_after_alloc num_blocks=[7] 0",
|
||||
"build_connector_meta",
|
||||
]
|
||||
assert events["storage2-WORKER"][:8] == [
|
||||
@@ -315,9 +315,9 @@ def test_multi_example_connector_consistency():
|
||||
|
||||
events = get_connector_events()
|
||||
# get_num_new_matched_tokens will return new tokens from the first
|
||||
# connector so update_state_after_alloc will be with allocated blocks
|
||||
# on that one but with zero blocks for others (first nonzero match is
|
||||
# chosen).
|
||||
# connector (first nonzero match is chosen), so update_state_after_alloc
|
||||
# will report those external tokens on that one. Other connectors still
|
||||
# receive the request's real blocks but with 0 external tokens.
|
||||
storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"])
|
||||
storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"])
|
||||
assert storage1_scheduler_events[:4] == [
|
||||
@@ -329,7 +329,7 @@ def test_multi_example_connector_consistency():
|
||||
assert storage2_scheduler_events[:4] == [
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[0] 0",
|
||||
"update_state_after_alloc num_blocks=[7] 0",
|
||||
"build_connector_meta",
|
||||
]
|
||||
|
||||
@@ -345,15 +345,15 @@ def test_multi_example_connector_consistency():
|
||||
|
||||
events = get_connector_events()
|
||||
# get_num_new_matched_tokens will be called for both connectors but will
|
||||
# return 0 from the first connector, but the second connector should have
|
||||
# a hit, so update_state_after_alloc will only be called with allocated
|
||||
# blocks for the second connector.
|
||||
# return 0 from the first connector, while the second connector has a hit.
|
||||
# Both connectors receive the request's real blocks, but only the chosen
|
||||
# (second) connector reports external tokens.
|
||||
storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"])
|
||||
storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"])
|
||||
assert storage1_scheduler_events[:4] == [
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[0] 0",
|
||||
"update_state_after_alloc num_blocks=[7] 0",
|
||||
"build_connector_meta",
|
||||
]
|
||||
assert storage2_scheduler_events[:4] == [
|
||||
|
||||
@@ -498,11 +498,16 @@ class KVConnectorBase_V1(ABC):
|
||||
asynchronously loaded into, and second when any additional blocks
|
||||
are allocated, after the load/transfer is complete.
|
||||
|
||||
Decide whether to load based on ``num_external_tokens``, not on
|
||||
whether ``blocks`` is empty: ``blocks`` may be non-empty even when
|
||||
``num_external_tokens == 0`` (e.g. a non-chosen sub-connector of
|
||||
MultiConnector still receives the request's real blocks).
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
blocks (KVCacheBlocks): the blocks allocated for the request.
|
||||
num_external_tokens (int): the number of tokens that will be
|
||||
loaded from the external KV cache.
|
||||
num_external_tokens (int): the number of tokens to load from the
|
||||
external KV cache. 0 means nothing should be loaded.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
@@ -550,17 +550,19 @@ class MoRIIOConnectorScheduler:
|
||||
# remote_engine_id is returned by the prefill's request_finished.
|
||||
# host/ports come from the request_id (parsed in add_new_req).
|
||||
if "remote_engine_id" in params:
|
||||
# If remote_blocks and num_external_tokens = 0, we have
|
||||
# a full prefix cache hit on the D worker. We need to call
|
||||
# send_notify in _read_blocks to free the memory on the P.
|
||||
|
||||
# Get unhashed blocks to pull from remote.
|
||||
local_block_ids = blocks.get_block_ids()[0]
|
||||
assert len(local_block_ids) <= len(remote_block_ids)
|
||||
if len(local_block_ids) == len(remote_block_ids):
|
||||
pass
|
||||
if num_external_tokens > 0:
|
||||
# Get unhashed blocks to pull from remote.
|
||||
local_block_ids = blocks.get_block_ids()[0]
|
||||
assert len(local_block_ids) <= len(remote_block_ids)
|
||||
if len(local_block_ids) != len(remote_block_ids):
|
||||
local_block_ids = remote_block_ids[
|
||||
-len(local_block_ids) :
|
||||
]
|
||||
else:
|
||||
local_block_ids = remote_block_ids[-len(local_block_ids) :]
|
||||
# If remote_blocks and num_external_tokens = 0, we have
|
||||
# a full prefix cache hit on the D worker. We need to call
|
||||
# send_notify in _read_blocks to free the memory on the P.
|
||||
local_block_ids = []
|
||||
|
||||
self._reqs_need_recv[request.request_id] = (
|
||||
request,
|
||||
@@ -593,7 +595,11 @@ class MoRIIOConnectorScheduler:
|
||||
)
|
||||
remote_notify_port = int(remote_notify_port)
|
||||
|
||||
block_ids = blocks.get_block_ids()[0]
|
||||
# num_external_tokens == 0: nothing to push, so don't tell the
|
||||
# producer to write into these blocks.
|
||||
block_notify_list = (
|
||||
blocks.get_block_ids()[0] if num_external_tokens > 0 else []
|
||||
)
|
||||
|
||||
for tp_index in range(self.tp_size):
|
||||
target_port = remote_notify_port + get_port_offset(
|
||||
@@ -603,7 +609,7 @@ class MoRIIOConnectorScheduler:
|
||||
self.send_notify_block(
|
||||
req_id=request.request_id,
|
||||
transfer_id=request.kv_transfer_params["transfer_id"],
|
||||
block_notify_list=block_ids,
|
||||
block_notify_list=block_notify_list,
|
||||
host=remote_host,
|
||||
port=target_port,
|
||||
)
|
||||
|
||||
@@ -403,14 +403,13 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
chosen_connector = self._requests_to_connector.get(request.request_id, -1)
|
||||
empty_blocks = blocks.new_empty()
|
||||
for i, c in enumerate(self._connectors):
|
||||
if i == chosen_connector:
|
||||
# Forward call to the chosen connector (if any).
|
||||
c.update_state_after_alloc(request, blocks, num_external_tokens)
|
||||
else:
|
||||
# Call with empty blocks for other connectors.
|
||||
c.update_state_after_alloc(request, empty_blocks, 0)
|
||||
# Other connectors still receive the request's real blocks
|
||||
c.update_state_after_alloc(request, blocks, 0)
|
||||
|
||||
def on_new_request(self, request: "Request") -> None:
|
||||
for c in self._connectors:
|
||||
|
||||
Reference in New Issue
Block a user