[MoRIIO] Require pending remote prefill for READ cache hits (#60638)
Signed-off-by: whx-sjtu <xiaowang990929@gmail.com>
This commit is contained in:
@@ -806,12 +806,50 @@ def test_truncate_mamba_request_noop_for_single_token_prompt():
|
||||
# --------------------------------------------------------------------------
|
||||
# get_num_new_matched_tokens (hybrid N-1 accounting)
|
||||
# --------------------------------------------------------------------------
|
||||
@pytest.mark.parametrize(
|
||||
"params,expected",
|
||||
[
|
||||
(None, (0, False)),
|
||||
({}, (0, False)),
|
||||
({"do_remote_prefill": False}, (0, False)),
|
||||
({"do_remote_prefill": True}, (31, False)),
|
||||
],
|
||||
)
|
||||
def test_read_matches_only_requests_with_pending_remote_prefill(params, expected):
|
||||
scheduler = _FakeScheduler(is_producer=False, mode=MoRIIOMode.READ)
|
||||
request = SimpleNamespace(num_prompt_tokens=32, kv_transfer_params=params)
|
||||
|
||||
assert scheduler.get_num_new_matched_tokens(request, 0) == expected
|
||||
|
||||
|
||||
def test_read_rematch_after_alloc_keeps_queued_transfer():
|
||||
"""Clearing the flag blocks another external hit, not the queued READ."""
|
||||
sched = _make_read_scheduler()
|
||||
request = _make_read_request([[10, 11], [91]])
|
||||
request.num_prompt_tokens = 32
|
||||
assert sched.get_num_new_matched_tokens(request, 0) == (31, False)
|
||||
|
||||
sched.update_state_after_alloc(
|
||||
request, _FakeBlocks(all_groups=([100, 101], [200])), num_external_tokens=31
|
||||
)
|
||||
|
||||
assert sched._reqs_need_recv["req"][1] == [[100, 101], [200]]
|
||||
assert request.kv_transfer_params["do_remote_prefill"] is False
|
||||
|
||||
meta = sched.build_connector_meta(SchedulerOutput.make_empty())
|
||||
queued = meta.reqs_to_recv["req"]
|
||||
assert queued.transfer_id == "tx"
|
||||
assert queued.local_block_ids == [[100, 101], [200]]
|
||||
assert queued.remote_block_ids == [[10, 11], [91]]
|
||||
assert sched.get_num_new_matched_tokens(request, 0) == (0, False)
|
||||
|
||||
|
||||
def test_get_num_new_matched_tokens_read_recomputes_last_token():
|
||||
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.READ, _has_mamba=True)
|
||||
req = SimpleNamespace(
|
||||
num_prompt_tokens=10,
|
||||
prompt_token_ids=list(range(10)),
|
||||
kv_transfer_params=None,
|
||||
kv_transfer_params={"do_remote_prefill": True},
|
||||
)
|
||||
n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=0)
|
||||
# READ always recomputes the final token locally: N-1 - computed.
|
||||
|
||||
@@ -776,10 +776,10 @@ class MoRIIOConnectorScheduler:
|
||||
|
||||
num_prompt_tokens = request.num_prompt_tokens
|
||||
num_external_tokens = max(num_prompt_tokens - num_computed_tokens, 0)
|
||||
params = request.kv_transfer_params or {}
|
||||
if not params.get("do_remote_prefill"):
|
||||
return 0, False
|
||||
if self.mode == MoRIIOMode.WRITE:
|
||||
params = request.kv_transfer_params or {}
|
||||
if not params.get("do_remote_prefill"):
|
||||
return 0, False
|
||||
# MoriiO in write mode, no remote prefill. Hybrid models never get
|
||||
# here: register_kv_caches refuses WRITE mode for them, so there is
|
||||
# no recurrent-state accounting to do.
|
||||
|
||||
Reference in New Issue
Block a user