[Bugfix][MoRIIO] Prevent remote KV reload after decoder preemption (#53250)
Signed-off-by: Jaebok Lee <jaebok9541@naver.com>
This commit is contained in:
@@ -195,6 +195,17 @@ def _build_meta(
|
|||||||
return scheduler.build_connector_meta(scheduler_output)
|
return scheduler.build_connector_meta(scheduler_output)
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_mode_does_not_reload_remote_kv_after_preemption():
|
||||||
|
scheduler = _write_consumer_scheduler_for_finished_request()
|
||||||
|
request = create_request(request_id=1, num_tokens=32, do_remote_prefill=True)
|
||||||
|
|
||||||
|
assert scheduler.get_num_new_matched_tokens(request, 0) == (32, True)
|
||||||
|
|
||||||
|
assert request.kv_transfer_params is not None
|
||||||
|
request.kv_transfer_params["do_remote_prefill"] = False
|
||||||
|
assert scheduler.get_num_new_matched_tokens(request, 0) == (0, False)
|
||||||
|
|
||||||
|
|
||||||
class FakeMoRIIOWrapper:
|
class FakeMoRIIOWrapper:
|
||||||
# A fake MoRIIOWrapper for testing purposes
|
# A fake MoRIIOWrapper for testing purposes
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
|
|||||||
@@ -824,7 +824,7 @@ def test_get_num_new_matched_tokens_write_plain_keeps_all_tokens():
|
|||||||
req = SimpleNamespace(
|
req = SimpleNamespace(
|
||||||
num_prompt_tokens=10,
|
num_prompt_tokens=10,
|
||||||
prompt_token_ids=list(range(10)),
|
prompt_token_ids=list(range(10)),
|
||||||
kv_transfer_params=None,
|
kv_transfer_params={"do_remote_prefill": True},
|
||||||
)
|
)
|
||||||
n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=2)
|
n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=2)
|
||||||
# Pure-attention WRITE: no N-1 drop; full length minus already-computed.
|
# Pure-attention WRITE: no N-1 drop; full length minus already-computed.
|
||||||
@@ -848,7 +848,7 @@ def test_get_num_new_matched_tokens_supports_embeds_only_prompts(
|
|||||||
num_prompt_tokens=10,
|
num_prompt_tokens=10,
|
||||||
prompt_token_ids=None,
|
prompt_token_ids=None,
|
||||||
prompt_embeds=object(),
|
prompt_embeds=object(),
|
||||||
kv_transfer_params=None,
|
kv_transfer_params={"do_remote_prefill": True},
|
||||||
)
|
)
|
||||||
|
|
||||||
assert sched.get_num_new_matched_tokens(req, num_computed_tokens) == (
|
assert sched.get_num_new_matched_tokens(req, num_computed_tokens) == (
|
||||||
|
|||||||
@@ -769,6 +769,9 @@ class MoRIIOConnectorScheduler:
|
|||||||
num_prompt_tokens = request.num_prompt_tokens
|
num_prompt_tokens = request.num_prompt_tokens
|
||||||
num_external_tokens = max(num_prompt_tokens - num_computed_tokens, 0)
|
num_external_tokens = max(num_prompt_tokens - num_computed_tokens, 0)
|
||||||
if self.mode == MoRIIOMode.WRITE:
|
if self.mode == MoRIIOMode.WRITE:
|
||||||
|
params = request.kv_transfer_params or {}
|
||||||
|
if not params.get("do_remote_prefill"):
|
||||||
|
return 0, False
|
||||||
# MoriiO in write mode, no remote prefill. Hybrid models never get
|
# MoriiO in write mode, no remote prefill. Hybrid models never get
|
||||||
# here: register_kv_caches refuses WRITE mode for them, so there is
|
# here: register_kv_caches refuses WRITE mode for them, so there is
|
||||||
# no recurrent-state accounting to do.
|
# no recurrent-state accounting to do.
|
||||||
|
|||||||
Reference in New Issue
Block a user