[Bugfix][MoRIIO] Prevent remote KV reload after decoder preemption (#53250)
Signed-off-by: Jaebok Lee <jaebok9541@naver.com>
This commit is contained in:
@@ -195,6 +195,17 @@ def _build_meta(
|
||||
return scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
|
||||
def test_write_mode_does_not_reload_remote_kv_after_preemption():
|
||||
scheduler = _write_consumer_scheduler_for_finished_request()
|
||||
request = create_request(request_id=1, num_tokens=32, do_remote_prefill=True)
|
||||
|
||||
assert scheduler.get_num_new_matched_tokens(request, 0) == (32, True)
|
||||
|
||||
assert request.kv_transfer_params is not None
|
||||
request.kv_transfer_params["do_remote_prefill"] = False
|
||||
assert scheduler.get_num_new_matched_tokens(request, 0) == (0, False)
|
||||
|
||||
|
||||
class FakeMoRIIOWrapper:
|
||||
# A fake MoRIIOWrapper for testing purposes
|
||||
def __init__(self, *args, **kwargs):
|
||||
|
||||
@@ -824,7 +824,7 @@ def test_get_num_new_matched_tokens_write_plain_keeps_all_tokens():
|
||||
req = SimpleNamespace(
|
||||
num_prompt_tokens=10,
|
||||
prompt_token_ids=list(range(10)),
|
||||
kv_transfer_params=None,
|
||||
kv_transfer_params={"do_remote_prefill": True},
|
||||
)
|
||||
n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=2)
|
||||
# Pure-attention WRITE: no N-1 drop; full length minus already-computed.
|
||||
@@ -848,7 +848,7 @@ def test_get_num_new_matched_tokens_supports_embeds_only_prompts(
|
||||
num_prompt_tokens=10,
|
||||
prompt_token_ids=None,
|
||||
prompt_embeds=object(),
|
||||
kv_transfer_params=None,
|
||||
kv_transfer_params={"do_remote_prefill": True},
|
||||
)
|
||||
|
||||
assert sched.get_num_new_matched_tokens(req, num_computed_tokens) == (
|
||||
|
||||
Reference in New Issue
Block a user