[Bugfix][MoRIIO] Prevent remote KV reload after decoder preemption (#53250)

Signed-off-by: Jaebok Lee <jaebok9541@naver.com>
This commit is contained in:
zzaebok
2026-10-08 22:28:16 +08:00
committed by GitHub
parent daa9085143
commit 2f7f05c256
3 changed files with 16 additions and 2 deletions
@@ -195,6 +195,17 @@ def _build_meta(
return scheduler.build_connector_meta(scheduler_output) return scheduler.build_connector_meta(scheduler_output)
def test_write_mode_does_not_reload_remote_kv_after_preemption():
scheduler = _write_consumer_scheduler_for_finished_request()
request = create_request(request_id=1, num_tokens=32, do_remote_prefill=True)
assert scheduler.get_num_new_matched_tokens(request, 0) == (32, True)
assert request.kv_transfer_params is not None
request.kv_transfer_params["do_remote_prefill"] = False
assert scheduler.get_num_new_matched_tokens(request, 0) == (0, False)
class FakeMoRIIOWrapper: class FakeMoRIIOWrapper:
# A fake MoRIIOWrapper for testing purposes # A fake MoRIIOWrapper for testing purposes
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
@@ -824,7 +824,7 @@ def test_get_num_new_matched_tokens_write_plain_keeps_all_tokens():
req = SimpleNamespace( req = SimpleNamespace(
num_prompt_tokens=10, num_prompt_tokens=10,
prompt_token_ids=list(range(10)), prompt_token_ids=list(range(10)),
kv_transfer_params=None, kv_transfer_params={"do_remote_prefill": True},
) )
n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=2) n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=2)
# Pure-attention WRITE: no N-1 drop; full length minus already-computed. # Pure-attention WRITE: no N-1 drop; full length minus already-computed.
@@ -848,7 +848,7 @@ def test_get_num_new_matched_tokens_supports_embeds_only_prompts(
num_prompt_tokens=10, num_prompt_tokens=10,
prompt_token_ids=None, prompt_token_ids=None,
prompt_embeds=object(), prompt_embeds=object(),
kv_transfer_params=None, kv_transfer_params={"do_remote_prefill": True},
) )
assert sched.get_num_new_matched_tokens(req, num_computed_tokens) == ( assert sched.get_num_new_matched_tokens(req, num_computed_tokens) == (
@@ -769,6 +769,9 @@ class MoRIIOConnectorScheduler:
num_prompt_tokens = request.num_prompt_tokens num_prompt_tokens = request.num_prompt_tokens
num_external_tokens = max(num_prompt_tokens - num_computed_tokens, 0) num_external_tokens = max(num_prompt_tokens - num_computed_tokens, 0)
if self.mode == MoRIIOMode.WRITE: if self.mode == MoRIIOMode.WRITE:
params = request.kv_transfer_params or {}
if not params.get("do_remote_prefill"):
return 0, False
# MoriiO in write mode, no remote prefill. Hybrid models never get # MoriiO in write mode, no remote prefill. Hybrid models never get
# here: register_kv_caches refuses WRITE mode for them, so there is # here: register_kv_caches refuses WRITE mode for them, so there is
# no recurrent-state accounting to do. # no recurrent-state accounting to do.