[Bugfix][DSv4.1] Skip SWA bounded replay for KV loads that carry the window (#59197)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yifan Qiao
2026-10-07 15:28:42 -07:00
committed by GitHub
co-authored by Codex Claude Opus 5.5
parent d169c5a903
commit ef63c23d35
9 changed files with 161 additions and 31 deletions
+4
View File
@@ -594,8 +594,12 @@ def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens):
failed_recving_kv_req_ids=set(),
finished_recving_kv_req_ids={request.request_id},
prefix_replay_tokens=0,
prefix_replay_group_ids=(),
)
scheduler._mark_prefix_replay = MethodType(Scheduler._mark_prefix_replay, scheduler)
scheduler._load_restores_replay_window = MethodType(
Scheduler._load_restores_replay_window, scheduler
)
Scheduler._update_waiting_for_remote_kv(scheduler, request)
assert request.num_tokens - request.num_computed_tokens == 1
assert _allocate_scheduled(manager, request, num_new_tokens=1) is not None
+27
View File
@@ -247,6 +247,33 @@ def test_remote_kv_hit_is_taken_in_whole_blocks():
assert swa_blocks[new_req.replay_start // BLOCK_SIZE] is not swa_manager._null_block
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.parametrize("matched", [WINDOW, 4 * BLOCK_SIZE - 1])
def test_kv_load_carrying_window_does_not_replay(is_async, matched):
"""A load of the request's own blocks (P/D) restores every transfer group,
window included, so the hit is taken as is: no replay, no cut to whole
blocks, and a hit no longer than the window is kept."""
scheduler = _replay_scheduler(
use_kv_connector=MockKVConfig(matched_tokens=matched, is_async=is_async)
)
request = create_requests(
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
)[0]
scheduler.connector.get_loaded_kv_cache_group_ids = lambda request: (FULL, SWA)
scheduler.add_request(request)
out = scheduler.schedule()
if is_async:
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
scheduler.update_from_output(
out, create_model_runner_output([], finished_recving={request.request_id})
)
out = scheduler.schedule()
new_req = _new_req_data(out, request)
assert new_req.num_computed_tokens == matched
assert new_req.replay_start == 0
assert out.num_scheduled_tokens[request.request_id] == NUM_PROMPT_TOKENS - matched
@pytest.mark.parametrize("via_connector", [False, True])
def test_hit_no_longer_than_window_is_ignored(via_connector):
"""Such a hit would be recomputed in full anyway. It is not adopted (an