[Bugfix][DSv4.1] Skip SWA bounded replay for KV loads that carry the window (#59197)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
Claude Opus 5.5
parent
d169c5a903
commit
ef63c23d35
@@ -594,8 +594,12 @@ def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens):
|
||||
failed_recving_kv_req_ids=set(),
|
||||
finished_recving_kv_req_ids={request.request_id},
|
||||
prefix_replay_tokens=0,
|
||||
prefix_replay_group_ids=(),
|
||||
)
|
||||
scheduler._mark_prefix_replay = MethodType(Scheduler._mark_prefix_replay, scheduler)
|
||||
scheduler._load_restores_replay_window = MethodType(
|
||||
Scheduler._load_restores_replay_window, scheduler
|
||||
)
|
||||
Scheduler._update_waiting_for_remote_kv(scheduler, request)
|
||||
assert request.num_tokens - request.num_computed_tokens == 1
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=1) is not None
|
||||
|
||||
@@ -247,6 +247,33 @@ def test_remote_kv_hit_is_taken_in_whole_blocks():
|
||||
assert swa_blocks[new_req.replay_start // BLOCK_SIZE] is not swa_manager._null_block
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
@pytest.mark.parametrize("matched", [WINDOW, 4 * BLOCK_SIZE - 1])
|
||||
def test_kv_load_carrying_window_does_not_replay(is_async, matched):
|
||||
"""A load of the request's own blocks (P/D) restores every transfer group,
|
||||
window included, so the hit is taken as is: no replay, no cut to whole
|
||||
blocks, and a hit no longer than the window is kept."""
|
||||
scheduler = _replay_scheduler(
|
||||
use_kv_connector=MockKVConfig(matched_tokens=matched, is_async=is_async)
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
)[0]
|
||||
scheduler.connector.get_loaded_kv_cache_group_ids = lambda request: (FULL, SWA)
|
||||
scheduler.add_request(request)
|
||||
out = scheduler.schedule()
|
||||
if is_async:
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
scheduler.update_from_output(
|
||||
out, create_model_runner_output([], finished_recving={request.request_id})
|
||||
)
|
||||
out = scheduler.schedule()
|
||||
new_req = _new_req_data(out, request)
|
||||
assert new_req.num_computed_tokens == matched
|
||||
assert new_req.replay_start == 0
|
||||
assert out.num_scheduled_tokens[request.request_id] == NUM_PROMPT_TOKENS - matched
|
||||
|
||||
|
||||
@pytest.mark.parametrize("via_connector", [False, True])
|
||||
def test_hit_no_longer_than_window_is_ignored(via_connector):
|
||||
"""Such a hit would be recomputed in full anyway. It is not adopted (an
|
||||
|
||||
Reference in New Issue
Block a user