[Bugfix][DSv4.1] Skip SWA bounded replay for KV loads that carry the window (#59197)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yifan Qiao
2026-10-07 15:28:42 -07:00
committed by GitHub
co-authored by Codex Claude Opus 5.5
parent d169c5a903
commit ef63c23d35
9 changed files with 161 additions and 31 deletions
@@ -96,6 +96,10 @@ class _SchedulerRequest:
default_factory=lambda: SimpleNamespace(routed_experts_prompt_start=0)
)
@property
def num_prompt_tokens(self) -> int:
return self.num_tokens - self.num_output_tokens
def is_finished(self) -> bool:
return self.finished
@@ -1439,6 +1443,18 @@ def test_scheduler_starts_worker_output_at_requested_prompt_token():
assert metadata.requests == {request.request_id: 2}
# A P/D prefiller cuts its prompt short of the last token, which can pass
# the requested start; the rows then start at the end of the cut prompt.
connector = _make_connector()
request = _scheduler_request(
"cut", [b"a" * 32], num_tokens=3, num_output_tokens=0, prompt_start=4
)
metadata = connector.build_connector_meta(
_step_output([request.request_id], [0], [3]),
{request.request_id: request},
)
assert metadata.requests == {request.request_id: 3}
def test_scheduler_connector_preserves_request_finish_order():
connector = _make_connector()
+4
View File
@@ -594,8 +594,12 @@ def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens):
failed_recving_kv_req_ids=set(),
finished_recving_kv_req_ids={request.request_id},
prefix_replay_tokens=0,
prefix_replay_group_ids=(),
)
scheduler._mark_prefix_replay = MethodType(Scheduler._mark_prefix_replay, scheduler)
scheduler._load_restores_replay_window = MethodType(
Scheduler._load_restores_replay_window, scheduler
)
Scheduler._update_waiting_for_remote_kv(scheduler, request)
assert request.num_tokens - request.num_computed_tokens == 1
assert _allocate_scheduled(manager, request, num_new_tokens=1) is not None
+27
View File
@@ -247,6 +247,33 @@ def test_remote_kv_hit_is_taken_in_whole_blocks():
assert swa_blocks[new_req.replay_start // BLOCK_SIZE] is not swa_manager._null_block
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.parametrize("matched", [WINDOW, 4 * BLOCK_SIZE - 1])
def test_kv_load_carrying_window_does_not_replay(is_async, matched):
"""A load of the request's own blocks (P/D) restores every transfer group,
window included, so the hit is taken as is: no replay, no cut to whole
blocks, and a hit no longer than the window is kept."""
scheduler = _replay_scheduler(
use_kv_connector=MockKVConfig(matched_tokens=matched, is_async=is_async)
)
request = create_requests(
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
)[0]
scheduler.connector.get_loaded_kv_cache_group_ids = lambda request: (FULL, SWA)
scheduler.add_request(request)
out = scheduler.schedule()
if is_async:
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
scheduler.update_from_output(
out, create_model_runner_output([], finished_recving={request.request_id})
)
out = scheduler.schedule()
new_req = _new_req_data(out, request)
assert new_req.num_computed_tokens == matched
assert new_req.replay_start == 0
assert out.num_scheduled_tokens[request.request_id] == NUM_PROMPT_TOKENS - matched
@pytest.mark.parametrize("via_connector", [False, True])
def test_hit_no_longer_than_window_is_ignored(via_connector):
"""Such a hit would be recomputed in full anyway. It is not adopted (an
@@ -1559,17 +1559,22 @@ def test_failed_load_rezeroes_unwritten_skipped_blocks():
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"has_mamba,is_hma_required,expected_count",
"has_mamba,is_hma_required,bounded_replay,expected_count",
[
(True, True, 9),
(False, False, 10),
(False, True, 10),
(True, True, False, 9),
(False, False, False, 10),
(False, True, False, 10),
(False, True, True, 9),
],
ids=["mamba", "fa_only", "swa_only"],
ids=["mamba", "fa_only", "swa_only", "swa_bounded_replay"],
)
def test_mamba_n1_d_side(has_mamba, is_hma_required, expected_count):
"""D-side: Mamba gets N-1 matched tokens, non-Mamba gets N."""
sched = make_nixl_scheduler(has_mamba=has_mamba, is_hma_required=is_hma_required)
def test_mamba_n1_d_side(has_mamba, is_hma_required, bounded_replay, expected_count):
"""D-side: Mamba and SWA bounded replay get N-1 matched tokens, others N."""
sched = make_nixl_scheduler(
has_mamba=has_mamba,
is_hma_required=is_hma_required,
bounded_replay=bounded_replay,
)
req = create_request(num_tokens=10, do_remote_prefill=True)
count, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=0)
@@ -1638,6 +1643,20 @@ def test_mamba_n1_p_side_truncation():
fa_sched.on_new_request(fa_req)
assert len(fa_req.prompt_token_ids) == fa_original
# SWA bounded replay: the window is not replayed after the load, so the
# prefiller stops short of the last token too.
swa_sched = make_nixl_scheduler(is_hma_required=True, bounded_replay=True)
swa_req = create_request(num_tokens=10, do_remote_decode=True)
swa_sched.on_new_request(swa_req)
assert len(swa_req.prompt_token_ids) == 9
# A request that skips reading the prefix cache (prompt logprobs) is not
# cut: the decoder loads nothing and recomputes its whole prompt.
logprobs_req = create_request(num_tokens=10, do_remote_decode=True)
logprobs_req.sampling_params.skip_reading_prefix_cache = True
swa_sched.on_new_request(logprobs_req)
assert len(logprobs_req.prompt_token_ids) == 10
@pytest.mark.cpu_test
@pytest.mark.parametrize(
+3
View File
@@ -532,6 +532,7 @@ def make_kv_cache_config(
def make_nixl_scheduler(
has_mamba: bool = False,
is_hma_required: bool = False,
bounded_replay: bool = False,
heartbeat: bool = False,
kv_lease_duration: int = 30,
):
@@ -548,6 +549,7 @@ def make_nixl_scheduler(
sched = object.__new__(NixlConnectorScheduler)
sched._has_mamba = has_mamba
sched._bounded_replay = bounded_replay
sched._is_hma_required = is_hma_required
sched.kv_cache_config = make_kv_cache_config(
block_size=16,
@@ -615,6 +617,7 @@ def make_nixl_push_scheduler(
sched.side_channel_port = 5600
sched.is_bidirectional_kv_xfer_enabled = is_bidirectional_kv_xfer_enabled
sched._has_mamba = has_mamba
sched._bounded_replay = False
sched.kv_cache_config = make_kv_cache_config(
block_size=16,
mamba_enabled=has_mamba,