[Bugfix][DSv4.1] Skip SWA bounded replay for KV loads that carry the window (#59197)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
Claude Opus 5.5
parent
d169c5a903
commit
ef63c23d35
@@ -96,6 +96,10 @@ class _SchedulerRequest:
|
||||
default_factory=lambda: SimpleNamespace(routed_experts_prompt_start=0)
|
||||
)
|
||||
|
||||
@property
|
||||
def num_prompt_tokens(self) -> int:
|
||||
return self.num_tokens - self.num_output_tokens
|
||||
|
||||
def is_finished(self) -> bool:
|
||||
return self.finished
|
||||
|
||||
@@ -1439,6 +1443,18 @@ def test_scheduler_starts_worker_output_at_requested_prompt_token():
|
||||
|
||||
assert metadata.requests == {request.request_id: 2}
|
||||
|
||||
# A P/D prefiller cuts its prompt short of the last token, which can pass
|
||||
# the requested start; the rows then start at the end of the cut prompt.
|
||||
connector = _make_connector()
|
||||
request = _scheduler_request(
|
||||
"cut", [b"a" * 32], num_tokens=3, num_output_tokens=0, prompt_start=4
|
||||
)
|
||||
metadata = connector.build_connector_meta(
|
||||
_step_output([request.request_id], [0], [3]),
|
||||
{request.request_id: request},
|
||||
)
|
||||
assert metadata.requests == {request.request_id: 3}
|
||||
|
||||
|
||||
def test_scheduler_connector_preserves_request_finish_order():
|
||||
connector = _make_connector()
|
||||
|
||||
@@ -594,8 +594,12 @@ def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens):
|
||||
failed_recving_kv_req_ids=set(),
|
||||
finished_recving_kv_req_ids={request.request_id},
|
||||
prefix_replay_tokens=0,
|
||||
prefix_replay_group_ids=(),
|
||||
)
|
||||
scheduler._mark_prefix_replay = MethodType(Scheduler._mark_prefix_replay, scheduler)
|
||||
scheduler._load_restores_replay_window = MethodType(
|
||||
Scheduler._load_restores_replay_window, scheduler
|
||||
)
|
||||
Scheduler._update_waiting_for_remote_kv(scheduler, request)
|
||||
assert request.num_tokens - request.num_computed_tokens == 1
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=1) is not None
|
||||
|
||||
@@ -247,6 +247,33 @@ def test_remote_kv_hit_is_taken_in_whole_blocks():
|
||||
assert swa_blocks[new_req.replay_start // BLOCK_SIZE] is not swa_manager._null_block
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
@pytest.mark.parametrize("matched", [WINDOW, 4 * BLOCK_SIZE - 1])
|
||||
def test_kv_load_carrying_window_does_not_replay(is_async, matched):
|
||||
"""A load of the request's own blocks (P/D) restores every transfer group,
|
||||
window included, so the hit is taken as is: no replay, no cut to whole
|
||||
blocks, and a hit no longer than the window is kept."""
|
||||
scheduler = _replay_scheduler(
|
||||
use_kv_connector=MockKVConfig(matched_tokens=matched, is_async=is_async)
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
)[0]
|
||||
scheduler.connector.get_loaded_kv_cache_group_ids = lambda request: (FULL, SWA)
|
||||
scheduler.add_request(request)
|
||||
out = scheduler.schedule()
|
||||
if is_async:
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
scheduler.update_from_output(
|
||||
out, create_model_runner_output([], finished_recving={request.request_id})
|
||||
)
|
||||
out = scheduler.schedule()
|
||||
new_req = _new_req_data(out, request)
|
||||
assert new_req.num_computed_tokens == matched
|
||||
assert new_req.replay_start == 0
|
||||
assert out.num_scheduled_tokens[request.request_id] == NUM_PROMPT_TOKENS - matched
|
||||
|
||||
|
||||
@pytest.mark.parametrize("via_connector", [False, True])
|
||||
def test_hit_no_longer_than_window_is_ignored(via_connector):
|
||||
"""Such a hit would be recomputed in full anyway. It is not adopted (an
|
||||
|
||||
@@ -1559,17 +1559,22 @@ def test_failed_load_rezeroes_unwritten_skipped_blocks():
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
"has_mamba,is_hma_required,expected_count",
|
||||
"has_mamba,is_hma_required,bounded_replay,expected_count",
|
||||
[
|
||||
(True, True, 9),
|
||||
(False, False, 10),
|
||||
(False, True, 10),
|
||||
(True, True, False, 9),
|
||||
(False, False, False, 10),
|
||||
(False, True, False, 10),
|
||||
(False, True, True, 9),
|
||||
],
|
||||
ids=["mamba", "fa_only", "swa_only"],
|
||||
ids=["mamba", "fa_only", "swa_only", "swa_bounded_replay"],
|
||||
)
|
||||
def test_mamba_n1_d_side(has_mamba, is_hma_required, expected_count):
|
||||
"""D-side: Mamba gets N-1 matched tokens, non-Mamba gets N."""
|
||||
sched = make_nixl_scheduler(has_mamba=has_mamba, is_hma_required=is_hma_required)
|
||||
def test_mamba_n1_d_side(has_mamba, is_hma_required, bounded_replay, expected_count):
|
||||
"""D-side: Mamba and SWA bounded replay get N-1 matched tokens, others N."""
|
||||
sched = make_nixl_scheduler(
|
||||
has_mamba=has_mamba,
|
||||
is_hma_required=is_hma_required,
|
||||
bounded_replay=bounded_replay,
|
||||
)
|
||||
req = create_request(num_tokens=10, do_remote_prefill=True)
|
||||
|
||||
count, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=0)
|
||||
@@ -1638,6 +1643,20 @@ def test_mamba_n1_p_side_truncation():
|
||||
fa_sched.on_new_request(fa_req)
|
||||
assert len(fa_req.prompt_token_ids) == fa_original
|
||||
|
||||
# SWA bounded replay: the window is not replayed after the load, so the
|
||||
# prefiller stops short of the last token too.
|
||||
swa_sched = make_nixl_scheduler(is_hma_required=True, bounded_replay=True)
|
||||
swa_req = create_request(num_tokens=10, do_remote_decode=True)
|
||||
swa_sched.on_new_request(swa_req)
|
||||
assert len(swa_req.prompt_token_ids) == 9
|
||||
|
||||
# A request that skips reading the prefix cache (prompt logprobs) is not
|
||||
# cut: the decoder loads nothing and recomputes its whole prompt.
|
||||
logprobs_req = create_request(num_tokens=10, do_remote_decode=True)
|
||||
logprobs_req.sampling_params.skip_reading_prefix_cache = True
|
||||
swa_sched.on_new_request(logprobs_req)
|
||||
assert len(logprobs_req.prompt_token_ids) == 10
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -532,6 +532,7 @@ def make_kv_cache_config(
|
||||
def make_nixl_scheduler(
|
||||
has_mamba: bool = False,
|
||||
is_hma_required: bool = False,
|
||||
bounded_replay: bool = False,
|
||||
heartbeat: bool = False,
|
||||
kv_lease_duration: int = 30,
|
||||
):
|
||||
@@ -548,6 +549,7 @@ def make_nixl_scheduler(
|
||||
|
||||
sched = object.__new__(NixlConnectorScheduler)
|
||||
sched._has_mamba = has_mamba
|
||||
sched._bounded_replay = bounded_replay
|
||||
sched._is_hma_required = is_hma_required
|
||||
sched.kv_cache_config = make_kv_cache_config(
|
||||
block_size=16,
|
||||
@@ -615,6 +617,7 @@ def make_nixl_push_scheduler(
|
||||
sched.side_channel_port = 5600
|
||||
sched.is_bidirectional_kv_xfer_enabled = is_bidirectional_kv_xfer_enabled
|
||||
sched._has_mamba = has_mamba
|
||||
sched._bounded_replay = False
|
||||
sched.kv_cache_config = make_kv_cache_config(
|
||||
block_size=16,
|
||||
mamba_enabled=has_mamba,
|
||||
|
||||
Reference in New Issue
Block a user