[Bugfix][DSv4.1] Skip SWA bounded replay for KV loads that carry the window (#59197)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
Claude Opus 5.5
parent
d169c5a903
commit
ef63c23d35
@@ -96,6 +96,10 @@ class _SchedulerRequest:
|
|||||||
default_factory=lambda: SimpleNamespace(routed_experts_prompt_start=0)
|
default_factory=lambda: SimpleNamespace(routed_experts_prompt_start=0)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def num_prompt_tokens(self) -> int:
|
||||||
|
return self.num_tokens - self.num_output_tokens
|
||||||
|
|
||||||
def is_finished(self) -> bool:
|
def is_finished(self) -> bool:
|
||||||
return self.finished
|
return self.finished
|
||||||
|
|
||||||
@@ -1439,6 +1443,18 @@ def test_scheduler_starts_worker_output_at_requested_prompt_token():
|
|||||||
|
|
||||||
assert metadata.requests == {request.request_id: 2}
|
assert metadata.requests == {request.request_id: 2}
|
||||||
|
|
||||||
|
# A P/D prefiller cuts its prompt short of the last token, which can pass
|
||||||
|
# the requested start; the rows then start at the end of the cut prompt.
|
||||||
|
connector = _make_connector()
|
||||||
|
request = _scheduler_request(
|
||||||
|
"cut", [b"a" * 32], num_tokens=3, num_output_tokens=0, prompt_start=4
|
||||||
|
)
|
||||||
|
metadata = connector.build_connector_meta(
|
||||||
|
_step_output([request.request_id], [0], [3]),
|
||||||
|
{request.request_id: request},
|
||||||
|
)
|
||||||
|
assert metadata.requests == {request.request_id: 3}
|
||||||
|
|
||||||
|
|
||||||
def test_scheduler_connector_preserves_request_finish_order():
|
def test_scheduler_connector_preserves_request_finish_order():
|
||||||
connector = _make_connector()
|
connector = _make_connector()
|
||||||
|
|||||||
@@ -594,8 +594,12 @@ def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens):
|
|||||||
failed_recving_kv_req_ids=set(),
|
failed_recving_kv_req_ids=set(),
|
||||||
finished_recving_kv_req_ids={request.request_id},
|
finished_recving_kv_req_ids={request.request_id},
|
||||||
prefix_replay_tokens=0,
|
prefix_replay_tokens=0,
|
||||||
|
prefix_replay_group_ids=(),
|
||||||
)
|
)
|
||||||
scheduler._mark_prefix_replay = MethodType(Scheduler._mark_prefix_replay, scheduler)
|
scheduler._mark_prefix_replay = MethodType(Scheduler._mark_prefix_replay, scheduler)
|
||||||
|
scheduler._load_restores_replay_window = MethodType(
|
||||||
|
Scheduler._load_restores_replay_window, scheduler
|
||||||
|
)
|
||||||
Scheduler._update_waiting_for_remote_kv(scheduler, request)
|
Scheduler._update_waiting_for_remote_kv(scheduler, request)
|
||||||
assert request.num_tokens - request.num_computed_tokens == 1
|
assert request.num_tokens - request.num_computed_tokens == 1
|
||||||
assert _allocate_scheduled(manager, request, num_new_tokens=1) is not None
|
assert _allocate_scheduled(manager, request, num_new_tokens=1) is not None
|
||||||
|
|||||||
@@ -247,6 +247,33 @@ def test_remote_kv_hit_is_taken_in_whole_blocks():
|
|||||||
assert swa_blocks[new_req.replay_start // BLOCK_SIZE] is not swa_manager._null_block
|
assert swa_blocks[new_req.replay_start // BLOCK_SIZE] is not swa_manager._null_block
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("is_async", [False, True])
|
||||||
|
@pytest.mark.parametrize("matched", [WINDOW, 4 * BLOCK_SIZE - 1])
|
||||||
|
def test_kv_load_carrying_window_does_not_replay(is_async, matched):
|
||||||
|
"""A load of the request's own blocks (P/D) restores every transfer group,
|
||||||
|
window included, so the hit is taken as is: no replay, no cut to whole
|
||||||
|
blocks, and a hit no longer than the window is kept."""
|
||||||
|
scheduler = _replay_scheduler(
|
||||||
|
use_kv_connector=MockKVConfig(matched_tokens=matched, is_async=is_async)
|
||||||
|
)
|
||||||
|
request = create_requests(
|
||||||
|
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||||
|
)[0]
|
||||||
|
scheduler.connector.get_loaded_kv_cache_group_ids = lambda request: (FULL, SWA)
|
||||||
|
scheduler.add_request(request)
|
||||||
|
out = scheduler.schedule()
|
||||||
|
if is_async:
|
||||||
|
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||||
|
scheduler.update_from_output(
|
||||||
|
out, create_model_runner_output([], finished_recving={request.request_id})
|
||||||
|
)
|
||||||
|
out = scheduler.schedule()
|
||||||
|
new_req = _new_req_data(out, request)
|
||||||
|
assert new_req.num_computed_tokens == matched
|
||||||
|
assert new_req.replay_start == 0
|
||||||
|
assert out.num_scheduled_tokens[request.request_id] == NUM_PROMPT_TOKENS - matched
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("via_connector", [False, True])
|
@pytest.mark.parametrize("via_connector", [False, True])
|
||||||
def test_hit_no_longer_than_window_is_ignored(via_connector):
|
def test_hit_no_longer_than_window_is_ignored(via_connector):
|
||||||
"""Such a hit would be recomputed in full anyway. It is not adopted (an
|
"""Such a hit would be recomputed in full anyway. It is not adopted (an
|
||||||
|
|||||||
@@ -1559,17 +1559,22 @@ def test_failed_load_rezeroes_unwritten_skipped_blocks():
|
|||||||
|
|
||||||
@pytest.mark.cpu_test
|
@pytest.mark.cpu_test
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"has_mamba,is_hma_required,expected_count",
|
"has_mamba,is_hma_required,bounded_replay,expected_count",
|
||||||
[
|
[
|
||||||
(True, True, 9),
|
(True, True, False, 9),
|
||||||
(False, False, 10),
|
(False, False, False, 10),
|
||||||
(False, True, 10),
|
(False, True, False, 10),
|
||||||
|
(False, True, True, 9),
|
||||||
],
|
],
|
||||||
ids=["mamba", "fa_only", "swa_only"],
|
ids=["mamba", "fa_only", "swa_only", "swa_bounded_replay"],
|
||||||
)
|
)
|
||||||
def test_mamba_n1_d_side(has_mamba, is_hma_required, expected_count):
|
def test_mamba_n1_d_side(has_mamba, is_hma_required, bounded_replay, expected_count):
|
||||||
"""D-side: Mamba gets N-1 matched tokens, non-Mamba gets N."""
|
"""D-side: Mamba and SWA bounded replay get N-1 matched tokens, others N."""
|
||||||
sched = make_nixl_scheduler(has_mamba=has_mamba, is_hma_required=is_hma_required)
|
sched = make_nixl_scheduler(
|
||||||
|
has_mamba=has_mamba,
|
||||||
|
is_hma_required=is_hma_required,
|
||||||
|
bounded_replay=bounded_replay,
|
||||||
|
)
|
||||||
req = create_request(num_tokens=10, do_remote_prefill=True)
|
req = create_request(num_tokens=10, do_remote_prefill=True)
|
||||||
|
|
||||||
count, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=0)
|
count, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=0)
|
||||||
@@ -1638,6 +1643,20 @@ def test_mamba_n1_p_side_truncation():
|
|||||||
fa_sched.on_new_request(fa_req)
|
fa_sched.on_new_request(fa_req)
|
||||||
assert len(fa_req.prompt_token_ids) == fa_original
|
assert len(fa_req.prompt_token_ids) == fa_original
|
||||||
|
|
||||||
|
# SWA bounded replay: the window is not replayed after the load, so the
|
||||||
|
# prefiller stops short of the last token too.
|
||||||
|
swa_sched = make_nixl_scheduler(is_hma_required=True, bounded_replay=True)
|
||||||
|
swa_req = create_request(num_tokens=10, do_remote_decode=True)
|
||||||
|
swa_sched.on_new_request(swa_req)
|
||||||
|
assert len(swa_req.prompt_token_ids) == 9
|
||||||
|
|
||||||
|
# A request that skips reading the prefix cache (prompt logprobs) is not
|
||||||
|
# cut: the decoder loads nothing and recomputes its whole prompt.
|
||||||
|
logprobs_req = create_request(num_tokens=10, do_remote_decode=True)
|
||||||
|
logprobs_req.sampling_params.skip_reading_prefix_cache = True
|
||||||
|
swa_sched.on_new_request(logprobs_req)
|
||||||
|
assert len(logprobs_req.prompt_token_ids) == 10
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.cpu_test
|
@pytest.mark.cpu_test
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
|
|||||||
@@ -532,6 +532,7 @@ def make_kv_cache_config(
|
|||||||
def make_nixl_scheduler(
|
def make_nixl_scheduler(
|
||||||
has_mamba: bool = False,
|
has_mamba: bool = False,
|
||||||
is_hma_required: bool = False,
|
is_hma_required: bool = False,
|
||||||
|
bounded_replay: bool = False,
|
||||||
heartbeat: bool = False,
|
heartbeat: bool = False,
|
||||||
kv_lease_duration: int = 30,
|
kv_lease_duration: int = 30,
|
||||||
):
|
):
|
||||||
@@ -548,6 +549,7 @@ def make_nixl_scheduler(
|
|||||||
|
|
||||||
sched = object.__new__(NixlConnectorScheduler)
|
sched = object.__new__(NixlConnectorScheduler)
|
||||||
sched._has_mamba = has_mamba
|
sched._has_mamba = has_mamba
|
||||||
|
sched._bounded_replay = bounded_replay
|
||||||
sched._is_hma_required = is_hma_required
|
sched._is_hma_required = is_hma_required
|
||||||
sched.kv_cache_config = make_kv_cache_config(
|
sched.kv_cache_config = make_kv_cache_config(
|
||||||
block_size=16,
|
block_size=16,
|
||||||
@@ -615,6 +617,7 @@ def make_nixl_push_scheduler(
|
|||||||
sched.side_channel_port = 5600
|
sched.side_channel_port = 5600
|
||||||
sched.is_bidirectional_kv_xfer_enabled = is_bidirectional_kv_xfer_enabled
|
sched.is_bidirectional_kv_xfer_enabled = is_bidirectional_kv_xfer_enabled
|
||||||
sched._has_mamba = has_mamba
|
sched._has_mamba = has_mamba
|
||||||
|
sched._bounded_replay = False
|
||||||
sched.kv_cache_config = make_kv_cache_config(
|
sched.kv_cache_config = make_kv_cache_config(
|
||||||
block_size=16,
|
block_size=16,
|
||||||
mamba_enabled=has_mamba,
|
mamba_enabled=has_mamba,
|
||||||
|
|||||||
@@ -67,8 +67,12 @@ class AuxOutputSchedulerConnector:
|
|||||||
block_hashes_by_request[request_id] = packed
|
block_hashes_by_request[request_id] = packed
|
||||||
self._sent_hash_counts[request_id] = len(request.block_hashes)
|
self._sent_hash_counts[request_id] = len(request.block_hashes)
|
||||||
assert request.sampling_params is not None
|
assert request.sampling_params is not None
|
||||||
|
# A P/D prefiller may cut the prompt short of the requested start.
|
||||||
scheduled_requests[request_id] = max(
|
scheduled_requests[request_id] = max(
|
||||||
request.sampling_params.routed_experts_prompt_start,
|
min(
|
||||||
|
request.sampling_params.routed_experts_prompt_start,
|
||||||
|
request.num_prompt_tokens,
|
||||||
|
),
|
||||||
0 if request.num_output_tokens == 0 else request.num_tokens - 1,
|
0 if request.num_output_tokens == 0 else request.num_tokens - 1,
|
||||||
)
|
)
|
||||||
# A settled token can complete a hash block after the next async schedule
|
# A settled token can complete a hash block after the next async schedule
|
||||||
|
|||||||
@@ -897,6 +897,10 @@ class MooncakeConnectorScheduler:
|
|||||||
# GDN is represented as a MambaSpec in vLLM. This Mooncake MambaSpec
|
# GDN is represented as a MambaSpec in vLLM. This Mooncake MambaSpec
|
||||||
# path is currently tested with GDN; Mamba2 is not validated yet.
|
# path is currently tested with GDN; Mamba2 is not validated yet.
|
||||||
self._has_mamba = kv_cache_config.has_mamba_layers
|
self._has_mamba = kv_cache_config.has_mamba_layers
|
||||||
|
self._bounded_replay = any(
|
||||||
|
g.kv_cache_spec.prefix_replay_tokens > 0
|
||||||
|
for g in kv_cache_config.kv_cache_groups
|
||||||
|
)
|
||||||
|
|
||||||
# Requests that need to start recv/send.
|
# Requests that need to start recv/send.
|
||||||
# New requests are added by update_state_after_alloc in
|
# New requests are added by update_state_after_alloc in
|
||||||
@@ -941,8 +945,10 @@ class MooncakeConnectorScheduler:
|
|||||||
|
|
||||||
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
|
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
|
||||||
"""D-side only. Returns N-1 for Mamba models since the decoder
|
"""D-side only. Returns N-1 for Mamba models since the decoder
|
||||||
always recomputes the last token and must start from h(N-1)."""
|
always recomputes the last token and must start from h(N-1), and for
|
||||||
if self._has_mamba and num_prompt_tokens > 1:
|
DSV41 SWA bounded replay, whose window is not replayed after a P/D load
|
||||||
|
and must be laid out for that token."""
|
||||||
|
if (self._has_mamba or self._bounded_replay) and num_prompt_tokens > 1:
|
||||||
return num_prompt_tokens - 1
|
return num_prompt_tokens - 1
|
||||||
return num_prompt_tokens
|
return num_prompt_tokens
|
||||||
|
|
||||||
@@ -958,6 +964,9 @@ class MooncakeConnectorScheduler:
|
|||||||
params is not None
|
params is not None
|
||||||
and not params.get("_p_side_truncated")
|
and not params.get("_p_side_truncated")
|
||||||
and request.num_prompt_tokens > 1
|
and request.num_prompt_tokens > 1
|
||||||
|
# A request that skips reading the prefix cache (prompt logprobs)
|
||||||
|
# loads nothing on the decoder, which recomputes its whole prompt.
|
||||||
|
and not request.get_skip_reading_prefix_cache()
|
||||||
):
|
):
|
||||||
if request.prompt_token_ids is not None:
|
if request.prompt_token_ids is not None:
|
||||||
request.prompt_token_ids.pop()
|
request.prompt_token_ids.pop()
|
||||||
@@ -973,7 +982,11 @@ class MooncakeConnectorScheduler:
|
|||||||
|
|
||||||
def on_new_request(self, request: "Request") -> None:
|
def on_new_request(self, request: "Request") -> None:
|
||||||
params = request.kv_transfer_params
|
params = request.kv_transfer_params
|
||||||
if params is not None and params.get("do_remote_decode") and self._has_mamba:
|
if (
|
||||||
|
params is not None
|
||||||
|
and params.get("do_remote_decode")
|
||||||
|
and (self._has_mamba or self._bounded_replay)
|
||||||
|
):
|
||||||
self._truncate_mamba_request_for_prefill(request)
|
self._truncate_mamba_request_for_prefill(request)
|
||||||
|
|
||||||
def get_num_new_matched_tokens(
|
def get_num_new_matched_tokens(
|
||||||
|
|||||||
@@ -101,6 +101,10 @@ class NixlBaseConnectorScheduler:
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
self._has_mamba = kv_cache_config.has_mamba_layers
|
self._has_mamba = kv_cache_config.has_mamba_layers
|
||||||
|
self._bounded_replay = any(
|
||||||
|
g.kv_cache_spec.prefix_replay_tokens > 0
|
||||||
|
for g in kv_cache_config.kv_cache_groups
|
||||||
|
)
|
||||||
|
|
||||||
logger.info("Initializing NIXL Scheduler %s", engine_id)
|
logger.info("Initializing NIXL Scheduler %s", engine_id)
|
||||||
if vllm_config.scheduler_config.disable_hybrid_kv_cache_manager:
|
if vllm_config.scheduler_config.disable_hybrid_kv_cache_manager:
|
||||||
@@ -376,14 +380,16 @@ class NixlBaseConnectorScheduler:
|
|||||||
"""Trailing prompt tokens the prefiller must not compute; the decoder
|
"""Trailing prompt tokens the prefiller must not compute; the decoder
|
||||||
recomputes them locally.
|
recomputes them locally.
|
||||||
|
|
||||||
Mamba needs h(N-1) so the decoder can derive h(N) itself. Multi-module
|
Mamba needs h(N-1) so the decoder can derive h(N) itself. DSV41's sliding
|
||||||
|
window under bounded replay is not replayed after a P/D load, so it must
|
||||||
|
be laid out for the token the decoder recomputes. Multi-module
|
||||||
MTP needs to keep its whole lookahead window off of the prefiller, which
|
MTP needs to keep its whole lookahead window off of the prefiller, which
|
||||||
would otherwise embed the unverified drafts in the MTP layer's KV cache. The
|
would otherwise embed the unverified drafts in the MTP layer's KV cache. The
|
||||||
decoder would never rebuild them, because the update is sized by the rejection
|
decoder would never rebuild them, because the update is sized by the rejection
|
||||||
count, which is zero for the first decode.
|
count, which is zero for the first decode.
|
||||||
"""
|
"""
|
||||||
return max(
|
return max(
|
||||||
1 if self._has_mamba else 0,
|
1 if self._has_mamba or self._bounded_replay else 0,
|
||||||
self.vllm_config.num_prefill_lookahead_tokens - 1,
|
self.vllm_config.num_prefill_lookahead_tokens - 1,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -412,6 +418,9 @@ class NixlBaseConnectorScheduler:
|
|||||||
# Guard against repeated truncation after preemption/reschedule.
|
# Guard against repeated truncation after preemption/reschedule.
|
||||||
and not params.get("_p_side_truncated")
|
and not params.get("_p_side_truncated")
|
||||||
and request.num_prompt_tokens > backoff
|
and request.num_prompt_tokens > backoff
|
||||||
|
# A request that skips reading the prefix cache (prompt logprobs)
|
||||||
|
# loads nothing on the decoder, which recomputes its whole prompt.
|
||||||
|
and not request.get_skip_reading_prefix_cache()
|
||||||
):
|
):
|
||||||
if request.prompt_token_ids is not None:
|
if request.prompt_token_ids is not None:
|
||||||
del request.prompt_token_ids[-backoff:]
|
del request.prompt_token_ids[-backoff:]
|
||||||
|
|||||||
@@ -290,10 +290,14 @@ class Scheduler(SchedulerInterface):
|
|||||||
# DSV41 SWA bounded replay: groups that declare a replay window are rebuilt
|
# DSV41 SWA bounded replay: groups that declare a replay window are rebuilt
|
||||||
# after a prefix hit by recomputing its trailing tokens. One window
|
# after a prefix hit by recomputing its trailing tokens. One window
|
||||||
# for all such groups, so the rewind matches every group's allocation.
|
# for all such groups, so the rewind matches every group's allocation.
|
||||||
replay_windows = {
|
self.prefix_replay_group_ids = tuple(
|
||||||
group.kv_cache_spec.prefix_replay_tokens
|
group_id
|
||||||
for group in kv_cache_config.kv_cache_groups
|
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
|
||||||
if group.kv_cache_spec.prefix_replay_tokens > 0
|
if group.kv_cache_spec.prefix_replay_tokens > 0
|
||||||
|
)
|
||||||
|
replay_windows = {
|
||||||
|
kv_cache_config.kv_cache_groups[group_id].kv_cache_spec.prefix_replay_tokens
|
||||||
|
for group_id in self.prefix_replay_group_ids
|
||||||
}
|
}
|
||||||
assert len(replay_windows) <= 1, (
|
assert len(replay_windows) <= 1, (
|
||||||
f"Prefix replay windows should agree: {sorted(replay_windows)}"
|
f"Prefix replay windows should agree: {sorted(replay_windows)}"
|
||||||
@@ -973,14 +977,17 @@ class Scheduler(SchedulerInterface):
|
|||||||
skip_request(request_queue)
|
skip_request(request_queue)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if self.prefix_replay_tokens:
|
if self.prefix_replay_tokens and not (
|
||||||
# SWA bounded replay recomputes the hit's last
|
ext_tokens and self._load_restores_replay_window(request)
|
||||||
|
):
|
||||||
|
# DSV41 SWA bounded replay recomputes the hit's last
|
||||||
# window, from the block holding its first
|
# window, from the block holding its first
|
||||||
# token; the sliding-window groups retire
|
# token; the sliding-window groups retire
|
||||||
# whole blocks below the window of the hit's
|
# whole blocks below the window of the hit's
|
||||||
# next token. Hits end on a block boundary, or
|
# next token. Hits end on a block boundary, or
|
||||||
# a hit ending one token short of one would
|
# a hit ending one token short of one would
|
||||||
# retire the block the replay starts in.
|
# retire the block the replay starts in. A load
|
||||||
|
# that carries the window replays nothing.
|
||||||
ext_tokens -= ext_tokens % self.block_size
|
ext_tokens -= ext_tokens % self.block_size
|
||||||
load_kv_async = load_kv_async and ext_tokens > 0
|
load_kv_async = load_kv_async and ext_tokens > 0
|
||||||
|
|
||||||
@@ -1026,9 +1033,17 @@ class Scheduler(SchedulerInterface):
|
|||||||
num_new_local_computed_tokens + num_external_computed_tokens
|
num_new_local_computed_tokens + num_external_computed_tokens
|
||||||
)
|
)
|
||||||
assert num_computed_tokens <= request.num_tokens
|
assert num_computed_tokens <= request.num_tokens
|
||||||
if 0 < num_computed_tokens <= self.prefix_replay_tokens:
|
# DSV41 SWA bounded replay: a KV load that carries the
|
||||||
# SWA bounded replay: a hit no longer than the replayed
|
# window is taken as is; any other hit no longer than the
|
||||||
# window would be recomputed in full and save nothing.
|
# replayed window would be recomputed in full and save
|
||||||
|
# nothing.
|
||||||
|
window_loaded = bool(
|
||||||
|
num_external_computed_tokens
|
||||||
|
) and self._load_restores_replay_window(request)
|
||||||
|
if (
|
||||||
|
not window_loaded
|
||||||
|
and 0 < num_computed_tokens <= self.prefix_replay_tokens
|
||||||
|
):
|
||||||
new_computed_blocks = (
|
new_computed_blocks = (
|
||||||
self.kv_cache_manager.empty_kv_cache_blocks
|
self.kv_cache_manager.empty_kv_cache_blocks
|
||||||
)
|
)
|
||||||
@@ -1091,7 +1106,7 @@ class Scheduler(SchedulerInterface):
|
|||||||
if did_prefix_cache_lookup:
|
if did_prefix_cache_lookup:
|
||||||
# Fresh admission: local and/or sync external hit.
|
# Fresh admission: local and/or sync external hit.
|
||||||
num_replay_tokens = self._mark_prefix_replay(
|
num_replay_tokens = self._mark_prefix_replay(
|
||||||
request, num_computed_tokens
|
request, num_computed_tokens, window_loaded
|
||||||
)
|
)
|
||||||
num_computed_tokens -= num_replay_tokens
|
num_computed_tokens -= num_replay_tokens
|
||||||
# Number of tokens to be scheduled.
|
# Number of tokens to be scheduled.
|
||||||
@@ -3083,16 +3098,31 @@ class Scheduler(SchedulerInterface):
|
|||||||
self._request_remaining_blocks(req) for req in self._inflight_prefills
|
self._request_remaining_blocks(req) for req in self._inflight_prefills
|
||||||
)
|
)
|
||||||
|
|
||||||
def _mark_prefix_replay(self, request: Request, num_hit_tokens: int) -> int:
|
def _load_restores_replay_window(self, request: Request) -> bool:
|
||||||
|
"""Whether the KV connector's load for ``request`` restores every DSV41
|
||||||
|
SWA bounded-replay group, as a P/D transfer of the request's own blocks
|
||||||
|
does; a prefix-cache store holds no window."""
|
||||||
|
assert self.connector is not None
|
||||||
|
return bool(self.prefix_replay_group_ids) and set(
|
||||||
|
self.prefix_replay_group_ids
|
||||||
|
).issubset(self.connector.get_loaded_kv_cache_group_ids(request))
|
||||||
|
|
||||||
|
def _mark_prefix_replay(
|
||||||
|
self, request: Request, num_hit_tokens: int, window_loaded: bool
|
||||||
|
) -> int:
|
||||||
"""Record where a prefix hit's replay starts and return the number of
|
"""Record where a prefix hit's replay starts and return the number of
|
||||||
hit tokens to recompute; the caller rewinds the computed count by it.
|
hit tokens to recompute; the caller rewinds the computed count by it.
|
||||||
|
|
||||||
The replayed tokens are the hit's last window (see
|
The replayed tokens are the hit's last window (see
|
||||||
``Request.replay_start``): the worker rebuilds their sliding-window KV
|
``Request.replay_start``): the worker rebuilds their sliding-window KV
|
||||||
and leaves their cached KV alone.
|
and leaves their cached KV alone. A hit whose KV load carries the
|
||||||
|
window replays nothing.
|
||||||
"""
|
"""
|
||||||
if not self.prefix_replay_tokens:
|
if not self.prefix_replay_tokens:
|
||||||
return 0
|
return 0
|
||||||
|
if window_loaded:
|
||||||
|
request.replay_start = 0
|
||||||
|
return 0
|
||||||
num_replay_tokens = min(self.prefix_replay_tokens, num_hit_tokens)
|
num_replay_tokens = min(self.prefix_replay_tokens, num_hit_tokens)
|
||||||
request.replay_start = num_hit_tokens - num_replay_tokens
|
request.replay_start = num_hit_tokens - num_replay_tokens
|
||||||
return num_replay_tokens
|
return num_replay_tokens
|
||||||
@@ -3106,7 +3136,8 @@ class Scheduler(SchedulerInterface):
|
|||||||
"""
|
"""
|
||||||
assert self.connector is not None
|
assert self.connector is not None
|
||||||
|
|
||||||
if request.request_id in self.failed_recving_kv_req_ids:
|
load_failed = request.request_id in self.failed_recving_kv_req_ids
|
||||||
|
if load_failed:
|
||||||
# Request had KV load failures; num_computed_tokens was already
|
# Request had KV load failures; num_computed_tokens was already
|
||||||
# updated in _update_requests_with_invalid_blocks
|
# updated in _update_requests_with_invalid_blocks
|
||||||
if request.num_computed_tokens:
|
if request.num_computed_tokens:
|
||||||
@@ -3132,11 +3163,15 @@ class Scheduler(SchedulerInterface):
|
|||||||
# This will cache the blocks iff caching is enabled.
|
# This will cache the blocks iff caching is enabled.
|
||||||
self.kv_cache_manager.cache_blocks(request, request.num_computed_tokens)
|
self.kv_cache_manager.cache_blocks(request, request.num_computed_tokens)
|
||||||
|
|
||||||
# SWA bounded replay recomputes the tail of the hit, which covers the
|
# DSV41 SWA bounded replay recomputes the tail of a hit loaded without
|
||||||
# last token; otherwise a full prompt hit re-computes that token so the
|
# its window, or of a failed load, which may have lost it; the replay
|
||||||
# next one can be sampled.
|
# covers the last token. Otherwise a full prompt hit re-computes that
|
||||||
|
# token so the next one can be sampled.
|
||||||
num_replay_tokens = self._mark_prefix_replay(
|
num_replay_tokens = self._mark_prefix_replay(
|
||||||
request, request.num_computed_tokens
|
request,
|
||||||
|
request.num_computed_tokens,
|
||||||
|
window_loaded=not load_failed
|
||||||
|
and self._load_restores_replay_window(request),
|
||||||
)
|
)
|
||||||
if num_replay_tokens > 0:
|
if num_replay_tokens > 0:
|
||||||
request.num_computed_tokens -= num_replay_tokens
|
request.num_computed_tokens -= num_replay_tokens
|
||||||
|
|||||||
Reference in New Issue
Block a user