[Bugfix][DSv4.1] Skip SWA bounded replay for KV loads that carry the window (#59197)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
Claude Opus 5.5
parent
d169c5a903
commit
ef63c23d35
@@ -67,8 +67,12 @@ class AuxOutputSchedulerConnector:
|
||||
block_hashes_by_request[request_id] = packed
|
||||
self._sent_hash_counts[request_id] = len(request.block_hashes)
|
||||
assert request.sampling_params is not None
|
||||
# A P/D prefiller may cut the prompt short of the requested start.
|
||||
scheduled_requests[request_id] = max(
|
||||
request.sampling_params.routed_experts_prompt_start,
|
||||
min(
|
||||
request.sampling_params.routed_experts_prompt_start,
|
||||
request.num_prompt_tokens,
|
||||
),
|
||||
0 if request.num_output_tokens == 0 else request.num_tokens - 1,
|
||||
)
|
||||
# A settled token can complete a hash block after the next async schedule
|
||||
|
||||
@@ -897,6 +897,10 @@ class MooncakeConnectorScheduler:
|
||||
# GDN is represented as a MambaSpec in vLLM. This Mooncake MambaSpec
|
||||
# path is currently tested with GDN; Mamba2 is not validated yet.
|
||||
self._has_mamba = kv_cache_config.has_mamba_layers
|
||||
self._bounded_replay = any(
|
||||
g.kv_cache_spec.prefix_replay_tokens > 0
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
|
||||
# Requests that need to start recv/send.
|
||||
# New requests are added by update_state_after_alloc in
|
||||
@@ -941,8 +945,10 @@ class MooncakeConnectorScheduler:
|
||||
|
||||
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
|
||||
"""D-side only. Returns N-1 for Mamba models since the decoder
|
||||
always recomputes the last token and must start from h(N-1)."""
|
||||
if self._has_mamba and num_prompt_tokens > 1:
|
||||
always recomputes the last token and must start from h(N-1), and for
|
||||
DSV41 SWA bounded replay, whose window is not replayed after a P/D load
|
||||
and must be laid out for that token."""
|
||||
if (self._has_mamba or self._bounded_replay) and num_prompt_tokens > 1:
|
||||
return num_prompt_tokens - 1
|
||||
return num_prompt_tokens
|
||||
|
||||
@@ -958,6 +964,9 @@ class MooncakeConnectorScheduler:
|
||||
params is not None
|
||||
and not params.get("_p_side_truncated")
|
||||
and request.num_prompt_tokens > 1
|
||||
# A request that skips reading the prefix cache (prompt logprobs)
|
||||
# loads nothing on the decoder, which recomputes its whole prompt.
|
||||
and not request.get_skip_reading_prefix_cache()
|
||||
):
|
||||
if request.prompt_token_ids is not None:
|
||||
request.prompt_token_ids.pop()
|
||||
@@ -973,7 +982,11 @@ class MooncakeConnectorScheduler:
|
||||
|
||||
def on_new_request(self, request: "Request") -> None:
|
||||
params = request.kv_transfer_params
|
||||
if params is not None and params.get("do_remote_decode") and self._has_mamba:
|
||||
if (
|
||||
params is not None
|
||||
and params.get("do_remote_decode")
|
||||
and (self._has_mamba or self._bounded_replay)
|
||||
):
|
||||
self._truncate_mamba_request_for_prefill(request)
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
|
||||
@@ -101,6 +101,10 @@ class NixlBaseConnectorScheduler:
|
||||
)
|
||||
)
|
||||
self._has_mamba = kv_cache_config.has_mamba_layers
|
||||
self._bounded_replay = any(
|
||||
g.kv_cache_spec.prefix_replay_tokens > 0
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
|
||||
logger.info("Initializing NIXL Scheduler %s", engine_id)
|
||||
if vllm_config.scheduler_config.disable_hybrid_kv_cache_manager:
|
||||
@@ -376,14 +380,16 @@ class NixlBaseConnectorScheduler:
|
||||
"""Trailing prompt tokens the prefiller must not compute; the decoder
|
||||
recomputes them locally.
|
||||
|
||||
Mamba needs h(N-1) so the decoder can derive h(N) itself. Multi-module
|
||||
Mamba needs h(N-1) so the decoder can derive h(N) itself. DSV41's sliding
|
||||
window under bounded replay is not replayed after a P/D load, so it must
|
||||
be laid out for the token the decoder recomputes. Multi-module
|
||||
MTP needs to keep its whole lookahead window off of the prefiller, which
|
||||
would otherwise embed the unverified drafts in the MTP layer's KV cache. The
|
||||
decoder would never rebuild them, because the update is sized by the rejection
|
||||
count, which is zero for the first decode.
|
||||
"""
|
||||
return max(
|
||||
1 if self._has_mamba else 0,
|
||||
1 if self._has_mamba or self._bounded_replay else 0,
|
||||
self.vllm_config.num_prefill_lookahead_tokens - 1,
|
||||
)
|
||||
|
||||
@@ -412,6 +418,9 @@ class NixlBaseConnectorScheduler:
|
||||
# Guard against repeated truncation after preemption/reschedule.
|
||||
and not params.get("_p_side_truncated")
|
||||
and request.num_prompt_tokens > backoff
|
||||
# A request that skips reading the prefix cache (prompt logprobs)
|
||||
# loads nothing on the decoder, which recomputes its whole prompt.
|
||||
and not request.get_skip_reading_prefix_cache()
|
||||
):
|
||||
if request.prompt_token_ids is not None:
|
||||
del request.prompt_token_ids[-backoff:]
|
||||
|
||||
Reference in New Issue
Block a user