[Bugfix][DSv4.1] Skip SWA bounded replay for KV loads that carry the window (#59197)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yifan Qiao
2026-10-07 15:28:42 -07:00
committed by GitHub
co-authored by Codex Claude Opus 5.5
parent d169c5a903
commit ef63c23d35
9 changed files with 161 additions and 31 deletions
@@ -67,8 +67,12 @@ class AuxOutputSchedulerConnector:
block_hashes_by_request[request_id] = packed
self._sent_hash_counts[request_id] = len(request.block_hashes)
assert request.sampling_params is not None
# A P/D prefiller may cut the prompt short of the requested start.
scheduled_requests[request_id] = max(
request.sampling_params.routed_experts_prompt_start,
min(
request.sampling_params.routed_experts_prompt_start,
request.num_prompt_tokens,
),
0 if request.num_output_tokens == 0 else request.num_tokens - 1,
)
# A settled token can complete a hash block after the next async schedule
@@ -897,6 +897,10 @@ class MooncakeConnectorScheduler:
# GDN is represented as a MambaSpec in vLLM. This Mooncake MambaSpec
# path is currently tested with GDN; Mamba2 is not validated yet.
self._has_mamba = kv_cache_config.has_mamba_layers
self._bounded_replay = any(
g.kv_cache_spec.prefix_replay_tokens > 0
for g in kv_cache_config.kv_cache_groups
)
# Requests that need to start recv/send.
# New requests are added by update_state_after_alloc in
@@ -941,8 +945,10 @@ class MooncakeConnectorScheduler:
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
"""D-side only. Returns N-1 for Mamba models since the decoder
always recomputes the last token and must start from h(N-1)."""
if self._has_mamba and num_prompt_tokens > 1:
always recomputes the last token and must start from h(N-1), and for
DSV41 SWA bounded replay, whose window is not replayed after a P/D load
and must be laid out for that token."""
if (self._has_mamba or self._bounded_replay) and num_prompt_tokens > 1:
return num_prompt_tokens - 1
return num_prompt_tokens
@@ -958,6 +964,9 @@ class MooncakeConnectorScheduler:
params is not None
and not params.get("_p_side_truncated")
and request.num_prompt_tokens > 1
# A request that skips reading the prefix cache (prompt logprobs)
# loads nothing on the decoder, which recomputes its whole prompt.
and not request.get_skip_reading_prefix_cache()
):
if request.prompt_token_ids is not None:
request.prompt_token_ids.pop()
@@ -973,7 +982,11 @@ class MooncakeConnectorScheduler:
def on_new_request(self, request: "Request") -> None:
params = request.kv_transfer_params
if params is not None and params.get("do_remote_decode") and self._has_mamba:
if (
params is not None
and params.get("do_remote_decode")
and (self._has_mamba or self._bounded_replay)
):
self._truncate_mamba_request_for_prefill(request)
def get_num_new_matched_tokens(
@@ -101,6 +101,10 @@ class NixlBaseConnectorScheduler:
)
)
self._has_mamba = kv_cache_config.has_mamba_layers
self._bounded_replay = any(
g.kv_cache_spec.prefix_replay_tokens > 0
for g in kv_cache_config.kv_cache_groups
)
logger.info("Initializing NIXL Scheduler %s", engine_id)
if vllm_config.scheduler_config.disable_hybrid_kv_cache_manager:
@@ -376,14 +380,16 @@ class NixlBaseConnectorScheduler:
"""Trailing prompt tokens the prefiller must not compute; the decoder
recomputes them locally.
Mamba needs h(N-1) so the decoder can derive h(N) itself. Multi-module
Mamba needs h(N-1) so the decoder can derive h(N) itself. DSV41's sliding
window under bounded replay is not replayed after a P/D load, so it must
be laid out for the token the decoder recomputes. Multi-module
MTP needs to keep its whole lookahead window off of the prefiller, which
would otherwise embed the unverified drafts in the MTP layer's KV cache. The
decoder would never rebuild them, because the update is sized by the rejection
count, which is zero for the first decode.
"""
return max(
1 if self._has_mamba else 0,
1 if self._has_mamba or self._bounded_replay else 0,
self.vllm_config.num_prefill_lookahead_tokens - 1,
)
@@ -412,6 +418,9 @@ class NixlBaseConnectorScheduler:
# Guard against repeated truncation after preemption/reschedule.
and not params.get("_p_side_truncated")
and request.num_prompt_tokens > backoff
# A request that skips reading the prefix cache (prompt logprobs)
# loads nothing on the decoder, which recomputes its whole prompt.
and not request.get_skip_reading_prefix_cache()
):
if request.prompt_token_ids is not None:
del request.prompt_token_ids[-backoff:]