[BugFix][Model Runner V2][Spec Decode] Fix decode instance's multi-layer MTP kv caches during P/D (#55055)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-09-15 17:31:25 -07:00
committed by GitHub
parent 031f5810c1
commit 85c1f58d50
7 changed files with 69 additions and 46 deletions
@@ -138,7 +138,7 @@ class TestPushScheduler:
with patch.object(
sched,
"_truncate_mamba_request_for_prefill",
"_truncate_request_for_prefill",
side_effect=AssertionError("must not truncate after cache lookup"),
):
assert sched.get_num_new_matched_tokens(request, 0) == (0, False)
+6 -1
View File
@@ -521,6 +521,8 @@ def make_nixl_scheduler(
Only sets the flags needed by the tests. When *heartbeat=True* the
scheduler-side heartbeat bookkeeping fields are also initialised.
"""
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import (
NixlConnectorScheduler,
)
@@ -532,6 +534,7 @@ def make_nixl_scheduler(
block_size=16,
mamba_enabled=has_mamba,
)
sched.vllm_config = SimpleNamespace(num_prefill_lookahead_tokens=0)
if heartbeat:
sched._heartbeat_by_engine = {}
@@ -598,9 +601,11 @@ def make_nixl_push_scheduler(
mamba_enabled=has_mamba,
)
# vllm_config is consulted for parallel_config.tensor_parallel_size.
# vllm_config is consulted for parallel_config.tensor_parallel_size, and by
# `_prefill_backoff` on both the P and D prefill paths.
vllm_config = MagicMock()
vllm_config.parallel_config.tensor_parallel_size = 1
vllm_config.num_prefill_lookahead_tokens = 0
sched.vllm_config = vllm_config
# Push-specific state.
+23
View File
@@ -650,6 +650,29 @@ class VllmConfig:
return self.num_speculative_tokens
return 0
@property
def num_prefill_lookahead_tokens(self) -> int:
"""Prefill tokens past the computed range that the drafter reads.
Mid-prefill the drafter consumes tokens the target model has not been
scheduled for yet, so every component that has to keep them available
must apply this margin: the scheduler, which never ends a chunk within
it and shifts encoder scheduling by it, and the KV cache manager, which
treats the trailing `this - 1` tokens as re-prefillable rather than
finalized. Consumers must read this property rather than re-deriving
their own per-method lookahead, so those components cannot drift apart.
"""
speculative_config = self.speculative_config
if speculative_config is None or not speculative_config.use_eagle():
return 0
if speculative_config.use_multi_module_mtp():
# Each MTP module reads one token further ahead than the one before
# it, so the chain needs num_speculative_tokens of runway at a
# chunked-prefill boundary.
return self.num_speculative_tokens
# Eagle-family drafters read only the immediate next token.
return 1
@property
def uniform_decode_query_len(self) -> int:
"""Query length of every request in a uniform decode batch.
@@ -199,8 +199,8 @@ class NixlBaseConnectorScheduler:
def on_new_request(self, request: "Request") -> None:
"""Track a request that may need heartbeats."""
params = request.kv_transfer_params
if params is not None and params.get("do_remote_decode") and self._has_mamba:
self._truncate_mamba_request_for_prefill(request)
if params is not None and params.get("do_remote_decode"):
self._truncate_request_for_prefill(request)
# NOTE (NickLucche) This excludes request meant for P, ie heartbeats are
# effectively disabled for Bidirectional KV transfer.
@@ -382,36 +382,56 @@ class NixlBaseConnectorScheduler:
(identity, b"", encoded_data[(target_pp_rank, target_tp_rank)], ts)
)
def _prefill_backoff(self) -> int:
"""Trailing prompt tokens the prefiller must not compute; the decoder
recomputes them locally.
Mamba needs h(N-1) so the decoder can derive h(N) itself. Multi-module
MTP needs to keep its whole lookahead window off of the prefiller, which
would otherwise embed the unverified drafts in the MTP layer's KV cache. The
decoder would never rebuild them, because the update is sized by the rejection
count, which is zero for the first decode.
"""
return max(
1 if self._has_mamba else 0,
self.vllm_config.num_prefill_lookahead_tokens - 1,
)
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
"""D-side only. Returns N-1 for Mamba models since the decoder
always recomputes the last token and must start from h(N-1)."""
if self._has_mamba and num_prompt_tokens > 1:
return num_prompt_tokens - 1
"""D-side only. The number of prompt tokens to load from the prefiller.
Stops short of the trailing ``_prefill_backoff()`` tokens that the decoder
will recompute locally."""
backoff = self._prefill_backoff()
if backoff and num_prompt_tokens > backoff:
return num_prompt_tokens - backoff
return num_prompt_tokens
def _truncate_mamba_request_for_prefill(self, request: "Request") -> None:
"""P-side only: drop the last prompt token so the prefiller computes
h(N-1) instead of h(N). The decoder recomputes the last token to
derive h(N) correctly.
def _truncate_request_for_prefill(self, request: "Request") -> None:
"""P-side only: drop the trailing ``_prefill_backoff()`` prompt tokens
so the prefiller stops short of what the decoder recomputes locally.
For Mamba that is the single token needed to yield h(N-1); for
multi-module MTP it is the drafter's whole lookahead window.
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
request is preempted and rescheduled."""
backoff = self._prefill_backoff()
params = request.kv_transfer_params
if (
params is not None
backoff
and params is not None
# Guard against repeated truncation after preemption/reschedule.
and not params.get("_p_side_truncated")
and request.num_prompt_tokens > 1
and request.num_prompt_tokens > backoff
):
if request.prompt_token_ids is not None:
request.prompt_token_ids.pop()
del request.prompt_token_ids[-backoff:]
elif request.prompt_embeds is not None:
request.prompt_embeds = request.prompt_embeds[:-1]
request.prompt_embeds = request.prompt_embeds[:-backoff]
else:
return
request._all_token_ids.pop()
request.num_prompt_tokens -= 1
del request._all_token_ids[-backoff:]
request.num_prompt_tokens -= backoff
request.max_tokens = 1
params["_p_side_truncated"] = True
+1 -6
View File
@@ -2678,12 +2678,7 @@ def get_kv_cache_configs(
# When speculating with more than 1 speculative module (e.g. multi-layered MTP)
# tag every SlidingWindowSpec with how many extra tokens to retain in the window.
extra_retained_tokens = (
vllm_config.speculative_config.num_speculative_tokens - 1
if vllm_config.speculative_config is not None
and vllm_config.speculative_config.use_multi_module_mtp()
else 0
)
extra_retained_tokens = max(0, vllm_config.num_prefill_lookahead_tokens - 1)
for layer_name, layer_spec in merged_kv_cache_specs.items():
if isinstance(layer_spec, SlidingWindowSpec):
merged_kv_cache_specs[layer_name] = replace(
+1 -13
View File
@@ -262,13 +262,7 @@ class Scheduler(SchedulerInterface):
self.use_eagle_block_drop = False
self.num_spec_tokens = vllm_config.num_speculative_tokens
self.num_lookahead_tokens = vllm_config.num_lookahead_tokens
# Positions past the computed tokens that the drafter reads mid-prefill.
# Eagle-family drafters read 1 ahead, but multi-module MTP reads
# num_spec_tokens ahead at chunked-prefill boundaries. Determines the
# encoder scheduling shift, the deferred encoder free, the KV cache
# manager's re-prefillable window (this minus 1), and how many tokens to
# reserve between a chunk boundary and the prefill end.
self.num_prefill_lookahead = 0
self.num_prefill_lookahead = vllm_config.num_prefill_lookahead_tokens
self.dynamic_sd_lookup: list[int] | None = None
if speculative_config is not None:
if speculative_config.num_speculative_tokens_per_batch_size:
@@ -278,12 +272,6 @@ class Scheduler(SchedulerInterface):
vllm_num_speculative_tokens=self.num_spec_tokens,
)
self.use_eagle = speculative_config.use_eagle()
if self.use_eagle:
self.num_prefill_lookahead = (
self.num_spec_tokens
if speculative_config.use_multi_module_mtp()
else 1
)
self.use_eagle_block_drop = speculative_config.use_eagle_block_drop()
if self.use_eagle and not self.use_eagle_block_drop:
logger.warning(
+1 -9
View File
@@ -296,15 +296,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.is_pooling_model = self.model_config.runner_type == "pooling"
self.pooling_runner: PoolingRunner | None = None
# Multi-module MTP feeds its modules the next num_speculative_steps prefill
# tokens during chunked prefill. Other speculators only read the immediate
# next one.
num_prefill_lookahead = (
self.num_speculative_steps
if self.speculative_config is not None
and self.speculative_config.use_multi_module_mtp()
else 1
)
num_prefill_lookahead = max(1, vllm_config.num_prefill_lookahead_tokens)
self.step_timing = StepTimingCollector()