From 85c1f58d50b80a6668f1f8158f0d8ef7b1d9112a Mon Sep 17 00:00:00 2001 From: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Date: Tue, 15 Sep 2026 17:31:25 -0700 Subject: [PATCH] [BugFix][Model Runner V2][Spec Decode] Fix decode instance's multi-layer MTP kv caches during P/D (#55055) Signed-off-by: Giancarlo Delfin --- .../unit/test_nixl_push_connector.py | 2 +- tests/v1/kv_connector/unit/utils.py | 7 ++- vllm/config/vllm.py | 23 ++++++++ .../kv_connector/v1/nixl/base_scheduler.py | 52 +++++++++++++------ vllm/v1/core/kv_cache_utils.py | 7 +-- vllm/v1/core/sched/scheduler.py | 14 +---- vllm/v1/worker/gpu/model_runner.py | 10 +--- 7 files changed, 69 insertions(+), 46 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index a908d32462..7ce84bf685 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -138,7 +138,7 @@ class TestPushScheduler: with patch.object( sched, - "_truncate_mamba_request_for_prefill", + "_truncate_request_for_prefill", side_effect=AssertionError("must not truncate after cache lookup"), ): assert sched.get_num_new_matched_tokens(request, 0) == (0, False) diff --git a/tests/v1/kv_connector/unit/utils.py b/tests/v1/kv_connector/unit/utils.py index 1ee1e81776..98633c233b 100644 --- a/tests/v1/kv_connector/unit/utils.py +++ b/tests/v1/kv_connector/unit/utils.py @@ -521,6 +521,8 @@ def make_nixl_scheduler( Only sets the flags needed by the tests. When *heartbeat=True* the scheduler-side heartbeat bookkeeping fields are also initialised. """ + from types import SimpleNamespace + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import ( NixlConnectorScheduler, ) @@ -532,6 +534,7 @@ def make_nixl_scheduler( block_size=16, mamba_enabled=has_mamba, ) + sched.vllm_config = SimpleNamespace(num_prefill_lookahead_tokens=0) if heartbeat: sched._heartbeat_by_engine = {} @@ -598,9 +601,11 @@ def make_nixl_push_scheduler( mamba_enabled=has_mamba, ) - # vllm_config is consulted for parallel_config.tensor_parallel_size. + # vllm_config is consulted for parallel_config.tensor_parallel_size, and by + # `_prefill_backoff` on both the P and D prefill paths. vllm_config = MagicMock() vllm_config.parallel_config.tensor_parallel_size = 1 + vllm_config.num_prefill_lookahead_tokens = 0 sched.vllm_config = vllm_config # Push-specific state. diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 8e0846982b..7167ca5a90 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -650,6 +650,29 @@ class VllmConfig: return self.num_speculative_tokens return 0 + @property + def num_prefill_lookahead_tokens(self) -> int: + """Prefill tokens past the computed range that the drafter reads. + + Mid-prefill the drafter consumes tokens the target model has not been + scheduled for yet, so every component that has to keep them available + must apply this margin: the scheduler, which never ends a chunk within + it and shifts encoder scheduling by it, and the KV cache manager, which + treats the trailing `this - 1` tokens as re-prefillable rather than + finalized. Consumers must read this property rather than re-deriving + their own per-method lookahead, so those components cannot drift apart. + """ + speculative_config = self.speculative_config + if speculative_config is None or not speculative_config.use_eagle(): + return 0 + if speculative_config.use_multi_module_mtp(): + # Each MTP module reads one token further ahead than the one before + # it, so the chain needs num_speculative_tokens of runway at a + # chunked-prefill boundary. + return self.num_speculative_tokens + # Eagle-family drafters read only the immediate next token. + return 1 + @property def uniform_decode_query_len(self) -> int: """Query length of every request in a uniform decode batch. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py index 9cfd100730..54184aa339 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py @@ -199,8 +199,8 @@ class NixlBaseConnectorScheduler: def on_new_request(self, request: "Request") -> None: """Track a request that may need heartbeats.""" params = request.kv_transfer_params - if params is not None and params.get("do_remote_decode") and self._has_mamba: - self._truncate_mamba_request_for_prefill(request) + if params is not None and params.get("do_remote_decode"): + self._truncate_request_for_prefill(request) # NOTE (NickLucche) This excludes request meant for P, ie heartbeats are # effectively disabled for Bidirectional KV transfer. @@ -382,36 +382,56 @@ class NixlBaseConnectorScheduler: (identity, b"", encoded_data[(target_pp_rank, target_tp_rank)], ts) ) + def _prefill_backoff(self) -> int: + """Trailing prompt tokens the prefiller must not compute; the decoder + recomputes them locally. + + Mamba needs h(N-1) so the decoder can derive h(N) itself. Multi-module + MTP needs to keep its whole lookahead window off of the prefiller, which + would otherwise embed the unverified drafts in the MTP layer's KV cache. The + decoder would never rebuild them, because the update is sized by the rejection + count, which is zero for the first decode. + """ + return max( + 1 if self._has_mamba else 0, + self.vllm_config.num_prefill_lookahead_tokens - 1, + ) + def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int: - """D-side only. Returns N-1 for Mamba models since the decoder - always recomputes the last token and must start from h(N-1).""" - if self._has_mamba and num_prompt_tokens > 1: - return num_prompt_tokens - 1 + """D-side only. The number of prompt tokens to load from the prefiller. + Stops short of the trailing ``_prefill_backoff()`` tokens that the decoder + will recompute locally.""" + backoff = self._prefill_backoff() + if backoff and num_prompt_tokens > backoff: + return num_prompt_tokens - backoff return num_prompt_tokens - def _truncate_mamba_request_for_prefill(self, request: "Request") -> None: - """P-side only: drop the last prompt token so the prefiller computes - h(N-1) instead of h(N). The decoder recomputes the last token to - derive h(N) correctly. + def _truncate_request_for_prefill(self, request: "Request") -> None: + """P-side only: drop the trailing ``_prefill_backoff()`` prompt tokens + so the prefiller stops short of what the decoder recomputes locally. + For Mamba that is the single token needed to yield h(N-1); for + multi-module MTP it is the drafter's whole lookahead window. Guarded by ``_p_side_truncated`` to avoid repeated truncation if the request is preempted and rescheduled.""" + backoff = self._prefill_backoff() params = request.kv_transfer_params if ( - params is not None + backoff + and params is not None # Guard against repeated truncation after preemption/reschedule. and not params.get("_p_side_truncated") - and request.num_prompt_tokens > 1 + and request.num_prompt_tokens > backoff ): if request.prompt_token_ids is not None: - request.prompt_token_ids.pop() + del request.prompt_token_ids[-backoff:] elif request.prompt_embeds is not None: - request.prompt_embeds = request.prompt_embeds[:-1] + request.prompt_embeds = request.prompt_embeds[:-backoff] else: return - request._all_token_ids.pop() - request.num_prompt_tokens -= 1 + del request._all_token_ids[-backoff:] + request.num_prompt_tokens -= backoff request.max_tokens = 1 params["_p_side_truncated"] = True diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 1c6e102c16..c55c06abf1 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -2678,12 +2678,7 @@ def get_kv_cache_configs( # When speculating with more than 1 speculative module (e.g. multi-layered MTP) # tag every SlidingWindowSpec with how many extra tokens to retain in the window. - extra_retained_tokens = ( - vllm_config.speculative_config.num_speculative_tokens - 1 - if vllm_config.speculative_config is not None - and vllm_config.speculative_config.use_multi_module_mtp() - else 0 - ) + extra_retained_tokens = max(0, vllm_config.num_prefill_lookahead_tokens - 1) for layer_name, layer_spec in merged_kv_cache_specs.items(): if isinstance(layer_spec, SlidingWindowSpec): merged_kv_cache_specs[layer_name] = replace( diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index daf2b316d0..f1e11e9b1f 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -262,13 +262,7 @@ class Scheduler(SchedulerInterface): self.use_eagle_block_drop = False self.num_spec_tokens = vllm_config.num_speculative_tokens self.num_lookahead_tokens = vllm_config.num_lookahead_tokens - # Positions past the computed tokens that the drafter reads mid-prefill. - # Eagle-family drafters read 1 ahead, but multi-module MTP reads - # num_spec_tokens ahead at chunked-prefill boundaries. Determines the - # encoder scheduling shift, the deferred encoder free, the KV cache - # manager's re-prefillable window (this minus 1), and how many tokens to - # reserve between a chunk boundary and the prefill end. - self.num_prefill_lookahead = 0 + self.num_prefill_lookahead = vllm_config.num_prefill_lookahead_tokens self.dynamic_sd_lookup: list[int] | None = None if speculative_config is not None: if speculative_config.num_speculative_tokens_per_batch_size: @@ -278,12 +272,6 @@ class Scheduler(SchedulerInterface): vllm_num_speculative_tokens=self.num_spec_tokens, ) self.use_eagle = speculative_config.use_eagle() - if self.use_eagle: - self.num_prefill_lookahead = ( - self.num_spec_tokens - if speculative_config.use_multi_module_mtp() - else 1 - ) self.use_eagle_block_drop = speculative_config.use_eagle_block_drop() if self.use_eagle and not self.use_eagle_block_drop: logger.warning( diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index b66d552cb4..82cc697e73 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -296,15 +296,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.is_pooling_model = self.model_config.runner_type == "pooling" self.pooling_runner: PoolingRunner | None = None - # Multi-module MTP feeds its modules the next num_speculative_steps prefill - # tokens during chunked prefill. Other speculators only read the immediate - # next one. - num_prefill_lookahead = ( - self.num_speculative_steps - if self.speculative_config is not None - and self.speculative_config.use_multi_module_mtp() - else 1 - ) + num_prefill_lookahead = max(1, vllm_config.num_prefill_lookahead_tokens) self.step_timing = StepTimingCollector()