[BugFix][Model Runner V2][Spec Decode] Fix decode instance's multi-layer MTP kv caches during P/D (#55055)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -138,7 +138,7 @@ class TestPushScheduler:
|
||||
|
||||
with patch.object(
|
||||
sched,
|
||||
"_truncate_mamba_request_for_prefill",
|
||||
"_truncate_request_for_prefill",
|
||||
side_effect=AssertionError("must not truncate after cache lookup"),
|
||||
):
|
||||
assert sched.get_num_new_matched_tokens(request, 0) == (0, False)
|
||||
|
||||
@@ -521,6 +521,8 @@ def make_nixl_scheduler(
|
||||
Only sets the flags needed by the tests. When *heartbeat=True* the
|
||||
scheduler-side heartbeat bookkeeping fields are also initialised.
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import (
|
||||
NixlConnectorScheduler,
|
||||
)
|
||||
@@ -532,6 +534,7 @@ def make_nixl_scheduler(
|
||||
block_size=16,
|
||||
mamba_enabled=has_mamba,
|
||||
)
|
||||
sched.vllm_config = SimpleNamespace(num_prefill_lookahead_tokens=0)
|
||||
|
||||
if heartbeat:
|
||||
sched._heartbeat_by_engine = {}
|
||||
@@ -598,9 +601,11 @@ def make_nixl_push_scheduler(
|
||||
mamba_enabled=has_mamba,
|
||||
)
|
||||
|
||||
# vllm_config is consulted for parallel_config.tensor_parallel_size.
|
||||
# vllm_config is consulted for parallel_config.tensor_parallel_size, and by
|
||||
# `_prefill_backoff` on both the P and D prefill paths.
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.parallel_config.tensor_parallel_size = 1
|
||||
vllm_config.num_prefill_lookahead_tokens = 0
|
||||
sched.vllm_config = vllm_config
|
||||
|
||||
# Push-specific state.
|
||||
|
||||
@@ -650,6 +650,29 @@ class VllmConfig:
|
||||
return self.num_speculative_tokens
|
||||
return 0
|
||||
|
||||
@property
|
||||
def num_prefill_lookahead_tokens(self) -> int:
|
||||
"""Prefill tokens past the computed range that the drafter reads.
|
||||
|
||||
Mid-prefill the drafter consumes tokens the target model has not been
|
||||
scheduled for yet, so every component that has to keep them available
|
||||
must apply this margin: the scheduler, which never ends a chunk within
|
||||
it and shifts encoder scheduling by it, and the KV cache manager, which
|
||||
treats the trailing `this - 1` tokens as re-prefillable rather than
|
||||
finalized. Consumers must read this property rather than re-deriving
|
||||
their own per-method lookahead, so those components cannot drift apart.
|
||||
"""
|
||||
speculative_config = self.speculative_config
|
||||
if speculative_config is None or not speculative_config.use_eagle():
|
||||
return 0
|
||||
if speculative_config.use_multi_module_mtp():
|
||||
# Each MTP module reads one token further ahead than the one before
|
||||
# it, so the chain needs num_speculative_tokens of runway at a
|
||||
# chunked-prefill boundary.
|
||||
return self.num_speculative_tokens
|
||||
# Eagle-family drafters read only the immediate next token.
|
||||
return 1
|
||||
|
||||
@property
|
||||
def uniform_decode_query_len(self) -> int:
|
||||
"""Query length of every request in a uniform decode batch.
|
||||
|
||||
@@ -199,8 +199,8 @@ class NixlBaseConnectorScheduler:
|
||||
def on_new_request(self, request: "Request") -> None:
|
||||
"""Track a request that may need heartbeats."""
|
||||
params = request.kv_transfer_params
|
||||
if params is not None and params.get("do_remote_decode") and self._has_mamba:
|
||||
self._truncate_mamba_request_for_prefill(request)
|
||||
if params is not None and params.get("do_remote_decode"):
|
||||
self._truncate_request_for_prefill(request)
|
||||
|
||||
# NOTE (NickLucche) This excludes request meant for P, ie heartbeats are
|
||||
# effectively disabled for Bidirectional KV transfer.
|
||||
@@ -382,36 +382,56 @@ class NixlBaseConnectorScheduler:
|
||||
(identity, b"", encoded_data[(target_pp_rank, target_tp_rank)], ts)
|
||||
)
|
||||
|
||||
def _prefill_backoff(self) -> int:
|
||||
"""Trailing prompt tokens the prefiller must not compute; the decoder
|
||||
recomputes them locally.
|
||||
|
||||
Mamba needs h(N-1) so the decoder can derive h(N) itself. Multi-module
|
||||
MTP needs to keep its whole lookahead window off of the prefiller, which
|
||||
would otherwise embed the unverified drafts in the MTP layer's KV cache. The
|
||||
decoder would never rebuild them, because the update is sized by the rejection
|
||||
count, which is zero for the first decode.
|
||||
"""
|
||||
return max(
|
||||
1 if self._has_mamba else 0,
|
||||
self.vllm_config.num_prefill_lookahead_tokens - 1,
|
||||
)
|
||||
|
||||
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
|
||||
"""D-side only. Returns N-1 for Mamba models since the decoder
|
||||
always recomputes the last token and must start from h(N-1)."""
|
||||
if self._has_mamba and num_prompt_tokens > 1:
|
||||
return num_prompt_tokens - 1
|
||||
"""D-side only. The number of prompt tokens to load from the prefiller.
|
||||
Stops short of the trailing ``_prefill_backoff()`` tokens that the decoder
|
||||
will recompute locally."""
|
||||
backoff = self._prefill_backoff()
|
||||
if backoff and num_prompt_tokens > backoff:
|
||||
return num_prompt_tokens - backoff
|
||||
return num_prompt_tokens
|
||||
|
||||
def _truncate_mamba_request_for_prefill(self, request: "Request") -> None:
|
||||
"""P-side only: drop the last prompt token so the prefiller computes
|
||||
h(N-1) instead of h(N). The decoder recomputes the last token to
|
||||
derive h(N) correctly.
|
||||
def _truncate_request_for_prefill(self, request: "Request") -> None:
|
||||
"""P-side only: drop the trailing ``_prefill_backoff()`` prompt tokens
|
||||
so the prefiller stops short of what the decoder recomputes locally.
|
||||
For Mamba that is the single token needed to yield h(N-1); for
|
||||
multi-module MTP it is the drafter's whole lookahead window.
|
||||
|
||||
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
|
||||
request is preempted and rescheduled."""
|
||||
backoff = self._prefill_backoff()
|
||||
params = request.kv_transfer_params
|
||||
if (
|
||||
params is not None
|
||||
backoff
|
||||
and params is not None
|
||||
# Guard against repeated truncation after preemption/reschedule.
|
||||
and not params.get("_p_side_truncated")
|
||||
and request.num_prompt_tokens > 1
|
||||
and request.num_prompt_tokens > backoff
|
||||
):
|
||||
if request.prompt_token_ids is not None:
|
||||
request.prompt_token_ids.pop()
|
||||
del request.prompt_token_ids[-backoff:]
|
||||
elif request.prompt_embeds is not None:
|
||||
request.prompt_embeds = request.prompt_embeds[:-1]
|
||||
request.prompt_embeds = request.prompt_embeds[:-backoff]
|
||||
else:
|
||||
return
|
||||
|
||||
request._all_token_ids.pop()
|
||||
request.num_prompt_tokens -= 1
|
||||
del request._all_token_ids[-backoff:]
|
||||
request.num_prompt_tokens -= backoff
|
||||
request.max_tokens = 1
|
||||
params["_p_side_truncated"] = True
|
||||
|
||||
|
||||
@@ -2678,12 +2678,7 @@ def get_kv_cache_configs(
|
||||
|
||||
# When speculating with more than 1 speculative module (e.g. multi-layered MTP)
|
||||
# tag every SlidingWindowSpec with how many extra tokens to retain in the window.
|
||||
extra_retained_tokens = (
|
||||
vllm_config.speculative_config.num_speculative_tokens - 1
|
||||
if vllm_config.speculative_config is not None
|
||||
and vllm_config.speculative_config.use_multi_module_mtp()
|
||||
else 0
|
||||
)
|
||||
extra_retained_tokens = max(0, vllm_config.num_prefill_lookahead_tokens - 1)
|
||||
for layer_name, layer_spec in merged_kv_cache_specs.items():
|
||||
if isinstance(layer_spec, SlidingWindowSpec):
|
||||
merged_kv_cache_specs[layer_name] = replace(
|
||||
|
||||
@@ -262,13 +262,7 @@ class Scheduler(SchedulerInterface):
|
||||
self.use_eagle_block_drop = False
|
||||
self.num_spec_tokens = vllm_config.num_speculative_tokens
|
||||
self.num_lookahead_tokens = vllm_config.num_lookahead_tokens
|
||||
# Positions past the computed tokens that the drafter reads mid-prefill.
|
||||
# Eagle-family drafters read 1 ahead, but multi-module MTP reads
|
||||
# num_spec_tokens ahead at chunked-prefill boundaries. Determines the
|
||||
# encoder scheduling shift, the deferred encoder free, the KV cache
|
||||
# manager's re-prefillable window (this minus 1), and how many tokens to
|
||||
# reserve between a chunk boundary and the prefill end.
|
||||
self.num_prefill_lookahead = 0
|
||||
self.num_prefill_lookahead = vllm_config.num_prefill_lookahead_tokens
|
||||
self.dynamic_sd_lookup: list[int] | None = None
|
||||
if speculative_config is not None:
|
||||
if speculative_config.num_speculative_tokens_per_batch_size:
|
||||
@@ -278,12 +272,6 @@ class Scheduler(SchedulerInterface):
|
||||
vllm_num_speculative_tokens=self.num_spec_tokens,
|
||||
)
|
||||
self.use_eagle = speculative_config.use_eagle()
|
||||
if self.use_eagle:
|
||||
self.num_prefill_lookahead = (
|
||||
self.num_spec_tokens
|
||||
if speculative_config.use_multi_module_mtp()
|
||||
else 1
|
||||
)
|
||||
self.use_eagle_block_drop = speculative_config.use_eagle_block_drop()
|
||||
if self.use_eagle and not self.use_eagle_block_drop:
|
||||
logger.warning(
|
||||
|
||||
@@ -296,15 +296,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.is_pooling_model = self.model_config.runner_type == "pooling"
|
||||
self.pooling_runner: PoolingRunner | None = None
|
||||
|
||||
# Multi-module MTP feeds its modules the next num_speculative_steps prefill
|
||||
# tokens during chunked prefill. Other speculators only read the immediate
|
||||
# next one.
|
||||
num_prefill_lookahead = (
|
||||
self.num_speculative_steps
|
||||
if self.speculative_config is not None
|
||||
and self.speculative_config.use_multi_module_mtp()
|
||||
else 1
|
||||
)
|
||||
num_prefill_lookahead = max(1, vllm_config.num_prefill_lookahead_tokens)
|
||||
|
||||
self.step_timing = StepTimingCollector()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user