[Model Runner V2][Spec Decode] Add KV cache support for multi-layer MTP (#50062)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Giancarlo Delfin
2026-08-13 16:22:29 -07:00
committed by GitHub
co-authored by Claude Fable 5
parent 71b0da7c4e
commit f80b66f548
10 changed files with 196 additions and 32 deletions
@@ -86,7 +86,7 @@ def test_arch_mapping_applies_before_callable_override():
@pytest.mark.cpu_test
def test_inkling_override_exposes_only_first_mtp_depth():
def test_inkling_override_exposes_all_mtp_depths():
text_config = _make_hf_config(
architectures=["InklingForCausalLM"],
model_type="inkling_model",
@@ -107,7 +107,9 @@ def test_inkling_override_exposes_only_first_mtp_depth():
assert out is text_config
assert out.model_type == "inkling_mtp"
assert out.architectures == ["InklingMTPModel"]
assert out.n_predict == 1
# Multi-module MTP: every checkpoint depth is exposed (module i drafts
# speculative token i), no longer clamped to the first depth.
assert out.n_predict == 8
assert out.num_nextn_predict_layers == 8
assert out.chain_hidden_post_norm is False
assert out.local_layer_ids == [0, 2, 4]
+3 -1
View File
@@ -3701,6 +3701,7 @@ def test_mamba_align_eagle_schedules_encoder_at_boundary():
)
scheduler.need_mamba_block_aligned_split = True
scheduler.use_eagle = True
scheduler.num_prefill_lookahead = 1
scheduler.max_num_encoder_input_tokens = 2048
scheduler.encoder_cache_manager = EncoderCacheManager(cache_size=2048)
@@ -5341,8 +5342,9 @@ def test_free_encoder_inputs_defers_for_eagle_lookahead():
worker-side token-embedding fallback is only a backstop."""
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
# create_scheduler only builds ngram spec configs; force the eagle path that
# _free_encoder_inputs keys off (self.use_eagle).
# _free_encoder_inputs keys off (its read-ahead deferral).
scheduler.use_eagle = True
scheduler.num_prefill_lookahead = 1
mm_positions = [[PlaceholderRange(offset=50, length=100)]]
request = create_requests(
num_requests=1,
+1 -10
View File
@@ -630,8 +630,7 @@ class SpeculativeConfig:
hf_config.model_type = "inkling_mtp"
hf_config.update(
{
# Inkling currently exposes only the first checkpoint depth.
"n_predict": 1,
"n_predict": checkpoint_depths,
"num_nextn_predict_layers": checkpoint_depths,
"chain_hidden_post_norm": mtp_config.get(
"chain_hidden_post_norm", False
@@ -1083,14 +1082,6 @@ class SpeculativeConfig:
"`num_speculative_tokens` was not provided"
)
if (
self.draft_model_config.hf_config.model_type == "inkling_mtp"
and self.num_speculative_tokens != 1
):
raise ValueError(
"Inkling MTP currently supports exactly one speculative token"
)
if self.dspark_draft_topk is not None and self.method != "dspark":
raise ValueError("dspark_draft_topk is only supported by DSpark")
+53 -3
View File
@@ -80,6 +80,7 @@ class KVCacheCoordinator(ABC):
scheduler_block_size: int,
hash_block_size: int,
metrics_collector: KVCacheMetricsCollector | None = None,
num_prefill_lookahead: int = 0,
):
self.kv_cache_config = kv_cache_config
self.max_model_len = max_model_len
@@ -91,6 +92,7 @@ class KVCacheCoordinator(ABC):
for g in kv_cache_config.kv_cache_groups
)
self.scheduler_block_size = scheduler_block_size
self.num_reprefillable_tokens = max(0, num_prefill_lookahead - 1)
self.block_pool = BlockPool(
num_gpu_blocks=kv_cache_config.num_blocks,
@@ -108,6 +110,28 @@ class KVCacheCoordinator(ABC):
if use_eagle and not self.eagle_group_ids:
self.eagle_group_ids = set(range(len(kv_cache_config.kv_cache_groups)))
# During chunked prefill with EAGLE, the single next prefill lookahead
# token past the chunk boundary is combined with the final hidden state
# and written to the KV cache. Therefore, the final chunk token must be
# excluded from prefix cache hits to prevent requests from acquiring the
# KV cache slot polluted with the next prefill token, which may or may not
# be present after the matching prefix. The last-block drop handles this
# edge case. During multi-module MTP, the issue generalizes to a prefill
# lookahead of num_speculative_tokens, so the dropped tail must be large
# enough to contain them. Hits land on scheduler-block boundaries (see
# `_cache_hit_alignment_tokens`), so the excluded tail is
# scheduler_block_size, not the group's own block size.
if (
enable_caching
and self.eagle_group_ids
and scheduler_block_size < num_prefill_lookahead
):
raise ValueError(
f"Multi-module MTP with prefix caching requires scheduler_block_size"
f" (={scheduler_block_size}) >= num_speculative_tokens"
f" (={num_prefill_lookahead})."
)
self.single_type_managers = tuple(
get_manager_for_kv_cache_spec(
kv_cache_spec=kv_cache_group.kv_cache_spec,
@@ -286,9 +310,14 @@ class KVCacheCoordinator(ABC):
(including tokens that are already cached).
"""
for manager in self.single_type_managers:
# Only cache tokens with finalized KV. The last num_reprefillable_tokens
# tokens can be re-prefilled during multi-module MTP.
num_tokens_to_cache = max(
0, num_computed_tokens - self.num_reprefillable_tokens
)
manager.cache_blocks(
request,
num_computed_tokens,
num_tokens_to_cache,
retention_interval=self.retention_interval,
)
@@ -407,6 +436,7 @@ class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator):
scheduler_block_size: int,
hash_block_size: int,
metrics_collector: KVCacheMetricsCollector | None = None,
num_prefill_lookahead: int = 0,
):
super().__init__(
kv_cache_config,
@@ -420,6 +450,7 @@ class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator):
scheduler_block_size=scheduler_block_size,
hash_block_size=hash_block_size,
metrics_collector=metrics_collector,
num_prefill_lookahead=num_prefill_lookahead,
)
self.num_single_type_manager = len(self.single_type_managers)
@@ -457,6 +488,7 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator):
scheduler_block_size: int,
hash_block_size: int,
metrics_collector: KVCacheMetricsCollector | None = None,
num_prefill_lookahead: int = 0,
):
super().__init__(
kv_cache_config,
@@ -470,6 +502,7 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator):
scheduler_block_size=scheduler_block_size,
hash_block_size=hash_block_size,
metrics_collector=metrics_collector,
num_prefill_lookahead=num_prefill_lookahead,
)
self.kv_cache_spec = self.kv_cache_config.kv_cache_groups[0].kv_cache_spec
self.block_size = self.kv_cache_spec.block_size
@@ -542,6 +575,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
scheduler_block_size: int,
hash_block_size: int,
metrics_collector: KVCacheMetricsCollector | None = None,
num_prefill_lookahead: int = 0,
):
super().__init__(
kv_cache_config,
@@ -555,6 +589,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
scheduler_block_size=scheduler_block_size,
hash_block_size=hash_block_size,
metrics_collector=metrics_collector,
num_prefill_lookahead=num_prefill_lookahead,
)
# hash_block_size: the block size used to compute block hashes.
# The actual block size usually equals hash_block_size, but in cases where
@@ -688,9 +723,20 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
# EAGLE groups match one block past each aligned boundary and drop
# it, so make that lookahead block eligible to be cached.
if manager.use_eagle and aligned_num_computed_tokens > 0:
# Only cache tokens with finalized KV. The last
# num_reprefillable_tokens tokens can be re-prefilled during
# multi-module MTP.
num_finalized_computed_tokens = max(
0, num_computed_tokens - self.num_reprefillable_tokens
)
aligned_num_finalized_computed_tokens = (
num_finalized_computed_tokens
// self.scheduler_block_size
* self.scheduler_block_size
)
num_tokens_to_cache = min(
num_computed_tokens,
aligned_num_computed_tokens + manager.block_size,
num_finalized_computed_tokens,
aligned_num_finalized_computed_tokens + manager.block_size,
)
# The manager already knows the fine hit granularity
# (``scheduler_block_size``); retention is passed separately so it
@@ -880,6 +926,7 @@ def get_kv_cache_coordinator(
scheduler_block_size: int,
hash_block_size: int,
metrics_collector: KVCacheMetricsCollector | None = None,
num_prefill_lookahead: int = 0,
) -> KVCacheCoordinator:
if not enable_caching:
return KVCacheCoordinatorNoPrefixCache(
@@ -893,6 +940,7 @@ def get_kv_cache_coordinator(
scheduler_block_size=scheduler_block_size,
hash_block_size=hash_block_size,
metrics_collector=metrics_collector,
num_prefill_lookahead=num_prefill_lookahead,
)
if len(kv_cache_config.kv_cache_groups) == 1:
return UnitaryKVCacheCoordinator(
@@ -907,6 +955,7 @@ def get_kv_cache_coordinator(
scheduler_block_size=scheduler_block_size,
hash_block_size=hash_block_size,
metrics_collector=metrics_collector,
num_prefill_lookahead=num_prefill_lookahead,
)
return HybridKVCacheCoordinator(
kv_cache_config,
@@ -920,4 +969,5 @@ def get_kv_cache_coordinator(
scheduler_block_size=scheduler_block_size,
hash_block_size=hash_block_size,
metrics_collector=metrics_collector,
num_prefill_lookahead=num_prefill_lookahead,
)
+2
View File
@@ -125,6 +125,7 @@ class KVCacheManager:
max_in_flight_tokens: int | None = None,
enable_caching: bool = True,
use_eagle: bool = False,
num_prefill_lookahead: int = 0,
log_stats: bool = False,
enable_kv_cache_events: bool = False,
dcp_world_size: int = 1,
@@ -161,6 +162,7 @@ class KVCacheManager:
scheduler_block_size=scheduler_block_size,
hash_block_size=hash_block_size,
metrics_collector=self.metrics_collector,
num_prefill_lookahead=num_prefill_lookahead,
)
self.num_kv_cache_groups = len(kv_cache_config.kv_cache_groups)
self.block_pool = self.coordinator.block_pool
+18
View File
@@ -1468,6 +1468,9 @@ def _promote_local_kv_cache_specs(
promoted_specs[layer_name] = replace_as(
spec,
target_cls,
# Promoted specs allocate blocks for all tokens and never free
# below the window, so the trailing-edge extension is moot.
drop=("extra_retained_tokens",),
block_size=block_size,
page_size_padded=promoted_page_size_padded(spec, block_size),
)
@@ -2128,6 +2131,21 @@ def get_kv_cache_configs(
# Check if the KV cache specs are registered correctly.
# This is to prevent that some layers are initialized with unregistered specs.
KVCacheSpecRegistry.check_kv_cache_spec_registry(merged_kv_cache_specs)
# When speculating with more than 1 speculative module (e.g. multi-layered MTP)
# tag every SlidingWindowSpec with how many extra tokens to retain in the window.
extra_retained_tokens = (
vllm_config.speculative_config.num_speculative_tokens - 1
if vllm_config.speculative_config is not None
and vllm_config.speculative_config.use_multi_module_mtp()
else 0
)
for layer_name, layer_spec in merged_kv_cache_specs.items():
if isinstance(layer_spec, SlidingWindowSpec):
merged_kv_cache_specs[layer_name] = replace(
layer_spec, extra_retained_tokens=extra_retained_tokens
)
# Get global KV cache groups. This also handles spec unification for
# hybrid models when disable_hybrid_kv_cache_manager is enabled.
# After this call, merged_kv_cache_specs may be modified in-place.
+58 -8
View File
@@ -247,6 +247,13 @@ class Scheduler(SchedulerInterface):
self.use_eagle = False
self.num_spec_tokens = vllm_config.num_speculative_tokens
self.num_lookahead_tokens = vllm_config.num_lookahead_tokens
# Positions past the computed tokens that the drafter reads mid-prefill.
# Eagle-family drafters read 1 ahead, but multi-module MTP reads
# num_spec_tokens ahead at chunked-prefill boundaries. Determines the
# encoder scheduling shift, the deferred encoder free, the KV cache
# manager's re-prefillable window (this minus 1), and how many tokens to
# reserve between a chunk boundary and the prefill end.
self.num_prefill_lookahead = 0
self.dynamic_sd_lookup: list[int] | None = None
if speculative_config is not None:
if speculative_config.num_speculative_tokens_per_batch_size:
@@ -256,6 +263,12 @@ class Scheduler(SchedulerInterface):
vllm_num_speculative_tokens=self.num_spec_tokens,
)
self.use_eagle = speculative_config.use_eagle()
if self.use_eagle:
self.num_prefill_lookahead = (
self.num_spec_tokens
if speculative_config.use_multi_module_mtp()
else 1
)
# Create the KV cache manager.
if hash_block_size is None:
@@ -267,6 +280,7 @@ class Scheduler(SchedulerInterface):
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
enable_caching=self.cache_config.enable_prefix_caching,
use_eagle=self.use_eagle,
num_prefill_lookahead=self.num_prefill_lookahead,
log_stats=self.log_stats,
enable_kv_cache_events=self.enable_kv_cache_events,
dcp_world_size=self.dcp_world_size,
@@ -438,6 +452,27 @@ class Scheduler(SchedulerInterface):
)
return blocks, num_local, shared_prefix_boundary, False
def _reserve_prefill_lookahead(
self,
request: Request,
num_computed_tokens: int,
num_new_tokens: int,
) -> int:
"""Never end a prefill chunk within num_prefill_lookahead of the
prefill end.
At a chunked-prefill boundary, the multi-module MTP drafter consumes
the next num_prefill_lookahead known prefill tokens as draft inputs. A
boundary closer to the end than that would make it fall back to
sampled drafts, permanently polluting the trailing modules' KV caches.
Either finish the prefill or leave at least num_prefill_lookahead for
the next chunk. No-op for eagle-family drafters (lookahead 1).
"""
remaining = request.num_tokens - num_computed_tokens - num_new_tokens
if 0 < remaining < self.num_prefill_lookahead:
num_new_tokens -= self.num_prefill_lookahead - remaining
return max(num_new_tokens, 0)
def schedule(self, throttle_prefills: bool = False) -> SchedulerOutput:
self.current_step += 1
# NOTE(woosuk) on the scheduling algorithm:
@@ -561,9 +596,15 @@ class Scheduler(SchedulerInterface):
request.num_computed_tokens,
num_new_tokens,
encoder_compute_budget,
shift_computed_tokens=1 if self.use_eagle else 0,
shift_computed_tokens=self.num_prefill_lookahead,
)
# Multi-module MTP: avoid ending a prefill chunk within
# num_prefill_lookahead of the prefill end.
num_new_tokens = self._reserve_prefill_lookahead(
request, request.num_computed_tokens, num_new_tokens
)
if num_new_tokens == 0:
# The request cannot be scheduled because one of the following
# reasons:
@@ -576,6 +617,8 @@ class Scheduler(SchedulerInterface):
# 3. The encoder cache is exhausted.
# 4. Insufficient budget for a block-aligned chunk in hybrid
# models with mamba cache mode \"align\".
# 5. Insufficient budget to keep a multi-module MTP prefill
# chunk out of the prefill-lookahead window.
# NOTE(woosuk): Here, by doing `continue` instead of `break`,
# we do not strictly follow the FCFS scheduling policy and
# allow the lower-priority requests to be scheduled.
@@ -946,11 +989,18 @@ class Scheduler(SchedulerInterface):
num_computed_tokens,
num_new_tokens,
encoder_compute_budget,
shift_computed_tokens=1 if self.use_eagle else 0,
shift_computed_tokens=self.num_prefill_lookahead,
)
if num_new_tokens == 0:
# The request cannot be scheduled.
break
# Multi-module MTP: avoid ending a prefill chunk within
# num_prefill_lookahead of the prefill end.
num_new_tokens = self._reserve_prefill_lookahead(
request, num_computed_tokens, num_new_tokens
)
if num_new_tokens == 0:
# The request cannot be scheduled.
break
# During async KV load, no forward pass is run yet.
# Allocate speculative lookahead slots later to avoid
@@ -2157,9 +2207,9 @@ class Scheduler(SchedulerInterface):
return
# Defer the free by the drafter's look-ahead so an entry stays
# referenced until the drafter's +1 read has also passed it, mirroring
# the shift the encoder scheduling path applies.
spec_lookahead = 1 if self.use_eagle else 0
# referenced until the drafter's read-ahead has also passed it,
# mirroring the shift the encoder scheduling path applies.
spec_lookahead = self.num_prefill_lookahead
# Here, we use list(set) to avoid modifying the set while iterating
# over it.
+19 -1
View File
@@ -561,6 +561,11 @@ class SingleTypeKVCacheManager(ABC):
return an empty list.
If eagle is enabled, drop the last matched block to force recompute the
last block to get the required hidden states for eagle drafting head.
For multi-module MTP, this recompute also rewrites the dropped block's
draft-layer KVs, which depend on up to num_speculative_tokens - 1
tokens past the matched prefix (i.e. on the cache writer's
continuation, which the block hash does not cover); the coordinator
asserts the block size covers that window.
Need to be customized for each attention type.
Args:
@@ -877,6 +882,10 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
def __init__(self, kv_cache_spec: SlidingWindowSpec, **kwargs) -> None:
super().__init__(kv_cache_spec, **kwargs)
self.sliding_window = kv_cache_spec.sliding_window
# Extra trailing tokens to retain below the window (never attended) so a
# multi-module MTP store-side lag can still reconstruct the window from
# cached blocks.
self.extra_retained_tokens = kv_cache_spec.extra_retained_tokens
@classmethod
def _contiguous_blocks_for_hit(
@@ -1072,13 +1081,22 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
attention computation since they are outside the sliding window.
Thus, get_num_skipped_tokens(7) == 4.
The trailing edge of the window is extended by ``extra_retained_tokens``
so that those extra trailing tokens' blocks are retained (but not
attended). This is needed for multi-module spec decoding which can
re-prefill the last num_spec_prefill_tokens - 1 tokens from the end
of the sequence, and thus needs to delay freeing/caching of blocks.
Args:
num_computed_tokens: The number of tokens that have been computed.
Returns:
The number of tokens that will be skipped for attention computation.
"""
return max(0, num_computed_tokens - self.sliding_window + 1)
return max(
0,
num_computed_tokens - self.sliding_window + 1 - self.extra_retained_tokens,
)
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
"""
+32 -7
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import copy
from collections import Counter
from collections.abc import Collection
from dataclasses import dataclass, fields, replace
from enum import Enum, IntEnum
from math import prod
@@ -88,14 +89,24 @@ def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
return get_kv_quant_mode(kv_cache_dtype) != KVQuantMode.NONE
def replace_as(spec: KVCacheSpec, target_cls: type[_SpecT], **changes) -> _SpecT:
def replace_as(
spec: KVCacheSpec,
target_cls: type[_SpecT],
*,
drop: Collection[str] = (),
**changes,
) -> _SpecT:
"""``dataclasses.replace``, but rebuilding *spec* as *target_cls*
e.g. ``SlidingWindowSpec`` -> ``FullAttentionSpec``
Every field of *spec* must exist on *target_cls*; fields only *target_cls* has keep
their default values.
Every field of *spec* must exist on *target_cls* unless named in *drop*;
fields only *target_cls* has keep their default values.
"""
kwargs = {f.name: getattr(spec, f.name) for f in fields(spec) if f.init}
kwargs = {
f.name: getattr(spec, f.name)
for f in fields(spec)
if f.init and f.name not in drop
}
kwargs.update(changes)
return target_cls(**kwargs)
@@ -573,6 +584,12 @@ class ChunkedLocalAttentionSpec(AttentionSpec):
class SlidingWindowSpec(AttentionSpec):
sliding_window: int
head_size_v: int = None # type: ignore[assignment]
# The trailing edge of the window is extended by ``extra_retained_tokens``
# so that those extra trailing tokens' blocks are retained (but not
# attended). This is needed for multi-module spec decoding which can
# re-prefill the last num_spec_prefill_tokens - 1 tokens from the end
# of the sequence, and thus needs to delay freeing/caching of blocks.
extra_retained_tokens: int = 0
def __post_init__(self):
if self.head_size_v is None:
@@ -614,8 +631,13 @@ class SlidingWindowSpec(AttentionSpec):
"""
# During chunked prefill, we hold KV for the last `sliding_window-1`
# computed tokens plus the in-flight tokens (frees happen on the
# processed-token basis); never more than `max_model_len`.
num_tokens = min(self.sliding_window - 1 + max_in_flight_tokens, max_model_len)
# processed-token basis); never more than `max_model_len`. An additional
# `extra_retained_tokens` trailing tokens are kept alive below the
# window for multi-module spec decoding, and must be accounted here too.
num_tokens = min(
self.sliding_window - 1 + self.extra_retained_tokens + max_in_flight_tokens,
max_model_len,
)
# +1 because the sliding window may not start from the beginning of
# the block. E.g. block size 4 and num_token 4 needs two blocks
# [XXCD][EF] to store the 6-token window [CDEF].
@@ -686,16 +708,18 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
model_version_set = set(spec.model_version for spec in specs)
sliding_window_set = set(spec.sliding_window for spec in specs)
block_stride_set = set(spec.indexes_kv_by_block_stride for spec in specs)
extra_retained_set = set(spec.extra_retained_tokens for spec in specs)
assert (
len(cache_dtype_str_set) == 1
and len(compress_ratio_set) == 1
and len(model_version_set) == 1
and len(sliding_window_set) == 1
and len(block_stride_set) == 1
and len(extra_retained_set) == 1
), (
"All attention layers in the same KV cache group must use the same "
"quantization method, compress ratio, model version, sliding "
"window size, and KV block stride indexing."
"window size, KV block stride indexing, and retained token count."
)
return cls(
block_size=specs[0].block_size,
@@ -705,6 +729,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
page_size_padded=specs[0].page_size_padded,
indexes_kv_by_block_stride=block_stride_set.pop(),
sliding_window=sliding_window_set.pop(),
extra_retained_tokens=extra_retained_set.pop(),
cache_dtype_str=cache_dtype_str_set.pop(),
compress_ratio=compress_ratio_set.pop(),
model_version=model_version_set.pop(),
@@ -26,6 +26,12 @@ def init_speculator(vllm_config: VllmConfig, device: torch.device):
)
return Gemma4Speculator(vllm_config, device)
elif speculative_config.use_multi_module_mtp():
from vllm.v1.worker.gpu.spec_decode.multi_module_mtp.speculator import (
MultiModuleMTPSpeculator,
)
return MultiModuleMTPSpeculator(vllm_config, device)
elif speculative_config.method == "mtp":
from vllm.v1.worker.gpu.spec_decode.mtp.speculator import MTPSpeculator