[Model Runner V2][Spec Decode] Add KV cache support for multi-layer MTP (#50062)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Giancarlo Delfin
2026-08-13 16:22:29 -07:00
committed by GitHub
co-authored by Claude Fable 5
parent 71b0da7c4e
commit f80b66f548
10 changed files with 196 additions and 32 deletions
@@ -86,7 +86,7 @@ def test_arch_mapping_applies_before_callable_override():
@pytest.mark.cpu_test
def test_inkling_override_exposes_only_first_mtp_depth():
def test_inkling_override_exposes_all_mtp_depths():
text_config = _make_hf_config(
architectures=["InklingForCausalLM"],
model_type="inkling_model",
@@ -107,7 +107,9 @@ def test_inkling_override_exposes_only_first_mtp_depth():
assert out is text_config
assert out.model_type == "inkling_mtp"
assert out.architectures == ["InklingMTPModel"]
assert out.n_predict == 1
# Multi-module MTP: every checkpoint depth is exposed (module i drafts
# speculative token i), no longer clamped to the first depth.
assert out.n_predict == 8
assert out.num_nextn_predict_layers == 8
assert out.chain_hidden_post_norm is False
assert out.local_layer_ids == [0, 2, 4]
+3 -1
View File
@@ -3701,6 +3701,7 @@ def test_mamba_align_eagle_schedules_encoder_at_boundary():
)
scheduler.need_mamba_block_aligned_split = True
scheduler.use_eagle = True
scheduler.num_prefill_lookahead = 1
scheduler.max_num_encoder_input_tokens = 2048
scheduler.encoder_cache_manager = EncoderCacheManager(cache_size=2048)
@@ -5341,8 +5342,9 @@ def test_free_encoder_inputs_defers_for_eagle_lookahead():
worker-side token-embedding fallback is only a backstop."""
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
# create_scheduler only builds ngram spec configs; force the eagle path that
# _free_encoder_inputs keys off (self.use_eagle).
# _free_encoder_inputs keys off (its read-ahead deferral).
scheduler.use_eagle = True
scheduler.num_prefill_lookahead = 1
mm_positions = [[PlaceholderRange(offset=50, length=100)]]
request = create_requests(
num_requests=1,