[Model Runner V2][Spec Decode] Add KV cache support for multi-layer MTP (#50062)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
71b0da7c4e
commit
f80b66f548
@@ -86,7 +86,7 @@ def test_arch_mapping_applies_before_callable_override():
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_inkling_override_exposes_only_first_mtp_depth():
|
||||
def test_inkling_override_exposes_all_mtp_depths():
|
||||
text_config = _make_hf_config(
|
||||
architectures=["InklingForCausalLM"],
|
||||
model_type="inkling_model",
|
||||
@@ -107,7 +107,9 @@ def test_inkling_override_exposes_only_first_mtp_depth():
|
||||
assert out is text_config
|
||||
assert out.model_type == "inkling_mtp"
|
||||
assert out.architectures == ["InklingMTPModel"]
|
||||
assert out.n_predict == 1
|
||||
# Multi-module MTP: every checkpoint depth is exposed (module i drafts
|
||||
# speculative token i), no longer clamped to the first depth.
|
||||
assert out.n_predict == 8
|
||||
assert out.num_nextn_predict_layers == 8
|
||||
assert out.chain_hidden_post_norm is False
|
||||
assert out.local_layer_ids == [0, 2, 4]
|
||||
|
||||
@@ -3701,6 +3701,7 @@ def test_mamba_align_eagle_schedules_encoder_at_boundary():
|
||||
)
|
||||
scheduler.need_mamba_block_aligned_split = True
|
||||
scheduler.use_eagle = True
|
||||
scheduler.num_prefill_lookahead = 1
|
||||
scheduler.max_num_encoder_input_tokens = 2048
|
||||
scheduler.encoder_cache_manager = EncoderCacheManager(cache_size=2048)
|
||||
|
||||
@@ -5341,8 +5342,9 @@ def test_free_encoder_inputs_defers_for_eagle_lookahead():
|
||||
worker-side token-embedding fallback is only a backstop."""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
# create_scheduler only builds ngram spec configs; force the eagle path that
|
||||
# _free_encoder_inputs keys off (self.use_eagle).
|
||||
# _free_encoder_inputs keys off (its read-ahead deferral).
|
||||
scheduler.use_eagle = True
|
||||
scheduler.num_prefill_lookahead = 1
|
||||
mm_positions = [[PlaceholderRange(offset=50, length=100)]]
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
|
||||
Reference in New Issue
Block a user