diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 368133720e..1e6255a4c2 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -4,6 +4,7 @@ import copy import hashlib import importlib from collections.abc import Callable +from dataclasses import replace from types import SimpleNamespace from typing import Any @@ -2214,13 +2215,14 @@ def new_mla_spec(cache_dtype_str=None, block_size=16): ) -def new_swa_mla_spec(head_size=576, sliding_window=128): +def new_swa_mla_spec(head_size=576, sliding_window=128, model_version=None): return SlidingWindowMLASpec( block_size=16, num_kv_heads=1, head_size=head_size, dtype=torch.float32, sliding_window=sliding_window, + model_version=model_version, ) @@ -3277,3 +3279,143 @@ def test_iter_layer_specs_returns_group_members(): block_size=4, kv_cache_specs={"a": full, "b": mla} ) assert list(iter_layer_specs(wrapped)) == [full, mla] + + +def _spec_decode_grouping_config(method="dspark"): + """Grouping config with an EAGLE-family speculative method enabled.""" + return SimpleNamespace( + scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False), + speculative_config=SimpleNamespace( + method=method, + use_eagle=lambda: True, + ), + ) + + +def _hybrid_specs_with_draft(draft: bool, draft_shares_target_spec: bool = False): + """A K3-shaped hybrid: MLA full attention + Mamba, optionally plus a + DSpark-style draft MLA layer marked non_causal_multi_token_decode. + + The target's fp8 KV dtype is what keeps the draft in its own bucket, as it + does on Kimi-K3 (target `--kv-cache-dtype fp8_e4m3`, draft `auto`). Pass + draft_shares_target_spec to collapse them into one group instead. + """ + target_dtype = None if draft_shares_target_spec else "fp8_e4m3" + specs = { + "target.attn.0": new_mla_spec(block_size=64, cache_dtype_str=target_dtype), + "target.attn.1": new_mla_spec(block_size=64, cache_dtype_str=target_dtype), + "target.mamba.0": new_mamba_spec(block_size=64, mamba_cache_mode="align"), + "target.mamba.1": new_mamba_spec(block_size=64, mamba_cache_mode="align"), + } + if draft: + draft_spec = new_mla_spec(block_size=64) + specs["draft.attn.0"] = replace(draft_spec, non_causal_multi_token_decode=True) + return specs + + +def test_draft_group_annotated_on_hybrid_general_path(): + # A drafter's MLA layer carries non_causal_multi_token_decode, so its group + # is identifiable without keying off a model version. + groups = get_kv_cache_groups( + _spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True) + ) + + flagged = [g for g in groups if g.is_eagle_group] + assert len(flagged) == 1 + assert flagged[0].layer_names == ["draft.attn.0"] + + +def test_mamba_groups_never_flagged_even_when_draft_shares_a_group(): + # MLAAttentionSpec.merge ORs non_causal_multi_token_decode rather than + # requiring equality, so a draft layer can share a group with target + # layers; that combined group still holds volatile draft KV and must be + # flagged. What must never happen is a Mamba group being flagged: that + # widens its lookup window to two consecutive chunks, which align-mode + # checkpointing never produces, zeroing every lookup. + groups = get_kv_cache_groups( + _spec_decode_grouping_config(), + _hybrid_specs_with_draft(draft=True, draft_shares_target_spec=True), + ) + + for group in groups: + if "draft.attn.0" in group.layer_names: + assert group.is_eagle_group + if isinstance(group.kv_cache_spec, MambaSpec): + assert not group.is_eagle_group + + +def test_draft_group_not_annotated_without_spec_decode(): + # The marker alone must not flag anything; the eagle semantics only apply + # when a speculative method is actually enabled. + config = SimpleNamespace( + scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False), + speculative_config=None, + ) + groups = get_kv_cache_groups(config, _hybrid_specs_with_draft(draft=True)) + + assert not any(g.is_eagle_group for g in groups) + + +def test_unidentifiable_draft_with_mamba_warns(caplog_vllm): + # No group carries the draft marker, so every consumer falls back to + # flagging all groups -- including Mamba ones, which then can never report + # a hit. That is silent today; it must at least be visible. + groups = get_kv_cache_groups( + _spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=False) + ) + + assert not any(g.is_eagle_group for g in groups) + assert "no KV cache group could be identified as the draft model's" in ( + caplog_vllm.text + ) + assert "Mamba groups" in caplog_vllm.text + + +def test_no_warning_when_draft_group_is_identified(caplog_vllm): + get_kv_cache_groups( + _spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True) + ) + + assert "could be identified as the draft model's" not in caplog_vllm.text + + +def _deepseek_v4_specs(model_version="deepseek_v4"): + """DeepseekV4-shaped specs: full MLA layers plus sliding-window MLA layers + at differing page sizes, with the MTP draft layer registered last.""" + return { + "model.layers.0.self_attn.attn": new_mla_spec(), + "model.layers.1.self_attn.attn": new_mla_spec(), + "model.layers.2.self_attn.attn": new_swa_mla_spec( + head_size=1024, model_version=model_version + ), + # The MTP block registers last, and its sliding-window size differs, so + # it lands in a group of its own. + "model.layers.3.self_attn.attn": new_swa_mla_spec( + head_size=1024, sliding_window=256, model_version=model_version + ), + } + + +def test_deepseek_v4_draft_group_annotated_on_group_and_unify_path(): + # DeepseekV4's MTP block reuses the target's decoder layer, so its spec + # carries no draft marker and only the positional rule can find it. This + # pins the pre-existing behaviour that the unified annotator must preserve. + groups = get_kv_cache_groups( + _spec_decode_grouping_config(method="mtp"), _deepseek_v4_specs() + ) + + flagged = [g for g in groups if g.is_eagle_group] + assert len(flagged) == 1 + assert "model.layers.3.self_attn.attn" in flagged[0].layer_names + + +def test_deepseek_v4_annotation_requires_model_version(): + # The positional rule is only sound for DeepseekV4, where the draft layer + # is known to be registered last. Without that model gate nothing may be + # flagged, however the grouping happens to fall out. + groups = get_kv_cache_groups( + _spec_decode_grouping_config(method="mtp"), + _deepseek_v4_specs(model_version=None), + ) + + assert not any(g.is_eagle_group for g in groups) diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 9b77fbd20a..d578b4300a 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -1742,23 +1742,56 @@ def _get_kv_cache_groups_uniform_groups( return [full_mla_group, *swa_mla_groups] -def _annotate_eagle_groups_deepseek_v4( +def _annotate_eagle_groups( vllm_config: VllmConfig, kv_cache_spec: dict[str, KVCacheSpec], kv_cache_groups: list[KVCacheGroupSpec], + use_deepseek_v4_fallback: bool = False, ) -> None: + """Flag the KV cache groups that hold drafter attention layers. + + Two detection rules, in order of preference: + + 1. Spec-driven. ``non_causal_multi_token_decode`` is declared on + MLAAttentionSpec and set by drafter attention layers that run a + non-causal multi-token decode (today only Kimi-K3 DSpark). It survives + MLAAttentionSpec.merge, so it still identifies a group after per-group + spec merging, wherever grouping happens to land. It is sufficient but + not necessary: a drafter whose spec is indistinguishable from the + target's cannot be found this way. + 2. Model-scoped positional fallback for DeepseekV4, whose MTP block reuses + the target's own decoder layer and so carries no spec marker. Its draft + attention layer is always the last registered layer, so flag whichever + group holds it. This rule is only valid where the groups partition + exactly the layers of ``kv_cache_spec``, which is true on the + group_and_unify path and not in general; other callers must leave + ``use_deepseek_v4_fallback`` False. + FIXME(yifan): avoid/generalize this hacky check. + + Args: + vllm_config: Config supplying the speculative method, if any. + kv_cache_spec: The kv cache spec of each attention layer, in layer + registration order. Only read by rule 2. + kv_cache_groups: Groups to annotate in place. + use_deepseek_v4_fallback: Enable rule 2. Only the group_and_unify path + may set this. + """ spec_config = vllm_config.speculative_config if spec_config is None or not spec_config.use_eagle(): return + + for group in kv_cache_groups: + if getattr(group.kv_cache_spec, "non_causal_multi_token_decode", False): + group.is_eagle_group = True + + if not use_deepseek_v4_fallback: + return # Detection uses the merged MLA spec's model_version. if not any( getattr(spec, "model_version", None) == "deepseek_v4" for spec in kv_cache_spec.values() ): return - # DeepseekV4's MTP attention layer is always the last layer, and we flag whichever - # group contains it. - # FIXME(yifan): avoid/generalize this hacky check. last_layer = next(reversed(kv_cache_spec)) for group in kv_cache_groups: if last_layer in group.layer_names: @@ -1766,6 +1799,45 @@ def _annotate_eagle_groups_deepseek_v4( break +def _warn_if_unannotated_eagle_mamba( + vllm_config: VllmConfig, + kv_cache_groups: list[KVCacheGroupSpec], +) -> None: + """Warn when the flag-all eagle fallback will silently disable reuse. + + With no group annotated, consumers flag every group as a draft group. That + widens a Mamba group's required lookup window to two consecutive chunks, + which align-mode checkpointing never produces, so reuse drops to zero with + no error and no metric to show it. + + Args: + vllm_config: Config supplying the speculative method, if any. + kv_cache_groups: Groups as they will be handed to consumers. + """ + spec_config = vllm_config.speculative_config + if spec_config is None or not spec_config.use_eagle(): + return + if any(group.is_eagle_group for group in kv_cache_groups): + return + mamba_groups = [ + idx + for idx, group in enumerate(kv_cache_groups) + if isinstance(group.kv_cache_spec, MambaSpec) + ] + if not mamba_groups: + return + logger.warning( + "Speculative decoding (method=%s) is enabled but no KV cache group " + "could be identified as the draft model's, so every group -- " + "including Mamba groups %s -- will be treated as a draft group. A " + "Mamba group cannot satisfy the widened lookup window that implies, " + "so prefix-cache reuse across requests will be disabled and any " + "external KV offload tier will store without ever serving a hit.", + spec_config.method, + mamba_groups, + ) + + def _largest_divisor_at_most(value: int, limit: int) -> int: for candidate in range(min(value, limit), 0, -1): if value % candidate == 0: @@ -1810,7 +1882,12 @@ def get_kv_cache_groups( # attention in different sizes. Need to group layers into multiple # UniformTypeKVCacheSpecs. kv_cache_groups = _get_kv_cache_groups_uniform_groups(grouped_specs) - _annotate_eagle_groups_deepseek_v4(vllm_config, kv_cache_spec, kv_cache_groups) + _annotate_eagle_groups( + vllm_config, + kv_cache_spec, + kv_cache_groups, + use_deepseek_v4_fallback=True, + ) return kv_cache_groups # Pull HiddenStateCacheSpec layers out before the general multi-group @@ -1855,6 +1932,8 @@ def get_kv_cache_groups( aligned = replace(spec, block_size=new_bs, page_size_padded=common_page) groups.append(KVCacheGroupSpec([name], aligned)) + _annotate_eagle_groups(vllm_config, kv_cache_spec, groups) + _warn_if_unannotated_eagle_mamba(vllm_config, groups) return groups