[Bugfix] Annotate MTP draft KV cache groups positionally on the hybrid grouping path (#55390)

Signed-off-by: Navjot Singh <navjot.singh@shopify.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Navjot Singh
2026-09-22 00:31:26 +00:00
committed by GitHub
co-authored by Codex
parent a369a7becc
commit 0bce411a07
2 changed files with 133 additions and 21 deletions
+90 -6
View File
@@ -4275,12 +4275,16 @@ def _deepseek_v4_specs(model_version="deepseek_v4"):
} }
def test_deepseek_v4_draft_group_annotated_on_packed_path(): @pytest.mark.parametrize(
("method", "model_type"),
[("mtp", "deepseek_v4"), ("dspark", "deepseek_v4"), ("dspark", "deepseek_v41")],
)
def test_deepseek_v4_draft_group_annotated_on_packed_path(method, model_type):
# DeepseekV4's MTP block reuses the target's decoder layer, so its spec # DeepseekV4's MTP block reuses the target's decoder layer, so its spec
# carries no draft marker and only the positional rule can find it. This # carries no draft marker and only the positional rule can find it. This
# pins the pre-existing behaviour that the unified annotator must preserve. # pins the pre-existing behaviour that the unified annotator must preserve.
groups = get_kv_cache_groups( groups = get_kv_cache_groups(
_spec_decode_grouping_config(method="mtp", model_type="deepseek_v4"), _spec_decode_grouping_config(method=method, model_type=model_type),
_deepseek_v4_specs(model_version=None), _deepseek_v4_specs(model_version=None),
) )
@@ -4289,13 +4293,93 @@ def test_deepseek_v4_draft_group_annotated_on_packed_path():
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
def test_deepseek_v4_annotation_requires_model_type(): def test_trailing_layer_fallback_applies_to_any_mtp_model():
# The positional rule is only sound for DeepseekV4, where the draft layer # The positional rule is sound for every MTP drafter, not just DeepseekV4:
# is known to be registered last. Without that model gate nothing may be # MTP blocks reuse the target's decoder layer (no spec marker) and always
# flagged, however the grouping happens to fall out. # register after every target layer. The model_type must not gate it.
groups = get_kv_cache_groups( groups = get_kv_cache_groups(
_spec_decode_grouping_config(method="mtp", model_type="other"), _spec_decode_grouping_config(method="mtp", model_type="other"),
_deepseek_v4_specs(), _deepseek_v4_specs(),
) )
flagged = [g for g in groups if g.is_eagle_group]
assert len(flagged) == 1
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
def _qwen3_5_hybrid_specs(with_mtp_layer: bool):
"""Qwen3.5-shaped hybrid: repeating [GDN x3, full-attn x1] blocks, with
the MTP drafter's full-attn layer (spec-identical to the target's)
registered last."""
specs = {}
idx = 0
for _ in range(2):
for _ in range(3):
specs[f"model.layers.{idx}.linear_attn"] = new_mamba_spec(
mamba_cache_mode="align"
)
idx += 1
specs[f"model.layers.{idx}.self_attn.attn"] = new_kv_cache_spec()
idx += 1
if with_mtp_layer:
specs["mtp.layers.0.self_attn.attn"] = new_kv_cache_spec()
return specs
def test_qwen3_5_mtp_draft_group_annotated_on_hybrid_path(caplog_vllm):
# A hybrid mamba + full-attention model with an MTP drafter that is
# spec-indistinguishable from the target reaches the general multi-group
# path. The trailing-layer rule must locate the draft group there so the
# Mamba groups are not swept up by the flag-all consumer fallback.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method="mtp", model_type="qwen3_5"),
_qwen3_5_hybrid_specs(with_mtp_layer=True),
)
flagged = [g for g in groups if g.is_eagle_group]
assert len(flagged) == 1
assert "mtp.layers.0.self_attn.attn" in flagged[0].layer_names
for group in groups:
if any(
isinstance(spec, MambaSpec)
for spec in iter_layer_specs(group.kv_cache_spec)
):
assert not group.is_eagle_group
assert "could be identified as the draft model's" not in caplog_vllm.text
@pytest.mark.parametrize("method", ["eagle", "eagle3", "dspark"])
def test_non_mtp_eagle_hybrid_still_warns(caplog_vllm, method):
# Other drafters are not covered by the trailing-layer rule, so an
# unidentifiable hybrid draft must still warn.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method=method, model_type="qwen3_5"),
_qwen3_5_hybrid_specs(with_mtp_layer=True),
)
assert not any(g.is_eagle_group for g in groups) assert not any(g.is_eagle_group for g in groups)
assert "could be identified as the draft model's" in caplog_vllm.text
def test_trailing_layer_fallback_requires_exact_partition():
# If the groups do not partition the layers exactly (e.g. a caller that
# dropped or duplicated layers), the positional rule is meaningless and
# must not fire.
from vllm.v1.core.kv_cache_utils import _annotate_eagle_groups
specs = _qwen3_5_hybrid_specs(with_mtp_layer=True)
config = _spec_decode_grouping_config(method="mtp", model_type="qwen3_5")
groups = get_kv_cache_groups(config, specs)
for g in groups:
g.is_eagle_group = False
# Remove one layer from its group: no longer an exact partition.
trimmed = [
KVCacheGroupSpec(
[n for n in g.layer_names if n != "model.layers.0.linear_attn"],
g.kv_cache_spec,
)
for g in groups
]
_annotate_eagle_groups(config, specs, trimmed, use_trailing_layer_fallback=True)
assert not any(g.is_eagle_group for g in trimmed)
+43 -15
View File
@@ -2117,16 +2117,24 @@ def _get_packed_kv_cache_groups(
vllm_config, vllm_config,
kv_cache_spec, kv_cache_spec,
groups, groups,
use_deepseek_v4_fallback=_is_deepseek_v4_eagle(vllm_config), use_trailing_layer_fallback=_uses_trailing_mtp_layers(vllm_config),
) )
_warn_if_unannotated_eagle_mamba(vllm_config, groups) _warn_if_unannotated_eagle_mamba(vllm_config, groups)
return groups return groups
def _is_deepseek_v4_eagle(vllm_config: VllmConfig) -> bool: def _uses_trailing_mtp_layers(vllm_config: VllmConfig) -> bool:
"""Whether the drafter's KV layers can be located positionally.
MTP drafters register their KV layers after the target layers, without a
distinguishing spec marker. DeepseekV4/V4.1 do this with ``dspark`` too.
The annotator separately checks that the groups partition the layers exactly.
"""
spec_config = vllm_config.speculative_config spec_config = vllm_config.speculative_config
if spec_config is None or not spec_config.use_eagle(): if spec_config is None or not spec_config.use_eagle():
return False return False
if spec_config.method == "mtp":
return True
model_config = vllm_config.model_config model_config = vllm_config.model_config
return model_config is not None and model_config.hf_config.model_type in ( return model_config is not None and model_config.hf_config.model_type in (
"deepseek_v4", "deepseek_v4",
@@ -2134,11 +2142,24 @@ def _is_deepseek_v4_eagle(vllm_config: VllmConfig) -> bool:
) )
def _groups_partition_layers_exactly(
kv_cache_spec: dict[str, KVCacheSpec],
kv_cache_groups: list[KVCacheGroupSpec],
) -> bool:
"""Whether the groups cover every layer of ``kv_cache_spec`` exactly once.
The trailing-layer draft fallback is only meaningful when the last
registered layer is guaranteed to appear in exactly one group.
"""
names = [name for group in kv_cache_groups for name in group.layer_names]
return len(names) == len(kv_cache_spec) and set(names) == set(kv_cache_spec)
def _annotate_eagle_groups( def _annotate_eagle_groups(
vllm_config: VllmConfig, vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec], kv_cache_spec: dict[str, KVCacheSpec],
kv_cache_groups: list[KVCacheGroupSpec], kv_cache_groups: list[KVCacheGroupSpec],
use_deepseek_v4_fallback: bool = False, use_trailing_layer_fallback: bool = False,
) -> None: ) -> None:
"""Flag the KV cache groups that hold drafter attention layers. """Flag the KV cache groups that hold drafter attention layers.
@@ -2151,14 +2172,14 @@ def _annotate_eagle_groups(
spec merging, wherever grouping happens to land. It is sufficient but spec merging, wherever grouping happens to land. It is sufficient but
not necessary: a drafter whose spec is indistinguishable from the not necessary: a drafter whose spec is indistinguishable from the
target's cannot be found this way. target's cannot be found this way.
2. Model-scoped positional fallback for DeepseekV4/V4.1, whose MTP block 2. Positional fallback for MTP drafters (including DeepseekV4/V4.1 DSpark), whose
reuses the target's own decoder layer and so carries no spec marker. Its MTP block reuses the target's own decoder layer and so carries no spec
draft attention layer is always the last registered layer, so flag whichever marker. Their draft attention layers always register after every
group holds it. This rule is only valid where the groups partition target layer, so flag whichever group holds the last registered layer.
exactly the layers of ``kv_cache_spec``, which is true on the packed This rule is only valid where the groups partition exactly the layers
grouping path and not in general; other callers must leave of ``kv_cache_spec``, which is re-checked here before applying it.
``use_deepseek_v4_fallback`` False. The caller gates this fallback on When a drafter's trailing caches span several groups, this rule flags
the configured model type. only the group holding the very last layer and must be generalized.
FIXME(yifan): avoid/generalize this hacky check. FIXME(yifan): avoid/generalize this hacky check.
Args: Args:
@@ -2166,8 +2187,8 @@ def _annotate_eagle_groups(
kv_cache_spec: The kv cache spec of each attention layer, in layer kv_cache_spec: The kv cache spec of each attention layer, in layer
registration order. Only read by rule 2. registration order. Only read by rule 2.
kv_cache_groups: Groups to annotate in place. kv_cache_groups: Groups to annotate in place.
use_deepseek_v4_fallback: Enable rule 2 for a DeepseekV4/V4.1 packed use_trailing_layer_fallback: Enable rule 2. Callers gate this on
group. ``_uses_trailing_mtp_layers``.
""" """
spec_config = vllm_config.speculative_config spec_config = vllm_config.speculative_config
@@ -2181,7 +2202,9 @@ def _annotate_eagle_groups(
): ):
group.is_eagle_group = True group.is_eagle_group = True
if not use_deepseek_v4_fallback: if not use_trailing_layer_fallback:
return
if not _groups_partition_layers_exactly(kv_cache_spec, kv_cache_groups):
return return
last_layer = next(reversed(kv_cache_spec)) last_layer = next(reversed(kv_cache_spec))
for group in kv_cache_groups: for group in kv_cache_groups:
@@ -2352,7 +2375,12 @@ def get_kv_cache_groups(
aligned = replace(spec, block_size=new_bs, page_size_padded=common_page) aligned = replace(spec, block_size=new_bs, page_size_padded=common_page)
groups.append(KVCacheGroupSpec([name], aligned)) groups.append(KVCacheGroupSpec([name], aligned))
_annotate_eagle_groups(vllm_config, kv_cache_spec, groups) _annotate_eagle_groups(
vllm_config,
kv_cache_spec,
groups,
use_trailing_layer_fallback=_uses_trailing_mtp_layers(vllm_config),
)
_warn_if_unannotated_eagle_mamba(vllm_config, groups) _warn_if_unannotated_eagle_mamba(vllm_config, groups)
return groups return groups