[Bugfix][AMD] Annotate draft KV cache groups on the hybrid grouping path (#52047)

Signed-off-by: okorzh-amd <okorzh-amd@users.noreply.github.com>
Co-authored-by: okorzh-amd <okorzh-amd@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Oxana Korzh
2026-08-28 21:53:46 -05:00
committed by GitHub
co-authored by okorzh-amd Claude Opus 5 Douglas Lehr
parent 6b110badbb
commit 93ab92be0c
2 changed files with 227 additions and 6 deletions
+143 -1
View File
@@ -4,6 +4,7 @@ import copy
import hashlib
import importlib
from collections.abc import Callable
from dataclasses import replace
from types import SimpleNamespace
from typing import Any
@@ -2214,13 +2215,14 @@ def new_mla_spec(cache_dtype_str=None, block_size=16):
)
def new_swa_mla_spec(head_size=576, sliding_window=128):
def new_swa_mla_spec(head_size=576, sliding_window=128, model_version=None):
return SlidingWindowMLASpec(
block_size=16,
num_kv_heads=1,
head_size=head_size,
dtype=torch.float32,
sliding_window=sliding_window,
model_version=model_version,
)
@@ -3277,3 +3279,143 @@ def test_iter_layer_specs_returns_group_members():
block_size=4, kv_cache_specs={"a": full, "b": mla}
)
assert list(iter_layer_specs(wrapped)) == [full, mla]
def _spec_decode_grouping_config(method="dspark"):
"""Grouping config with an EAGLE-family speculative method enabled."""
return SimpleNamespace(
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
speculative_config=SimpleNamespace(
method=method,
use_eagle=lambda: True,
),
)
def _hybrid_specs_with_draft(draft: bool, draft_shares_target_spec: bool = False):
"""A K3-shaped hybrid: MLA full attention + Mamba, optionally plus a
DSpark-style draft MLA layer marked non_causal_multi_token_decode.
The target's fp8 KV dtype is what keeps the draft in its own bucket, as it
does on Kimi-K3 (target `--kv-cache-dtype fp8_e4m3`, draft `auto`). Pass
draft_shares_target_spec to collapse them into one group instead.
"""
target_dtype = None if draft_shares_target_spec else "fp8_e4m3"
specs = {
"target.attn.0": new_mla_spec(block_size=64, cache_dtype_str=target_dtype),
"target.attn.1": new_mla_spec(block_size=64, cache_dtype_str=target_dtype),
"target.mamba.0": new_mamba_spec(block_size=64, mamba_cache_mode="align"),
"target.mamba.1": new_mamba_spec(block_size=64, mamba_cache_mode="align"),
}
if draft:
draft_spec = new_mla_spec(block_size=64)
specs["draft.attn.0"] = replace(draft_spec, non_causal_multi_token_decode=True)
return specs
def test_draft_group_annotated_on_hybrid_general_path():
# A drafter's MLA layer carries non_causal_multi_token_decode, so its group
# is identifiable without keying off a model version.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True)
)
flagged = [g for g in groups if g.is_eagle_group]
assert len(flagged) == 1
assert flagged[0].layer_names == ["draft.attn.0"]
def test_mamba_groups_never_flagged_even_when_draft_shares_a_group():
# MLAAttentionSpec.merge ORs non_causal_multi_token_decode rather than
# requiring equality, so a draft layer can share a group with target
# layers; that combined group still holds volatile draft KV and must be
# flagged. What must never happen is a Mamba group being flagged: that
# widens its lookup window to two consecutive chunks, which align-mode
# checkpointing never produces, zeroing every lookup.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(),
_hybrid_specs_with_draft(draft=True, draft_shares_target_spec=True),
)
for group in groups:
if "draft.attn.0" in group.layer_names:
assert group.is_eagle_group
if isinstance(group.kv_cache_spec, MambaSpec):
assert not group.is_eagle_group
def test_draft_group_not_annotated_without_spec_decode():
# The marker alone must not flag anything; the eagle semantics only apply
# when a speculative method is actually enabled.
config = SimpleNamespace(
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
speculative_config=None,
)
groups = get_kv_cache_groups(config, _hybrid_specs_with_draft(draft=True))
assert not any(g.is_eagle_group for g in groups)
def test_unidentifiable_draft_with_mamba_warns(caplog_vllm):
# No group carries the draft marker, so every consumer falls back to
# flagging all groups -- including Mamba ones, which then can never report
# a hit. That is silent today; it must at least be visible.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=False)
)
assert not any(g.is_eagle_group for g in groups)
assert "no KV cache group could be identified as the draft model's" in (
caplog_vllm.text
)
assert "Mamba groups" in caplog_vllm.text
def test_no_warning_when_draft_group_is_identified(caplog_vllm):
get_kv_cache_groups(
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True)
)
assert "could be identified as the draft model's" not in caplog_vllm.text
def _deepseek_v4_specs(model_version="deepseek_v4"):
"""DeepseekV4-shaped specs: full MLA layers plus sliding-window MLA layers
at differing page sizes, with the MTP draft layer registered last."""
return {
"model.layers.0.self_attn.attn": new_mla_spec(),
"model.layers.1.self_attn.attn": new_mla_spec(),
"model.layers.2.self_attn.attn": new_swa_mla_spec(
head_size=1024, model_version=model_version
),
# The MTP block registers last, and its sliding-window size differs, so
# it lands in a group of its own.
"model.layers.3.self_attn.attn": new_swa_mla_spec(
head_size=1024, sliding_window=256, model_version=model_version
),
}
def test_deepseek_v4_draft_group_annotated_on_group_and_unify_path():
# DeepseekV4's MTP block reuses the target's decoder layer, so its spec
# carries no draft marker and only the positional rule can find it. This
# pins the pre-existing behaviour that the unified annotator must preserve.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method="mtp"), _deepseek_v4_specs()
)
flagged = [g for g in groups if g.is_eagle_group]
assert len(flagged) == 1
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
def test_deepseek_v4_annotation_requires_model_version():
# The positional rule is only sound for DeepseekV4, where the draft layer
# is known to be registered last. Without that model gate nothing may be
# flagged, however the grouping happens to fall out.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method="mtp"),
_deepseek_v4_specs(model_version=None),
)
assert not any(g.is_eagle_group for g in groups)
+84 -5
View File
@@ -1742,23 +1742,56 @@ def _get_kv_cache_groups_uniform_groups(
return [full_mla_group, *swa_mla_groups]
def _annotate_eagle_groups_deepseek_v4(
def _annotate_eagle_groups(
vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec],
kv_cache_groups: list[KVCacheGroupSpec],
use_deepseek_v4_fallback: bool = False,
) -> None:
"""Flag the KV cache groups that hold drafter attention layers.
Two detection rules, in order of preference:
1. Spec-driven. ``non_causal_multi_token_decode`` is declared on
MLAAttentionSpec and set by drafter attention layers that run a
non-causal multi-token decode (today only Kimi-K3 DSpark). It survives
MLAAttentionSpec.merge, so it still identifies a group after per-group
spec merging, wherever grouping happens to land. It is sufficient but
not necessary: a drafter whose spec is indistinguishable from the
target's cannot be found this way.
2. Model-scoped positional fallback for DeepseekV4, whose MTP block reuses
the target's own decoder layer and so carries no spec marker. Its draft
attention layer is always the last registered layer, so flag whichever
group holds it. This rule is only valid where the groups partition
exactly the layers of ``kv_cache_spec``, which is true on the
group_and_unify path and not in general; other callers must leave
``use_deepseek_v4_fallback`` False.
FIXME(yifan): avoid/generalize this hacky check.
Args:
vllm_config: Config supplying the speculative method, if any.
kv_cache_spec: The kv cache spec of each attention layer, in layer
registration order. Only read by rule 2.
kv_cache_groups: Groups to annotate in place.
use_deepseek_v4_fallback: Enable rule 2. Only the group_and_unify path
may set this.
"""
spec_config = vllm_config.speculative_config
if spec_config is None or not spec_config.use_eagle():
return
for group in kv_cache_groups:
if getattr(group.kv_cache_spec, "non_causal_multi_token_decode", False):
group.is_eagle_group = True
if not use_deepseek_v4_fallback:
return
# Detection uses the merged MLA spec's model_version.
if not any(
getattr(spec, "model_version", None) == "deepseek_v4"
for spec in kv_cache_spec.values()
):
return
# DeepseekV4's MTP attention layer is always the last layer, and we flag whichever
# group contains it.
# FIXME(yifan): avoid/generalize this hacky check.
last_layer = next(reversed(kv_cache_spec))
for group in kv_cache_groups:
if last_layer in group.layer_names:
@@ -1766,6 +1799,45 @@ def _annotate_eagle_groups_deepseek_v4(
break
def _warn_if_unannotated_eagle_mamba(
vllm_config: VllmConfig,
kv_cache_groups: list[KVCacheGroupSpec],
) -> None:
"""Warn when the flag-all eagle fallback will silently disable reuse.
With no group annotated, consumers flag every group as a draft group. That
widens a Mamba group's required lookup window to two consecutive chunks,
which align-mode checkpointing never produces, so reuse drops to zero with
no error and no metric to show it.
Args:
vllm_config: Config supplying the speculative method, if any.
kv_cache_groups: Groups as they will be handed to consumers.
"""
spec_config = vllm_config.speculative_config
if spec_config is None or not spec_config.use_eagle():
return
if any(group.is_eagle_group for group in kv_cache_groups):
return
mamba_groups = [
idx
for idx, group in enumerate(kv_cache_groups)
if isinstance(group.kv_cache_spec, MambaSpec)
]
if not mamba_groups:
return
logger.warning(
"Speculative decoding (method=%s) is enabled but no KV cache group "
"could be identified as the draft model's, so every group -- "
"including Mamba groups %s -- will be treated as a draft group. A "
"Mamba group cannot satisfy the widened lookup window that implies, "
"so prefix-cache reuse across requests will be disabled and any "
"external KV offload tier will store without ever serving a hit.",
spec_config.method,
mamba_groups,
)
def _largest_divisor_at_most(value: int, limit: int) -> int:
for candidate in range(min(value, limit), 0, -1):
if value % candidate == 0:
@@ -1810,7 +1882,12 @@ def get_kv_cache_groups(
# attention in different sizes. Need to group layers into multiple
# UniformTypeKVCacheSpecs.
kv_cache_groups = _get_kv_cache_groups_uniform_groups(grouped_specs)
_annotate_eagle_groups_deepseek_v4(vllm_config, kv_cache_spec, kv_cache_groups)
_annotate_eagle_groups(
vllm_config,
kv_cache_spec,
kv_cache_groups,
use_deepseek_v4_fallback=True,
)
return kv_cache_groups
# Pull HiddenStateCacheSpec layers out before the general multi-group
@@ -1855,6 +1932,8 @@ def get_kv_cache_groups(
aligned = replace(spec, block_size=new_bs, page_size_padded=common_page)
groups.append(KVCacheGroupSpec([name], aligned))
_annotate_eagle_groups(vllm_config, kv_cache_spec, groups)
_warn_if_unannotated_eagle_mamba(vllm_config, groups)
return groups