[Bugfix][AMD] Annotate draft KV cache groups on the hybrid grouping path (#52047)

Signed-off-by: okorzh-amd <okorzh-amd@users.noreply.github.com>
Co-authored-by: okorzh-amd <okorzh-amd@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Oxana Korzh
2026-08-28 21:53:46 -05:00
committed by GitHub
co-authored by okorzh-amd Claude Opus 5 Douglas Lehr
parent 6b110badbb
commit 93ab92be0c
2 changed files with 227 additions and 6 deletions
+143 -1
View File
@@ -4,6 +4,7 @@ import copy
import hashlib
import importlib
from collections.abc import Callable
from dataclasses import replace
from types import SimpleNamespace
from typing import Any
@@ -2214,13 +2215,14 @@ def new_mla_spec(cache_dtype_str=None, block_size=16):
)
def new_swa_mla_spec(head_size=576, sliding_window=128):
def new_swa_mla_spec(head_size=576, sliding_window=128, model_version=None):
return SlidingWindowMLASpec(
block_size=16,
num_kv_heads=1,
head_size=head_size,
dtype=torch.float32,
sliding_window=sliding_window,
model_version=model_version,
)
@@ -3277,3 +3279,143 @@ def test_iter_layer_specs_returns_group_members():
block_size=4, kv_cache_specs={"a": full, "b": mla}
)
assert list(iter_layer_specs(wrapped)) == [full, mla]
def _spec_decode_grouping_config(method="dspark"):
"""Grouping config with an EAGLE-family speculative method enabled."""
return SimpleNamespace(
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
speculative_config=SimpleNamespace(
method=method,
use_eagle=lambda: True,
),
)
def _hybrid_specs_with_draft(draft: bool, draft_shares_target_spec: bool = False):
"""A K3-shaped hybrid: MLA full attention + Mamba, optionally plus a
DSpark-style draft MLA layer marked non_causal_multi_token_decode.
The target's fp8 KV dtype is what keeps the draft in its own bucket, as it
does on Kimi-K3 (target `--kv-cache-dtype fp8_e4m3`, draft `auto`). Pass
draft_shares_target_spec to collapse them into one group instead.
"""
target_dtype = None if draft_shares_target_spec else "fp8_e4m3"
specs = {
"target.attn.0": new_mla_spec(block_size=64, cache_dtype_str=target_dtype),
"target.attn.1": new_mla_spec(block_size=64, cache_dtype_str=target_dtype),
"target.mamba.0": new_mamba_spec(block_size=64, mamba_cache_mode="align"),
"target.mamba.1": new_mamba_spec(block_size=64, mamba_cache_mode="align"),
}
if draft:
draft_spec = new_mla_spec(block_size=64)
specs["draft.attn.0"] = replace(draft_spec, non_causal_multi_token_decode=True)
return specs
def test_draft_group_annotated_on_hybrid_general_path():
# A drafter's MLA layer carries non_causal_multi_token_decode, so its group
# is identifiable without keying off a model version.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True)
)
flagged = [g for g in groups if g.is_eagle_group]
assert len(flagged) == 1
assert flagged[0].layer_names == ["draft.attn.0"]
def test_mamba_groups_never_flagged_even_when_draft_shares_a_group():
# MLAAttentionSpec.merge ORs non_causal_multi_token_decode rather than
# requiring equality, so a draft layer can share a group with target
# layers; that combined group still holds volatile draft KV and must be
# flagged. What must never happen is a Mamba group being flagged: that
# widens its lookup window to two consecutive chunks, which align-mode
# checkpointing never produces, zeroing every lookup.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(),
_hybrid_specs_with_draft(draft=True, draft_shares_target_spec=True),
)
for group in groups:
if "draft.attn.0" in group.layer_names:
assert group.is_eagle_group
if isinstance(group.kv_cache_spec, MambaSpec):
assert not group.is_eagle_group
def test_draft_group_not_annotated_without_spec_decode():
# The marker alone must not flag anything; the eagle semantics only apply
# when a speculative method is actually enabled.
config = SimpleNamespace(
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
speculative_config=None,
)
groups = get_kv_cache_groups(config, _hybrid_specs_with_draft(draft=True))
assert not any(g.is_eagle_group for g in groups)
def test_unidentifiable_draft_with_mamba_warns(caplog_vllm):
# No group carries the draft marker, so every consumer falls back to
# flagging all groups -- including Mamba ones, which then can never report
# a hit. That is silent today; it must at least be visible.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=False)
)
assert not any(g.is_eagle_group for g in groups)
assert "no KV cache group could be identified as the draft model's" in (
caplog_vllm.text
)
assert "Mamba groups" in caplog_vllm.text
def test_no_warning_when_draft_group_is_identified(caplog_vllm):
get_kv_cache_groups(
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True)
)
assert "could be identified as the draft model's" not in caplog_vllm.text
def _deepseek_v4_specs(model_version="deepseek_v4"):
"""DeepseekV4-shaped specs: full MLA layers plus sliding-window MLA layers
at differing page sizes, with the MTP draft layer registered last."""
return {
"model.layers.0.self_attn.attn": new_mla_spec(),
"model.layers.1.self_attn.attn": new_mla_spec(),
"model.layers.2.self_attn.attn": new_swa_mla_spec(
head_size=1024, model_version=model_version
),
# The MTP block registers last, and its sliding-window size differs, so
# it lands in a group of its own.
"model.layers.3.self_attn.attn": new_swa_mla_spec(
head_size=1024, sliding_window=256, model_version=model_version
),
}
def test_deepseek_v4_draft_group_annotated_on_group_and_unify_path():
# DeepseekV4's MTP block reuses the target's decoder layer, so its spec
# carries no draft marker and only the positional rule can find it. This
# pins the pre-existing behaviour that the unified annotator must preserve.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method="mtp"), _deepseek_v4_specs()
)
flagged = [g for g in groups if g.is_eagle_group]
assert len(flagged) == 1
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
def test_deepseek_v4_annotation_requires_model_version():
# The positional rule is only sound for DeepseekV4, where the draft layer
# is known to be registered last. Without that model gate nothing may be
# flagged, however the grouping happens to fall out.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method="mtp"),
_deepseek_v4_specs(model_version=None),
)
assert not any(g.is_eagle_group for g in groups)