[Bugfix][AMD] Annotate draft KV cache groups on the hybrid grouping path (#52047)
Signed-off-by: okorzh-amd <okorzh-amd@users.noreply.github.com> Co-authored-by: okorzh-amd <okorzh-amd@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
okorzh-amd
Claude Opus 5
Douglas Lehr
parent
6b110badbb
commit
93ab92be0c
@@ -4,6 +4,7 @@ import copy
|
||||
import hashlib
|
||||
import importlib
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
@@ -2214,13 +2215,14 @@ def new_mla_spec(cache_dtype_str=None, block_size=16):
|
||||
)
|
||||
|
||||
|
||||
def new_swa_mla_spec(head_size=576, sliding_window=128):
|
||||
def new_swa_mla_spec(head_size=576, sliding_window=128, model_version=None):
|
||||
return SlidingWindowMLASpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=head_size,
|
||||
dtype=torch.float32,
|
||||
sliding_window=sliding_window,
|
||||
model_version=model_version,
|
||||
)
|
||||
|
||||
|
||||
@@ -3277,3 +3279,143 @@ def test_iter_layer_specs_returns_group_members():
|
||||
block_size=4, kv_cache_specs={"a": full, "b": mla}
|
||||
)
|
||||
assert list(iter_layer_specs(wrapped)) == [full, mla]
|
||||
|
||||
|
||||
def _spec_decode_grouping_config(method="dspark"):
|
||||
"""Grouping config with an EAGLE-family speculative method enabled."""
|
||||
return SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
speculative_config=SimpleNamespace(
|
||||
method=method,
|
||||
use_eagle=lambda: True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _hybrid_specs_with_draft(draft: bool, draft_shares_target_spec: bool = False):
|
||||
"""A K3-shaped hybrid: MLA full attention + Mamba, optionally plus a
|
||||
DSpark-style draft MLA layer marked non_causal_multi_token_decode.
|
||||
|
||||
The target's fp8 KV dtype is what keeps the draft in its own bucket, as it
|
||||
does on Kimi-K3 (target `--kv-cache-dtype fp8_e4m3`, draft `auto`). Pass
|
||||
draft_shares_target_spec to collapse them into one group instead.
|
||||
"""
|
||||
target_dtype = None if draft_shares_target_spec else "fp8_e4m3"
|
||||
specs = {
|
||||
"target.attn.0": new_mla_spec(block_size=64, cache_dtype_str=target_dtype),
|
||||
"target.attn.1": new_mla_spec(block_size=64, cache_dtype_str=target_dtype),
|
||||
"target.mamba.0": new_mamba_spec(block_size=64, mamba_cache_mode="align"),
|
||||
"target.mamba.1": new_mamba_spec(block_size=64, mamba_cache_mode="align"),
|
||||
}
|
||||
if draft:
|
||||
draft_spec = new_mla_spec(block_size=64)
|
||||
specs["draft.attn.0"] = replace(draft_spec, non_causal_multi_token_decode=True)
|
||||
return specs
|
||||
|
||||
|
||||
def test_draft_group_annotated_on_hybrid_general_path():
|
||||
# A drafter's MLA layer carries non_causal_multi_token_decode, so its group
|
||||
# is identifiable without keying off a model version.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True)
|
||||
)
|
||||
|
||||
flagged = [g for g in groups if g.is_eagle_group]
|
||||
assert len(flagged) == 1
|
||||
assert flagged[0].layer_names == ["draft.attn.0"]
|
||||
|
||||
|
||||
def test_mamba_groups_never_flagged_even_when_draft_shares_a_group():
|
||||
# MLAAttentionSpec.merge ORs non_causal_multi_token_decode rather than
|
||||
# requiring equality, so a draft layer can share a group with target
|
||||
# layers; that combined group still holds volatile draft KV and must be
|
||||
# flagged. What must never happen is a Mamba group being flagged: that
|
||||
# widens its lookup window to two consecutive chunks, which align-mode
|
||||
# checkpointing never produces, zeroing every lookup.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(),
|
||||
_hybrid_specs_with_draft(draft=True, draft_shares_target_spec=True),
|
||||
)
|
||||
|
||||
for group in groups:
|
||||
if "draft.attn.0" in group.layer_names:
|
||||
assert group.is_eagle_group
|
||||
if isinstance(group.kv_cache_spec, MambaSpec):
|
||||
assert not group.is_eagle_group
|
||||
|
||||
|
||||
def test_draft_group_not_annotated_without_spec_decode():
|
||||
# The marker alone must not flag anything; the eagle semantics only apply
|
||||
# when a speculative method is actually enabled.
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
speculative_config=None,
|
||||
)
|
||||
groups = get_kv_cache_groups(config, _hybrid_specs_with_draft(draft=True))
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
|
||||
|
||||
def test_unidentifiable_draft_with_mamba_warns(caplog_vllm):
|
||||
# No group carries the draft marker, so every consumer falls back to
|
||||
# flagging all groups -- including Mamba ones, which then can never report
|
||||
# a hit. That is silent today; it must at least be visible.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=False)
|
||||
)
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
assert "no KV cache group could be identified as the draft model's" in (
|
||||
caplog_vllm.text
|
||||
)
|
||||
assert "Mamba groups" in caplog_vllm.text
|
||||
|
||||
|
||||
def test_no_warning_when_draft_group_is_identified(caplog_vllm):
|
||||
get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True)
|
||||
)
|
||||
|
||||
assert "could be identified as the draft model's" not in caplog_vllm.text
|
||||
|
||||
|
||||
def _deepseek_v4_specs(model_version="deepseek_v4"):
|
||||
"""DeepseekV4-shaped specs: full MLA layers plus sliding-window MLA layers
|
||||
at differing page sizes, with the MTP draft layer registered last."""
|
||||
return {
|
||||
"model.layers.0.self_attn.attn": new_mla_spec(),
|
||||
"model.layers.1.self_attn.attn": new_mla_spec(),
|
||||
"model.layers.2.self_attn.attn": new_swa_mla_spec(
|
||||
head_size=1024, model_version=model_version
|
||||
),
|
||||
# The MTP block registers last, and its sliding-window size differs, so
|
||||
# it lands in a group of its own.
|
||||
"model.layers.3.self_attn.attn": new_swa_mla_spec(
|
||||
head_size=1024, sliding_window=256, model_version=model_version
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def test_deepseek_v4_draft_group_annotated_on_group_and_unify_path():
|
||||
# DeepseekV4's MTP block reuses the target's decoder layer, so its spec
|
||||
# carries no draft marker and only the positional rule can find it. This
|
||||
# pins the pre-existing behaviour that the unified annotator must preserve.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(method="mtp"), _deepseek_v4_specs()
|
||||
)
|
||||
|
||||
flagged = [g for g in groups if g.is_eagle_group]
|
||||
assert len(flagged) == 1
|
||||
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
|
||||
|
||||
|
||||
def test_deepseek_v4_annotation_requires_model_version():
|
||||
# The positional rule is only sound for DeepseekV4, where the draft layer
|
||||
# is known to be registered last. Without that model gate nothing may be
|
||||
# flagged, however the grouping happens to fall out.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(method="mtp"),
|
||||
_deepseek_v4_specs(model_version=None),
|
||||
)
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
|
||||
Reference in New Issue
Block a user