[Bugfix][AMD] Annotate draft KV cache groups on the hybrid grouping path (#52047)
Signed-off-by: okorzh-amd <okorzh-amd@users.noreply.github.com> Co-authored-by: okorzh-amd <okorzh-amd@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
okorzh-amd
Claude Opus 5
Douglas Lehr
parent
6b110badbb
commit
93ab92be0c
@@ -4,6 +4,7 @@ import copy
|
||||
import hashlib
|
||||
import importlib
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
@@ -2214,13 +2215,14 @@ def new_mla_spec(cache_dtype_str=None, block_size=16):
|
||||
)
|
||||
|
||||
|
||||
def new_swa_mla_spec(head_size=576, sliding_window=128):
|
||||
def new_swa_mla_spec(head_size=576, sliding_window=128, model_version=None):
|
||||
return SlidingWindowMLASpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=head_size,
|
||||
dtype=torch.float32,
|
||||
sliding_window=sliding_window,
|
||||
model_version=model_version,
|
||||
)
|
||||
|
||||
|
||||
@@ -3277,3 +3279,143 @@ def test_iter_layer_specs_returns_group_members():
|
||||
block_size=4, kv_cache_specs={"a": full, "b": mla}
|
||||
)
|
||||
assert list(iter_layer_specs(wrapped)) == [full, mla]
|
||||
|
||||
|
||||
def _spec_decode_grouping_config(method="dspark"):
|
||||
"""Grouping config with an EAGLE-family speculative method enabled."""
|
||||
return SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
speculative_config=SimpleNamespace(
|
||||
method=method,
|
||||
use_eagle=lambda: True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _hybrid_specs_with_draft(draft: bool, draft_shares_target_spec: bool = False):
|
||||
"""A K3-shaped hybrid: MLA full attention + Mamba, optionally plus a
|
||||
DSpark-style draft MLA layer marked non_causal_multi_token_decode.
|
||||
|
||||
The target's fp8 KV dtype is what keeps the draft in its own bucket, as it
|
||||
does on Kimi-K3 (target `--kv-cache-dtype fp8_e4m3`, draft `auto`). Pass
|
||||
draft_shares_target_spec to collapse them into one group instead.
|
||||
"""
|
||||
target_dtype = None if draft_shares_target_spec else "fp8_e4m3"
|
||||
specs = {
|
||||
"target.attn.0": new_mla_spec(block_size=64, cache_dtype_str=target_dtype),
|
||||
"target.attn.1": new_mla_spec(block_size=64, cache_dtype_str=target_dtype),
|
||||
"target.mamba.0": new_mamba_spec(block_size=64, mamba_cache_mode="align"),
|
||||
"target.mamba.1": new_mamba_spec(block_size=64, mamba_cache_mode="align"),
|
||||
}
|
||||
if draft:
|
||||
draft_spec = new_mla_spec(block_size=64)
|
||||
specs["draft.attn.0"] = replace(draft_spec, non_causal_multi_token_decode=True)
|
||||
return specs
|
||||
|
||||
|
||||
def test_draft_group_annotated_on_hybrid_general_path():
|
||||
# A drafter's MLA layer carries non_causal_multi_token_decode, so its group
|
||||
# is identifiable without keying off a model version.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True)
|
||||
)
|
||||
|
||||
flagged = [g for g in groups if g.is_eagle_group]
|
||||
assert len(flagged) == 1
|
||||
assert flagged[0].layer_names == ["draft.attn.0"]
|
||||
|
||||
|
||||
def test_mamba_groups_never_flagged_even_when_draft_shares_a_group():
|
||||
# MLAAttentionSpec.merge ORs non_causal_multi_token_decode rather than
|
||||
# requiring equality, so a draft layer can share a group with target
|
||||
# layers; that combined group still holds volatile draft KV and must be
|
||||
# flagged. What must never happen is a Mamba group being flagged: that
|
||||
# widens its lookup window to two consecutive chunks, which align-mode
|
||||
# checkpointing never produces, zeroing every lookup.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(),
|
||||
_hybrid_specs_with_draft(draft=True, draft_shares_target_spec=True),
|
||||
)
|
||||
|
||||
for group in groups:
|
||||
if "draft.attn.0" in group.layer_names:
|
||||
assert group.is_eagle_group
|
||||
if isinstance(group.kv_cache_spec, MambaSpec):
|
||||
assert not group.is_eagle_group
|
||||
|
||||
|
||||
def test_draft_group_not_annotated_without_spec_decode():
|
||||
# The marker alone must not flag anything; the eagle semantics only apply
|
||||
# when a speculative method is actually enabled.
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
speculative_config=None,
|
||||
)
|
||||
groups = get_kv_cache_groups(config, _hybrid_specs_with_draft(draft=True))
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
|
||||
|
||||
def test_unidentifiable_draft_with_mamba_warns(caplog_vllm):
|
||||
# No group carries the draft marker, so every consumer falls back to
|
||||
# flagging all groups -- including Mamba ones, which then can never report
|
||||
# a hit. That is silent today; it must at least be visible.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=False)
|
||||
)
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
assert "no KV cache group could be identified as the draft model's" in (
|
||||
caplog_vllm.text
|
||||
)
|
||||
assert "Mamba groups" in caplog_vllm.text
|
||||
|
||||
|
||||
def test_no_warning_when_draft_group_is_identified(caplog_vllm):
|
||||
get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True)
|
||||
)
|
||||
|
||||
assert "could be identified as the draft model's" not in caplog_vllm.text
|
||||
|
||||
|
||||
def _deepseek_v4_specs(model_version="deepseek_v4"):
|
||||
"""DeepseekV4-shaped specs: full MLA layers plus sliding-window MLA layers
|
||||
at differing page sizes, with the MTP draft layer registered last."""
|
||||
return {
|
||||
"model.layers.0.self_attn.attn": new_mla_spec(),
|
||||
"model.layers.1.self_attn.attn": new_mla_spec(),
|
||||
"model.layers.2.self_attn.attn": new_swa_mla_spec(
|
||||
head_size=1024, model_version=model_version
|
||||
),
|
||||
# The MTP block registers last, and its sliding-window size differs, so
|
||||
# it lands in a group of its own.
|
||||
"model.layers.3.self_attn.attn": new_swa_mla_spec(
|
||||
head_size=1024, sliding_window=256, model_version=model_version
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def test_deepseek_v4_draft_group_annotated_on_group_and_unify_path():
|
||||
# DeepseekV4's MTP block reuses the target's decoder layer, so its spec
|
||||
# carries no draft marker and only the positional rule can find it. This
|
||||
# pins the pre-existing behaviour that the unified annotator must preserve.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(method="mtp"), _deepseek_v4_specs()
|
||||
)
|
||||
|
||||
flagged = [g for g in groups if g.is_eagle_group]
|
||||
assert len(flagged) == 1
|
||||
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
|
||||
|
||||
|
||||
def test_deepseek_v4_annotation_requires_model_version():
|
||||
# The positional rule is only sound for DeepseekV4, where the draft layer
|
||||
# is known to be registered last. Without that model gate nothing may be
|
||||
# flagged, however the grouping happens to fall out.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(method="mtp"),
|
||||
_deepseek_v4_specs(model_version=None),
|
||||
)
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
|
||||
@@ -1742,23 +1742,56 @@ def _get_kv_cache_groups_uniform_groups(
|
||||
return [full_mla_group, *swa_mla_groups]
|
||||
|
||||
|
||||
def _annotate_eagle_groups_deepseek_v4(
|
||||
def _annotate_eagle_groups(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
use_deepseek_v4_fallback: bool = False,
|
||||
) -> None:
|
||||
"""Flag the KV cache groups that hold drafter attention layers.
|
||||
|
||||
Two detection rules, in order of preference:
|
||||
|
||||
1. Spec-driven. ``non_causal_multi_token_decode`` is declared on
|
||||
MLAAttentionSpec and set by drafter attention layers that run a
|
||||
non-causal multi-token decode (today only Kimi-K3 DSpark). It survives
|
||||
MLAAttentionSpec.merge, so it still identifies a group after per-group
|
||||
spec merging, wherever grouping happens to land. It is sufficient but
|
||||
not necessary: a drafter whose spec is indistinguishable from the
|
||||
target's cannot be found this way.
|
||||
2. Model-scoped positional fallback for DeepseekV4, whose MTP block reuses
|
||||
the target's own decoder layer and so carries no spec marker. Its draft
|
||||
attention layer is always the last registered layer, so flag whichever
|
||||
group holds it. This rule is only valid where the groups partition
|
||||
exactly the layers of ``kv_cache_spec``, which is true on the
|
||||
group_and_unify path and not in general; other callers must leave
|
||||
``use_deepseek_v4_fallback`` False.
|
||||
FIXME(yifan): avoid/generalize this hacky check.
|
||||
|
||||
Args:
|
||||
vllm_config: Config supplying the speculative method, if any.
|
||||
kv_cache_spec: The kv cache spec of each attention layer, in layer
|
||||
registration order. Only read by rule 2.
|
||||
kv_cache_groups: Groups to annotate in place.
|
||||
use_deepseek_v4_fallback: Enable rule 2. Only the group_and_unify path
|
||||
may set this.
|
||||
"""
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is None or not spec_config.use_eagle():
|
||||
return
|
||||
|
||||
for group in kv_cache_groups:
|
||||
if getattr(group.kv_cache_spec, "non_causal_multi_token_decode", False):
|
||||
group.is_eagle_group = True
|
||||
|
||||
if not use_deepseek_v4_fallback:
|
||||
return
|
||||
# Detection uses the merged MLA spec's model_version.
|
||||
if not any(
|
||||
getattr(spec, "model_version", None) == "deepseek_v4"
|
||||
for spec in kv_cache_spec.values()
|
||||
):
|
||||
return
|
||||
# DeepseekV4's MTP attention layer is always the last layer, and we flag whichever
|
||||
# group contains it.
|
||||
# FIXME(yifan): avoid/generalize this hacky check.
|
||||
last_layer = next(reversed(kv_cache_spec))
|
||||
for group in kv_cache_groups:
|
||||
if last_layer in group.layer_names:
|
||||
@@ -1766,6 +1799,45 @@ def _annotate_eagle_groups_deepseek_v4(
|
||||
break
|
||||
|
||||
|
||||
def _warn_if_unannotated_eagle_mamba(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> None:
|
||||
"""Warn when the flag-all eagle fallback will silently disable reuse.
|
||||
|
||||
With no group annotated, consumers flag every group as a draft group. That
|
||||
widens a Mamba group's required lookup window to two consecutive chunks,
|
||||
which align-mode checkpointing never produces, so reuse drops to zero with
|
||||
no error and no metric to show it.
|
||||
|
||||
Args:
|
||||
vllm_config: Config supplying the speculative method, if any.
|
||||
kv_cache_groups: Groups as they will be handed to consumers.
|
||||
"""
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is None or not spec_config.use_eagle():
|
||||
return
|
||||
if any(group.is_eagle_group for group in kv_cache_groups):
|
||||
return
|
||||
mamba_groups = [
|
||||
idx
|
||||
for idx, group in enumerate(kv_cache_groups)
|
||||
if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
if not mamba_groups:
|
||||
return
|
||||
logger.warning(
|
||||
"Speculative decoding (method=%s) is enabled but no KV cache group "
|
||||
"could be identified as the draft model's, so every group -- "
|
||||
"including Mamba groups %s -- will be treated as a draft group. A "
|
||||
"Mamba group cannot satisfy the widened lookup window that implies, "
|
||||
"so prefix-cache reuse across requests will be disabled and any "
|
||||
"external KV offload tier will store without ever serving a hit.",
|
||||
spec_config.method,
|
||||
mamba_groups,
|
||||
)
|
||||
|
||||
|
||||
def _largest_divisor_at_most(value: int, limit: int) -> int:
|
||||
for candidate in range(min(value, limit), 0, -1):
|
||||
if value % candidate == 0:
|
||||
@@ -1810,7 +1882,12 @@ def get_kv_cache_groups(
|
||||
# attention in different sizes. Need to group layers into multiple
|
||||
# UniformTypeKVCacheSpecs.
|
||||
kv_cache_groups = _get_kv_cache_groups_uniform_groups(grouped_specs)
|
||||
_annotate_eagle_groups_deepseek_v4(vllm_config, kv_cache_spec, kv_cache_groups)
|
||||
_annotate_eagle_groups(
|
||||
vllm_config,
|
||||
kv_cache_spec,
|
||||
kv_cache_groups,
|
||||
use_deepseek_v4_fallback=True,
|
||||
)
|
||||
return kv_cache_groups
|
||||
|
||||
# Pull HiddenStateCacheSpec layers out before the general multi-group
|
||||
@@ -1855,6 +1932,8 @@ def get_kv_cache_groups(
|
||||
aligned = replace(spec, block_size=new_bs, page_size_padded=common_page)
|
||||
groups.append(KVCacheGroupSpec([name], aligned))
|
||||
|
||||
_annotate_eagle_groups(vllm_config, kv_cache_spec, groups)
|
||||
_warn_if_unannotated_eagle_mamba(vllm_config, groups)
|
||||
return groups
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user