[Core][Spec Decode] Replace EAGLE group heuristics with explicit draft-layer tags (#61455)
Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Yifan Qiao
Claude Opus 5.5
parent
08a65e287e
commit
c3de0b37e2
@@ -236,10 +236,9 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
)
|
||||
if use_eagle_block_drop and not eagle_groups:
|
||||
# No group is annotated as holding drafter layers. This happens
|
||||
# for shared-group MTP models (e.g. Qwen3.5-style), whose drafter
|
||||
# is a regular decoder layer merged into the target's
|
||||
# full-attention group, and for separate-draft models that miss
|
||||
# annotation. Treating every group as a drafter group here makes
|
||||
# when the drafter owns no KV cache (e.g. KV-sharing drafters) or
|
||||
# the model runner does not tag draft layers' KV cache specs.
|
||||
# Treating every group as a drafter group here makes
|
||||
# the volatile-tail pop apply to ALL groups, which can zero the
|
||||
# whole request's offload hit whenever any group's servable
|
||||
# window is a single chunk (issue #52735). Drafter KV served one
|
||||
|
||||
@@ -9,7 +9,7 @@ from dataclasses import replace
|
||||
from functools import partial, reduce
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import SpeculativeConfig, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.mem_utils import format_gib
|
||||
@@ -1038,120 +1038,20 @@ def _get_packed_kv_cache_groups(
|
||||
assert group_spec is not None
|
||||
groups.append(KVCacheGroupSpec(group_layer_names, group_spec))
|
||||
|
||||
_annotate_eagle_groups(
|
||||
vllm_config,
|
||||
kv_cache_spec,
|
||||
groups,
|
||||
use_trailing_layer_fallback=_uses_trailing_mtp_layers(vllm_config),
|
||||
)
|
||||
_warn_if_unannotated_eagle_mamba(vllm_config, groups)
|
||||
return groups
|
||||
|
||||
|
||||
def _uses_trailing_mtp_layers(vllm_config: VllmConfig) -> bool:
|
||||
"""Whether the drafter's KV layers can be located positionally.
|
||||
|
||||
MTP drafters register their KV layers after the target layers, without a
|
||||
distinguishing spec marker. DeepseekV4/V4.1 do this with ``dspark`` too.
|
||||
The annotator separately checks that the groups partition the layers exactly.
|
||||
"""
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is None or not spec_config.use_eagle():
|
||||
return False
|
||||
if spec_config.method == "mtp":
|
||||
return True
|
||||
model_config = vllm_config.model_config
|
||||
return model_config is not None and model_config.hf_config.model_type in (
|
||||
"deepseek_v4",
|
||||
"deepseek_v41",
|
||||
)
|
||||
|
||||
|
||||
def _groups_partition_layers_exactly(
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> bool:
|
||||
"""Whether the groups cover every layer of ``kv_cache_spec`` exactly once.
|
||||
|
||||
The trailing-layer draft fallback is only meaningful when the last
|
||||
registered layer is guaranteed to appear in exactly one group.
|
||||
"""
|
||||
names = [name for group in kv_cache_groups for name in group.layer_names]
|
||||
return len(names) == len(kv_cache_spec) and set(names) == set(kv_cache_spec)
|
||||
|
||||
|
||||
def _annotate_eagle_groups(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
use_trailing_layer_fallback: bool = False,
|
||||
) -> None:
|
||||
"""Flag the KV cache groups that hold drafter attention layers.
|
||||
|
||||
Two detection rules, in order of preference:
|
||||
|
||||
1. Spec-driven. ``non_causal_multi_token_decode`` is declared on
|
||||
MLAAttentionSpec and set by drafter attention layers that run a
|
||||
non-causal multi-token decode (today only Kimi-K3 DSpark). It survives
|
||||
MLAAttentionSpec.merge, so it still identifies a group after per-group
|
||||
spec merging, wherever grouping happens to land. It is sufficient but
|
||||
not necessary: a drafter whose spec is indistinguishable from the
|
||||
target's cannot be found this way.
|
||||
2. Positional fallback for MTP drafters (including DeepseekV4/V4.1 DSpark), whose
|
||||
MTP block reuses the target's own decoder layer and so carries no spec
|
||||
marker. Their draft attention layers always register after every
|
||||
target layer, so flag whichever group holds the last registered layer.
|
||||
This rule is only valid where the groups partition exactly the layers
|
||||
of ``kv_cache_spec``, which is re-checked here before applying it.
|
||||
When a drafter's trailing caches span several groups, this rule flags
|
||||
only the group holding the very last layer and must be generalized.
|
||||
FIXME(yifan): avoid/generalize this hacky check.
|
||||
|
||||
Args:
|
||||
vllm_config: Config supplying the speculative method, if any.
|
||||
kv_cache_spec: The kv cache spec of each attention layer, in layer
|
||||
registration order. Only read by rule 2.
|
||||
kv_cache_groups: Groups to annotate in place.
|
||||
use_trailing_layer_fallback: Enable rule 2. Callers gate this on
|
||||
``_uses_trailing_mtp_layers``.
|
||||
|
||||
"""
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is None or not spec_config.use_eagle_block_drop():
|
||||
return
|
||||
|
||||
for group in kv_cache_groups:
|
||||
if any(
|
||||
getattr(spec, "non_causal_multi_token_decode", False)
|
||||
for spec in iter_layer_specs(group.kv_cache_spec)
|
||||
):
|
||||
group.is_eagle_group = True
|
||||
|
||||
if not use_trailing_layer_fallback:
|
||||
return
|
||||
if not _groups_partition_layers_exactly(kv_cache_spec, kv_cache_groups):
|
||||
return
|
||||
last_layer = next(reversed(kv_cache_spec))
|
||||
for group in kv_cache_groups:
|
||||
if last_layer in group.layer_names:
|
||||
group.is_eagle_group = True
|
||||
break
|
||||
|
||||
|
||||
def _warn_if_unannotated_eagle_mamba(
|
||||
vllm_config: VllmConfig,
|
||||
spec_config: SpeculativeConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> None:
|
||||
"""Warn when no KV cache group could be identified as the draft model's.
|
||||
|
||||
Args:
|
||||
vllm_config: Config supplying the speculative method, if any.
|
||||
spec_config: Config supplying the speculative method.
|
||||
kv_cache_groups: Groups as they will be handed to consumers.
|
||||
|
||||
"""
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is None or not spec_config.use_eagle():
|
||||
return
|
||||
if any(group.is_eagle_group for group in kv_cache_groups):
|
||||
return
|
||||
mamba_groups = [
|
||||
@@ -1297,13 +1197,6 @@ def get_kv_cache_groups(
|
||||
aligned = replace(spec, block_size=new_bs, page_size_padded=common_page)
|
||||
groups.append(KVCacheGroupSpec([name], aligned))
|
||||
|
||||
_annotate_eagle_groups(
|
||||
vllm_config,
|
||||
kv_cache_spec,
|
||||
groups,
|
||||
use_trailing_layer_fallback=_uses_trailing_mtp_layers(vllm_config),
|
||||
)
|
||||
_warn_if_unannotated_eagle_mamba(vllm_config, groups)
|
||||
return groups
|
||||
|
||||
|
||||
@@ -1556,7 +1449,7 @@ def _project_kv_cache_groups_to_worker(
|
||||
KVCacheGroupSpec(
|
||||
worker_layer_names,
|
||||
group_spec,
|
||||
is_eagle_group=group.is_eagle_group and bool(worker_layer_names),
|
||||
is_eagle_group=group.is_eagle_group,
|
||||
host_resident=group.host_resident,
|
||||
enable_kv_transfer=group.enable_kv_transfer,
|
||||
role=group.role,
|
||||
@@ -1636,6 +1529,14 @@ def get_kv_cache_configs(
|
||||
global_kv_cache_groups = get_kv_cache_groups_from_builder(
|
||||
builder, vllm_config, merged_kv_cache_specs
|
||||
)
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is not None and spec_config.use_eagle_block_drop():
|
||||
draft_layer_names = {
|
||||
name for name, spec in merged_kv_cache_specs.items() if spec.is_draft
|
||||
}
|
||||
for group in global_kv_cache_groups:
|
||||
group.is_eagle_group = not draft_layer_names.isdisjoint(group.layer_names)
|
||||
_warn_if_unannotated_eagle_mamba(spec_config, global_kv_cache_groups)
|
||||
|
||||
# If original_max_model_len was -1, automatically
|
||||
# determine the maximum model length that fits in available GPU memory.
|
||||
|
||||
@@ -175,6 +175,10 @@ class KVCacheSpec:
|
||||
``MultipleOf(n)``: whole kernel blocks, when blocks are split into kernel
|
||||
blocks of up to ``n`` tokens."""
|
||||
|
||||
is_draft: bool = field(default=False, kw_only=True, compare=False)
|
||||
"""Whether a draft model owns this layer; excluded from equality so it never
|
||||
affects grouping."""
|
||||
|
||||
def __post_init__(self):
|
||||
alignment = self.block_stride_alignment
|
||||
if isinstance(alignment, int) and alignment <= 0:
|
||||
@@ -660,6 +664,8 @@ class FullAttentionSpec(AttentionSpec):
|
||||
)
|
||||
for spec in specs:
|
||||
for f in fields(AttentionSpec):
|
||||
if not f.compare:
|
||||
continue
|
||||
assert getattr(spec, f.name) == getattr(merged_spec, f.name), (
|
||||
"All attention layers in the same KV cache group must have "
|
||||
"the same attention spec."
|
||||
@@ -747,6 +753,8 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
)
|
||||
for spec in specs:
|
||||
for f in fields(AttentionSpec):
|
||||
if not f.compare:
|
||||
continue
|
||||
assert getattr(spec, f.name) == getattr(merged_spec, f.name), (
|
||||
"All attention layers in the same KV cache group must have "
|
||||
"the same attention spec."
|
||||
@@ -1208,6 +1216,8 @@ class SinkFullAttentionSpec(FullAttentionSpec):
|
||||
)
|
||||
for spec in specs:
|
||||
for f in fields(AttentionSpec):
|
||||
if not f.compare:
|
||||
continue
|
||||
assert getattr(spec, f.name) == getattr(merged_spec, f.name), (
|
||||
"All attention layers in the same KV cache group must have "
|
||||
"the same attention spec."
|
||||
@@ -1445,7 +1455,7 @@ class KVCacheGroupSpec:
|
||||
layer_names: list[str]
|
||||
# The KV cache spec of this manager layer
|
||||
kv_cache_spec: KVCacheSpec
|
||||
# Whether this group contains EAGLE/MTP draft attention layers.
|
||||
# Whether this group holds draft layers and takes the EAGLE last-block drop.
|
||||
is_eagle_group: bool = False
|
||||
# Host groups use the dedicated HiSparse pool; others share the device pool.
|
||||
host_resident: bool = False
|
||||
|
||||
@@ -1330,10 +1330,13 @@ class SpecDecodeBaseProposer:
|
||||
self.vllm_config,
|
||||
AttentionLayerBase, # type: ignore[type-abstract]
|
||||
)
|
||||
draft_attn_layer_names = set(all_attn_layers) - target_attn_layer_names
|
||||
for name in draft_attn_layer_names:
|
||||
all_attn_layers[name].is_draft_layer = True
|
||||
# Filter to only layers that have KV cache specs.
|
||||
self._draft_attn_layer_names = {
|
||||
name
|
||||
for name in (set(all_attn_layers.keys()) - target_attn_layer_names)
|
||||
for name in draft_attn_layer_names
|
||||
if all_attn_layers[name].get_kv_cache_spec(self.vllm_config) is not None
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import numpy as np
|
||||
@@ -148,6 +148,8 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
|
||||
if spec := attn_module.get_kv_cache_spec(vllm_config):
|
||||
if isinstance(spec, AttentionSpec):
|
||||
spec = customize_attention_spec(attn_module.get_attn_backend(), spec)
|
||||
if attn_module.is_draft_layer:
|
||||
spec = replace(spec, is_draft=True)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
if vllm_config.attention_config.hisparse_config is not None:
|
||||
kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers)
|
||||
|
||||
@@ -7140,6 +7140,8 @@ class GPUModelRunner(
|
||||
spec = customize_attention_spec(
|
||||
attn_module.get_attn_backend(), spec
|
||||
)
|
||||
if attn_module.is_draft_layer:
|
||||
spec = replace(spec, is_draft=True)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
|
||||
return kv_cache_spec
|
||||
|
||||
Reference in New Issue
Block a user