[Core][Spec Decode] Replace EAGLE group heuristics with explicit draft-layer tags (#61455)
Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Yifan Qiao
Claude Opus 5.5
parent
08a65e287e
commit
c3de0b37e2
@@ -175,6 +175,10 @@ class KVCacheSpec:
|
||||
``MultipleOf(n)``: whole kernel blocks, when blocks are split into kernel
|
||||
blocks of up to ``n`` tokens."""
|
||||
|
||||
is_draft: bool = field(default=False, kw_only=True, compare=False)
|
||||
"""Whether a draft model owns this layer; excluded from equality so it never
|
||||
affects grouping."""
|
||||
|
||||
def __post_init__(self):
|
||||
alignment = self.block_stride_alignment
|
||||
if isinstance(alignment, int) and alignment <= 0:
|
||||
@@ -660,6 +664,8 @@ class FullAttentionSpec(AttentionSpec):
|
||||
)
|
||||
for spec in specs:
|
||||
for f in fields(AttentionSpec):
|
||||
if not f.compare:
|
||||
continue
|
||||
assert getattr(spec, f.name) == getattr(merged_spec, f.name), (
|
||||
"All attention layers in the same KV cache group must have "
|
||||
"the same attention spec."
|
||||
@@ -747,6 +753,8 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
)
|
||||
for spec in specs:
|
||||
for f in fields(AttentionSpec):
|
||||
if not f.compare:
|
||||
continue
|
||||
assert getattr(spec, f.name) == getattr(merged_spec, f.name), (
|
||||
"All attention layers in the same KV cache group must have "
|
||||
"the same attention spec."
|
||||
@@ -1208,6 +1216,8 @@ class SinkFullAttentionSpec(FullAttentionSpec):
|
||||
)
|
||||
for spec in specs:
|
||||
for f in fields(AttentionSpec):
|
||||
if not f.compare:
|
||||
continue
|
||||
assert getattr(spec, f.name) == getattr(merged_spec, f.name), (
|
||||
"All attention layers in the same KV cache group must have "
|
||||
"the same attention spec."
|
||||
@@ -1445,7 +1455,7 @@ class KVCacheGroupSpec:
|
||||
layer_names: list[str]
|
||||
# The KV cache spec of this manager layer
|
||||
kv_cache_spec: KVCacheSpec
|
||||
# Whether this group contains EAGLE/MTP draft attention layers.
|
||||
# Whether this group holds draft layers and takes the EAGLE last-block drop.
|
||||
is_eagle_group: bool = False
|
||||
# Host groups use the dedicated HiSparse pool; others share the device pool.
|
||||
host_resident: bool = False
|
||||
|
||||
Reference in New Issue
Block a user