[Bugfix][R3] Unwrap UniformTypeKVCacheSpecs when selecting the routed-experts KV group (#53240)

Signed-off-by: Hollow Man <hollowman@opensuse.org>
This commit is contained in:
ℍ𝕠𝕝𝕝𝕠𝕨 𝕄𝕒𝕟
2026-08-21 19:13:53 -07:00
committed by GitHub
parent 7f4a1b7e24
commit b2db227a7c
5 changed files with 119 additions and 23 deletions
+30 -7
View File
@@ -1020,6 +1020,35 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
)
def iter_layer_specs(kv_cache_spec: KVCacheSpec) -> Collection[KVCacheSpec]:
"""The per-layer specs a KV cache group spec covers.
``UniformTypeKVCacheSpecs`` groups keep one spec per layer; every other
spec describes its group on its own. Returns the layer specs either way so
callers do not have to special-case the wrapper.
"""
if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs):
return kv_cache_spec.kv_cache_specs.values()
return (kv_cache_spec,)
def is_full_attention_spec(kv_cache_spec: KVCacheSpec) -> bool:
"""Whether a KV cache group spec is (or wraps) full attention.
``UniformTypeKVCacheSpecs`` is not itself a ``FullAttentionSpec``, so a bare
isinstance check misses groups that carry the wrapper -- DeepSeek-V4's MLA
layers, or any model taking the ``UniformTypeKVCacheSpecs.from_specs`` path.
Every layer must be full attention: a group holding a recycling
(sliding-window) layer has no stable slot layout, so callers that key data
by slot cannot use it.
"""
layer_specs = iter_layer_specs(kv_cache_spec)
return len(layer_specs) > 0 and all(
isinstance(spec, FullAttentionSpec) for spec in layer_specs
)
def get_kv_cache_spec_kind(kv_cache_spec: KVCacheSpec) -> KVCacheSpecKind:
if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs):
inner_kinds = {
@@ -1144,15 +1173,9 @@ class KVCacheConfig:
"""Whether attention groups store their KV cache at more than one precision."""
kv_cache_precisions: set[tuple[torch.dtype, KVQuantMode]] = set()
for group in self.kv_cache_groups:
group_spec = group.kv_cache_spec
group_specs = (
list(group_spec.kv_cache_specs.values())
if isinstance(group_spec, UniformTypeKVCacheSpecs)
else [group_spec]
)
kv_cache_precisions.update(
(spec.dtype, spec.kv_quant_mode)
for spec in group_specs
for spec in iter_layer_specs(group.kv_cache_spec)
if isinstance(spec, AttentionSpec)
)
return len(kv_cache_precisions) > 1