[Bugfix] Report FULL_ATTENTION for uniform-base UniformTypeKVCacheSpecs groups instead of UNKNOWN (#51218)
Signed-off-by: Yifan Jiang <19356972+yifjiang@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Yifan Qiao
parent
caf9e8f7d5
commit
5936bac7d7
@@ -934,6 +934,15 @@ def get_kv_cache_spec_kind(kv_cache_spec: KVCacheSpec) -> KVCacheSpecKind:
|
||||
}
|
||||
if len(inner_kinds) == 1:
|
||||
return next(iter(inner_kinds))
|
||||
# A group is only formed when all members share one registered
|
||||
# uniform_type_base_spec, so UNKNOWN would discard what the merge
|
||||
# already established.
|
||||
base_specs = {
|
||||
KVCacheSpecRegistry.get_uniform_type_base_spec(spec)
|
||||
for spec in kv_cache_spec.kv_cache_specs.values()
|
||||
}
|
||||
if len(base_specs) == 1 and next(iter(base_specs)) is FullAttentionSpec:
|
||||
return KVCacheSpecKind.FULL_ATTENTION
|
||||
return KVCacheSpecKind.UNKNOWN
|
||||
# Keep subclass checks before base classes so specialized specs keep their
|
||||
# more precise kind.
|
||||
|
||||
Reference in New Issue
Block a user