[Bugfix] Detect mixed precision in packed KV cache specs (#49623)

Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Michael Goin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Michael Goin
2026-07-23 20:07:06 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent d65acd83d8
commit 275556c35c
2 changed files with 40 additions and 5 deletions
+13 -5
View File
@@ -975,11 +975,19 @@ class KVCacheConfig:
@property
def has_mixed_precision_kv_cache(self) -> bool:
"""Whether attention groups store their KV cache at more than one precision."""
kv_cache_precisions = {
(group.kv_cache_spec.dtype, group.kv_cache_spec.kv_quant_mode)
for group in self.kv_cache_groups
if isinstance(group.kv_cache_spec, AttentionSpec)
}
kv_cache_precisions: set[tuple[torch.dtype, KVQuantMode]] = set()
for group in self.kv_cache_groups:
group_spec = group.kv_cache_spec
group_specs = (
list(group_spec.kv_cache_specs.values())
if isinstance(group_spec, UniformTypeKVCacheSpecs)
else [group_spec]
)
kv_cache_precisions.update(
(spec.dtype, spec.kv_quant_mode)
for spec in group_specs
if isinstance(spec, AttentionSpec)
)
return len(kv_cache_precisions) > 1
@property