diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 23f22089f7..079694d505 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -2069,6 +2069,33 @@ def test_generate_scheduler_kv_cache_config(): ) +def test_mixed_precision_kv_cache_with_uniform_type_specs(): + fp8_spec = new_kv_cache_spec(dtype=torch.float8_e4m3fn) + bf16_spec = new_kv_cache_spec(dtype=torch.bfloat16) + worker_config = KVCacheConfig( + num_blocks=10, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["fp8_layer"], + UniformTypeKVCacheSpecs( + block_size=16, kv_cache_specs={"fp8_layer": fp8_spec} + ), + ), + KVCacheGroupSpec( + ["bf16_layer"], + UniformTypeKVCacheSpecs( + block_size=16, kv_cache_specs={"bf16_layer": bf16_spec} + ), + ), + ], + ) + scheduler_config = generate_scheduler_kv_cache_config([worker_config]) + + assert worker_config.needs_kv_cache_zeroing + assert scheduler_config.needs_kv_cache_zeroing + + def new_mla_spec(cache_dtype_str=None, block_size=16): # head_size = kv_lora_rank(512) + qk_rope_head_dim(64) = 576 return MLAAttentionSpec( diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index 7b694db051..164350c7fb 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -975,11 +975,19 @@ class KVCacheConfig: @property def has_mixed_precision_kv_cache(self) -> bool: """Whether attention groups store their KV cache at more than one precision.""" - kv_cache_precisions = { - (group.kv_cache_spec.dtype, group.kv_cache_spec.kv_quant_mode) - for group in self.kv_cache_groups - if isinstance(group.kv_cache_spec, AttentionSpec) - } + kv_cache_precisions: set[tuple[torch.dtype, KVQuantMode]] = set() + for group in self.kv_cache_groups: + group_spec = group.kv_cache_spec + group_specs = ( + list(group_spec.kv_cache_specs.values()) + if isinstance(group_spec, UniformTypeKVCacheSpecs) + else [group_spec] + ) + kv_cache_precisions.update( + (spec.dtype, spec.kv_quant_mode) + for spec in group_specs + if isinstance(spec, AttentionSpec) + ) return len(kv_cache_precisions) > 1 @property