[Bugfix] Detect mixed precision in packed KV cache specs (#49623)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
d65acd83d8
commit
275556c35c
@@ -2069,6 +2069,33 @@ def test_generate_scheduler_kv_cache_config():
|
||||
)
|
||||
|
||||
|
||||
def test_mixed_precision_kv_cache_with_uniform_type_specs():
|
||||
fp8_spec = new_kv_cache_spec(dtype=torch.float8_e4m3fn)
|
||||
bf16_spec = new_kv_cache_spec(dtype=torch.bfloat16)
|
||||
worker_config = KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["fp8_layer"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=16, kv_cache_specs={"fp8_layer": fp8_spec}
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["bf16_layer"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=16, kv_cache_specs={"bf16_layer": bf16_spec}
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
scheduler_config = generate_scheduler_kv_cache_config([worker_config])
|
||||
|
||||
assert worker_config.needs_kv_cache_zeroing
|
||||
assert scheduler_config.needs_kv_cache_zeroing
|
||||
|
||||
|
||||
def new_mla_spec(cache_dtype_str=None, block_size=16):
|
||||
# head_size = kv_lora_rank(512) + qk_rope_head_dim(64) = 576
|
||||
return MLAAttentionSpec(
|
||||
|
||||
@@ -975,11 +975,19 @@ class KVCacheConfig:
|
||||
@property
|
||||
def has_mixed_precision_kv_cache(self) -> bool:
|
||||
"""Whether attention groups store their KV cache at more than one precision."""
|
||||
kv_cache_precisions = {
|
||||
(group.kv_cache_spec.dtype, group.kv_cache_spec.kv_quant_mode)
|
||||
for group in self.kv_cache_groups
|
||||
if isinstance(group.kv_cache_spec, AttentionSpec)
|
||||
}
|
||||
kv_cache_precisions: set[tuple[torch.dtype, KVQuantMode]] = set()
|
||||
for group in self.kv_cache_groups:
|
||||
group_spec = group.kv_cache_spec
|
||||
group_specs = (
|
||||
list(group_spec.kv_cache_specs.values())
|
||||
if isinstance(group_spec, UniformTypeKVCacheSpecs)
|
||||
else [group_spec]
|
||||
)
|
||||
kv_cache_precisions.update(
|
||||
(spec.dtype, spec.kv_quant_mode)
|
||||
for spec in group_specs
|
||||
if isinstance(spec, AttentionSpec)
|
||||
)
|
||||
return len(kv_cache_precisions) > 1
|
||||
|
||||
@property
|
||||
|
||||
Reference in New Issue
Block a user