[Bugfix] Detect mixed precision in packed KV cache specs (#49623)

Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Michael Goin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Michael Goin
2026-07-23 20:07:06 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent d65acd83d8
commit 275556c35c
2 changed files with 40 additions and 5 deletions
+27
View File
@@ -2069,6 +2069,33 @@ def test_generate_scheduler_kv_cache_config():
)
def test_mixed_precision_kv_cache_with_uniform_type_specs():
fp8_spec = new_kv_cache_spec(dtype=torch.float8_e4m3fn)
bf16_spec = new_kv_cache_spec(dtype=torch.bfloat16)
worker_config = KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["fp8_layer"],
UniformTypeKVCacheSpecs(
block_size=16, kv_cache_specs={"fp8_layer": fp8_spec}
),
),
KVCacheGroupSpec(
["bf16_layer"],
UniformTypeKVCacheSpecs(
block_size=16, kv_cache_specs={"bf16_layer": bf16_spec}
),
),
],
)
scheduler_config = generate_scheduler_kv_cache_config([worker_config])
assert worker_config.needs_kv_cache_zeroing
assert scheduler_config.needs_kv_cache_zeroing
def new_mla_spec(cache_dtype_str=None, block_size=16):
# head_size = kv_lora_rank(512) + qk_rope_head_dim(64) = 576
return MLAAttentionSpec(