[Bugfix][KV Cache] Don't route uniform-page-size MLA+SWA models into DeepseekV4 packing (#48256)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Nicolò Lucchesi
2026-07-13 08:16:24 +00:00
committed by GitHub
co-authored by Claude
parent bea70c7cfc
commit 56a357ed33
2 changed files with 48 additions and 0 deletions
+43
View File
@@ -32,6 +32,7 @@ from vllm.v1.core.kv_cache_utils import (
get_kv_cache_configs,
get_max_concurrency_for_kv_cache_config,
get_request_block_hasher,
group_and_unify_kv_cache_specs,
hash_block_tokens,
init_none_hash,
is_kv_cache_spec_uniform,
@@ -1951,6 +1952,48 @@ def new_mla_spec(cache_dtype_str=None):
)
def new_swa_mla_spec(head_size=576, sliding_window=128):
return SlidingWindowMLASpec(
block_size=16,
num_kv_heads=1,
head_size=head_size,
dtype=torch.float32,
sliding_window=sliding_window,
)
def test_group_and_unify_kv_cache_specs_no_swa_mla_returns_none():
# Without any SlidingWindowMLASpec the function does not apply.
specs = {"mla.0": new_mla_spec(), "mla.1": new_mla_spec()}
assert group_and_unify_kv_cache_specs(specs) is None
def test_group_and_unify_kv_cache_specs_uniform_page_size_returns_none():
# A non-DeepseekV4 model that mixes full MLA and sliding-window MLA layers
# with a uniform page size must not fall into the DeepseekV4 tuple-packing
# path; it should defer to the generic uniform-page-size grouping instead.
mla_spec = new_mla_spec()
swa_spec = new_swa_mla_spec()
assert mla_spec.page_size_bytes == swa_spec.page_size_bytes
specs = {"mla.0": mla_spec, "mla.1": new_mla_spec(), "swa.0": swa_spec}
assert group_and_unify_kv_cache_specs(specs) is None
def test_group_and_unify_kv_cache_specs_mixed_page_size_groups():
# DeepseekV4-style: differing page sizes across MLA and sliding-window MLA
# layers do require tuple packing, so grouping must still be produced.
mla_spec = new_mla_spec()
swa_spec = new_swa_mla_spec(head_size=1024)
assert mla_spec.page_size_bytes != swa_spec.page_size_bytes
specs = {"mla.0": mla_spec, "mla.1": new_mla_spec(), "swa.0": swa_spec}
grouped = group_and_unify_kv_cache_specs(specs)
assert grouped is not None
# One MLA group plus one sliding-window MLA group.
assert len(grouped) == 2
layer_names = {name for g in grouped for name in g.kv_cache_specs}
assert layer_names == {"mla.0", "mla.1", "swa.0"}
def test_get_kv_cache_spec_kind_prefers_specific_attention_subclasses():
assert get_kv_cache_spec_kind(new_mla_spec()) == KVCacheSpecKind.MLA_ATTENTION
+5
View File
@@ -1535,6 +1535,11 @@ def group_and_unify_kv_cache_specs(
):
return None
# SlidingWindowMLASpec models with uniform page sizes don't need tuple packing.
page_sizes = {spec.page_size_bytes for spec in kv_cache_spec.values()}
if len(page_sizes) <= 1:
return None
mla_specs: dict[str, KVCacheSpec] = {}
grouped_swa_mla_specs: dict[tuple[int, int], dict[str, KVCacheSpec]] = defaultdict(
dict